You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by tl...@apache.org on 2021/07/13 14:55:01 UTC
[ignite] branch sql-calcite updated: IGNITE-13159 Calcite
integration. Decorrelator support SOME and ALL operator (#9251)
This is an automated email from the ASF dual-hosted git repository.
tledkov pushed a commit to branch sql-calcite
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/sql-calcite by this push:
new 53da524 IGNITE-13159 Calcite integration. Decorrelator support SOME and ALL operator (#9251)
53da524 is described below
commit 53da5242370e3eeeb2c75ed7ce991bf06db6423b
Author: Evgeniy Stanilovskiy <st...@gmail.com>
AuthorDate: Tue Jul 13 17:54:31 2021 +0300
IGNITE-13159 Calcite integration. Decorrelator support SOME and ALL operator (#9251)
---
.../query/calcite/CalciteQueryProcessor.java | 1 +
.../calcite/exec/exp/ExpressionFactoryImpl.java | 9 ++++++
.../query/calcite/prepare/IgnitePlanner.java | 5 +--
.../calcite/rule/CorrelatedNestedLoopJoinRule.java | 28 +++++++++++-----
.../calcite/rule/LogicalScanConverterRule.java | 8 ++++-
.../rule/TableFunctionScanConverterRule.java | 22 +++++++++++++
.../query/calcite/CalciteQueryProcessorTest.java | 37 +++++++++++++++++++++-
.../processors/query/calcite/FunctionsTest.java | 5 ++-
...{test_neumann.test_ignore => test_neumann.test} | 1 -
9 files changed, 102 insertions(+), 14 deletions(-)
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java
index dd4914c..69bbbd4 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java
@@ -79,6 +79,7 @@ public class CalciteQueryProcessor extends GridProcessorAdapter implements Query
// TODO: remove this after IGNITE-14277
.withInSubQueryThreshold(Integer.MAX_VALUE)
.withDecorrelationEnabled(true)
+ .withExpand(false)
.withHintStrategyTable(
HintStrategyTable.builder()
.hintStrategy("DISABLE_RULE", (hint, rel) -> true)
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
index 472bd6c..0510a31 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
@@ -48,6 +48,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexProgram;
@@ -326,6 +327,14 @@ public class ExpressionFactoryImpl<Row> implements ExpressionFactory<Row> {
b.append(';');
b.append(nodes.get(i));
+
+ new RexShuttle() {
+ @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+ b.append(", fldIdx=").append(fieldAccess.getField().getIndex());
+
+ return super.visitFieldAccess(fieldAccess);
+ }
+ }.apply(nodes.get(i));
}
b.append(']');
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java
index 78c8929..ed79642 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java
@@ -216,10 +216,10 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander {
@Override public RelRoot rel(SqlNode sql) {
SqlToRelConverter sqlToRelConverter = sqlToRelConverter(validator(), catalogReader, sqlToRelConverterCfg);
RelRoot root = sqlToRelConverter.convertQuery(sql, false, true);
+
root = root.withRel(sqlToRelConverter.decorrelate(sql, root.rel));
- root = trimUnusedFields(root);
- return root;
+ return trimUnusedFields(root);
}
/** {@inheritDoc} */
@@ -338,6 +338,7 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander {
// near the leaves created by trim migrate past joins and seem to
// prevent join-reordering.
final SqlToRelConverter.Config config = sqlToRelConverterCfg
+ .withExpand(false)
.withTrimUnusedFields(RelOptUtil.countJoins(root.rel) < 2);
SqlToRelConverter converter = sqlToRelConverter(validator(), catalogReader, config);
boolean ordered = !root.collation.getFieldCollations().isEmpty();
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/CorrelatedNestedLoopJoinRule.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/CorrelatedNestedLoopJoinRule.java
index a1e2c66..87307da 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/CorrelatedNestedLoopJoinRule.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/CorrelatedNestedLoopJoinRule.java
@@ -26,6 +26,7 @@ import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
@@ -46,13 +47,14 @@ import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention;
import org.apache.ignite.internal.processors.query.calcite.rel.IgniteCorrelatedNestedLoopJoin;
import org.apache.ignite.internal.processors.query.calcite.trait.CorrelationTrait;
import org.apache.ignite.internal.processors.query.calcite.trait.RewindabilityTrait;
+import org.apache.ignite.internal.util.typedef.F;
/** */
public class CorrelatedNestedLoopJoinRule extends ConverterRule {
/** */
public static final RelOptRule INSTANCE = Config.DEFAULT.toRule();
- /** */
+ /** TODO: https://issues.apache.org/jira/browse/IGNITE-14757 */
public static final RelOptRule INSTANCE_BATCHED = Config.DEFAULT.withBatchSize(100).toRule();
/** */
@@ -84,10 +86,21 @@ public class CorrelatedNestedLoopJoinRule extends ConverterRule {
final Set<CorrelationId> correlationIds = new HashSet<>();
final ArrayList<RexNode> corrVar = new ArrayList<>();
- for (int i = 0; i < batchSize; i++) {
- CorrelationId correlationId = cluster.createCorrel();
- correlationIds.add(correlationId);
- corrVar.add(rexBuilder.makeCorrel(rel.getLeft().getRowType(), correlationId));
+ final Set<CorrelationId> corrIds = RelOptUtil.getVariablesUsed(rel.getRight());
+
+ // TODO: remove all near 'if' scope after https://issues.apache.org/jira/browse/CALCITE-4673 will be merged.
+ if (corrIds.size() > 1) {
+ CorrelationId corr0 = F.first(corrIds);
+ corrVar.add(rexBuilder.makeCorrel(rel.getLeft().getRowType(), corr0));
+ correlationIds.add(corr0);
+ }
+
+ if (corrVar.isEmpty()) {
+ for (int i = 0; i < batchSize; i++) {
+ CorrelationId correlationId = cluster.createCorrel();
+ correlationIds.add(correlationId);
+ corrVar.add(rexBuilder.makeCorrel(rel.getLeft().getRowType(), correlationId));
+ }
}
// Generate first condition
@@ -124,14 +137,13 @@ public class CorrelatedNestedLoopJoinRule extends ConverterRule {
RelNode right = relBuilder.build();
- CorrelationTrait corrTrait = CorrelationTrait.correlations(correlationIds);
- right = right.copy(filterInTraits.replace(corrTrait), right.getInputs());
-
JoinRelType joinType = rel.getJoinType();
RelTraitSet outTraits = cluster.traitSetOf(IgniteConvention.INSTANCE);
RelTraitSet leftInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE);
+ CorrelationTrait corrTrait = CorrelationTrait.correlations(correlationIds);
+
RelTraitSet rightInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE)
.replace(RewindabilityTrait.REWINDABLE)
.replace(corrTrait);
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/LogicalScanConverterRule.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/LogicalScanConverterRule.java
index 7c3226d..2fdbca4 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/LogicalScanConverterRule.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/LogicalScanConverterRule.java
@@ -43,9 +43,15 @@ public abstract class LogicalScanConverterRule<T extends ProjectableFilterableTa
RelMetadataQuery mq,
IgniteLogicalIndexScan rel
) {
+ RelTraitSet traits = rel.getTraitSet().replace(IgniteConvention.INSTANCE);
+
+ Set<CorrelationId> corrIds = RexUtils.extractCorrelationIds(rel.condition());
+ if (!corrIds.isEmpty())
+ traits = traits.replace(CorrelationTrait.correlations(corrIds));
+
return new IgniteIndexScan(
rel.getCluster(),
- rel.getTraitSet().replace(IgniteConvention.INSTANCE),
+ traits,
rel.getTable(),
rel.indexName(),
rel.projects(),
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableFunctionScanConverterRule.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableFunctionScanConverterRule.java
index 4a21c94..2235b0b 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableFunctionScanConverterRule.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableFunctionScanConverterRule.java
@@ -16,14 +16,20 @@
*/
package org.apache.ignite.internal.processors.query.calcite.rule;
+import java.util.List;
import java.util.Set;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.PhysicalNode;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.sql2rel.DeduplicateCorrelateVariables;
+import org.apache.calcite.util.Util;
import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention;
import org.apache.ignite.internal.processors.query.calcite.rel.IgniteTableFunctionScan;
import org.apache.ignite.internal.processors.query.calcite.trait.CorrelationTrait;
@@ -31,6 +37,7 @@ import org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistribut
import org.apache.ignite.internal.processors.query.calcite.trait.RewindabilityTrait;
import org.apache.ignite.internal.processors.query.calcite.util.RexUtils;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
/**
* Rule to convert a {@link LogicalTableFunctionScan} to an {@link IgniteTableFunctionScan}.
@@ -53,8 +60,23 @@ public class TableFunctionScanConverterRule extends AbstractIgniteConverterRule<
.replace(RewindabilityTrait.REWINDABLE)
.replace(IgniteDistributions.broadcast());
+ RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
+
Set<CorrelationId> corrIds = RexUtils.extractCorrelationIds(rel.getCall());
+ // TODO: remove all near 'if' scope after https://issues.apache.org/jira/browse/CALCITE-4673 will be merged.
+ if (corrIds.size() > 1) {
+ final List<CorrelationId> correlNames = U.arrayList(corrIds);
+
+ RelNode rel0 = DeduplicateCorrelateVariables.go(rexBuilder, correlNames.get(0), Util.skip(correlNames), rel);
+
+ corrIds = RelOptUtil.getVariablesUsed(rel0);
+
+ assert corrIds.size() == 1 : "Multiple correlates are applied: " + corrIds;
+
+ rel = (LogicalTableFunctionScan)rel0;
+ }
+
if (!corrIds.isEmpty())
traitSet = traitSet.replace(CorrelationTrait.correlations(corrIds));
diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessorTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessorTest.java
index 9839df8..5b95d37 100644
--- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessorTest.java
+++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessorTest.java
@@ -792,7 +792,7 @@ public class CalciteQueryProcessorTest extends GridCommonAbstractTest {
/** */
@Test
- public void query2() throws Exception {
+ public void query2() {
IgniteCache<Integer, Developer> developer = grid(1).getOrCreateCache(new CacheConfiguration<Integer, Developer>()
.setName("developer")
.setSqlSchema("PUBLIC")
@@ -1044,6 +1044,41 @@ public class CalciteQueryProcessorTest extends GridCommonAbstractTest {
assertThat(qCur.fieldsMeta().get(3).fieldTypeName(), equalTo(String.class.getName()));
}
+ /** Quantified predicates test. */
+ @Test
+ public void quantifiedCompTest() throws InterruptedException {
+ populateTables();
+
+ assertQuery(client, "select salary from account where salary > SOME (10, 11) ORDER BY salary")
+ .returns(11d)
+ .returns(12d)
+ .returns(13d)
+ .returns(13d)
+ .returns(13d)
+ .check();
+
+ assertQuery(client, "select salary from account where salary < SOME (12, 12) ORDER BY salary")
+ .returns(10d)
+ .returns(11d)
+ .check();
+
+ assertQuery(client, "select salary from account where salary < ANY (11, 12) ORDER BY salary")
+ .returns(10d)
+ .returns(11d)
+ .check();
+
+ assertQuery(client, "select salary from account where salary > ANY (12, 13) ORDER BY salary")
+ .returns(13d)
+ .returns(13d)
+ .returns(13d)
+ .check();
+
+ assertQuery(client, "select salary from account where salary <> ALL (12, 13) ORDER BY salary")
+ .returns(10d)
+ .returns(11d)
+ .check();
+ }
+
/** */
private static List<String> deriveColumnNamesFromCursor(FieldsQueryCursor cursor) {
List<String> names = new ArrayList<>(cursor.getColumnsCount());
diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/FunctionsTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/FunctionsTest.java
index e3c851a..7b2468c 100644
--- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/FunctionsTest.java
+++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/FunctionsTest.java
@@ -133,7 +133,10 @@ public class FunctionsTest extends GridCommonAbstractTest {
"Increment can't be 0");
}
- /** */
+ /**
+ * Important! Don`t change query call sequence in this test. This also tests correctness of
+ * {@link org.apache.ignite.internal.processors.query.calcite.exec.exp.ExpressionFactoryImpl#SCALAR_CACHE} usage.
+ */
@Test
public void testRangeWithCache() throws Exception {
IgniteCache<Integer, Integer> cache = grid(0).getOrCreateCache(
diff --git a/modules/calcite/src/test/sql/subquery/test_neumann.test_ignore b/modules/calcite/src/test/sql/subquery/test_neumann.test
similarity index 96%
rename from modules/calcite/src/test/sql/subquery/test_neumann.test_ignore
rename to modules/calcite/src/test/sql/subquery/test_neumann.test
index 00744d8..b925f63 100644
--- a/modules/calcite/src/test/sql/subquery/test_neumann.test_ignore
+++ b/modules/calcite/src/test/sql/subquery/test_neumann.test
@@ -1,7 +1,6 @@
# name: test/sql/subquery/test_neumann.test
# description: Test subqueries from the paper 'Unnesting Arbitrary Subqueries'
# group: [subquery]
-# Ignored: https://issues.apache.org/jira/browse/IGNITE-14537
statement ok
PRAGMA enable_verification