You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by ch...@apache.org on 2020/06/18 08:19:00 UTC
[calcite] branch master updated: [CALCITE-4020] Support Calc
operator in RelFieldTrimmer (Xu Zhaohui)
This is an automated email from the ASF dual-hosted git repository.
chunwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new d7eb31b [CALCITE-4020] Support Calc operator in RelFieldTrimmer (Xu Zhaohui)
d7eb31b is described below
commit d7eb31b91560e34269a097dc720cbe926983bb0c
Author: xzh <95...@qq.com>
AuthorDate: Fri May 22 08:26:43 2020 +0800
[CALCITE-4020] Support Calc operator in RelFieldTrimmer (Xu Zhaohui)
---
.../apache/calcite/sql2rel/RelFieldTrimmer.java | 89 +++++++++++++
.../calcite/sql2rel/RelFieldTrimmerTest.java | 147 +++++++++++++++++++++
2 files changed, 236 insertions(+)
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
index 0a4ec01..34a201e 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
@@ -26,6 +26,7 @@ import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Calc;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.Exchange;
import org.apache.calcite.rel.core.Filter;
@@ -52,6 +53,7 @@ import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPermuteInputsShuttle;
+import org.apache.calcite.rex.RexProgram;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitor;
import org.apache.calcite.sql.SqlExplainFormat;
@@ -71,10 +73,12 @@ import org.apache.calcite.util.mapping.Mappings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -352,6 +356,91 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
/**
* Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
+ * {@link org.apache.calcite.rel.logical.LogicalCalc}.
+ */
+ public TrimResult trimFields(
+ Calc calc,
+ ImmutableBitSet fieldsUsed,
+ Set<RelDataTypeField> extraFields) {
+ final RexProgram rexProgram = calc.getProgram();
+ final List<RexNode> projs = Lists.transform(rexProgram.getProjectList(),
+ rexProgram::expandLocalRef);
+
+ final RelDataType rowType = calc.getRowType();
+ final int fieldCount = rowType.getFieldCount();
+ final RelNode input = calc.getInput();
+
+ final Set<RelDataTypeField> inputExtraFields =
+ new HashSet<>(extraFields);
+ RelOptUtil.InputFinder inputFinder =
+ new RelOptUtil.InputFinder(inputExtraFields);
+ for (Ord<RexNode> ord : Ord.zip(projs)) {
+ if (fieldsUsed.get(ord.i)) {
+ ord.e.accept(inputFinder);
+ }
+ }
+ ImmutableBitSet inputFieldsUsed = inputFinder.build();
+
+ // Create input with trimmed columns.
+ TrimResult trimResult =
+ trimChild(calc, input, inputFieldsUsed, inputExtraFields);
+ RelNode newInput = trimResult.left;
+ final Mapping inputMapping = trimResult.right;
+
+ // If the input is unchanged, and we need to project all columns,
+ // there's nothing we can do.
+ if (newInput == input
+ && fieldsUsed.cardinality() == fieldCount) {
+ return result(calc, Mappings.createIdentity(fieldCount));
+ }
+
+ // Some parts of the system can't handle rows with zero fields, so
+ // pretend that one field is used.
+ if (fieldsUsed.cardinality() == 0) {
+ return dummyProject(fieldCount, newInput);
+ }
+
+ // Build new project expressions, and populate the mapping.
+ final List<RexNode> newProjects = new ArrayList<>();
+ final RexVisitor<RexNode> shuttle =
+ new RexPermuteInputsShuttle(
+ inputMapping, newInput);
+ final Mapping mapping =
+ Mappings.create(
+ MappingType.INVERSE_SURJECTION,
+ fieldCount,
+ fieldsUsed.cardinality());
+ for (Ord<RexNode> ord : Ord.zip(projs)) {
+ if (fieldsUsed.get(ord.i)) {
+ mapping.set(ord.i, newProjects.size());
+ RexNode newProjectExpr = ord.e.accept(shuttle);
+ newProjects.add(newProjectExpr);
+ }
+ }
+
+ final RelDataType newRowType =
+ RelOptUtil.permute(calc.getCluster().getTypeFactory(), rowType,
+ mapping);
+
+ final RelNode newInputRelNode = relBuilder.push(newInput).build();
+ RexNode newConditionExpr = null;
+ if (rexProgram.getCondition() != null) {
+ final List<RexNode> filter = Lists.transform(
+ ImmutableList.of(
+ rexProgram.getCondition()), rexProgram::expandLocalRef);
+ assert filter.size() == 1;
+ final RexNode conditionExpr = filter.get(0);
+ newConditionExpr = conditionExpr.accept(shuttle);
+ }
+ final RexProgram newRexProgram = RexProgram.create(newInputRelNode.getRowType(),
+ newProjects, newConditionExpr, newRowType.getFieldNames(),
+ newInputRelNode.getCluster().getRexBuilder());
+ final Calc newCalc = calc.copy(calc.getTraitSet(), newInputRelNode, newRexProgram);
+ return result(newCalc, mapping);
+ }
+
+ /**
+ * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
* {@link org.apache.calcite.rel.logical.LogicalProject}.
*/
public TrimResult trimFields(
diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
index 7c57f07..bee8796 100644
--- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
+++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
@@ -17,17 +17,25 @@
package org.apache.calcite.sql2rel;
import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelDistributions;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Calc;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.hint.HintPredicates;
import org.apache.calcite.rel.hint.HintStrategyTable;
import org.apache.calcite.rel.hint.RelHint;
+import org.apache.calcite.rel.rules.CalcMergeRule;
+import org.apache.calcite.rel.rules.FilterToCalcRule;
+import org.apache.calcite.rel.rules.ProjectToCalcRule;
import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.test.CalciteAssert;
import org.apache.calcite.tools.Frameworks;
@@ -306,4 +314,143 @@ class RelFieldTrimmerTest {
assertTrue(project.getHints().contains(projectHint));
}
+ @Test void testCalcFieldTrimmer0() {
+ final RelBuilder builder = RelBuilder.create(config().build());
+ final RelNode root =
+ builder.scan("EMP")
+ .project(builder.field("EMPNO"), builder.field("ENAME"), builder.field("DEPTNO"))
+ .exchange(RelDistributions.SINGLETON)
+ .project(builder.field("EMPNO"), builder.field("ENAME"))
+ .build();
+
+ final HepProgram hepProgram = new HepProgramBuilder().
+ addRuleInstance(ProjectToCalcRule.INSTANCE).build();
+
+ final HepPlanner hepPlanner = new HepPlanner(hepProgram);
+ hepPlanner.setRoot(root);
+ final RelNode relNode = hepPlanner.findBestExp();
+ final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder);
+ final RelNode trimmed = fieldTrimmer.trim(relNode);
+
+ final String expected = ""
+ + "LogicalCalc(expr#0..1=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " LogicalExchange(distribution=[single])\n"
+ + " LogicalCalc(expr#0..1=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " LogicalProject(EMPNO=[$0], ENAME=[$1])\n"
+ + " LogicalTableScan(table=[[scott, EMP]])\n";
+ assertThat(trimmed, hasTree(expected));
+ }
+
+ @Test void testCalcFieldTrimmer1() {
+ final RelBuilder builder = RelBuilder.create(config().build());
+ final RelNode root =
+ builder.scan("EMP")
+ .project(builder.field("EMPNO"), builder.field("ENAME"), builder.field("DEPTNO"))
+ .exchange(RelDistributions.SINGLETON)
+ .filter(
+ builder.call(SqlStdOperatorTable.GREATER_THAN,
+ builder.field("EMPNO"), builder.literal(100)))
+ .build();
+
+ final HepProgram hepProgram = new HepProgramBuilder()
+ .addRuleInstance(ProjectToCalcRule.INSTANCE)
+ .addRuleInstance(FilterToCalcRule.INSTANCE)
+ .build();
+
+ final HepPlanner hepPlanner = new HepPlanner(hepProgram);
+ hepPlanner.setRoot(root);
+ final RelNode relNode = hepPlanner.findBestExp();
+ final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder);
+ final RelNode trimmed = fieldTrimmer.trim(relNode);
+
+ final String expected = ""
+ + "LogicalCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[>($t0, $t3)], proj#0."
+ + ".2=[{exprs}], $condition=[$t4])\n"
+ + " LogicalExchange(distribution=[single])\n"
+ + " LogicalCalc(expr#0..2=[{inputs}], proj#0..2=[{exprs}])\n"
+ + " LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7])\n"
+ + " LogicalTableScan(table=[[scott, EMP]])\n";
+ assertThat(trimmed, hasTree(expected));
+ }
+
+ @Test void testCalcFieldTrimmer2() {
+ final RelBuilder builder = RelBuilder.create(config().build());
+ final RelNode root =
+ builder.scan("EMP")
+ .project(builder.field("EMPNO"), builder.field("ENAME"), builder.field("DEPTNO"))
+ .exchange(RelDistributions.SINGLETON)
+ .filter(
+ builder.call(SqlStdOperatorTable.GREATER_THAN,
+ builder.field("EMPNO"), builder.literal(100)))
+ .project(builder.field("EMPNO"), builder.field("ENAME"))
+ .build();
+
+ final HepProgram hepProgram = new HepProgramBuilder()
+ .addRuleInstance(ProjectToCalcRule.INSTANCE)
+ .addRuleInstance(FilterToCalcRule.INSTANCE)
+ .addRuleInstance(CalcMergeRule.INSTANCE).build();
+
+ final HepPlanner hepPlanner = new HepPlanner(hepProgram);
+ hepPlanner.setRoot(root);
+ final RelNode relNode = hepPlanner.findBestExp();
+ final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder);
+ final RelNode trimmed = fieldTrimmer.trim(relNode);
+
+ final String expected = ""
+ + "LogicalCalc(expr#0..1=[{inputs}], expr#2=[100], expr#3=[>($t0, $t2)], proj#0."
+ + ".1=[{exprs}], $condition=[$t3])\n"
+ + " LogicalExchange(distribution=[single])\n"
+ + " LogicalCalc(expr#0..1=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " LogicalProject(EMPNO=[$0], ENAME=[$1])\n"
+ + " LogicalTableScan(table=[[scott, EMP]])\n";
+ assertThat(trimmed, hasTree(expected));
+ }
+
+ @Test void testCalcWithHints() {
+ final RelHint calcHint = RelHint.builder("resource").build();
+ final RelBuilder builder = RelBuilder.create(config().build());
+ builder.getCluster().setHintStrategies(
+ HintStrategyTable.builder().hintStrategy("resource", HintPredicates.CALC).build());
+ final RelNode original =
+ builder.scan("EMP")
+ .project(
+ builder.field("EMPNO"),
+ builder.field("ENAME"),
+ builder.field("DEPTNO")
+ ).hints(calcHint)
+ .sort(builder.field("EMPNO"))
+ .project(builder.field("EMPNO"))
+ .build();
+
+ final HepProgram hepProgram = new HepProgramBuilder()
+ .addRuleInstance(ProjectToCalcRule.INSTANCE)
+ .build();
+ final HepPlanner hepPlanner = new HepPlanner(hepProgram);
+ hepPlanner.setRoot(original);
+ final RelNode relNode = hepPlanner.findBestExp();
+
+ final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder);
+ final RelNode trimmed = fieldTrimmer.trim(relNode);
+
+ final String expected = ""
+ + "LogicalCalc(expr#0=[{inputs}], EMPNO=[$t0])\n"
+ + " LogicalSort(sort0=[$0], dir0=[ASC])\n"
+ + " LogicalCalc(expr#0=[{inputs}], EMPNO=[$t0])\n"
+ + " LogicalProject(EMPNO=[$0])\n"
+ + " LogicalTableScan(table=[[scott, EMP]])\n";
+ assertThat(trimmed, hasTree(expected));
+
+ assertTrue(original.getInput(0).getInput(0) instanceof Project);
+ final Project originalProject = (Project) original.getInput(0).getInput(0);
+ assertTrue(originalProject.getHints().contains(calcHint));
+
+ assertTrue(relNode.getInput(0).getInput(0) instanceof Calc);
+ final Calc originalCalc = (Calc) relNode.getInput(0).getInput(0);
+ assertTrue(originalCalc.getHints().contains(calcHint));
+
+ assertTrue(trimmed.getInput(0).getInput(0) instanceof Calc);
+ final Calc calc = (Calc) trimmed.getInput(0).getInput(0);
+ assertTrue(calc.getHints().contains(calcHint));
+ }
+
}