You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2016/11/23 07:26:17 UTC
calcite git commit: [CALCITE-1501] EnumerableUnion should use array
comparator when row format is ARRAY (Dayue Gao)
Repository: calcite
Updated Branches:
refs/heads/master 9b62ab3dc -> ce2122ff2
[CALCITE-1501] EnumerableUnion should use array comparator when row format is ARRAY (Dayue Gao)
Also implement EnumerableIntersect and EnumerableMinus.
Close apache/calcite#326
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/ce2122ff
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/ce2122ff
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/ce2122ff
Branch: refs/heads/master
Commit: ce2122ff2562340333bfa0ba371872fc9a9c6251
Parents: 9b62ab3
Author: gaodayue <ga...@meituan.com>
Authored: Mon Nov 21 17:16:31 2016 +0800
Committer: Julian Hyde <jh...@apache.org>
Committed: Tue Nov 22 21:26:46 2016 -0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableIntersect.java | 10 ++--
.../adapter/enumerable/EnumerableMinus.java | 6 +-
.../adapter/enumerable/EnumerableUnion.java | 13 ++--
.../calcite/prepare/CalcitePrepareImpl.java | 3 +
.../org/apache/calcite/prepare/Prepare.java | 2 +-
.../java/org/apache/calcite/runtime/Hook.java | 3 +
.../java/org/apache/calcite/test/JdbcTest.java | 62 ++++++++++++++++++++
.../calcite/test/MaterializationTest.java | 27 +++------
core/src/test/resources/sql/misc.iq | 47 +++++++++++----
.../calcite/linq4j/EnumerableDefaults.java | 6 ++
10 files changed, 132 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java
index 6c74c17..8462f65 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java
@@ -57,12 +57,10 @@ public class EnumerableIntersect extends Intersect implements EnumerableRel {
intersectExp = childExp;
} else {
intersectExp =
- Expressions.call(
- intersectExp,
- all
- ? BuiltInMethod.CONCAT.method
- : BuiltInMethod.INTERSECT.method,
- childExp);
+ Expressions.call(intersectExp,
+ BuiltInMethod.INTERSECT.method,
+ Expressions.list(childExp)
+ .appendIfNotNull(result.physType.comparer()));
}
// Once the first input has chosen its format, ask for the same for
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java
index c783cab..7ed4edb 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java
@@ -57,10 +57,10 @@ public class EnumerableMinus extends Minus implements EnumerableRel {
minusExp = childExp;
} else {
minusExp =
- Expressions.call(
- minusExp,
+ Expressions.call(minusExp,
BuiltInMethod.EXCEPT.method,
- childExp);
+ Expressions.list(childExp)
+ .appendIfNotNull(result.physType.comparer()));
}
// Once the first input has chosen its format, ask for the same for
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUnion.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUnion.java
index 5162701..2df6550 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUnion.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUnion.java
@@ -55,13 +55,12 @@ public class EnumerableUnion extends Union implements EnumerableRel {
if (unionExp == null) {
unionExp = childExp;
} else {
- unionExp =
- Expressions.call(
- unionExp,
- all
- ? BuiltInMethod.CONCAT.method
- : BuiltInMethod.UNION.method,
- childExp);
+ unionExp = all
+ ? Expressions.call(unionExp, BuiltInMethod.CONCAT.method, childExp)
+ : Expressions.call(unionExp,
+ BuiltInMethod.UNION.method,
+ Expressions.list(childExp)
+ .appendIfNotNull(result.physType.comparer()));
}
// Once the first input has chosen its format, ask for the same for
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index 196254c..4d7daca 100644
--- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -552,6 +552,9 @@ public class CalcitePrepareImpl implements CalcitePrepare {
}
});
}
+
+ Hook.PLANNER.run(planner); // allow test to add or remove rules
+
return planner;
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/prepare/Prepare.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
index 3532ed3..2d97bb9 100644
--- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java
+++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
@@ -160,7 +160,7 @@ public abstract class Prepare {
}
private Program getProgram() {
- // Allow a test to override the planner.
+ // Allow a test to override the default program.
final List<Materialization> materializations = ImmutableList.of();
final Holder<Program> holder = Holder.of(null);
Hook.PROGRAM.run(Pair.of(materializations, holder));
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/main/java/org/apache/calcite/runtime/Hook.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/runtime/Hook.java b/core/src/main/java/org/apache/calcite/runtime/Hook.java
index efa9004..301ac94 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Hook.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Hook.java
@@ -54,6 +54,9 @@ public enum Hook {
/** Called with the output of sql-to-rel-converter. */
CONVERTED,
+ /** Called with the created planner. */
+ PLANNER,
+
/** Called after de-correlation and field trimming, but before
* optimization. */
TRIMMED,
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/test/java/org/apache/calcite/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
index eaa026b..bc214c2 100644
--- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java
+++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
@@ -43,6 +43,7 @@ import org.apache.calcite.linq4j.QueryProvider;
import org.apache.calcite.linq4j.Queryable;
import org.apache.calcite.linq4j.function.Function0;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.prepare.CalcitePrepareImpl;
@@ -50,6 +51,7 @@ import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rel.rules.IntersectToDistinctRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.runtime.FlatLists;
@@ -3456,6 +3458,66 @@ public class JdbcTest {
"C=0");
}
+ @Test public void testUnionAll() {
+ CalciteAssert.hr()
+ .query("select \"empid\", \"name\" from \"hr\".\"emps\" where \"deptno\"=10\n"
+ + "union all\n"
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"empid\">=150")
+ .explainContains(""
+ + "PLAN=EnumerableUnion(all=[true])")
+ .returnsUnordered("empid=100; name=Bill",
+ "empid=110; name=Theodore",
+ "empid=150; name=Sebastian",
+ "empid=150; name=Sebastian",
+ "empid=200; name=Eric");
+ }
+
+ @Test public void testUnion() {
+ final String sql = ""
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"deptno\"=10\n"
+ + "union\n"
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"empid\">=150";
+ CalciteAssert.hr()
+ .query(sql)
+ .explainContains(""
+ + "PLAN=EnumerableUnion(all=[false])")
+ .returnsUnordered("empid=100; name=Bill",
+ "empid=110; name=Theodore",
+ "empid=150; name=Sebastian",
+ "empid=200; name=Eric");
+ }
+
+ @Test public void testIntersect() {
+ final String sql = ""
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"deptno\"=10\n"
+ + "intersect\n"
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"empid\">=150";
+ CalciteAssert.hr()
+ .query(sql)
+ .withHook(Hook.PLANNER, new Function<RelOptPlanner, Void>() {
+ @Override public Void apply(RelOptPlanner planner) {
+ planner.removeRule(IntersectToDistinctRule.INSTANCE);
+ return null;
+ }
+ })
+ .explainContains(""
+ + "PLAN=EnumerableIntersect(all=[false])")
+ .returnsUnordered("empid=150; name=Sebastian");
+ }
+
+ @Test public void testExcept() {
+ final String sql = ""
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"deptno\"=10\n"
+ + "except\n"
+ + "select \"empid\", \"name\" from \"hr\".\"emps\" where \"empid\">=150";
+ CalciteAssert.hr()
+ .query(sql)
+ .explainContains(""
+ + "PLAN=EnumerableMinus(all=[false])")
+ .returnsUnordered("empid=100; name=Bill",
+ "empid=110; name=Theodore");
+ }
+
/** Tests that SUM and AVG over empty set return null. COUNT returns 0. */
@Test public void testAggregateEmpty() {
CalciteAssert.hr()
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index 6f891b9..b89854f 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -21,7 +21,6 @@ import org.apache.calcite.materialize.MaterializationService;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.plan.SubstitutionVisitor;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
@@ -37,13 +36,9 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.runtime.Hook;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.tools.Program;
-import org.apache.calcite.tools.Programs;
import org.apache.calcite.tools.RuleSet;
import org.apache.calcite.tools.RuleSets;
-import org.apache.calcite.util.Holder;
import org.apache.calcite.util.JsonBuilder;
-import org.apache.calcite.util.Pair;
import org.apache.calcite.util.TryThreadLocal;
import org.apache.calcite.util.Util;
@@ -187,22 +182,14 @@ public class MaterializationTest {
// Add any additional rules required for the test
if (rules.iterator().hasNext()) {
- that.withHook(Hook.PROGRAM,
- new Function<Pair<List<Prepare.Materialization>, Holder<Program>>,
- Void>() {
- public Void apply(Pair<List<Prepare.Materialization>, Holder<Program>> pair) {
- pair.right.set(new Program() {
- public RelNode run(RelOptPlanner planner, RelNode rel,
- RelTraitSet requiredOutputTraits) {
- for (RelOptRule rule : rules) {
- planner.addRule(rule);
- }
- return Programs.standard().run(planner, rel, requiredOutputTraits);
- }
- });
- return null;
+ that.withHook(Hook.PLANNER, new Function<RelOptPlanner, Void>() {
+ public Void apply(RelOptPlanner planner) {
+ for (RelOptRule rule : rules) {
+ planner.addRule(rule);
}
- });
+ return null;
+ }
+ });
}
that.explainMatches("", explainChecker)
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/core/src/test/resources/sql/misc.iq
----------------------------------------------------------------------
diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq
index a93d7df..0378e26 100644
--- a/core/src/test/resources/sql/misc.iq
+++ b/core/src/test/resources/sql/misc.iq
@@ -1261,6 +1261,36 @@ select 3, 'c';
!ok
+# Composite UNION
+select deptno, deptno from "scott".emp
+union
+select deptno, deptno from "scott".dept;
++--------+--------+
+| DEPTNO | DEPTNO |
++--------+--------+
+| 10 | 10 |
+| 20 | 20 |
+| 30 | 30 |
+| 40 | 40 |
++--------+--------+
+(4 rows)
+
+!ok
+
+# Composite MINUS
+select deptno, deptno from "scott".emp
+except
+select deptno, deptno from "scott".dept where deptno = 10;
++--------+--------+
+| DEPTNO | DEPTNO |
++--------+--------+
+| 20 | 20 |
+| 30 | 30 |
++--------+--------+
+(2 rows)
+
+!ok
+
select * from (
select 2 as a, 3 as b)
where a < b;
@@ -1338,20 +1368,17 @@ select comm from "scott".emp where deptno = 10;
!ok
# As above, composite
-# Disabled, due to bug, apparently in EnumerableIntersect
-!if (false) {
-select 10 as deptno, comm from "scott".emp where mgr = 7698
+select job, comm from "scott".emp where mgr = 7698
intersect
-select deptno, comm from "scott".emp where deptno = 10;
-+--------+------+
-| DEPTNO | COMM |
-+--------+------+
-| 10 | |
-+--------+------+
+select job, comm from "scott".emp where deptno = 10;
++-------+------+
+| JOB | COMM |
++-------+------+
+| CLERK | |
++-------+------+
(1 row)
!ok
-!}
# Make sure that IN does not compare the shorter of two strings.
# If it did, 'MARTIN' would be emitted.
http://git-wip-us.apache.org/repos/asf/calcite/blob/ce2122ff/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
----------------------------------------------------------------------
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index e715cdb..d29ef1d 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -556,6 +556,9 @@ public abstract class EnumerableDefaults {
public static <TSource> Enumerable<TSource> except(
Enumerable<TSource> source0, Enumerable<TSource> source1,
EqualityComparer<TSource> comparer) {
+ if (comparer == Functions.identityComparer()) {
+ return except(source0, source1);
+ }
Set<Wrapped<TSource>> set = new HashSet<>();
Function1<TSource, Wrapped<TSource>> wrapper = wrapperFor(comparer);
source0.select(wrapper).into(set);
@@ -994,6 +997,9 @@ public abstract class EnumerableDefaults {
public static <TSource> Enumerable<TSource> intersect(
Enumerable<TSource> source0, Enumerable<TSource> source1,
EqualityComparer<TSource> comparer) {
+ if (comparer == Functions.identityComparer()) {
+ return intersect(source0, source1);
+ }
Set<Wrapped<TSource>> set0 = new HashSet<>();
Function1<TSource, Wrapped<TSource>> wrapper = wrapperFor(comparer);
source0.select(wrapper).into(set0);