You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2015/01/03 02:53:06 UTC
drill git commit: DRILL-1905: There are 3 parts of the fix: (1) Check
compatibility during initial creation, not during copy() of Union rels,
(2) Added implementations of isDistinct() and getFamily() in Drill specific
derived classes, (3) In Calcite, add
Repository: drill
Updated Branches:
refs/heads/master c051bbd88 -> b491cdb37
DRILL-1905: There are 3 parts of the fix: (1) Check compatibility during initial creation, not during copy() of Union rels, (2) Added implementations of isDistinct() and getFamily() in Drill specific derived classes, (3) In Calcite, add processing of ANY type in leastRestrictiveSqlType().
Bump up the Calcite version to 0.9-drill-r16
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b491cdb3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b491cdb3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b491cdb3
Branch: refs/heads/master
Commit: b491cdb37ea066e2aa041fcca706cc62c8881767
Parents: c051bbd
Author: Aman Sinha <as...@maprtech.com>
Authored: Tue Dec 23 09:49:58 2014 -0800
Committer: Aman Sinha <as...@maprtech.com>
Committed: Fri Jan 2 17:37:03 2015 -0800
----------------------------------------------------------------------
.../drill/exec/planner/common/DrillUnionRelBase.java | 12 +++++++++---
.../drill/exec/planner/logical/DrillUnionRel.java | 7 ++++---
.../drill/exec/planner/logical/DrillUnionRule.java | 3 ++-
.../drill/exec/planner/physical/UnionAllPrel.java | 9 ++++++---
.../drill/exec/planner/physical/UnionAllPrule.java | 4 +++-
.../drill/exec/planner/physical/UnionDistinctPrel.java | 8 +++++---
.../drill/exec/planner/physical/UnionDistinctPrule.java | 4 +++-
.../apache/drill/exec/planner/physical/UnionPrel.java | 5 +++--
.../drill/exec/planner/types/RelDataTypeDrillImpl.java | 5 +++++
.../src/test/java/org/apache/drill/TestUnionAll.java | 10 ++++++++++
pom.xml | 2 +-
11 files changed, 51 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
index 127798f..932aa76 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
@@ -32,10 +32,10 @@ import org.eigenbase.reltype.RelDataType;
public abstract class DrillUnionRelBase extends UnionRelBase implements DrillRelNode {
public DrillUnionRelBase(RelOptCluster cluster, RelTraitSet traits,
- List<RelNode> inputs, boolean all) throws InvalidRelException {
+ List<RelNode> inputs, boolean all, boolean checkCompatibility) throws InvalidRelException {
super(cluster, traits, inputs, all);
- // if (! this.isHomogeneous(false /* don't compare names */)) {
- if (! this.isCompatible(false /* don't compare names */, true /* allow substrings */)) {
+ if (checkCompatibility &&
+ !this.isCompatible(false /* don't compare names */, true /* allow substrings */)) {
throw new InvalidRelException("Input row types of the Union are not compatible.");
}
}
@@ -50,4 +50,10 @@ public abstract class DrillUnionRelBase extends UnionRelBase implements DrillRel
}
return true;
}
+
+ @Override
+ public boolean isDistinct() {
+ return !this.all;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
index 1fa7001..39d5d4d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
@@ -38,15 +38,16 @@ import org.eigenbase.relopt.RelTraitSet;
public class DrillUnionRel extends DrillUnionRelBase implements DrillRel {
/** Creates a DrillUnionRel. */
public DrillUnionRel(RelOptCluster cluster, RelTraitSet traits,
- List<RelNode> inputs, boolean all) throws InvalidRelException {
- super(cluster, traits, inputs, all);
+ List<RelNode> inputs, boolean all, boolean checkCompatibility) throws InvalidRelException {
+ super(cluster, traits, inputs, all, checkCompatibility);
}
@Override
public DrillUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs,
boolean all) {
try {
- return new DrillUnionRel(getCluster(), traitSet, inputs, all);
+ return new DrillUnionRel(getCluster(), traitSet, inputs, all,
+ false /* don't check compatibility during copy */);
} catch (InvalidRelException e) {
throw new AssertionError(e) ;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
index 93a0391..f71d203 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
@@ -52,7 +52,8 @@ public class DrillUnionRule extends RelOptRule {
convertedInputs.add(convertedInput);
}
try {
- call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
+ call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all,
+ true /* check compatibility */));
} catch (InvalidRelException e) {
tracer.warning(e.toString()) ;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
index 33abb48..d5698f7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
@@ -38,15 +38,18 @@ import com.google.common.collect.Lists;
public class UnionAllPrel extends UnionPrel {
- public UnionAllPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
- super(cluster, traits, inputs, true /* all */);
+ public UnionAllPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs,
+ boolean checkCompatibility) throws InvalidRelException {
+ super(cluster, traits, inputs, true /* all */, checkCompatibility);
+
}
@Override
public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
try {
- return new UnionAllPrel(this.getCluster(), traitSet, inputs);
+ return new UnionAllPrel(this.getCluster(), traitSet, inputs,
+ false /* don't check compatibility during copy */);
}catch (InvalidRelException e) {
throw new AssertionError(e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
index 869493f..fa533f7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
@@ -60,7 +60,9 @@ public class UnionAllPrule extends Prule {
}
traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
- UnionAllPrel unionAll = new UnionAllPrel(union.getCluster(), traits, convertedInputList);
+ UnionAllPrel unionAll =
+ new UnionAllPrel(union.getCluster(), traits, convertedInputList,
+ false /* compatibility already checked during logical phase */);
call.transformTo(unionAll);
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
index 3fa5659..05086ed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
@@ -38,15 +38,17 @@ import com.google.common.collect.Lists;
public class UnionDistinctPrel extends UnionPrel {
- public UnionDistinctPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
- super(cluster, traits, inputs, false /* all = false */);
+ public UnionDistinctPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs,
+ boolean checkCompatibility) throws InvalidRelException {
+ super(cluster, traits, inputs, false /* all = false */, checkCompatibility);
}
@Override
public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
try {
- return new UnionDistinctPrel(this.getCluster(), traitSet, inputs);
+ return new UnionDistinctPrel(this.getCluster(), traitSet, inputs,
+ false /* don't check compatibility during copy */);
}catch (InvalidRelException e) {
throw new AssertionError(e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
index 67a817e..fd85448 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
@@ -60,7 +60,9 @@ public class UnionDistinctPrule extends Prule {
}
traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
- UnionDistinctPrel unionDistinct = new UnionDistinctPrel(union.getCluster(), traits, convertedInputList);
+ UnionDistinctPrel unionDistinct =
+ new UnionDistinctPrel(union.getCluster(), traits, convertedInputList,
+ false /* compatibility already checked during logical phase */);
call.transformTo(unionDistinct);
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
index a1f300b..6303489 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
@@ -30,8 +30,9 @@ import org.eigenbase.relopt.RelTraitSet;
public abstract class UnionPrel extends DrillUnionRelBase implements Prel{
- public UnionPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs, boolean all) throws InvalidRelException{
- super(cluster, traits, inputs, all);
+ public UnionPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs, boolean all,
+ boolean checkCompatibility) throws InvalidRelException {
+ super(cluster, traits, inputs, all, checkCompatibility);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
index eae3510..559cf68 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.reltype.RelDataTypeFamily;
import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.reltype.RelDataTypeImpl;
import org.eigenbase.reltype.RelDataTypePrecedenceList;
@@ -112,4 +113,8 @@ public class RelDataTypeDrillImpl extends RelDataTypeImpl {
return true;
}
+ @Override
+ public RelDataTypeFamily getFamily() {
+ return getSqlTypeName().getFamily();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index bf0eea0..36b062b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -17,10 +17,13 @@
*/
package org.apache.drill;
+import org.apache.drill.common.util.TestTools;
import org.junit.Test;
public class TestUnionAll extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionAll.class);
+ static final String WORKING_PATH = TestTools.getWorkingPath();
+ static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
@Test // Simple Union-All over two scans
public void testUnionAll1() throws Exception {
@@ -67,5 +70,12 @@ public class TestUnionAll extends BaseTestQuery{
test("select n_name from cp.`tpch/nation.parquet` union all select r_comment from cp.`tpch/region.parquet`");
}
+ @Test // DRILL-1905: Union-all of * column from JSON files in different directories
+ public void testUnionAll9() throws Exception {
+ String query1 = String.format("select * from dfs_test.`%s/multilevel/json/1994/Q1/orders_94_q1.json` " +
+ " union all select * from dfs_test.`%s/multilevel/json/1995/Q1/orders_95_q1.json`",
+ TEST_RES_PATH, TEST_RES_PATH);
+ test(query1);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b491cdb3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8cb47b..9a539e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -920,7 +920,7 @@
<dependency>
<groupId>net.hydromatic</groupId>
<artifactId>optiq-core</artifactId>
- <version>0.9-drill-r15</version>
+ <version>0.9-drill-r16</version>
<exclusions>
<exclusion>
<groupId>org.jgrapht</groupId>