You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2015/05/09 17:51:49 UTC
[3/3] drill git commit: DRILL-2897: Avoid parallelization when Drill
is running limit 0 query.
DRILL-2897: Avoid parallelization when Drill is running limit 0 query.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9d209622
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9d209622
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9d209622
Branch: refs/heads/master
Commit: 9d209622934cb2be929a1591fd6a059165e1ac39
Parents: 960f876
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Apr 26 22:35:57 2015 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat May 9 08:03:59 2015 -0700
----------------------------------------------------------------------
.../exec/planner/physical/PlannerSettings.java | 7 +-
.../planner/sql/handlers/DefaultSqlHandler.java | 57 +++----
.../planner/sql/handlers/FindLimit0Visitor.java | 119 +++++++++++++++
.../org/apache/drill/TestExampleQueries.java | 1 +
.../java/org/apache/drill/TestTpchLimit0.java | 149 +++++++++++++++++++
5 files changed, 305 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 7d8dd97..d4fb4f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -33,6 +33,7 @@ public class PlannerSettings implements Context{
private int numEndPoints = 0;
private boolean useDefaultCosting = false; // True: use default Optiq costing, False: use Drill costing
+ private boolean forceSingleMode;
public static final int MAX_BROADCAST_THRESHOLD = Integer.MAX_VALUE;
public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 1024;
@@ -81,7 +82,11 @@ public class PlannerSettings implements Context{
}
public boolean isSingleMode() {
- return options.getOption(EXCHANGE.getOptionName()).bool_val;
+ return forceSingleMode || options.getOption(EXCHANGE.getOptionName()).bool_val;
+ }
+
+ public void forceSingleMode() {
+ forceSingleMode = true;
}
public int numEndPoints() {
http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index eda1b5f..188aaa9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -22,19 +22,23 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.rules.ProjectRemoveRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.TypedSqlNode;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.tools.Planner;
import org.apache.calcite.tools.RelConversionException;
import org.apache.calcite.tools.ValidationException;
-
import org.apache.drill.common.JSONOptions;
import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
@@ -55,7 +59,6 @@ import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier;
@@ -77,13 +80,6 @@ import org.apache.drill.exec.util.Pointer;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.plan.hep.HepPlanner;
-import org.apache.calcite.sql.SqlExplainLevel;
-import org.apache.calcite.sql.SqlNode;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Preconditions;
@@ -215,23 +211,23 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
return rel;
}
- protected RelNode preprocessNode(RelNode rel) throws SqlUnsupportedException{
- /* Traverse the tree to do the following pre-processing tasks:
- * 1. replace the convert_from, convert_to function to actual implementations
- * Eg: convert_from(EXPR, 'JSON') be converted to convert_fromjson(EXPR);
- * TODO: Ideally all function rewrites would move here instead of DrillOptiq
+ protected RelNode preprocessNode(RelNode rel) throws SqlUnsupportedException {
+ /*
+ * Traverse the tree to do the following pre-processing tasks: 1. replace the convert_from, convert_to function to
+ * actual implementations Eg: convert_from(EXPR, 'JSON') be converted to convert_fromjson(EXPR); TODO: Ideally all
+ * function rewrites would move here instead of DrillOptiq.
*
- * 2. see where the tree contains unsupported functions;
- * throw SqlUnsupportedException if there is
+ * 2. see where the tree contains unsupported functions; throw SqlUnsupportedException if there is any.
*/
- PreProcessLogicalRel visitor = PreProcessLogicalRel.createVisitor(planner.getTypeFactory(), context.getDrillOperatorTable());
- try {
- rel = rel.accept(visitor);
- } catch(UnsupportedOperationException ex) {
- visitor.convertException();
- throw ex;
- }
+ PreProcessLogicalRel visitor = PreProcessLogicalRel.createVisitor(planner.getTypeFactory(),
+ context.getDrillOperatorTable());
+ try {
+ rel = rel.accept(visitor);
+ } catch (UnsupportedOperationException ex) {
+ visitor.convertException();
+ throw ex;
+ }
return rel;
}
@@ -243,9 +239,16 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
if (convertedRelNode instanceof DrillStoreRel) {
throw new UnsupportedOperationException();
} else {
+
+ // If the query contains a limit 0 clause, disable distributed mode since it is overkill for determining schema.
+ if (FindLimit0Visitor.containsLimit0(convertedRelNode)) {
+ context.getPlannerSettings().forceSingleMode();
+ }
+
// Put a non-trivial topProject to ensure the final output field name is preserved, when necessary.
- DrillRel topPreservedNameProj = addRenamedProject((DrillRel)convertedRelNode, validatedRowType);
- return new DrillScreenRel(topPreservedNameProj.getCluster(), topPreservedNameProj.getTraitSet(), topPreservedNameProj);
+ DrillRel topPreservedNameProj = addRenamedProject((DrillRel) convertedRelNode, validatedRowType);
+ return new DrillScreenRel(topPreservedNameProj.getCluster(), topPreservedNameProj.getTraitSet(),
+ topPreservedNameProj);
}
} catch (RelOptPlanner.CannotPlanException ex) {
logger.error(ex.getMessage());
http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
new file mode 100644
index 0000000..d2c5fa6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.handlers;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttleImpl;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.drill.exec.planner.logical.DrillLimitRel;
+
+/**
+ * Visitor that will identify whether the root portion of the RelNode tree contains a limit 0 pattern. In this case, we
+ * inform the planner settings that this plan should be run as a single node plan to reduce the overhead associated with
+ * executing a schema-only query.
+ */
+public class FindLimit0Visitor extends RelShuttleImpl {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
+
+ private boolean contains = false;
+
+ public static boolean containsLimit0(RelNode rel) {
+ FindLimit0Visitor visitor = new FindLimit0Visitor();
+ rel.accept(visitor);
+ return visitor.isContains();
+ }
+
+ private FindLimit0Visitor() {
+ }
+
+ boolean isContains() {
+ return contains;
+ }
+
+ private boolean isLimit0(RexNode fetch) {
+ if (fetch != null && fetch.isA(SqlKind.LITERAL)) {
+ RexLiteral l = (RexLiteral) fetch;
+ switch (l.getTypeName()) {
+ case BIGINT:
+ case INTEGER:
+ case DECIMAL:
+ if (((long) l.getValue2()) == 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public RelNode visit(LogicalSort sort) {
+ if (isLimit0(sort.fetch)) {
+ contains = true;
+ return sort;
+ }
+
+ return super.visit(sort);
+ }
+
+ @Override
+ public RelNode visit(RelNode other) {
+ if (other instanceof DrillLimitRel) {
+ if (isLimit0(((DrillLimitRel) other).getFetch())) {
+ contains = true;
+ return other;
+ }
+ }
+
+ return super.visit(other);
+ }
+
+ // The following set of RelNodes should terminate a search for the limit 0 pattern as they want convey its meaning.
+
+ @Override
+ public RelNode visit(LogicalAggregate aggregate) {
+ return aggregate;
+ }
+
+ @Override
+ public RelNode visit(LogicalIntersect intersect) {
+ return intersect;
+ }
+
+ @Override
+ public RelNode visit(LogicalJoin join) {
+ return join;
+ }
+
+ @Override
+ public RelNode visit(LogicalMinus minus) {
+ return minus;
+ }
+
+ @Override
+ public RelNode visit(LogicalUnion union) {
+ return union;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 7e07500..18336cf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -284,6 +284,7 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test
+ @Ignore("DRILL-3004")
public void testJoin() throws Exception{
test("alter session set `planner.enable_hashjoin` = false");
test("SELECT\n" +
http://git-wip-us.apache.org/repos/asf/drill/blob/9d209622/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
new file mode 100644
index 0000000..22471c8
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestTpchLimit0 extends BaseTestQuery{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchLimit0.class);
+
+ private void testLimitZero(String fileName) throws Exception {
+ String query = getFile(fileName);
+ query = "ALTER SESSION SET `planner.slice_target` = 1; select * from \n(" + query.replace(";", ")xyz limit 0;");
+ test(query);
+ }
+
+ @Test
+ public void tpch01() throws Exception{
+ testLimitZero("queries/tpch/01.sql");
+ }
+
+ @Test
+ @Ignore // DRILL-512
+ public void tpch02() throws Exception{
+ testLimitZero("queries/tpch/02.sql");
+ }
+
+ @Test
+ public void tpch03() throws Exception{
+ testLimitZero("queries/tpch/03.sql");
+ }
+
+ @Test
+ public void tpch04() throws Exception{
+ testLimitZero("queries/tpch/04.sql");
+ }
+
+ @Test
+ public void tpch05() throws Exception{
+ testLimitZero("queries/tpch/05.sql");
+ }
+
+ @Test
+ public void tpch06() throws Exception{
+ testLimitZero("queries/tpch/06.sql");
+ }
+
+ @Test
+ public void tpch07() throws Exception{
+ testLimitZero("queries/tpch/07.sql");
+ }
+
+ @Test
+ public void tpch08() throws Exception{
+ testLimitZero("queries/tpch/08.sql");
+ }
+
+ @Test
+ public void tpch09() throws Exception{
+ testLimitZero("queries/tpch/09.sql");
+ }
+
+ @Test
+ public void tpch10() throws Exception{
+ testLimitZero("queries/tpch/10.sql");
+ }
+
+ @Test
+ @Ignore // Cartesian problem
+ public void tpch11() throws Exception{
+ testLimitZero("queries/tpch/11.sql");
+ }
+
+ @Test
+ public void tpch12() throws Exception{
+ testLimitZero("queries/tpch/12.sql");
+ }
+
+ @Test
+ public void tpch13() throws Exception{
+ testLimitZero("queries/tpch/13.sql");
+ }
+
+ @Test
+ public void tpch14() throws Exception{
+ testLimitZero("queries/tpch/14.sql");
+ }
+
+ @Test
+ @Ignore //
+ public void tpch15() throws Exception{
+ testLimitZero("queries/tpch/15.sql");
+ }
+
+ @Test
+ @Ignore // invalid plan, due to Nulls value NOT IN sub-q
+ public void tpch16() throws Exception{
+ testLimitZero("queries/tpch/16.sql");
+ }
+
+ @Test
+ @Ignore //
+ public void tpch17() throws Exception{
+ testLimitZero("queries/tpch/17.sql");
+ }
+
+ @Test
+ public void tpch18() throws Exception{
+ testLimitZero("queries/tpch/18.sql");
+ }
+
+ @Test
+ @Ignore // DRILL-519
+ public void tpch19() throws Exception{
+ testLimitZero("queries/tpch/19.sql");
+ }
+
+ @Test
+ public void tpch20() throws Exception{
+ testLimitZero("queries/tpch/20.sql");
+ }
+
+ @Test
+ @Ignore
+ public void tpch21() throws Exception{
+ testLimitZero("queries/tpch/21.sql");
+ }
+
+ @Test
+ @Ignore // DRILL-518
+ public void tpch22() throws Exception{
+ testLimitZero("queries/tpch/22.sql");
+ }
+}