You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/08/31 22:24:43 UTC

[1/2] drill git commit: DRILL-3045: Make planning time memory limit configurable

Repository: drill
Updated Branches:
  refs/heads/master 07dc46c11 -> bcc01c05a


DRILL-3045: Make planning time memory limit configurable


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dfa47da6
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dfa47da6
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dfa47da6

Branch: refs/heads/master
Commit: dfa47da669dc2455389151c4e4071d405030c7a9
Parents: 07dc46c
Author: Mehant Baid <me...@gmail.com>
Authored: Fri Aug 28 11:23:54 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Sun Aug 30 21:43:13 2015 -0700

----------------------------------------------------------------------
 .../org/apache/drill/exec/ops/QueryContext.java   |  7 ++-----
 .../exec/planner/physical/PlannerSettings.java    | 18 ++++++++++++++++++
 .../exec/server/options/SystemOptionManager.java  |  1 +
 3 files changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index c3cd4e6..1cd67ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -57,9 +57,6 @@ import org.apache.drill.exec.util.Utilities;
 public class QueryContext implements AutoCloseable, OptimizerRulesContext {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
 
-  private static final int INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024;
-  private static final int MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024;
-
   private final DrillbitContext drillbitContext;
   private final UserSession session;
   private final OptionManager queryOptions;
@@ -94,8 +91,8 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext {
     contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo);
 
     try {
-      allocator = drillbitContext.getAllocator().getChildAllocator(null, INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES,
-          MAX_OFF_HEAP_ALLOCATION_IN_BYTES, false);
+      allocator = drillbitContext.getAllocator().getChildAllocator(null, plannerSettings.getInitialPlanningMemorySize(),
+          plannerSettings.getPlanningMemoryLimit(), false);
     } catch (OutOfMemoryException e) {
       throw new DrillRuntimeException("Error creating off-heap allocator for planning context.",e);
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 3355018..0a05d08 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.planner.physical;
 
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.ops.QueryContext;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValidator;
 import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
@@ -38,6 +39,13 @@ public class PlannerSettings implements Context{
   public static final int MAX_BROADCAST_THRESHOLD = Integer.MAX_VALUE;
   public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 1024;
 
+  // initial off heap memory allocation (1M)
+  private static final long INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024;
+  // default off heap memory for planning (256M)
+  private static final long DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024;
+  // max off heap memory for planning (16G)
+  private static final long MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 16l * 1024 * 1024 * 1024;
+
   public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", true);
   public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false);
   public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true);
@@ -65,6 +73,8 @@ public class PlannerSettings implements Context{
   public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type";
   public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false);
   public static final OptionValidator HEP_JOIN_OPT = new BooleanValidator("planner.enable_hep_join_opt", true);
+  public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit",
+      INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES);
 
   public static final OptionValidator IDENTIFIER_MAX_LENGTH =
       new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */,
@@ -188,6 +198,14 @@ public class PlannerSettings implements Context{
     return options.getOption(IDENTIFIER_MAX_LENGTH.getOptionName()).num_val;
   }
 
+  public long getPlanningMemoryLimit() {
+    return options.getOption(PLANNER_MEMORY_LIMIT.getOptionName()).num_val;
+  }
+
+  public static long getInitialPlanningMemorySize() {
+    return INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES;
+  }
+
   @Override
   public <T> T unwrap(Class<T> clazz) {
     if(clazz == PlannerSettings.class){

http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 2d41740..12db4d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -73,6 +73,7 @@ public class SystemOptionManager extends BaseOptionManager {
       PlannerSettings.PARTITION_SENDER_SET_THREADS,
       PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
       PlannerSettings.HEP_JOIN_OPT,
+      PlannerSettings.PLANNER_MEMORY_LIMIT,
       ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
       ExecConstants.OUTPUT_FORMAT_VALIDATOR,
       ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,


[2/2] drill git commit: DRILL-3719: Expand -expression as -1 * expression in visitCall of DrillOptiq

Posted by me...@apache.org.
DRILL-3719: Expand -expression as -1 * expression in visitCall of DrillOptiq


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bcc01c05
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bcc01c05
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bcc01c05

Branch: refs/heads/master
Commit: bcc01c05a4d593b1f9226121e64930251586b304
Parents: dfa47da
Author: Hsuan-Yi Chu <hs...@usc.edu>
Authored: Fri Aug 28 10:33:01 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Sun Aug 30 21:43:14 2015 -0700

----------------------------------------------------------------------
 .../drill/exec/planner/logical/DrillOptiq.java  | 17 +++++++++++++----
 .../org/apache/drill/TestExampleQueries.java    | 20 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/bcc01c05/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 11b9c9e..43d8b6b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -41,6 +41,7 @@ import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.planner.StarColumnHelper;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexCorrelVariable;
 import org.apache.calcite.rex.RexDynamicParam;
@@ -123,9 +124,18 @@ public class DrillOptiq {
         logger.debug("Prefix");
         LogicalExpression arg = call.getOperands().get(0).accept(this);
         switch(call.getKind()){
-        case NOT:
-          return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
-            ExpressionPosition.UNKNOWN, arg);
+          case NOT:
+            return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
+                ExpressionPosition.UNKNOWN, arg);
+          case MINUS_PREFIX:
+            final RexBuilder builder = input.getCluster().getRexBuilder();
+            final List<RexNode> operands = Lists.newArrayList();
+            operands.add(builder.makeExactLiteral(new BigDecimal(-1)));
+            operands.add(call.getOperands().get(0));
+
+            return visitCall((RexCall) builder.makeCall(
+                SqlStdOperatorTable.MULTIPLY,
+                    operands));
         }
         throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
       case SPECIAL:
@@ -243,7 +253,6 @@ public class DrillOptiq {
       return super.visitFieldAccess(fieldAccess);
     }
 
-
     private LogicalExpression getDrillCastFunctionFromOptiq(RexCall call){
       LogicalExpression arg = call.getOperands().get(0).accept(this);
       MajorType castType = null;

http://git-wip-us.apache.org/repos/asf/drill/blob/bcc01c05/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 6b74ecf..d15cff2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -1101,4 +1101,24 @@ public class TestExampleQueries extends BaseTestQuery {
     test(query1);
     test(query2);
   }
+
+  @Test
+  public void testNegativeExtractOperator() throws Exception {
+    String query = "select -EXTRACT(DAY FROM birth_date) as col \n" +
+        "from cp.`employee.json` \n" +
+        "order by col \n" +
+        "limit 5";
+
+    testBuilder()
+        .sqlQuery(query)
+        .ordered()
+        .baselineColumns("col")
+        .baselineValues(-27l)
+        .baselineValues(-27l)
+        .baselineValues(-27l)
+        .baselineValues(-26l)
+        .baselineValues(-26l)
+        .build()
+        .run();
+  }
 }