You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/08/31 22:24:43 UTC
[1/2] drill git commit: DRILL-3045: Make planning time memory limit
configurable
Repository: drill
Updated Branches:
refs/heads/master 07dc46c11 -> bcc01c05a
DRILL-3045: Make planning time memory limit configurable
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dfa47da6
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dfa47da6
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dfa47da6
Branch: refs/heads/master
Commit: dfa47da669dc2455389151c4e4071d405030c7a9
Parents: 07dc46c
Author: Mehant Baid <me...@gmail.com>
Authored: Fri Aug 28 11:23:54 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Sun Aug 30 21:43:13 2015 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/ops/QueryContext.java | 7 ++-----
.../exec/planner/physical/PlannerSettings.java | 18 ++++++++++++++++++
.../exec/server/options/SystemOptionManager.java | 1 +
3 files changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index c3cd4e6..1cd67ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -57,9 +57,6 @@ import org.apache.drill.exec.util.Utilities;
public class QueryContext implements AutoCloseable, OptimizerRulesContext {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
- private static final int INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024;
- private static final int MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024;
-
private final DrillbitContext drillbitContext;
private final UserSession session;
private final OptionManager queryOptions;
@@ -94,8 +91,8 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext {
contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo);
try {
- allocator = drillbitContext.getAllocator().getChildAllocator(null, INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES,
- MAX_OFF_HEAP_ALLOCATION_IN_BYTES, false);
+ allocator = drillbitContext.getAllocator().getChildAllocator(null, plannerSettings.getInitialPlanningMemorySize(),
+ plannerSettings.getPlanningMemoryLimit(), false);
} catch (OutOfMemoryException e) {
throw new DrillRuntimeException("Error creating off-heap allocator for planning context.",e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 3355018..0a05d08 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.planner.physical;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValidator;
import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
@@ -38,6 +39,13 @@ public class PlannerSettings implements Context{
public static final int MAX_BROADCAST_THRESHOLD = Integer.MAX_VALUE;
public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 1024;
+ // initial off heap memory allocation (1M)
+ private static final long INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024;
+ // default off heap memory for planning (256M)
+ private static final long DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 256 * 1024 * 1024;
+ // max off heap memory for planning (16G)
+ private static final long MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 16l * 1024 * 1024 * 1024;
+
public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", true);
public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false);
public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true);
@@ -65,6 +73,8 @@ public class PlannerSettings implements Context{
public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type";
public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false);
public static final OptionValidator HEP_JOIN_OPT = new BooleanValidator("planner.enable_hep_join_opt", true);
+ public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit",
+ INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES);
public static final OptionValidator IDENTIFIER_MAX_LENGTH =
new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */,
@@ -188,6 +198,14 @@ public class PlannerSettings implements Context{
return options.getOption(IDENTIFIER_MAX_LENGTH.getOptionName()).num_val;
}
+ public long getPlanningMemoryLimit() {
+ return options.getOption(PLANNER_MEMORY_LIMIT.getOptionName()).num_val;
+ }
+
+ public static long getInitialPlanningMemorySize() {
+ return INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES;
+ }
+
@Override
public <T> T unwrap(Class<T> clazz) {
if(clazz == PlannerSettings.class){
http://git-wip-us.apache.org/repos/asf/drill/blob/dfa47da6/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 2d41740..12db4d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -73,6 +73,7 @@ public class SystemOptionManager extends BaseOptionManager {
PlannerSettings.PARTITION_SENDER_SET_THREADS,
PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
PlannerSettings.HEP_JOIN_OPT,
+ PlannerSettings.PLANNER_MEMORY_LIMIT,
ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
[2/2] drill git commit: DRILL-3719: Expand -expression as -1 *
expression in visitCall of DrillOptiq
Posted by me...@apache.org.
DRILL-3719: Expand -expression as -1 * expression in visitCall of DrillOptiq
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bcc01c05
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bcc01c05
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bcc01c05
Branch: refs/heads/master
Commit: bcc01c05a4d593b1f9226121e64930251586b304
Parents: dfa47da
Author: Hsuan-Yi Chu <hs...@usc.edu>
Authored: Fri Aug 28 10:33:01 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Sun Aug 30 21:43:14 2015 -0700
----------------------------------------------------------------------
.../drill/exec/planner/logical/DrillOptiq.java | 17 +++++++++++++----
.../org/apache/drill/TestExampleQueries.java | 20 ++++++++++++++++++++
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/bcc01c05/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 11b9c9e..43d8b6b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -41,6 +41,7 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.exec.planner.StarColumnHelper;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexDynamicParam;
@@ -123,9 +124,18 @@ public class DrillOptiq {
logger.debug("Prefix");
LogicalExpression arg = call.getOperands().get(0).accept(this);
switch(call.getKind()){
- case NOT:
- return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
- ExpressionPosition.UNKNOWN, arg);
+ case NOT:
+ return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
+ ExpressionPosition.UNKNOWN, arg);
+ case MINUS_PREFIX:
+ final RexBuilder builder = input.getCluster().getRexBuilder();
+ final List<RexNode> operands = Lists.newArrayList();
+ operands.add(builder.makeExactLiteral(new BigDecimal(-1)));
+ operands.add(call.getOperands().get(0));
+
+ return visitCall((RexCall) builder.makeCall(
+ SqlStdOperatorTable.MULTIPLY,
+ operands));
}
throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
case SPECIAL:
@@ -243,7 +253,6 @@ public class DrillOptiq {
return super.visitFieldAccess(fieldAccess);
}
-
private LogicalExpression getDrillCastFunctionFromOptiq(RexCall call){
LogicalExpression arg = call.getOperands().get(0).accept(this);
MajorType castType = null;
http://git-wip-us.apache.org/repos/asf/drill/blob/bcc01c05/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 6b74ecf..d15cff2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -1101,4 +1101,24 @@ public class TestExampleQueries extends BaseTestQuery {
test(query1);
test(query2);
}
+
+ @Test
+ public void testNegativeExtractOperator() throws Exception {
+ String query = "select -EXTRACT(DAY FROM birth_date) as col \n" +
+ "from cp.`employee.json` \n" +
+ "order by col \n" +
+ "limit 5";
+
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("col")
+ .baselineValues(-27l)
+ .baselineValues(-27l)
+ .baselineValues(-27l)
+ .baselineValues(-26l)
+ .baselineValues(-26l)
+ .build()
+ .run();
+ }
}