You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2017/07/21 17:23:38 UTC
[2/5] drill git commit: DRILL-5669: Add a configurable option for
minimum memory allocation to buffered ops
DRILL-5669: Add a configurable option for minimum memory allocation to buffered ops
closes #879
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6b0e8378
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6b0e8378
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6b0e8378
Branch: refs/heads/master
Commit: 6b0e8378cc198c926f5b1cf95a07fd4c9fc092de
Parents: 35a7ad0
Author: Boaz Ben-Zvi <bo...@BBenZvi-E754-MBP13.local>
Authored: Mon Jul 17 14:21:29 2017 -0700
Committer: Arina Ielchiieva <ar...@gmail.com>
Committed: Fri Jul 21 16:32:26 2017 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/exec/ExecConstants.java | 9 +++++++++
.../drill/exec/server/options/SystemOptionManager.java | 1 +
.../apache/drill/exec/util/MemoryAllocationUtilities.java | 8 ++++++--
3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 5b82d1f..97cb321 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -343,6 +343,15 @@ public interface ExecConstants {
MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE, 2 * 1024 * 1024 * 1024L);
/**
+ * Minimum memory alocated to each buffered operator instance.
+ * <p/>
+ * DEFAULT: 40 MB
+ */
+ String MIN_MEMORY_PER_BUFFERED_OP_KEY = "planner.memory.min_memory_per_buffered_op";
+ LongValidator MIN_MEMORY_PER_BUFFERED_OP = new RangeLongValidator(
+ MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE, 40 * 1024 * 1024L);
+
+ /**
* Extra query memory per node for non-blocking operators.
* NOTE: This option is currently used only for memory estimation.
* <p/>
http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index c2a4d65..3392a21 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -144,6 +144,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
ExecConstants.EARLY_LIMIT0_OPT,
ExecConstants.ENABLE_MEMORY_ESTIMATION,
ExecConstants.MAX_QUERY_MEMORY_PER_NODE,
+ ExecConstants.MIN_MEMORY_PER_BUFFERED_OP,
ExecConstants.NON_BLOCKING_OPERATORS_MEMORY,
ExecConstants.HASH_JOIN_TABLE_FACTOR,
ExecConstants.HASH_AGG_TABLE_FACTOR,
http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
index 79b49e4..4580222 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
@@ -66,11 +66,15 @@ public class MemoryAllocationUtilities {
final long maxOperatorAlloc = maxAllocPerNode / (bufferedOpList.size() * maxWidthPerNode);
logger.debug("Max buffered operator alloc: {}", maxOperatorAlloc);
+ // User configurable option to allow forcing minimum memory.
+ // Ensure that the buffered ops receive the minimum memory needed to make progress.
+ // Without this, the math might work out to allocate too little memory.
+ final long opMinMem = queryContext.getOptions().getOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY).num_val;
+
for(final PhysicalOperator op : bufferedOpList) {
- // Ensure that the sort receives the minimum memory needed to make progress.
- // Without this, the math might work out to allocate too little memory.
long alloc = Math.max(maxOperatorAlloc, op.getInitialAllocation());
+ alloc = Math.max(alloc, opMinMem);
op.setMaxAllocation(alloc);
}
}