You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2015/06/27 23:03:02 UTC
drill git commit: DRILL-3307: Query with window function runs out of
memory
Repository: drill
Updated Branches:
refs/heads/master 8d7458958 -> e85cdc54c
DRILL-3307: Query with window function runs out of memory
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e85cdc54
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e85cdc54
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e85cdc54
Branch: refs/heads/master
Commit: e85cdc54cc21e0be2bf7169868ec00b8ef7c1f22
Parents: 8d74589
Author: Steven Phillips <sm...@apache.org>
Authored: Fri Jun 26 16:15:21 2015 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 27 11:58:46 2015 -0700
----------------------------------------------------------------------
.../exec/physical/impl/xsort/ExternalSortBatch.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e85cdc54/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 02a1c08..6da5582 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -406,7 +406,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
batchGroups.addAll(spilledBatchGroups);
logger.warn("Starting to merge. {} batch groups. Current allocated memory: {}", batchGroups.size(), oContext.getAllocator().getAllocatedMemory());
VectorContainer hyperBatch = constructHyperBatch(batchGroups);
- createCopier(hyperBatch, batchGroups, container);
+ createCopier(hyperBatch, batchGroups, container, false);
int estimatedRecordSize = 0;
for (VectorWrapper w : batchGroups.get(0)) {
@@ -474,7 +474,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
}
int targetRecordCount = Math.max(1, 250 * 1000 / estimatedRecordSize);
VectorContainer hyperBatch = constructHyperBatch(batchGroupList);
- createCopier(hyperBatch, batchGroupList, outputContainer);
+ createCopier(hyperBatch, batchGroupList, outputContainer, true);
int count = copier.next(targetRecordCount);
assert count > 0;
@@ -673,7 +673,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
g.getEvalBlock()._return(JExpr.lit(0));
}
- private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList, VectorContainer outputContainer) throws SchemaChangeException {
+ private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList, VectorContainer outputContainer, boolean spilling) throws SchemaChangeException {
try {
if (copier == null) {
CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION, context.getFunctionRegistry());
@@ -689,11 +689,12 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
copier.cleanup();
}
+ BufferAllocator allocator = spilling ? copierAllocator : oContext.getAllocator();
for (VectorWrapper<?> i : batch) {
- ValueVector v = TypeHelper.getNewVector(i.getField(), copierAllocator);
+ ValueVector v = TypeHelper.getNewVector(i.getField(), allocator);
outputContainer.add(v);
}
- copier.setup(context, copierAllocator, batch, batchGroupList, outputContainer);
+ copier.setup(context, allocator, batch, batchGroupList, outputContainer);
} catch (ClassTransformationException e) {
throw new RuntimeException(e);
} catch (IOException e) {