You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/06/25 07:30:39 UTC
[5/5] drill git commit: DRILL-3345: TestWindowFrame fails to properly
check cases involving multiple batches
DRILL-3345: TestWindowFrame fails to properly check cases involving multiple batches
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3f0d9221
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3f0d9221
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3f0d9221
Branch: refs/heads/master
Commit: 3f0d9221d3f96c20db10e868cc33c2e972318ba6
Parents: 00aa01f
Author: adeneche <ad...@gmail.com>
Authored: Wed Jun 24 08:58:39 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Wed Jun 24 16:54:36 2015 -0700
----------------------------------------------------------------------
.../impl/window/DefaultFrameTemplate.java | 6 +++---
.../exec/physical/impl/xsort/MSortTemplate.java | 21 +++++++++++---------
.../exec/record/selection/SelectionVector4.java | 16 +++++++++++----
3 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
index ada068b..535deaa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
@@ -242,11 +242,11 @@ public abstract class DefaultFrameTemplate implements WindowFramer {
final int lastSize = last.getRecordCount();
if (!isSamePartition(currentSize - 1, current, lastSize - 1, last)
- || !isPeer(currentSize - 1, current, lastSize - 1, last)) {
- logger.trace("frame changed, we are ready to process first saved batch");
+ /*|| !isPeer(currentSize - 1, current, lastSize - 1, last)*/) {
+ logger.trace("partition changed, we are ready to process first saved batch");
return true;
} else {
- logger.trace("frame didn't change, fetch next batch");
+ logger.trace("partition didn't change, fetch next batch");
return false;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
index 6686fbe..37529ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
@@ -47,6 +47,11 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
private Queue<Integer> newRunStarts;
private FragmentContext context;
+ /**
+ * This is only useful for debugging and/or unit testing. Controls the maximum size of batches exposed to downstream
+ */
+ private int desiredRecordBatchCount;
+
@Override
public void setup(final FragmentContext context, final BufferAllocator allocator, final SelectionVector4 vector4, final VectorContainer hyperBatch) throws SchemaChangeException{
// we pass in the local hyperBatch since that is where we'll be reading data.
@@ -71,15 +76,13 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
}
final DrillBuf drillBuf = allocator.buffer(4 * totalCount);
- // This is only useful for debugging: change the maximum size of batches exposed to downstream
- // when we don't spill to disk
- int MSORT_BATCH_MAXSIZE;
try {
- MSORT_BATCH_MAXSIZE = context.getConfig().getInt(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE);
+ desiredRecordBatchCount = context.getConfig().getInt(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE);
} catch(ConfigException.Missing e) {
- MSORT_BATCH_MAXSIZE = Character.MAX_VALUE;
+ // value not found, use default value instead
+ desiredRecordBatchCount = Character.MAX_VALUE;
}
- aux = new SelectionVector4(drillBuf, totalCount, MSORT_BATCH_MAXSIZE);
+ aux = new SelectionVector4(drillBuf, totalCount, desiredRecordBatchCount);
}
/**
@@ -150,11 +153,11 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
if (outIndex < vector4.getTotalCount()) {
copyRun(outIndex, vector4.getTotalCount());
}
- final SelectionVector4 tmp = aux.createNewWrapperCurrent();
+ final SelectionVector4 tmp = aux.createNewWrapperCurrent(desiredRecordBatchCount);
aux.clear();
- aux = this.vector4.createNewWrapperCurrent();
+ aux = this.vector4.createNewWrapperCurrent(desiredRecordBatchCount);
vector4.clear();
- this.vector4 = tmp.createNewWrapperCurrent();
+ this.vector4 = tmp.createNewWrapperCurrent(desiredRecordBatchCount);
tmp.clear();
runStarts = newRunStarts;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
index 69bc78f..8db0437 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.record.DeadBuf;
public class SelectionVector4 {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector4.class);
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector4.class);
private ByteBuf data;
private int recordCount;
@@ -67,13 +67,13 @@ public class SelectionVector4 {
/**
* Caution: This method shares the underlying buffer between this vector and the newly created one.
+ * @param batchRecordCount this will be used when creating the new vector
* @return Newly created single batch SelectionVector4.
- * @throws SchemaChangeException
*/
- public SelectionVector4 createNewWrapperCurrent() {
+ public SelectionVector4 createNewWrapperCurrent(int batchRecordCount) {
try {
data.retain();
- SelectionVector4 sv4 = new SelectionVector4(data, recordCount, length);
+ SelectionVector4 sv4 = new SelectionVector4(data, recordCount, batchRecordCount);
sv4.start = this.start;
return sv4;
} catch (SchemaChangeException e) {
@@ -81,6 +81,14 @@ public class SelectionVector4 {
}
}
+ /**
+ * Caution: This method shares the underlying buffer between this vector and the newly created one.
+ * @return Newly created single batch SelectionVector4.
+ */
+ public SelectionVector4 createNewWrapperCurrent() {
+ return createNewWrapperCurrent(length);
+ }
+
public boolean next() {
// logger.debug("Next called. Start: {}, Length: {}, recordCount: " + recordCount, start, length);