You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/02 00:22:05 UTC
[04/18] git commit: fix parquet query join errors
fix parquet query join errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cc329aa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cc329aa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cc329aa4
Branch: refs/heads/master
Commit: cc329aa4418cf6c363f7da7363594f5174655fe1
Parents: d553749
Author: Ben Becker <be...@gmail.com>
Authored: Fri Aug 30 14:59:08 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700
----------------------------------------------------------------------
.../exec/physical/impl/join/JoinStatus.java | 20 ++++----
.../exec/physical/impl/join/JoinTemplate.java | 2 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 51 ++------------------
3 files changed, 15 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
index c755e5f..84a11fc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
@@ -112,14 +112,15 @@ public final class JoinStatus {
* Side effect: advances to next left batch if current left batch size is exceeded.
*/
public final boolean isLeftPositionAllowed(){
- if(!isLeftPositionInCurrentBatch()){
+ if (lastLeft == IterOutcome.NONE)
+ return false;
+ if (!isLeftPositionInCurrentBatch()) {
leftPosition = 0;
lastLeft = left.next();
return lastLeft == IterOutcome.OK;
- }else{
- lastLeft = IterOutcome.OK;
- return true;
}
+ lastLeft = IterOutcome.OK;
+ return true;
}
/**
@@ -129,16 +130,15 @@ public final class JoinStatus {
public final boolean isRightPositionAllowed(){
if (rightSourceMode == RightSourceMode.SV4)
return svRightPosition < sv4.getCount();
-
- if(!isRightPositionInCurrentBatch()){
+ if (lastRight == IterOutcome.NONE)
+ return false;
+ if (!isRightPositionInCurrentBatch()) {
rightPosition = 0;
lastRight = right.next();
return lastRight == IterOutcome.OK;
- }else{
- lastRight = IterOutcome.OK;
- return true;
}
-
+ lastRight = IterOutcome.OK;
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index d6fc802..3d6dfce 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -119,7 +119,7 @@ public abstract class JoinTemplate implements JoinWorker {
// If the left key has duplicates and we're about to cross a boundary in the right batch, add the
// right table's record batch to the sv4 builder before calling next. These records will need to be
// copied again for each duplicate left key.
- if (status.isLeftRepeating() && !status.isNextRightPositionInCurrentBatch()) {
+ if (status.isLeftRepeating() && !status.isRightPositionInCurrentBatch()) {
status.outputBatch.addRightToBatchBuilder();
crossedBatchBoundaries = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 02a1961..5a62c1f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
import org.apache.drill.exec.record.*;
import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
/**
* A merge join combining to incoming in-order batches.
@@ -107,7 +108,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
// we do this in the here instead of the constructor because don't necessary want to start consuming on construction.
status.ensureInitial();
-
+
// loop so we can start over again if we find a new batch was created.
while(true){
@@ -367,13 +368,13 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
// add fields from both batches
for (VectorWrapper<?> w : left) {
ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
- getAllocator(w.getValueVector(), outgoingVector).alloc(left.getRecordCount() * 16);
+ VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / left.getRecordCount())).alloc(left.getRecordCount() * 16);
container.add(outgoingVector);
}
for (VectorWrapper<?> w : right) {
ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
- getAllocator(w.getValueVector(), outgoingVector).alloc(right.getRecordCount() * 16);
+ VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / right.getRecordCount())).alloc(right.getRecordCount() * 16);
container.add(outgoingVector);
}
@@ -381,48 +382,4 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
logger.debug("Built joined schema: {}", container.getSchema());
}
- private VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
- if(outgoing instanceof FixedWidthVector){
- return new FixedVectorAllocator((FixedWidthVector) outgoing);
- }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
- return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
- }else{
- throw new UnsupportedOperationException();
- }
- }
-
- private class FixedVectorAllocator implements VectorAllocator{
- FixedWidthVector out;
-
- public FixedVectorAllocator(FixedWidthVector out) {
- super();
- this.out = out;
- }
-
- public void alloc(int recordCount){
- out.allocateNew(recordCount);
- out.getMutator().setValueCount(recordCount);
- }
- }
-
- private class VariableVectorAllocator implements VectorAllocator{
- VariableWidthVector in;
- VariableWidthVector out;
-
- public VariableVectorAllocator(VariableWidthVector in, VariableWidthVector out) {
- super();
- this.in = in;
- this.out = out;
- }
-
- public void alloc(int recordCount){
- out.allocateNew(in.getByteCapacity() * 4, recordCount);
- out.getMutator().setValueCount(recordCount);
- }
- }
-
- public interface VectorAllocator{
- public void alloc(int recordCount);
- }
-
}