You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/02 00:22:05 UTC

[04/18] git commit: fix parquet query join errors

fix parquet query join errors


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cc329aa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cc329aa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cc329aa4

Branch: refs/heads/master
Commit: cc329aa4418cf6c363f7da7363594f5174655fe1
Parents: d553749
Author: Ben Becker <be...@gmail.com>
Authored: Fri Aug 30 14:59:08 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../exec/physical/impl/join/JoinStatus.java     | 20 ++++----
 .../exec/physical/impl/join/JoinTemplate.java   |  2 +-
 .../exec/physical/impl/join/MergeJoinBatch.java | 51 ++------------------
 3 files changed, 15 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
index c755e5f..84a11fc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
@@ -112,14 +112,15 @@ public final class JoinStatus {
    * Side effect: advances to next left batch if current left batch size is exceeded.
    */
   public final boolean isLeftPositionAllowed(){
-    if(!isLeftPositionInCurrentBatch()){
+    if (lastLeft == IterOutcome.NONE)
+      return false;
+    if (!isLeftPositionInCurrentBatch()) {
       leftPosition = 0;
       lastLeft = left.next();
       return lastLeft == IterOutcome.OK;
-    }else{
-      lastLeft = IterOutcome.OK;
-      return true;
     }
+    lastLeft = IterOutcome.OK;
+    return true;
   }
 
   /**
@@ -129,16 +130,15 @@ public final class JoinStatus {
   public final boolean isRightPositionAllowed(){
     if (rightSourceMode == RightSourceMode.SV4)
       return svRightPosition < sv4.getCount();
-
-    if(!isRightPositionInCurrentBatch()){
+    if (lastRight == IterOutcome.NONE)
+      return false;
+    if (!isRightPositionInCurrentBatch()) {
       rightPosition = 0;
       lastRight = right.next();
       return lastRight == IterOutcome.OK;
-    }else{
-      lastRight = IterOutcome.OK;
-      return true;
     }
-    
+    lastRight = IterOutcome.OK;
+    return true;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index d6fc802..3d6dfce 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -119,7 +119,7 @@ public abstract class JoinTemplate implements JoinWorker {
           // If the left key has duplicates and we're about to cross a boundary in the right batch, add the
           // right table's record batch to the sv4 builder before calling next.  These records will need to be
           // copied again for each duplicate left key.
-          if (status.isLeftRepeating() && !status.isNextRightPositionInCurrentBatch()) {
+          if (status.isLeftRepeating() && !status.isRightPositionInCurrentBatch()) {
             status.outputBatch.addRightToBatchBuilder();
             crossedBatchBoundaries = true;
           }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 02a1961..5a62c1f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
 import org.apache.drill.exec.record.*;
 import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
 /**
  * A merge join combining to incoming in-order batches.
@@ -107,7 +108,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     
     // we do this in the here instead of the constructor because don't necessary want to start consuming on construction.
     status.ensureInitial();
-
+    
     // loop so we can start over again if we find a new batch was created.
     while(true){
 
@@ -367,13 +368,13 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     // add fields from both batches
     for (VectorWrapper<?> w : left) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(left.getRecordCount() * 16);
+      VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / left.getRecordCount())).alloc(left.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
     for (VectorWrapper<?> w : right) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(right.getRecordCount() * 16);
+      VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / right.getRecordCount())).alloc(right.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
@@ -381,48 +382,4 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     logger.debug("Built joined schema: {}", container.getSchema());
   }
 
-  private VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
-    if(outgoing instanceof FixedWidthVector){
-      return new FixedVectorAllocator((FixedWidthVector) outgoing);
-    }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
-      return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
-    }else{
-      throw new UnsupportedOperationException();
-    }
-  }
-
-  private class FixedVectorAllocator implements VectorAllocator{
-    FixedWidthVector out;
-
-    public FixedVectorAllocator(FixedWidthVector out) {
-      super();
-      this.out = out;
-    }
-
-    public void alloc(int recordCount){
-      out.allocateNew(recordCount);
-      out.getMutator().setValueCount(recordCount);
-    }
-  }
-
-  private class VariableVectorAllocator implements VectorAllocator{
-    VariableWidthVector in;
-    VariableWidthVector out;
-
-    public VariableVectorAllocator(VariableWidthVector in, VariableWidthVector out) {
-      super();
-      this.in = in;
-      this.out = out;
-    }
-
-    public void alloc(int recordCount){
-      out.allocateNew(in.getByteCapacity() * 4, recordCount);
-      out.getMutator().setValueCount(recordCount);
-    }
-  }
-
-  public interface VectorAllocator{
-    public void alloc(int recordCount);
-  }
-
 }