You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/01/17 00:14:55 UTC

[1/5] drill git commit: DRILL-1969: Improve the error message displayed when querying Hive views from Drill

Repository: drill
Updated Branches:
  refs/heads/master d60850bcc -> bd4d669d1


DRILL-1969: Improve the error message displayed when querying Hive views from Drill


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fe6f7efe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fe6f7efe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fe6f7efe

Branch: refs/heads/master
Commit: fe6f7efe75c97e78392231128b1a6a4971a262c5
Parents: d60850b
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 10:39:45 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:31 2015 -0800

----------------------------------------------------------------------
 .../java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java | 3 ++-
 .../main/java/org/apache/drill/exec/work/foreman/Foreman.java    | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/fe6f7efe/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index a3d4583..f4baf3b 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -71,7 +71,8 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
     HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
     try {
       if (hiveReadEntry.getJdbcTableType() == TableType.VIEW) {
-        throw new UnsupportedOperationException("Querying Hive views from Drill is not supported in current version.");
+        throw new UnsupportedOperationException(
+            "Querying views created in Hive from Drill is not supported in current version.");
       }
 
       return new HiveScan(hiveReadEntry, this, columns);

http://git-wip-us.apache.org/repos/asf/drill/blob/fe6f7efe/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index e10a6aa..fcf5b91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -28,6 +28,7 @@ import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.logical.LogicalPlan;
@@ -438,7 +439,8 @@ public class Foreman implements Runnable, Closeable, Comparable<Object> {
         assert exception != null;
         recordNewState(QueryState.FAILED);
         cancelExecutingFragments();
-        DrillPBError error = ErrorHelper.logAndConvertError(context.getCurrentEndpoint(), "Query failed: "+ exception.getMessage(), exception, logger);
+        DrillPBError error = ErrorHelper.logAndConvertError(context.getCurrentEndpoint(),
+            ExceptionUtils.getRootCauseMessage(exception), exception, logger);
         QueryResult result = QueryResult //
           .newBuilder() //
           .addError(error) //


[4/5] drill git commit: DRILL-1993: Fix allocation issues in HashTable and HashAgg to reduce memory waste

Posted by ve...@apache.org.
DRILL-1993: Fix allocation issues in HashTable and HashAgg to reduce memory waste


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bd4d669d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bd4d669d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bd4d669d

Branch: refs/heads/master
Commit: bd4d669d1b836d6990eb3701e81c56f3d109db18
Parents: 8cdab2e
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 01:41:53 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800

----------------------------------------------------------------------
 .../impl/aggregate/HashAggTemplate.java         | 40 +++++++++++++++++---
 .../physical/impl/common/ChainedHashTable.java  |  1 -
 .../exec/physical/impl/common/HashTable.java    |  3 ++
 .../physical/impl/common/HashTableTemplate.java | 28 ++++++++++----
 .../apache/drill/exec/vector/ObjectVector.java  |  6 +++
 5 files changed, 63 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index dc2b05c..2fd5ce1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -44,7 +44,10 @@ import org.apache.drill.exec.record.RecordBatch.IterOutcome;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.FixedWidthVector;
+import org.apache.drill.exec.vector.ObjectVector;
 import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
 import javax.inject.Named;
@@ -128,7 +131,24 @@ public abstract class HashAggTemplate implements HashAggregator {
         MaterializedField outputField = materializedValueFields[i];
         // Create a type-specific ValueVector for this value
         vector = TypeHelper.getNewVector(outputField, allocator);
-        vector.allocateNew();
+
+        // Try to allocate space to store BATCH_SIZE records. Key stored at index i in HashTable has its workspace
+        // variables (such as count, sum etc) stored at index i in HashAgg. HashTable and HashAgg both have
+        // BatchHolders. Whenever a BatchHolder in HashAgg reaches its capacity, a new BatchHolder is added to
+        // HashTable. If HashAgg can't store BATCH_SIZE records in a BatchHolder, it leaves empty slots in current
+        // BatchHolder in HashTable, causing the HashTable to be space inefficient. So it is better to allocate space
+        // to fit as close to as BATCH_SIZE records.
+        if (vector instanceof FixedWidthVector) {
+          ((FixedWidthVector) vector).allocateNew(HashTable.BATCH_SIZE);
+        } else if (vector instanceof VariableWidthVector) {
+          ((VariableWidthVector) vector).allocateNew(HashTable.VARIABLE_WIDTH_VECTOR_SIZE * HashTable.BATCH_SIZE,
+              HashTable.BATCH_SIZE);
+        } else if (vector instanceof ObjectVector) {
+          ((ObjectVector)vector).allocateNew(HashTable.BATCH_SIZE);
+        } else {
+          vector.allocateNew();
+        }
+
         capacity = Math.min(capacity, vector.getValueCapacity());
 
         aggrValuesContainer.add(vector);
@@ -149,10 +169,14 @@ public abstract class HashAggTemplate implements HashAggregator {
       outStartIdxHolder.value = batchOutputCount;
       outNumRecordsHolder.value = 0;
       boolean status = true;
-      for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
-        if (outputRecordValues(i, batchOutputCount)) {
+
+      // Output records starting from 'batchOutputCount' in current batch until there are no more records
+      // or output vectors have no space left. In destination vectors, start filling records from 0th position.
+      while(batchOutputCount <= maxOccupiedIdx) {
+        if (outputRecordValues(batchOutputCount, outNumRecordsHolder.value)) {
           if (EXTRA_DEBUG_2) {
-            logger.debug("Outputting values to output index: {}", batchOutputCount);
+            logger.debug("Outputting values from input index {} to output index: {}",
+                batchOutputCount, outNumRecordsHolder.value);
           }
           batchOutputCount++;
           outNumRecordsHolder.value++;
@@ -256,7 +280,7 @@ public abstract class HashAggTemplate implements HashAggregator {
 
     numGroupByOutFields = groupByOutFieldIds.length;
     batchHolders = new ArrayList<BatchHolder>();
-    addBatchHolder();
+    // First BatchHolder is created when the first put request is received.
 
     doSetup(incoming);
   }
@@ -494,7 +518,11 @@ public abstract class HashAggTemplate implements HashAggregator {
       logger.debug("HashAggregate: Output current batch index {} with {} records.", outBatchIndex, numOutputRecords);
 
       lastBatchOutputCount = numOutputRecords;
-      outBatchIndex++;
+      // If there are no more records to output, go to the next batch. If there are any records left refer to the
+      // same BatchHolder. Remaining records will be outputted in next outputCurrentBatch() call(s).
+      if (batchHolders.get(outBatchIndex).getNumPendingOutput() == 0) {
+        outBatchIndex++;
+      }
       if (outBatchIndex == batchHolders.size()) {
         allFlushed = true;
 

http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 322fd1f..b5cfdca 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -163,7 +163,6 @@ public class ChainedHashTable {
       final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
       // create a type-specific ValueVector for this key
       ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
-      vv.allocateNew();
       htKeyFieldIds[i] = htContainerOrig.add(vv);
 
       i++;

http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index 6966ba1..1ec74bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -51,6 +51,9 @@ public interface HashTable {
   static final public int BATCH_SIZE = Character.MAX_VALUE + 1;
   static final public int BATCH_MASK = 0x0000FFFF;
 
+  /** Variable width vector size in bytes */
+  public static final int VARIABLE_WIDTH_VECTOR_SIZE = 50 * BATCH_SIZE;
+
   public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
       RecordBatch incomingBuild, RecordBatch incomingProbe,
       RecordBatch outgoing, VectorContainer htContainerOrig);

http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index c80e97a..ba980d7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -30,8 +30,10 @@ import org.apache.drill.exec.record.TransferPair;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.FixedWidthVector;
 import org.apache.drill.exec.vector.IntVector;
 import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
 
 import javax.inject.Named;
 import java.util.ArrayList;
@@ -116,15 +118,24 @@ public abstract class HashTableTemplate implements HashTable {
     private BatchHolder(int idx) {
       this.batchIndex = idx;
 
-      if (idx == 0) {  // first batch holder can use the original htContainer
-        htContainer = htContainerOrig;
-      } else { // otherwise create a new one using the original's fields
-        htContainer = new VectorContainer();
-        for (VectorWrapper<?> w : htContainerOrig) {
-          ValueVector vv = TypeHelper.getNewVector(w.getField(), allocator);
+      htContainer = new VectorContainer();
+      for (VectorWrapper<?> w : htContainerOrig) {
+        ValueVector vv = TypeHelper.getNewVector(w.getField(), allocator);
+
+        // Capacity for "hashValues" and "links" vectors is BATCH_SIZE records. It is better to allocate space for
+        // "key" vectors to store as close to as BATCH_SIZE records. A new BatchHolder is created when either BATCH_SIZE
+        // records are inserted or "key" vectors ran out of space. Allocating too less space for "key" vectors will
+        // result in unused space in "hashValues" and "links" vectors in the BatchHolder. Also for each new
+        // BatchHolder we create a SV4 vector of BATCH_SIZE in HashJoinHelper.
+        if (vv instanceof FixedWidthVector) {
+          ((FixedWidthVector) vv).allocateNew(BATCH_SIZE);
+        } else if (vv instanceof VariableWidthVector) {
+          ((VariableWidthVector) vv).allocateNew(VARIABLE_WIDTH_VECTOR_SIZE, BATCH_SIZE);
+        } else {
           vv.allocateNew();
-          htContainer.add(vv);
         }
+
+        htContainer.add(vv);
       }
 
       links = allocMetadataVector(HashTable.BATCH_SIZE, EMPTY_SLOT);
@@ -454,7 +465,7 @@ public abstract class HashTableTemplate implements HashTable {
 
     // Create the first batch holder
     batchHolders = new ArrayList<BatchHolder>();
-    addBatchHolder();
+    // First BatchHolder is created when the first put request is received.
 
     doSetup(incomingBuild, incomingProbe);
 
@@ -753,6 +764,7 @@ public abstract class HashTableTemplate implements HashTable {
 
   public boolean outputKeys(int batchIdx, VectorContainer outContainer, int outStartIndex, int numRecords) {
     assert batchIdx < batchHolders.size();
+
     if (!batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
       return false;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index b68f089..3c15db3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -97,6 +97,12 @@ public class ObjectVector extends BaseValueVector{
     addNewArray();
   }
 
+  public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
+    while (maxCount < valueCount) {
+      addNewArray();
+    }
+  }
+
   @Override
   public boolean allocateNewSafe() {
     allocateNew();


[5/5] drill git commit: DRILL-1990: Add peak memory allocation in a operator to OperatorStats.

Posted by ve...@apache.org.
DRILL-1990: Add peak memory allocation in a operator to OperatorStats.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8cdab2ed
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8cdab2ed
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8cdab2ed

Branch: refs/heads/master
Commit: 8cdab2ed7d4c0d57ea668f6387281acbc26f4890
Parents: 10fd9e1
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Dec 30 17:13:35 2014 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800

----------------------------------------------------------------------
 .../java/io/netty/buffer/FakeAllocator.java     |   5 +
 .../org/apache/drill/exec/memory/Accountor.java |  16 ++-
 .../drill/exec/memory/BufferAllocator.java      |   2 +
 .../drill/exec/memory/TopLevelAllocator.java    |  10 ++
 .../apache/drill/exec/ops/OperatorStats.java    |   2 +-
 .../drill/exec/proto/SchemaUserBitShared.java   |  10 +-
 .../apache/drill/exec/proto/UserBitShared.java  | 134 +++++++++----------
 .../drill/exec/proto/beans/OperatorProfile.java |  22 +--
 protocol/src/main/protobuf/UserBitShared.proto  |   2 +-
 9 files changed, 116 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
index df85e74..3de0a75 100644
--- a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
+++ b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -91,6 +91,11 @@ class FakeAllocator implements BufferAllocator {
     return 0;
   }
 
+  @Override
+  public long getPeakMemoryAllocation() {
+    return 0;
+  }
+
   static class FakeAccountor extends Accountor {
 
     public FakeAccountor() {

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
index 0a8c405..2b48ef0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -62,6 +62,8 @@ public class Accountor {
   private final FragmentContext fragmentContext;
   long fragmentLimit;
 
+  private long peakMemoryAllocation = 0;
+
   // The top level Allocator has an accountor that keeps track of all the FragmentContexts currently executing.
   // This enables the top level accountor to calculate a new fragment limit whenever necessary.
   private final List<FragmentContext> fragmentContexts;
@@ -132,14 +134,22 @@ public class Accountor {
     return remainder.getUsed();
   }
 
+  public long getPeakMemoryAllocation() {
+    return peakMemoryAllocation;
+  }
+
   public boolean reserve(long size) {
     logger.trace("Fragment:"+fragmentStr+" Reserved "+size+" bytes. Total Allocated: "+getAllocation());
-    return remainder.get(size, this.applyFragmentLimit);
+    boolean status = remainder.get(size, this.applyFragmentLimit);
+    peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+    return status;
   }
 
   public boolean forceAdditionalReservation(long size) {
     if (size > 0) {
-      return remainder.forceGet(size);
+      boolean status = remainder.forceGet(size);
+      peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+      return status;
     } else {
       return true;
     }
@@ -156,6 +166,8 @@ public class Accountor {
     if (ENABLE_ACCOUNTING) {
       buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
     }
+
+    peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
   }
 
 

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
index e072126..83d9d1e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -108,5 +108,7 @@ public interface BufferAllocator extends Closeable {
 
   public abstract long getAllocatedMemory();
 
+  public abstract long getPeakMemoryAllocation();
+
   public DrillBuf getEmpty();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 32ec37f..67e1fdb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -99,6 +99,11 @@ public class TopLevelAllocator implements BufferAllocator {
   }
 
   @Override
+  public long getPeakMemoryAllocation() {
+    return acct.getPeakMemoryAllocation();
+  }
+
+  @Override
   public ByteBufAllocator getUnderlyingAllocator() {
     return innerAllocator;
   }
@@ -290,6 +295,11 @@ public class TopLevelAllocator implements BufferAllocator {
     }
 
     @Override
+    public long getPeakMemoryAllocation() {
+      return childAcct.getPeakMemoryAllocation();
+    }
+
+    @Override
     public DrillBuf getEmpty() {
       return empty;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 8771ab8..0e9da0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -129,7 +129,7 @@ public class OperatorStats {
         .setWaitNanos(waitNanos);
 
     if(allocator != null){
-      b.setLocalMemoryAllocated(allocator.getAllocatedMemory());
+      b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation());
     }
 
 

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
index fdd003e..68e86db 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
@@ -2175,8 +2175,8 @@ public final class SchemaUserBitShared
                     output.writeInt64(5, message.getSetupNanos(), false);
                 if(message.hasProcessNanos())
                     output.writeInt64(6, message.getProcessNanos(), false);
-                if(message.hasLocalMemoryAllocated())
-                    output.writeInt64(7, message.getLocalMemoryAllocated(), false);
+                if(message.hasPeakLocalMemoryAllocated())
+                    output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false);
                 for(org.apache.drill.exec.proto.UserBitShared.MetricValue metric : message.getMetricList())
                     output.writeObject(8, metric, org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true);
 
@@ -2238,7 +2238,7 @@ public final class SchemaUserBitShared
                             builder.setProcessNanos(input.readInt64());
                             break;
                         case 7:
-                            builder.setLocalMemoryAllocated(input.readInt64());
+                            builder.setPeakLocalMemoryAllocated(input.readInt64());
                             break;
                         case 8:
                             builder.addMetric(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.MetricValue.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.MERGE));
@@ -2292,7 +2292,7 @@ public final class SchemaUserBitShared
                 case 4: return "operatorType";
                 case 5: return "setupNanos";
                 case 6: return "processNanos";
-                case 7: return "localMemoryAllocated";
+                case 7: return "peakLocalMemoryAllocated";
                 case 8: return "metric";
                 case 9: return "waitNanos";
                 default: return null;
@@ -2311,7 +2311,7 @@ public final class SchemaUserBitShared
             fieldMap.put("operatorType", 4);
             fieldMap.put("setupNanos", 5);
             fieldMap.put("processNanos", 6);
-            fieldMap.put("localMemoryAllocated", 7);
+            fieldMap.put("peakLocalMemoryAllocated", 7);
             fieldMap.put("metric", 8);
             fieldMap.put("waitNanos", 9);
         }

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index e0666f2..5682a3b 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -17137,15 +17137,15 @@ public final class UserBitShared {
      */
     long getProcessNanos();
 
-    // optional int64 local_memory_allocated = 7;
+    // optional int64 peak_local_memory_allocated = 7;
     /**
-     * <code>optional int64 local_memory_allocated = 7;</code>
+     * <code>optional int64 peak_local_memory_allocated = 7;</code>
      */
-    boolean hasLocalMemoryAllocated();
+    boolean hasPeakLocalMemoryAllocated();
     /**
-     * <code>optional int64 local_memory_allocated = 7;</code>
+     * <code>optional int64 peak_local_memory_allocated = 7;</code>
      */
-    long getLocalMemoryAllocated();
+    long getPeakLocalMemoryAllocated();
 
     // repeated .exec.shared.MetricValue metric = 8;
     /**
@@ -17263,7 +17263,7 @@ public final class UserBitShared {
             }
             case 56: {
               bitField0_ |= 0x00000010;
-              localMemoryAllocated_ = input.readInt64();
+              peakLocalMemoryAllocated_ = input.readInt64();
               break;
             }
             case 66: {
@@ -17425,20 +17425,20 @@ public final class UserBitShared {
       return processNanos_;
     }
 
-    // optional int64 local_memory_allocated = 7;
-    public static final int LOCAL_MEMORY_ALLOCATED_FIELD_NUMBER = 7;
-    private long localMemoryAllocated_;
+    // optional int64 peak_local_memory_allocated = 7;
+    public static final int PEAK_LOCAL_MEMORY_ALLOCATED_FIELD_NUMBER = 7;
+    private long peakLocalMemoryAllocated_;
     /**
-     * <code>optional int64 local_memory_allocated = 7;</code>
+     * <code>optional int64 peak_local_memory_allocated = 7;</code>
      */
-    public boolean hasLocalMemoryAllocated() {
+    public boolean hasPeakLocalMemoryAllocated() {
       return ((bitField0_ & 0x00000010) == 0x00000010);
     }
     /**
-     * <code>optional int64 local_memory_allocated = 7;</code>
+     * <code>optional int64 peak_local_memory_allocated = 7;</code>
      */
-    public long getLocalMemoryAllocated() {
-      return localMemoryAllocated_;
+    public long getPeakLocalMemoryAllocated() {
+      return peakLocalMemoryAllocated_;
     }
 
     // repeated .exec.shared.MetricValue metric = 8;
@@ -17499,7 +17499,7 @@ public final class UserBitShared {
       operatorType_ = 0;
       setupNanos_ = 0L;
       processNanos_ = 0L;
-      localMemoryAllocated_ = 0L;
+      peakLocalMemoryAllocated_ = 0L;
       metric_ = java.util.Collections.emptyList();
       waitNanos_ = 0L;
     }
@@ -17531,7 +17531,7 @@ public final class UserBitShared {
         output.writeInt64(6, processNanos_);
       }
       if (((bitField0_ & 0x00000010) == 0x00000010)) {
-        output.writeInt64(7, localMemoryAllocated_);
+        output.writeInt64(7, peakLocalMemoryAllocated_);
       }
       for (int i = 0; i < metric_.size(); i++) {
         output.writeMessage(8, metric_.get(i));
@@ -17570,7 +17570,7 @@ public final class UserBitShared {
       }
       if (((bitField0_ & 0x00000010) == 0x00000010)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeInt64Size(7, localMemoryAllocated_);
+          .computeInt64Size(7, peakLocalMemoryAllocated_);
       }
       for (int i = 0; i < metric_.size(); i++) {
         size += com.google.protobuf.CodedOutputStream
@@ -17712,7 +17712,7 @@ public final class UserBitShared {
         bitField0_ = (bitField0_ & ~0x00000008);
         processNanos_ = 0L;
         bitField0_ = (bitField0_ & ~0x00000010);
-        localMemoryAllocated_ = 0L;
+        peakLocalMemoryAllocated_ = 0L;
         bitField0_ = (bitField0_ & ~0x00000020);
         if (metricBuilder_ == null) {
           metric_ = java.util.Collections.emptyList();
@@ -17778,7 +17778,7 @@ public final class UserBitShared {
         if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
           to_bitField0_ |= 0x00000010;
         }
-        result.localMemoryAllocated_ = localMemoryAllocated_;
+        result.peakLocalMemoryAllocated_ = peakLocalMemoryAllocated_;
         if (metricBuilder_ == null) {
           if (((bitField0_ & 0x00000040) == 0x00000040)) {
             metric_ = java.util.Collections.unmodifiableList(metric_);
@@ -17846,8 +17846,8 @@ public final class UserBitShared {
         if (other.hasProcessNanos()) {
           setProcessNanos(other.getProcessNanos());
         }
-        if (other.hasLocalMemoryAllocated()) {
-          setLocalMemoryAllocated(other.getLocalMemoryAllocated());
+        if (other.hasPeakLocalMemoryAllocated()) {
+          setPeakLocalMemoryAllocated(other.getPeakLocalMemoryAllocated());
         }
         if (metricBuilder_ == null) {
           if (!other.metric_.isEmpty()) {
@@ -18277,35 +18277,35 @@ public final class UserBitShared {
         return this;
       }
 
-      // optional int64 local_memory_allocated = 7;
-      private long localMemoryAllocated_ ;
+      // optional int64 peak_local_memory_allocated = 7;
+      private long peakLocalMemoryAllocated_ ;
       /**
-       * <code>optional int64 local_memory_allocated = 7;</code>
+       * <code>optional int64 peak_local_memory_allocated = 7;</code>
        */
-      public boolean hasLocalMemoryAllocated() {
+      public boolean hasPeakLocalMemoryAllocated() {
         return ((bitField0_ & 0x00000020) == 0x00000020);
       }
       /**
-       * <code>optional int64 local_memory_allocated = 7;</code>
+       * <code>optional int64 peak_local_memory_allocated = 7;</code>
        */
-      public long getLocalMemoryAllocated() {
-        return localMemoryAllocated_;
+      public long getPeakLocalMemoryAllocated() {
+        return peakLocalMemoryAllocated_;
       }
       /**
-       * <code>optional int64 local_memory_allocated = 7;</code>
+       * <code>optional int64 peak_local_memory_allocated = 7;</code>
        */
-      public Builder setLocalMemoryAllocated(long value) {
+      public Builder setPeakLocalMemoryAllocated(long value) {
         bitField0_ |= 0x00000020;
-        localMemoryAllocated_ = value;
+        peakLocalMemoryAllocated_ = value;
         onChanged();
         return this;
       }
       /**
-       * <code>optional int64 local_memory_allocated = 7;</code>
+       * <code>optional int64 peak_local_memory_allocated = 7;</code>
        */
-      public Builder clearLocalMemoryAllocated() {
+      public Builder clearPeakLocalMemoryAllocated() {
         bitField0_ = (bitField0_ & ~0x00000020);
-        localMemoryAllocated_ = 0L;
+        peakLocalMemoryAllocated_ = 0L;
         onChanged();
         return this;
       }
@@ -19877,41 +19877,41 @@ public final class UserBitShared {
       ".exec.shared.OperatorProfile\022\022\n\nstart_ti" +
       "me\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_use" +
       "d\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010endp" +
-      "oint\030\t \001(\0132\026.exec.DrillbitEndpoint\"\372\001\n\017O" +
+      "oint\030\t \001(\0132\026.exec.DrillbitEndpoint\"\377\001\n\017O" +
       "peratorProfile\0221\n\rinput_profile\030\001 \003(\0132\032.",
       "exec.shared.StreamProfile\022\023\n\013operator_id" +
       "\030\003 \001(\005\022\025\n\roperator_type\030\004 \001(\005\022\023\n\013setup_n" +
-      "anos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022\036\n\026loc" +
-      "al_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(" +
-      "\0132\030.exec.shared.MetricValue\022\022\n\nwait_nano" +
-      "s\030\t \001(\003\"B\n\rStreamProfile\022\017\n\007records\030\001 \001(" +
-      "\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013M" +
-      "etricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong_va" +
-      "lue\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\nRpcCh" +
-      "annel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004",
-      "USER\020\002*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020" +
-      "\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmentState\022\013\n\007SEND" +
-      "ING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNIN" +
-      "G\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAIL" +
-      "ED\020\005*\264\005\n\020CoreOperatorType\022\021\n\rSINGLE_SEND" +
-      "ER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTER\020\002\022\022" +
-      "\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMER" +
-      "GE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDER\020\006\022\t\n\005" +
-      "LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030ORDERED" +
-      "_PARTITION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n\022UNOR",
-      "DERED_RECEIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022\n\n\006SC" +
-      "REEN\020\r\022\034\n\030SELECTION_VECTOR_REMOVER\020\016\022\027\n\023" +
-      "STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N_SORT\020\020\022\021\n" +
-      "\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNION\020\023\022\014" +
-      "\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GROUP_SCAN\020\025" +
-      "\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE_SCAN" +
-      "\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQUET_WRITER\020" +
-      "\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT_WRITER\020\033\022" +
-      "\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020\035\022\030\n" +
-      "\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COMPLEX_TO_JS",
-      "ON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n\016HBASE_SUB" +
-      "_SCAN\020!\022\n\n\006WINDOW\020\"B.\n\033org.apache.drill." +
-      "exec.protoB\rUserBitSharedH\001"
+      "anos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033pea" +
+      "k_local_memory_allocated\030\007 \001(\003\022(\n\006metric" +
+      "\030\010 \003(\0132\030.exec.shared.MetricValue\022\022\n\nwait" +
+      "_nanos\030\t \001(\003\"B\n\rStreamProfile\022\017\n\007records" +
+      "\030\001 \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003" +
+      "\"J\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlo" +
+      "ng_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\n" +
+      "RpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020",
+      "\001\022\010\n\004USER\020\002*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOG" +
+      "ICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmentState\022\013\n" +
+      "\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007R" +
+      "UNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n" +
+      "\006FAILED\020\005*\264\005\n\020CoreOperatorType\022\021\n\rSINGLE" +
+      "_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTE" +
+      "R\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016" +
+      "\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDER\020" +
+      "\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030OR" +
+      "DERED_PARTITION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n",
+      "\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022" +
+      "\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_REMOVER\020" +
+      "\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N_SORT" +
+      "\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNIO" +
+      "N\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GROUP_S" +
+      "CAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE" +
+      "_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQUET_WR" +
+      "ITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT_WRIT" +
+      "ER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN" +
+      "\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COMPLEX_",
+      "TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n\016HBAS" +
+      "E_SUB_SCAN\020!\022\n\n\006WINDOW\020\"B.\n\033org.apache.d" +
+      "rill.exec.protoB\rUserBitSharedH\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -20013,7 +20013,7 @@ public final class UserBitShared {
           internal_static_exec_shared_OperatorProfile_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_exec_shared_OperatorProfile_descriptor,
-              new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "LocalMemoryAllocated", "Metric", "WaitNanos", });
+              new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "PeakLocalMemoryAllocated", "Metric", "WaitNanos", });
           internal_static_exec_shared_StreamProfile_descriptor =
             getDescriptor().getMessageTypes().get(16);
           internal_static_exec_shared_StreamProfile_fieldAccessorTable = new

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
index f1b1acc..e197f54 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
@@ -54,7 +54,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
     private int operatorType;
     private long setupNanos;
     private long processNanos;
-    private long localMemoryAllocated;
+    private long peakLocalMemoryAllocated;
     private List<MetricValue> metric;
     private long waitNanos;
 
@@ -130,16 +130,16 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
         return this;
     }
 
-    // localMemoryAllocated
+    // peakLocalMemoryAllocated
 
-    public long getLocalMemoryAllocated()
+    public long getPeakLocalMemoryAllocated()
     {
-        return localMemoryAllocated;
+        return peakLocalMemoryAllocated;
     }
 
-    public OperatorProfile setLocalMemoryAllocated(long localMemoryAllocated)
+    public OperatorProfile setPeakLocalMemoryAllocated(long peakLocalMemoryAllocated)
     {
-        this.localMemoryAllocated = localMemoryAllocated;
+        this.peakLocalMemoryAllocated = peakLocalMemoryAllocated;
         return this;
     }
 
@@ -242,7 +242,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
                     message.processNanos = input.readInt64();
                     break;
                 case 7:
-                    message.localMemoryAllocated = input.readInt64();
+                    message.peakLocalMemoryAllocated = input.readInt64();
                     break;
                 case 8:
                     if(message.metric == null)
@@ -284,8 +284,8 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
         if(message.processNanos != 0)
             output.writeInt64(6, message.processNanos, false);
 
-        if(message.localMemoryAllocated != 0)
-            output.writeInt64(7, message.localMemoryAllocated, false);
+        if(message.peakLocalMemoryAllocated != 0)
+            output.writeInt64(7, message.peakLocalMemoryAllocated, false);
 
         if(message.metric != null)
         {
@@ -310,7 +310,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
             case 4: return "operatorType";
             case 5: return "setupNanos";
             case 6: return "processNanos";
-            case 7: return "localMemoryAllocated";
+            case 7: return "peakLocalMemoryAllocated";
             case 8: return "metric";
             case 9: return "waitNanos";
             default: return null;
@@ -331,7 +331,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
         __fieldMap.put("operatorType", 4);
         __fieldMap.put("setupNanos", 5);
         __fieldMap.put("processNanos", 6);
-        __fieldMap.put("localMemoryAllocated", 7);
+        __fieldMap.put("peakLocalMemoryAllocated", 7);
         __fieldMap.put("metric", 8);
         __fieldMap.put("waitNanos", 9);
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 3b85e9b..8f05c45 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -164,7 +164,7 @@ message OperatorProfile {
   optional int32 operator_type = 4;
   optional int64 setup_nanos = 5;
   optional int64 process_nanos = 6;
-  optional int64 local_memory_allocated = 7;
+  optional int64 peak_local_memory_allocated = 7;
   repeated MetricValue metric = 8;
   optional int64 wait_nanos = 9;
 }


[3/5] drill git commit: DRILL-1991: Code indendation and formatting cleanup for few files

Posted by ve...@apache.org.
DRILL-1991: Code indendation and formatting cleanup for few files

+ Reformatted using IDE except removing two unused static variables.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/10fd9e10
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/10fd9e10
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/10fd9e10

Branch: refs/heads/master
Commit: 10fd9e101207d5d861f023e45fb46c0787bf6da0
Parents: fe6f7ef
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 00:02:47 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800

----------------------------------------------------------------------
 .../physical/impl/aggregate/HashAggBatch.java   |  77 +-
 .../impl/aggregate/HashAggTemplate.java         | 110 +--
 .../physical/impl/aggregate/HashAggregator.java |  18 +-
 .../physical/impl/common/ChainedHashTable.java  | 166 +++--
 .../exec/physical/impl/common/HashTable.java    |  27 +-
 .../physical/impl/common/HashTableTemplate.java | 160 +++--
 .../exec/physical/impl/join/HashJoinBatch.java  | 705 +++++++++----------
 7 files changed, 654 insertions(+), 609 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index 113e883..35faf22 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -17,8 +17,8 @@
  */
 package org.apache.drill.exec.physical.impl.aggregate;
 
-import java.io.IOException;
-
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JVar;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
@@ -51,8 +51,7 @@ import org.apache.drill.exec.record.selection.SelectionVector2;
 import org.apache.drill.exec.record.selection.SelectionVector4;
 import org.apache.drill.exec.vector.ValueVector;
 
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JVar;
+import java.io.IOException;
 
 public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggBatch.class);
@@ -60,21 +59,25 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
   private HashAggregator aggregator;
   private final RecordBatch incoming;
   private LogicalExpression[] aggrExprs;
-  private TypedFieldId[] groupByOutFieldIds ;
-  private TypedFieldId[] aggrOutFieldIds ;      // field ids for the outgoing batch
+  private TypedFieldId[] groupByOutFieldIds;
+  private TypedFieldId[] aggrOutFieldIds;      // field ids for the outgoing batch
 
   private final GeneratorMapping UPDATE_AGGR_INSIDE =
-    GeneratorMapping.create("setupInterior" /* setup method */, "updateAggrValuesInternal" /* eval method */,
-                            "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+      GeneratorMapping.create("setupInterior" /* setup method */, "updateAggrValuesInternal" /* eval method */,
+          "resetValues" /* reset */, "cleanup" /* cleanup */);
 
   private final GeneratorMapping UPDATE_AGGR_OUTSIDE =
-    GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordValues" /* eval method */,
-                            "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+      GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordValues" /* eval method */,
+          "resetValues" /* reset */, "cleanup" /* cleanup */);
 
-  private final MappingSet UpdateAggrValuesMapping = new MappingSet("incomingRowIdx" /* read index */, "outRowIdx" /* write index */, "htRowIdx" /* workspace index */, "incoming" /* read container */, "outgoing" /* write container */, "aggrValuesContainer" /* workspace container */, UPDATE_AGGR_INSIDE, UPDATE_AGGR_OUTSIDE, UPDATE_AGGR_INSIDE);
+  private final MappingSet UpdateAggrValuesMapping =
+      new MappingSet("incomingRowIdx" /* read index */, "outRowIdx" /* write index */,
+          "htRowIdx" /* workspace index */, "incoming" /* read container */, "outgoing" /* write container */,
+          "aggrValuesContainer" /* workspace container */, UPDATE_AGGR_INSIDE, UPDATE_AGGR_OUTSIDE, UPDATE_AGGR_INSIDE);
 
 
-  public HashAggBatch(HashAggregate popConfig, RecordBatch incoming, FragmentContext context) throws ExecutionSetupException {
+  public HashAggBatch(HashAggregate popConfig, RecordBatch incoming, FragmentContext context) throws
+      ExecutionSetupException {
     super(popConfig, context);
     this.incoming = incoming;
   }
@@ -122,7 +125,7 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
       logger.debug("Next outcome of {}", outcome);
       switch (outcome) {
       case NONE:
-//        throw new UnsupportedOperationException("Received NONE on first batch");
+        //        throw new UnsupportedOperationException("Received NONE on first batch");
         return outcome;
       case NOT_YET:
       case STOP:
@@ -144,13 +147,13 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
       return IterOutcome.NONE;
     }
 
-  if (aggregator.buildComplete() && ! aggregator.allFlushed()) {
-    // aggregation is complete and not all records have been output yet
-    IterOutcome outcome = aggregator.outputCurrentBatch();
-    return outcome;
-  }
+    if (aggregator.buildComplete() && !aggregator.allFlushed()) {
+      // aggregation is complete and not all records have been output yet
+      IterOutcome outcome = aggregator.outputCurrentBatch();
+      return outcome;
+    }
 
-  logger.debug("Starting aggregator doWork; incoming record count = {} ", incoming.getRecordCount());
+    logger.debug("Starting aggregator doWork; incoming record count = {} ", incoming.getRecordCount());
 
     while (true) {
       AggOutcome out = aggregator.doWork();
@@ -198,8 +201,10 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
     }
   }
 
-  private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
-    CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+  private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException,
+      IOException {
+    CodeGenerator<HashAggregator> top =
+        CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry());
     ClassGenerator<HashAggregator> cg = top.getRoot();
     ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
 
@@ -217,7 +222,8 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
 
     for (i = 0; i < numGroupByExprs; i++) {
       NamedExpression ne = popConfig.getGroupByExprs()[i];
-      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+      final LogicalExpression expr =
+          ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
       if (expr == null) {
         continue;
       }
@@ -231,7 +237,8 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
 
     for (i = 0; i < numAggrExprs; i++) {
       NamedExpression ne = popConfig.getAggrExprs()[i];
-      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+      final LogicalExpression expr =
+          ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
 
       if (collector.hasErrors()) {
         throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
@@ -255,17 +262,12 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
     container.buildSchema(SelectionVectorMode.NONE);
     HashAggregator agg = context.getImplementationClass(top);
 
-    HashTableConfig htConfig = new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
-                                                   HashTable.DEFAULT_LOAD_FACTOR,
-                                                   popConfig.getGroupByExprs(),
-                                                   null /* no probe exprs */) ;
+    HashTableConfig htConfig =
+        new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
+            HashTable.DEFAULT_LOAD_FACTOR, popConfig.getGroupByExprs(), null /* no probe exprs */);
 
-    agg.setup(popConfig, htConfig, context, this.stats,
-              oContext.getAllocator(), incoming, this,
-              aggrExprs,
-              cgInner.getWorkspaceTypes(),
-              groupByOutFieldIds,
-              this.container);
+    agg.setup(popConfig, htConfig, context, this.stats, oContext.getAllocator(), incoming, this, aggrExprs,
+        cgInner.getWorkspaceTypes(), groupByOutFieldIds, this.container);
 
     return agg;
   }
@@ -286,25 +288,23 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
     case FOUR_BYTE: {
       JVar var = cg.declareClassField("sv4_", cg.getModel()._ref(SelectionVector4.class));
       cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
-      cg.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+      cg.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));
       return;
     }
     case NONE: {
-      cg.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+      cg.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));
       return;
     }
     case TWO_BYTE: {
       JVar var = cg.declareClassField("sv2_", cg.getModel()._ref(SelectionVector2.class));
       cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
-      cg.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));;
+      cg.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));
       return;
     }
 
     default:
       throw new IllegalStateException();
-
     }
-
   }
 
   @Override
@@ -320,5 +320,4 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
   protected void killIncoming(boolean sendUpstream) {
     incoming.kill(sendUpstream);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index d7cf904..dc2b05c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -17,13 +17,7 @@
  */
 package org.apache.drill.exec.physical.impl.aggregate;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.inject.Named;
-
+import com.google.common.collect.Lists;
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.ExpressionPosition;
@@ -53,17 +47,19 @@ import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
-import com.google.common.collect.Lists;
+import javax.inject.Named;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 public abstract class HashAggTemplate implements HashAggregator {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggregator.class);
 
-  private static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024;
-  private static final long ALLOCATOR_MAX_RESERVATION = 20L*1000*1000*1000;
-
   private static final boolean EXTRA_DEBUG_1 = false;
   private static final boolean EXTRA_DEBUG_2 = false;
-  private static final String TOO_BIG_ERROR = "Couldn't add value to an empty batch.  This likely means that a single value is too long for a varlen field.";
+  private static final String TOO_BIG_ERROR =
+      "Couldn't add value to an empty batch.  This likely means that a single value is too long for a varlen field.";
   private boolean first = true;
   private boolean newSchema = false;
   private int underlyingIndex = 0;
@@ -93,7 +89,7 @@ public abstract class HashAggTemplate implements HashAggregator {
 
   private MaterializedField[] materializedValueFields;
   private boolean allFlushed = false;
-  private boolean  buildComplete = false;
+  private boolean buildComplete = false;
 
   private OperatorStats stats = null;
   private HashTableStats htStats = new HashTableStats();
@@ -113,7 +109,6 @@ public abstract class HashAggTemplate implements HashAggregator {
     }
   }
 
-
   public class BatchHolder {
 
     private VectorContainer aggrValuesContainer; // container for aggr values (workspace variables)
@@ -127,16 +122,16 @@ public abstract class HashAggTemplate implements HashAggregator {
 
       aggrValuesContainer = new VectorContainer();
 
-      ValueVector vector ;
+      ValueVector vector;
 
-      for(int i = 0; i < materializedValueFields.length; i++) {
+      for (int i = 0; i < materializedValueFields.length; i++) {
         MaterializedField outputField = materializedValueFields[i];
         // Create a type-specific ValueVector for this value
-        vector = TypeHelper.getNewVector(outputField, allocator) ;
+        vector = TypeHelper.getNewVector(outputField, allocator);
         vector.allocateNew();
         capacity = Math.min(capacity, vector.getValueCapacity());
 
-        aggrValuesContainer.add(vector) ;
+        aggrValuesContainer.add(vector);
       }
     }
 
@@ -155,9 +150,9 @@ public abstract class HashAggTemplate implements HashAggregator {
       outNumRecordsHolder.value = 0;
       boolean status = true;
       for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
-        if (outputRecordValues(i, batchOutputCount) ) {
+        if (outputRecordValues(i, batchOutputCount)) {
           if (EXTRA_DEBUG_2) {
-            logger.debug("Outputting values to output index: {}", batchOutputCount) ;
+            logger.debug("Outputting values to output index: {}", batchOutputCount);
           }
           batchOutputCount++;
           outNumRecordsHolder.value++;
@@ -190,26 +185,26 @@ public abstract class HashAggTemplate implements HashAggregator {
     // Code-generated methods (implemented in HashAggBatch)
 
     @RuntimeOverridden
-    public void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing, @Named("aggrValuesContainer") VectorContainer aggrValuesContainer) {}
+    public void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing,
+        @Named("aggrValuesContainer") VectorContainer aggrValuesContainer) {
+    }
 
     @RuntimeOverridden
-    public void updateAggrValuesInternal(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {}
+    public void updateAggrValuesInternal(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+    }
 
     @RuntimeOverridden
-    public boolean outputRecordValues(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return true;}
+    public boolean outputRecordValues(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {
+      return true;
+    }
   }
 
 
   @Override
-  public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig,
-                    FragmentContext context,
-                    OperatorStats stats,
-                    BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
-                    LogicalExpression[] valueExprs,
-                    List<TypedFieldId> valueFieldIds,
-                    TypedFieldId[] groupByOutFieldIds,
-                    VectorContainer outContainer)
-    throws SchemaChangeException, ClassTransformationException, IOException {
+  public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig, FragmentContext context,
+      OperatorStats stats, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
+      LogicalExpression[] valueExprs, List<TypedFieldId> valueFieldIds, TypedFieldId[] groupByOutFieldIds,
+      VectorContainer outContainer) throws SchemaChangeException, ClassTransformationException, IOException {
 
     if (valueExprs == null || valueFieldIds == null) {
       throw new IllegalArgumentException("Invalid aggr value exprs or workspace variables.");
@@ -235,7 +230,8 @@ public abstract class HashAggTemplate implements HashAggregator {
     // we need to build a hash table on the aggregation column a1.
     // TODO:  This functionality will be added later.
     if (hashAggrConfig.getGroupByExprs().length == 0) {
-      throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by expressions.");
+      throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by " +
+          "expressions.");
     }
 
     this.htIdxHolder = new IndexPointer();
@@ -246,15 +242,17 @@ public abstract class HashAggTemplate implements HashAggregator {
 
     if (valueFieldIds.size() > 0) {
       int i = 0;
-      FieldReference ref = new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getIntermediateType());
+      FieldReference ref =
+          new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getIntermediateType());
       for (TypedFieldId id : valueFieldIds) {
         materializedValueFields[i++] = MaterializedField.create(ref, id.getIntermediateType());
       }
     }
 
-    ChainedHashTable ht = new ChainedHashTable(htConfig, context, allocator, incoming,
-        null /* no incoming probe */, outgoing, true /* nulls are equal */) ;
-    this.htable = ht.createAndSetupHashTable(groupByOutFieldIds) ;
+    ChainedHashTable ht =
+        new ChainedHashTable(htConfig, context, allocator, incoming, null /* no incoming probe */, outgoing,
+            true /* nulls are equal */);
+    this.htable = ht.createAndSetupHashTable(groupByOutFieldIds);
 
     numGroupByOutFields = groupByOutFieldIds.length;
     batchHolders = new ArrayList<BatchHolder>();
@@ -265,19 +263,20 @@ public abstract class HashAggTemplate implements HashAggregator {
 
   @Override
   public AggOutcome doWork() {
-    try{
+    try {
       // Note: Keeping the outer and inner try blocks here to maintain some similarity with
       // StreamingAggregate which does somethings conditionally in the outer try block.
       // In the future HashAggregate may also need to perform some actions conditionally
       // in the outer try block.
 
-      outside: while(true) {
+      outside:
+      while (true) {
         // loop through existing records, aggregating the values as necessary.
         if (EXTRA_DEBUG_1) {
-          logger.debug ("Starting outer loop of doWork()...");
+          logger.debug("Starting outer loop of doWork()...");
         }
         for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
-          if(EXTRA_DEBUG_2) {
+          if (EXTRA_DEBUG_2) {
             logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
           }
           boolean success = checkGroupAndAggrValues(currentIndex);
@@ -369,7 +368,7 @@ public abstract class HashAggTemplate implements HashAggregator {
     // Skip the keys and only allocate for outputting the workspace values
     // (keys will be output through splitAndTransfer)
     Iterator<VectorWrapper<?>> outgoingIter = outContainer.iterator();
-    for (int i=0; i < numGroupByOutFields; i++) {
+    for (int i = 0; i < numGroupByOutFields; i++) {
       outgoingIter.next();
     }
     while (outgoingIter.hasNext()) {
@@ -469,16 +468,18 @@ public abstract class HashAggTemplate implements HashAggregator {
     int numOutputRecords = outNumRecordsHolder.value;
 
     if (EXTRA_DEBUG_1) {
-      logger.debug("After output values: outStartIdx = {}, outNumRecords = {}", outStartIdxHolder.value, outNumRecordsHolder.value);
+      logger.debug("After output values: outStartIdx = {}, outNumRecords = {}", outStartIdxHolder.value,
+          outNumRecordsHolder.value);
     }
     if (outputValuesStatus) {
-      outputKeysStatus = this.htable.outputKeys(outBatchIndex, this.outContainer, outStartIdxHolder.value, outNumRecordsHolder.value) ;
+      outputKeysStatus =
+          this.htable.outputKeys(outBatchIndex, this.outContainer, outStartIdxHolder.value, outNumRecordsHolder.value);
     }
 
     if (outputKeysStatus && outputValuesStatus) {
 
       // set the value count for outgoing batch value vectors
-      for(VectorWrapper<?> v : outgoing) {
+      for (VectorWrapper<?> v : outgoing) {
         v.getValueVector().getMutator().setValueCount(numOutputRecords);
       }
 
@@ -506,14 +507,16 @@ public abstract class HashAggTemplate implements HashAggregator {
       if (!outputKeysStatus) {
         logger.debug("Failed to output keys for current batch index: {} ", outBatchIndex);
         for (VectorWrapper<?> v : outContainer) {
-          logger.debug("At the time of failure, size of valuevector in outContainer = {}.", v.getValueVector().getValueCapacity());
+          logger.debug("At the time of failure, size of valuevector in outContainer = {}.",
+              v.getValueVector().getValueCapacity());
         }
         context.fail(new Exception("Failed to output keys for current batch !"));
       }
       if (!outputValuesStatus) {
         logger.debug("Failed to output values for current batch index: {} ", outBatchIndex);
         for (VectorWrapper<?> v : outContainer) {
-          logger.debug("At the time of failure, size of valuevector in outContainer = {}.", v.getValueVector().getValueCapacity());
+          logger.debug("At the time of failure, size of valuevector in outContainer = {}.",
+              v.getValueVector().getValueCapacity());
         }
         context.fail(new Exception("Failed to output values for current batch !"));
       }
@@ -555,7 +558,7 @@ public abstract class HashAggTemplate implements HashAggregator {
     }
     */
 
-    HashTable.PutStatus putStatus = htable.put(incomingRowIdx, htIdxHolder, 1 /* retry count */) ;
+    HashTable.PutStatus putStatus = htable.put(incomingRowIdx, htIdxHolder, 1 /* retry count */);
 
     if (putStatus != HashTable.PutStatus.PUT_FAILED) {
       int currentIdx = htIdxHolder.value;
@@ -564,11 +567,11 @@ public abstract class HashAggTemplate implements HashAggregator {
       if (currentIdx >= batchHolders.size() * HashTable.BATCH_SIZE) {
         addBatchHolder();
       }
-      BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & HashTable.BATCH_MASK);
+      BatchHolder bh = batchHolders.get((currentIdx >>> 16) & HashTable.BATCH_MASK);
       int idxWithinBatch = currentIdx & HashTable.BATCH_MASK;
 
       // Check if we have almost filled up the workspace vectors and add a batch if necessary
-      if ((idxWithinBatch ==  (bh.capacity - 1)) && (bh.allocatedNextBatch == false)) {
+      if ((idxWithinBatch == (bh.capacity - 1)) && (bh.allocatedNextBatch == false)) {
         htable.addNewKeyBatch();
         addBatchHolder();
         bh.allocatedNextBatch = true;
@@ -585,10 +588,9 @@ public abstract class HashAggTemplate implements HashAggregator {
         //  logger.debug("group-by key = {} already present at hash table index = {}", holder.value, currentIdx) ;
         //}
 
-      }
-      else if (putStatus == HashTable.PutStatus.KEY_ADDED) {
+      } else if (putStatus == HashTable.PutStatus.KEY_ADDED) {
         if (EXTRA_DEBUG_2) {
-          logger.debug("Group-by key was added to hash table, inserting new aggregate values") ;
+          logger.debug("Group-by key was added to hash table, inserting new aggregate values");
         }
 
         // debugging
@@ -617,7 +619,9 @@ public abstract class HashAggTemplate implements HashAggregator {
 
   // Code-generated methods (implemented in HashAggBatch)
   public abstract void doSetup(@Named("incoming") RecordBatch incoming);
+
   public abstract int getVectorIndex(@Named("recordIndex") int recordIndex);
+
   public abstract boolean resetValues();
 
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
index 238242b..0f7f394 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
@@ -17,9 +17,6 @@
  */
 package org.apache.drill.exec.physical.impl.aggregate;
 
-import java.io.IOException;
-import java.util.List;
-
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.exec.compile.TemplateClassDefinition;
 import org.apache.drill.exec.exception.ClassTransformationException;
@@ -34,21 +31,22 @@ import org.apache.drill.exec.record.RecordBatch.IterOutcome;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorContainer;
 
+import java.io.IOException;
+import java.util.List;
+
 public interface HashAggregator {
 
-  public static TemplateClassDefinition<HashAggregator> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashAggregator>(HashAggregator.class, HashAggTemplate.class);
+  public static TemplateClassDefinition<HashAggregator> TEMPLATE_DEFINITION =
+      new TemplateClassDefinition<HashAggregator>(HashAggregator.class, HashAggTemplate.class);
 
   public static enum AggOutcome {
     RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR
   }
 
   public abstract void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig, FragmentContext context,
-                             OperatorStats stats, BufferAllocator allocator, RecordBatch incoming,
-                             HashAggBatch outgoing, LogicalExpression[] valueExprs,
-                             List<TypedFieldId> valueFieldIds,
-                             TypedFieldId[] keyFieldIds,
-                             VectorContainer outContainer)
-    throws SchemaChangeException, IOException, ClassTransformationException;
+      OperatorStats stats, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
+      LogicalExpression[] valueExprs, List<TypedFieldId> valueFieldIds, TypedFieldId[] keyFieldIds,
+      VectorContainer outContainer) throws SchemaChangeException, IOException, ClassTransformationException;
 
   public abstract IterOutcome getOutcome();
 

http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 0502f7e..322fd1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -17,19 +17,13 @@
  */
 package org.apache.drill.exec.physical.impl.common;
 
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.ValueExpressions;
 import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.compile.sig.GeneratorMapping;
@@ -53,56 +47,68 @@ import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.resolver.TypeCastRules;
 import org.apache.drill.exec.vector.ValueVector;
 
-import com.sun.codemodel.JConditional;
-import com.sun.codemodel.JExpr;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
 
 
 public class ChainedHashTable {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ChainedHashTable.class);
 
   private static final GeneratorMapping KEY_MATCH_BUILD =
-    GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalBuild" /* eval method */,
-                            null /* reset */, null /* cleanup */);
+      GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalBuild" /* eval method */,
+          null /* reset */, null /* cleanup */);
 
   private static final GeneratorMapping KEY_MATCH_PROBE =
-    GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalProbe" /* eval method */,
-                            null /* reset */, null /* cleanup */);
+      GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalProbe" /* eval method */,
+          null /* reset */, null /* cleanup */);
 
   private static final GeneratorMapping GET_HASH_BUILD =
-    GeneratorMapping.create("doSetup" /* setup method */, "getHashBuild" /* eval method */,
-                            null /* reset */, null /* cleanup */);
+      GeneratorMapping.create("doSetup" /* setup method */, "getHashBuild" /* eval method */, null /* reset */,
+          null /* cleanup */);
 
   private static final GeneratorMapping GET_HASH_PROBE =
-    GeneratorMapping.create("doSetup" /* setup method */, "getHashProbe" /* eval method */,
-                            null /* reset */, null /* cleanup */);
+      GeneratorMapping.create("doSetup" /* setup method */, "getHashProbe" /* eval method */, null /* reset */,
+          null /* cleanup */);
 
   private static final GeneratorMapping SET_VALUE =
-    GeneratorMapping.create("setupInterior" /* setup method */, "setValue" /* eval method */,
-                            null /* reset */, null /* cleanup */);
+      GeneratorMapping.create("setupInterior" /* setup method */, "setValue" /* eval method */, null /* reset */,
+          null /* cleanup */);
 
   private static final GeneratorMapping OUTPUT_KEYS =
-    GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordKeys" /* eval method */,
-                            null /* reset */, null /* cleanup */) ;
+      GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordKeys" /* eval method */,
+          null /* reset */, null /* cleanup */);
 
   // GM for putting constant expression into method "setupInterior"
   private static final GeneratorMapping SETUP_INTERIOR_CONSTANT =
       GeneratorMapping.create("setupInterior" /* setup method */, "setupInterior" /* eval method */,
-                              null /* reset */, null /* cleanup */);
+          null /* reset */, null /* cleanup */);
 
   // GM for putting constant expression into method "doSetup"
   private static final GeneratorMapping DO_SETUP_CONSTANT =
-      GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */,
-                              null /* reset */, null /* cleanup */);
-
-  private final MappingSet KeyMatchIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
-  private final MappingSet KeyMatchIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
-  private final MappingSet KeyMatchHtableMapping = new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
-  private final MappingSet KeyMatchHtableProbeMapping = new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
-  private final MappingSet GetHashIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, DO_SETUP_CONSTANT, GET_HASH_BUILD);
-  private final MappingSet GetHashIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, DO_SETUP_CONSTANT, GET_HASH_PROBE);
-  private final MappingSet SetValueMapping = new MappingSet("incomingRowIdx" /* read index */, "htRowIdx" /* write index */, "incomingBuild" /* read container */, "htContainer" /* write container */, SETUP_INTERIOR_CONSTANT, SET_VALUE);
-
-  private final MappingSet OutputRecordKeysMapping = new MappingSet("htRowIdx" /* read index */, "outRowIdx" /* write index */, "htContainer" /* read container */, "outgoing" /* write container */, SETUP_INTERIOR_CONSTANT, OUTPUT_KEYS);
+      GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */, null /* reset */,
+          null /* cleanup */);
+
+  private final MappingSet KeyMatchIncomingBuildMapping =
+      new MappingSet("incomingRowIdx", null, "incomingBuild", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
+  private final MappingSet KeyMatchIncomingProbeMapping =
+      new MappingSet("incomingRowIdx", null, "incomingProbe", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
+  private final MappingSet KeyMatchHtableMapping =
+      new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
+  private final MappingSet KeyMatchHtableProbeMapping =
+      new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
+  private final MappingSet GetHashIncomingBuildMapping =
+      new MappingSet("incomingRowIdx", null, "incomingBuild", null, DO_SETUP_CONSTANT, GET_HASH_BUILD);
+  private final MappingSet GetHashIncomingProbeMapping =
+      new MappingSet("incomingRowIdx", null, "incomingProbe", null, DO_SETUP_CONSTANT, GET_HASH_PROBE);
+  private final MappingSet SetValueMapping =
+      new MappingSet("incomingRowIdx" /* read index */, "htRowIdx" /* write index */,
+          "incomingBuild" /* read container */, "htContainer" /* write container */, SETUP_INTERIOR_CONSTANT,
+          SET_VALUE);
+
+  private final MappingSet OutputRecordKeysMapping =
+      new MappingSet("htRowIdx" /* read index */, "outRowIdx" /* write index */, "htContainer" /* read container */,
+          "outgoing" /* write container */, SETUP_INTERIOR_CONSTANT, OUTPUT_KEYS);
 
   private HashTableConfig htConfig;
   private final FragmentContext context;
@@ -112,13 +118,8 @@ public class ChainedHashTable {
   private final RecordBatch outgoing;
   private final boolean areNullsEqual;
 
-  public ChainedHashTable(HashTableConfig htConfig,
-                          FragmentContext context,
-                          BufferAllocator allocator,
-                          RecordBatch incomingBuild,
-                          RecordBatch incomingProbe,
-                          RecordBatch outgoing,
-                          boolean areNullsEqual)  {
+  public ChainedHashTable(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
+      RecordBatch incomingBuild, RecordBatch incomingProbe, RecordBatch outgoing, boolean areNullsEqual) {
 
     this.htConfig = htConfig;
     this.context = context;
@@ -129,14 +130,15 @@ public class ChainedHashTable {
     this.areNullsEqual = areNullsEqual;
   }
 
-  public HashTable createAndSetupHashTable (TypedFieldId[] outKeyFieldIds) throws ClassTransformationException, IOException, SchemaChangeException {
+  public HashTable createAndSetupHashTable(TypedFieldId[] outKeyFieldIds) throws ClassTransformationException,
+      IOException, SchemaChangeException {
     CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getFunctionRegistry());
     ClassGenerator<HashTable> cg = top.getRoot();
     ClassGenerator<HashTable> cgInner = cg.getInnerGenerator("BatchHolder");
 
     LogicalExpression[] keyExprsBuild = new LogicalExpression[htConfig.getKeyExprsBuild().length];
     LogicalExpression[] keyExprsProbe = null;
-    boolean isProbe = (htConfig.getKeyExprsProbe() != null) ;
+    boolean isProbe = (htConfig.getKeyExprsProbe() != null);
     if (isProbe) {
       keyExprsProbe = new LogicalExpression[htConfig.getKeyExprsProbe().length];
     }
@@ -148,7 +150,8 @@ public class ChainedHashTable {
 
     int i = 0;
     for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
-      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
+      final LogicalExpression expr =
+          ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
       if (collector.hasErrors()) {
         throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
       }
@@ -169,7 +172,9 @@ public class ChainedHashTable {
     if (isProbe) {
       i = 0;
       for (NamedExpression ne : htConfig.getKeyExprsProbe()) {
-        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector, context.getFunctionRegistry());
+        final LogicalExpression expr =
+            ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector,
+                context.getFunctionRegistry());
         if (collector.hasErrors()) {
           throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
         }
@@ -183,7 +188,8 @@ public class ChainedHashTable {
 
     // generate code for isKeyMatch(), setValue(), getHash() and outputRecordKeys()
     setupIsKeyMatchInternal(cgInner, KeyMatchIncomingBuildMapping, KeyMatchHtableMapping, keyExprsBuild, htKeyFieldIds);
-    setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableProbeMapping, keyExprsProbe, htKeyFieldIds) ;
+    setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableProbeMapping, keyExprsProbe,
+        htKeyFieldIds);
 
     setupSetValue(cgInner, keyExprsBuild, htKeyFieldIds);
     if (outgoing != null) {
@@ -203,8 +209,8 @@ public class ChainedHashTable {
      */
     addLeastRestrictiveCasts(keyExprsBuild, keyExprsProbe);
 
-    setupGetHash(cg /* use top level code generator for getHash */,  GetHashIncomingBuildMapping, keyExprsBuild, false);
-    setupGetHash(cg /* use top level code generator for getHash */,  GetHashIncomingProbeMapping, keyExprsProbe, true);
+    setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, keyExprsBuild, false);
+    setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, keyExprsProbe, true);
 
     HashTable ht = context.getImplementationClass(top);
     ht.setup(htConfig, context, allocator, incomingBuild, incomingProbe, outgoing, htContainerOrig);
@@ -213,9 +219,9 @@ public class ChainedHashTable {
   }
 
 
-  private void setupIsKeyMatchInternal(ClassGenerator<HashTable> cg, MappingSet incomingMapping, MappingSet htableMapping,
-                                       LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
-    throws SchemaChangeException {
+  private void setupIsKeyMatchInternal(ClassGenerator<HashTable> cg, MappingSet incomingMapping,
+      MappingSet htableMapping, LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds) throws
+      SchemaChangeException {
     cg.setMappingSet(incomingMapping);
 
     if (keyExprs == null || keyExprs.length == 0) {
@@ -255,24 +261,24 @@ public class ChainedHashTable {
     cg.getEvalBlock()._return(JExpr.TRUE);
   }
 
-  private void setupSetValue(ClassGenerator<HashTable> cg, LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
-    throws SchemaChangeException {
+  private void setupSetValue(ClassGenerator<HashTable> cg, LogicalExpression[] keyExprs,
+      TypedFieldId[] htKeyFieldIds) throws SchemaChangeException {
 
     cg.setMappingSet(SetValueMapping);
 
     int i = 0;
     for (LogicalExpression expr : keyExprs) {
-      ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(htKeyFieldIds[i++], expr, true) ;
+      ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(htKeyFieldIds[i++], expr, true);
 
       HoldingContainer hc = cg.addExpr(vvwExpr, false); // this will write to the htContainer at htRowIdx
       cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
     }
 
     cg.getEvalBlock()._return(JExpr.TRUE);
-
   }
 
-  private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds, TypedFieldId[] outKeyFieldIds) {
+  private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds,
+      TypedFieldId[] outKeyFieldIds) {
 
     cg.setMappingSet(OutputRecordKeysMapping);
 
@@ -314,24 +320,34 @@ public class ChainedHashTable {
         ErrorCollector errorCollector = new ErrorCollectorImpl();
 
         if (result == null) {
-          throw new DrillRuntimeException(String.format("Join conditions cannot be compared failing build expression: %s failing probe expression: %s",
-              buildExpr.getMajorType().toString(), probeExpr.getMajorType().toString()));
-        }
-        else if (result != buildType) {
+          throw new DrillRuntimeException(String.format("Join conditions cannot be compared failing build " +
+              "expression:" + " %s failing probe expression: %s", buildExpr.getMajorType().toString(),
+              probeExpr.getMajorType().toString()));
+        } else if (result != buildType) {
           // Add a cast expression on top of the build expression
-          LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(buildExpr, probeExpr.getMajorType(), context.getFunctionRegistry(), errorCollector);
+          LogicalExpression castExpr =
+              ExpressionTreeMaterializer.addCastExpression(buildExpr, probeExpr.getMajorType(),
+                  context.getFunctionRegistry(), errorCollector);
           // Store the newly casted expression
-          keyExprsBuild[i] = ExpressionTreeMaterializer.materialize(castExpr, incomingBuild, errorCollector, context.getFunctionRegistry());
+          keyExprsBuild[i] =
+              ExpressionTreeMaterializer.materialize(castExpr, incomingBuild, errorCollector,
+                  context.getFunctionRegistry());
         } else if (result != probeType) {
           // Add a cast expression on top of the probe expression
-          LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(probeExpr, buildExpr.getMajorType(), context.getFunctionRegistry(), errorCollector);
+          LogicalExpression castExpr =
+              ExpressionTreeMaterializer.addCastExpression(probeExpr, buildExpr.getMajorType(),
+                  context.getFunctionRegistry(), errorCollector);
           // store the newly casted expression
-          keyExprsProbe[i] = ExpressionTreeMaterializer.materialize(castExpr, incomingProbe, errorCollector, context.getFunctionRegistry());
+          keyExprsProbe[i] =
+              ExpressionTreeMaterializer.materialize(castExpr, incomingProbe, errorCollector,
+                  context.getFunctionRegistry());
         }
       }
     }
   }
-  private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs, boolean isProbe) throws SchemaChangeException {
+
+  private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs,
+      boolean isProbe) throws SchemaChangeException {
 
     cg.setMappingSet(incomingMapping);
 
@@ -349,24 +365,26 @@ public class ChainedHashTable {
       HoldingContainer input = cg.addExpr(expr, false);
 
       // compute the hash(expr)
-      LogicalExpression hashfunc = FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT), context.getFunctionRegistry(), input);
+      LogicalExpression hashfunc =
+          FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT),
+              context.getFunctionRegistry(), input);
       HoldingContainer hashValue = cg.addExpr(hashfunc, false);
 
       if (i == 0) {
         combinedHashValue = hashValue; // first expression..just use the hash value
-      }
-      else {
+      } else {
 
         // compute the combined hash value using XOR
-        LogicalExpression xorfunc = FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT), context.getFunctionRegistry(), hashValue, combinedHashValue);
+        LogicalExpression xorfunc =
+            FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT),
+                context.getFunctionRegistry(), hashValue, combinedHashValue);
         combinedHashValue = cg.addExpr(xorfunc, false);
       }
     }
 
     if (combinedHashValue != null) {
-      cg.getEvalBlock()._return(combinedHashValue.getValue()) ;
-    }
-    else {
+      cg.getEvalBlock()._return(combinedHashValue.getValue());
+    } else {
       cg.getEvalBlock()._return(JExpr.lit(0));
     }
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index e8ccd62..6966ba1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -25,26 +25,35 @@ import org.apache.drill.exec.record.VectorContainer;
 
 public interface HashTable {
 
-  public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
+  public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION =
+      new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
 
-  /** The initial default capacity of the hash table (in terms of number of buckets). */
+  /**
+   * The initial default capacity of the hash table (in terms of number of buckets).
+   */
   static final public int DEFAULT_INITIAL_CAPACITY = 1 << 16;
 
-  /** The maximum capacity of the hash table (in terms of number of buckets). */
+  /**
+   * The maximum capacity of the hash table (in terms of number of buckets).
+   */
   static final public int MAXIMUM_CAPACITY = 1 << 30;
 
-  /** The default load factor of a hash table. */
+  /**
+   * The default load factor of a hash table.
+   */
   static final public float DEFAULT_LOAD_FACTOR = 0.75f;
 
-  static public enum PutStatus {KEY_PRESENT, KEY_ADDED, PUT_FAILED ;}
+  static public enum PutStatus {KEY_PRESENT, KEY_ADDED, PUT_FAILED;}
 
-  /** The batch size used for internal batch holders */
-  static final public int BATCH_SIZE = Character.MAX_VALUE+1;
+  /**
+   * The batch size used for internal batch holders
+   */
+  static final public int BATCH_SIZE = Character.MAX_VALUE + 1;
   static final public int BATCH_MASK = 0x0000FFFF;
 
   public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
-                    RecordBatch incomingBuild, RecordBatch incomingProbe,
-                    RecordBatch outgoing, VectorContainer htContainerOrig);
+      RecordBatch incomingBuild, RecordBatch incomingProbe,
+      RecordBatch outgoing, VectorContainer htContainerOrig);
 
   public void updateBatches();
 

http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index 5b56f8e..c80e97a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -17,11 +17,6 @@
  */
 package org.apache.drill.exec.physical.impl.common;
 
-import java.util.ArrayList;
-import java.util.Iterator;
-
-import javax.inject.Named;
-
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
@@ -38,6 +33,10 @@ import org.apache.drill.exec.vector.BigIntVector;
 import org.apache.drill.exec.vector.IntVector;
 import org.apache.drill.exec.vector.ValueVector;
 
+import javax.inject.Named;
+import java.util.ArrayList;
+import java.util.Iterator;
+
 public abstract class HashTableTemplate implements HashTable {
 
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTable.class);
@@ -115,7 +114,6 @@ public abstract class HashTableTemplate implements HashTable {
     private int batchIndex = 0;
 
     private BatchHolder(int idx) {
-
       this.batchIndex = idx;
 
       if (idx == 0) {  // first batch holder can use the original htContainer
@@ -134,10 +132,10 @@ public abstract class HashTableTemplate implements HashTable {
     }
 
     private void init(IntVector links, IntVector hashValues, int size) {
-      for (int i=0; i < size; i++) {
+      for (int i = 0; i < size; i++) {
         links.getMutator().setSafe(i, EMPTY_SLOT);
       }
-      for (int i=0; i < size; i++) {
+      for (int i = 0; i < size; i++) {
         hashValues.getMutator().setSafe(i, 0);
       }
       links.getMutator().setValueCount(size);
@@ -159,7 +157,8 @@ public abstract class HashTableTemplate implements HashTable {
       boolean match = false;
 
       if (currentIdxWithinBatch >= HashTable.BATCH_SIZE) {
-        logger.debug("Batch size = {}, incomingRowIdx = {}, currentIdxWithinBatch = {}.", HashTable.BATCH_SIZE, incomingRowIdx, currentIdxWithinBatch);
+        logger.debug("Batch size = {}, incomingRowIdx = {}, currentIdxWithinBatch = {}.", HashTable.BATCH_SIZE,
+            incomingRowIdx, currentIdxWithinBatch);
       }
       assert (currentIdxWithinBatch < HashTable.BATCH_SIZE);
       assert (incomingRowIdx < HashTable.BATCH_SIZE);
@@ -170,7 +169,7 @@ public abstract class HashTableTemplate implements HashTable {
         match = isKeyMatchInternalBuild(incomingRowIdx, currentIdxWithinBatch);
       }
 
-      if (! match) {
+      if (!match) {
         currentIdxHolder.value = links.getAccessor().get(currentIdxWithinBatch);
       }
       return match;
@@ -178,10 +177,11 @@ public abstract class HashTableTemplate implements HashTable {
 
     // Insert a new <key1, key2...keyN> entry coming from the incoming batch into the hash table
     // container at the specified index
-    private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdxWithinBatch) {
+    private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch,
+        int lastEntryIdxWithinBatch) {
       int currentIdxWithinBatch = currentIdx & BATCH_MASK;
 
-      if (! setValue(incomingRowIdx, currentIdxWithinBatch)) {
+      if (!setValue(incomingRowIdx, currentIdxWithinBatch)) {
         return false;
       }
 
@@ -198,7 +198,8 @@ public abstract class HashTableTemplate implements HashTable {
       maxOccupiedIdx = Math.max(maxOccupiedIdx, currentIdxWithinBatch);
 
       if (EXTRA_DEBUG) {
-        logger.debug("BatchHolder: inserted key at incomingRowIdx = {}, currentIdx = {}, hash value = {}.", incomingRowIdx, currentIdx, hashValue);
+        logger.debug("BatchHolder: inserted key at incomingRowIdx = {}, currentIdx = {}, hash value = {}.",
+            incomingRowIdx, currentIdx, hashValue);
       }
 
       return true;
@@ -210,7 +211,8 @@ public abstract class HashTableTemplate implements HashTable {
 
     private void rehash(int numbuckets, IntVector newStartIndices, int batchStartIdx) {
 
-      logger.debug("Rehashing entries within the batch: {}; batchStartIdx = {}, total numBuckets in hash table = {}.", batchIndex, batchStartIdx, numbuckets);
+      logger.debug("Rehashing entries within the batch: {}; batchStartIdx = {}, total numBuckets in hash table = {}" +
+          ".", batchIndex, batchStartIdx, numbuckets);
 
       int size = links.getAccessor().getValueCount();
       IntVector newLinks = allocMetadataVector(size, EMPTY_SLOT);
@@ -229,7 +231,10 @@ public abstract class HashTableTemplate implements HashTable {
           newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
 
           if (EXTRA_DEBUG) {
-            logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+            logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, " +
+                "hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx),
+                entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch),
+                newHashValues.getAccessor().get(entryIdxWithinBatch));
           }
 
         } else {
@@ -251,17 +256,26 @@ public abstract class HashTableTemplate implements HashTable {
               newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
 
               if (EXTRA_DEBUG) {
-                logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+                logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, " +
+                    "newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch,
+                    newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch,
+                    newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get
+                        (entryIdxWithinBatch));
               }
 
               break;
             } else if (bh != this && bh.links.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
               bh.links.getMutator().setSafe(idxWithinBatch, entryIdx); // update the link in the other batch
-              newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this batch to mark end of the hash chain
-              newHashValues.getMutator().setSafe(entryIdxWithinBatch,  hash);
+              newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this
+              // batch to mark end of the hash chain
+              newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
 
               if (EXTRA_DEBUG) {
-                logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+                logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, " +
+                    "newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch,
+                    newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch,
+                    newLinks.getAccessor().get(entryIdxWithinBatch),
+                    newHashValues.getAccessor().get(entryIdxWithinBatch));
               }
 
               break;
@@ -287,8 +301,8 @@ public abstract class HashTableTemplate implements HashTable {
     private boolean outputKeys(VectorContainer outContainer, int outStartIndex, int numRecords) {
 
       /** for debugging
-        BigIntVector vv0 = getValueVector(0);
-        BigIntHolder holder = new BigIntHolder();
+      BigIntVector vv0 = getValueVector(0);
+      BigIntHolder holder = new BigIntHolder();
       */
 
       // set the value count for htContainer's value vectors before the transfer ..
@@ -304,7 +318,8 @@ public abstract class HashTableTemplate implements HashTable {
       }
 
 /*
-      logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.", this.batchIndex, 0, maxOccupiedIdx);
+      logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.",
+      this.batchIndex, 0, maxOccupiedIdx);
       for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
         if (outputRecordKeys(i, batchOutputCount) ) {
           if (EXTRA_DEBUG) logger.debug("Outputting keys to output index: {}", batchOutputCount) ;
@@ -312,7 +327,8 @@ public abstract class HashTableTemplate implements HashTable {
           // debugging
           // holder.value = vv0.getAccessor().get(i);
           // if (holder.value == 100018 || holder.value == 100021) {
-          //  logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i, batchOutputCount);
+          //  logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i,
+          //      batchOutputCount);
           // }
 
           batchOutputCount++;
@@ -337,14 +353,15 @@ public abstract class HashTableTemplate implements HashTable {
         if (idxWithinBatch == EMPTY_SLOT) {
           break;
         } else {
-          logger.debug("links[ {} ] = {}, hashValues[ {} ] = {}.", idxWithinBatch, links.getAccessor().get(idxWithinBatch), idxWithinBatch, hashValues.getAccessor().get(idxWithinBatch));
+          logger.debug("links[ {} ] = {}, hashValues[ {} ] = {}.", idxWithinBatch,
+              links.getAccessor().get(idxWithinBatch), idxWithinBatch, hashValues.getAccessor().get(idxWithinBatch));
           idx = links.getAccessor().get(idxWithinBatch);
         }
       }
     }
 
     private void clear() {
-      htContainer.clear();;
+      htContainer.clear();
       links.clear();
       hashValues.clear();
     }
@@ -363,30 +380,41 @@ public abstract class HashTableTemplate implements HashTable {
     // These methods will be code-generated
 
     @RuntimeOverridden
-    protected void setupInterior(@Named("incomingBuild") RecordBatch incomingBuild,
-                                 @Named("incomingProbe") RecordBatch incomingProbe,
-                                 @Named("outgoing") RecordBatch outgoing,
-                                 @Named("htContainer") VectorContainer htContainer) {}
+    protected void setupInterior(
+        @Named("incomingBuild") RecordBatch incomingBuild,
+        @Named("incomingProbe") RecordBatch incomingProbe,
+        @Named("outgoing") RecordBatch outgoing,
+        @Named("htContainer") VectorContainer htContainer) {
+    }
 
     @RuntimeOverridden
-    protected boolean isKeyMatchInternalBuild(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+    protected boolean isKeyMatchInternalBuild(
+        @Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+      return false;
+    }
 
     @RuntimeOverridden
-    protected boolean isKeyMatchInternalProbe(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+    protected boolean isKeyMatchInternalProbe(
+        @Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+      return false;
+    }
 
     @RuntimeOverridden
-    protected boolean setValue(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+    protected boolean setValue(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+      return false;
+    }
 
     @RuntimeOverridden
-    protected boolean outputRecordKeys(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return false;}
-
+    protected boolean outputRecordKeys(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {
+      return false;
+    }
   } // class BatchHolder
 
 
   @Override
   public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
-                    RecordBatch incomingBuild, RecordBatch incomingProbe,
-                    RecordBatch outgoing, VectorContainer htContainerOrig) {
+      RecordBatch incomingBuild, RecordBatch incomingProbe,
+      RecordBatch outgoing, VectorContainer htContainerOrig) {
     float loadf = htConfig.getLoadFactor();
     int initialCap = htConfig.getInitialCapacity();
 
@@ -483,16 +511,16 @@ public abstract class HashTableTemplate implements HashTable {
 
   private static int roundUpToPowerOf2(int number) {
     int rounded = number >= MAXIMUM_CAPACITY
-           ? MAXIMUM_CAPACITY
-           : (rounded = Integer.highestOneBit(number)) != 0
-               ? (Integer.bitCount(number) > 1) ? rounded << 1 : rounded
-               : 1;
+        ? MAXIMUM_CAPACITY
+        : (rounded = Integer.highestOneBit(number)) != 0
+        ? (Integer.bitCount(number) > 1) ? rounded << 1 : rounded
+        : 1;
 
-        return rounded;
+    return rounded;
   }
 
   public PutStatus put(int incomingRowIdx, IndexPointer htIdxHolder, int retryCount) {
-    HashTable.PutStatus putStatus = put(incomingRowIdx, htIdxHolder) ;
+    HashTable.PutStatus putStatus = put(incomingRowIdx, htIdxHolder);
     int count = retryCount;
     int numBatchHolders;
     while (putStatus == PutStatus.PUT_FAILED && count > 0) {
@@ -526,7 +554,8 @@ public abstract class HashTableTemplate implements HashTable {
       addBatchIfNeeded(currentIdx);
 
       if (EXTRA_DEBUG) {
-        logger.debug("Empty bucket index = {}. incomingRowIdx = {}; inserting new entry at currentIdx = {}.", i, incomingRowIdx, currentIdx);
+        logger.debug("Empty bucket index = {}. incomingRowIdx = {}; inserting new entry at currentIdx = {}.", i,
+            incomingRowIdx, currentIdx);
       }
 
       if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
@@ -542,7 +571,7 @@ public abstract class HashTableTemplate implements HashTable {
     currentIdx = startIdx;
     boolean found = false;
 
-    bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+    bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
     currentIdxHolder.value = currentIdx;
 
     // if startIdx is non-empty, follow the hash chain links until we find a matching
@@ -554,13 +583,12 @@ public abstract class HashTableTemplate implements HashTable {
         htIdxHolder.value = currentIdxHolder.value;
         found = true;
         break;
-      }
-      else if (currentIdxHolder.value == EMPTY_SLOT) {
+      } else if (currentIdxHolder.value == EMPTY_SLOT) {
         lastEntryBatch = bh;
         lastEntryIdxWithinBatch = currentIdxWithinBatch;
         break;
       } else {
-        bh = batchHolders.get( (currentIdxHolder.value >>> 16) & HashTable.BATCH_MASK);
+        bh = batchHolders.get((currentIdxHolder.value >>> 16) & HashTable.BATCH_MASK);
         lastEntryBatch = bh;
       }
     }
@@ -571,7 +599,8 @@ public abstract class HashTableTemplate implements HashTable {
       addBatchIfNeeded(currentIdx);
 
       if (EXTRA_DEBUG) {
-        logger.debug("No match was found for incomingRowIdx = {}; inserting new entry at currentIdx = {}.", incomingRowIdx, currentIdx);
+        logger.debug("No match was found for incomingRowIdx = {}; inserting new entry at currentIdx = {}.",
+            incomingRowIdx, currentIdx);
       }
 
       if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
@@ -582,17 +611,18 @@ public abstract class HashTableTemplate implements HashTable {
       }
     }
 
-    return found ? PutStatus.KEY_PRESENT : PutStatus.KEY_ADDED ;
+    return found ? PutStatus.KEY_PRESENT : PutStatus.KEY_ADDED;
   }
 
-  private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdx) {
+  private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch,
+      int lastEntryIdx) {
 
     addBatchIfNeeded(currentIdx);
 
-    BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+    BatchHolder bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
 
     if (bh.insertEntry(incomingRowIdx, currentIdx, hashValue, lastEntryBatch, lastEntryIdx)) {
-      numEntries++ ;
+      numEntries++;
 
       /* Resize hash table if needed and transfer the metadata
        * Resize only after inserting the current entry into the hash table
@@ -617,10 +647,10 @@ public abstract class HashTableTemplate implements HashTable {
     int currentIdx = startIndices.getAccessor().get(i);
 
     if (currentIdx == EMPTY_SLOT) {
-        return -1;
+      return -1;
     }
 
-    BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+    BatchHolder bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
     currentIdxHolder.value = currentIdx;
 
     boolean found = false;
@@ -632,7 +662,7 @@ public abstract class HashTableTemplate implements HashTable {
       } else if (currentIdxHolder.value == EMPTY_SLOT) {
         break;
       } else {
-        bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
+        bh = batchHolders.get((currentIdxHolder.value >>> 16) & BATCH_MASK);
       }
     }
 
@@ -651,8 +681,7 @@ public abstract class HashTableTemplate implements HashTable {
         logger.debug("HashTable: Added new batch. Num batches = {}.", batchHolders.size());
       }
       return bh;
-    }
-    else {
+    } else {
       return batchHolders.get(batchHolders.size() - 1);
     }
   }
@@ -700,7 +729,7 @@ public abstract class HashTableTemplate implements HashTable {
     IntVector newStartIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
 
     for (int i = 0; i < batchHolders.size(); i++) {
-      BatchHolder bh = batchHolders.get(i) ;
+      BatchHolder bh = batchHolders.get(i);
       int batchStartIdx = i * BATCH_SIZE;
       bh.rehash(tableSize, newStartIndices, batchStartIdx);
     }
@@ -714,7 +743,7 @@ public abstract class HashTableTemplate implements HashTable {
       for (int i = 0; i < startIndices.getAccessor().getValueCount(); i++) {
         logger.debug("Bucket: {}, startIdx[ {} ] = {}.", i, i, startIndices.getAccessor().get(i));
         int idx = startIndices.getAccessor().get(i);
-        BatchHolder bh = batchHolders.get( (idx >>> 16) & BATCH_MASK);
+        BatchHolder bh = batchHolders.get((idx >>> 16) & BATCH_MASK);
         bh.dump(idx);
       }
     }
@@ -724,7 +753,7 @@ public abstract class HashTableTemplate implements HashTable {
 
   public boolean outputKeys(int batchIdx, VectorContainer outContainer, int outStartIndex, int numRecords) {
     assert batchIdx < batchHolders.size();
-    if (! batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
+    if (!batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
       return false;
     }
     return true;
@@ -733,7 +762,7 @@ public abstract class HashTableTemplate implements HashTable {
   private IntVector allocMetadataVector(int size, int initialValue) {
     IntVector vector = (IntVector) TypeHelper.getNewVector(dummyIntField, allocator);
     vector.allocateNew(size);
-    for (int i=0; i < size; i++) {
+    for (int i = 0; i < size; i++) {
       vector.getMutator().setSafe(i, initialValue);
     }
     vector.getMutator().setValueCount(size);
@@ -747,8 +776,11 @@ public abstract class HashTableTemplate implements HashTable {
   }
 
   // These methods will be code-generated in the context of the outer class
-  protected abstract void doSetup(@Named("incomingBuild") RecordBatch incomingBuild, @Named("incomingProbe") RecordBatch incomingProbe);
-  protected abstract int getHashBuild(@Named("incomingRowIdx") int incomingRowIdx) ;
-  protected abstract int getHashProbe(@Named("incomingRowIdx") int incomingRowIdx) ;
+  protected abstract void doSetup(@Named("incomingBuild") RecordBatch incomingBuild,
+      @Named("incomingProbe") RecordBatch incomingProbe);
+
+  protected abstract int getHashBuild(@Named("incomingRowIdx") int incomingRowIdx);
+
+  protected abstract int getHashProbe(@Named("incomingRowIdx") int incomingRowIdx);
 
 }


[2/5] drill git commit: DRILL-1991: Code indendation and formatting cleanup for few files

Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 5deb67f..f370dc7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -17,9 +17,9 @@
  */
 package org.apache.drill.exec.physical.impl.join;
 
-import java.io.IOException;
-import java.util.List;
-
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JVar;
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.common.logical.data.NamedExpression;
@@ -49,7 +49,6 @@ import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
 import org.apache.drill.exec.record.ExpandableHyperContainer;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.TransferPair;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.VectorWrapper;
@@ -57,118 +56,105 @@ import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.AbstractContainerVector;
 import org.eigenbase.rel.JoinRelType;
 
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JExpression;
-import com.sun.codemodel.JVar;
+import java.io.IOException;
+import java.util.List;
 
 public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
+  // Probe side record batch
+  private final RecordBatch left;
 
-  public static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024;
-  public static final long ALLOCATOR_MAX_RESERVATION = 20L*1000*1000*1000;
-
-    // Probe side record batch
-    private final RecordBatch left;
-
-    // Build side record batch
-    private final RecordBatch right;
+  // Build side record batch
+  private final RecordBatch right;
 
-    // Join type, INNER, LEFT, RIGHT or OUTER
-    private final JoinRelType joinType;
+  // Join type, INNER, LEFT, RIGHT or OUTER
+  private final JoinRelType joinType;
 
-    // Join conditions
-    private final List<JoinCondition> conditions;
+  // Join conditions
+  private final List<JoinCondition> conditions;
 
-    // Runtime generated class implementing HashJoinProbe interface
-    private HashJoinProbe hashJoinProbe = null;
+  // Runtime generated class implementing HashJoinProbe interface
+  private HashJoinProbe hashJoinProbe = null;
 
-    /* Helper class
-     * Maintains linked list of build side records with the same key
-     * Keeps information about which build records have a corresponding
-     * matching key in the probe side (for outer, right joins)
-     */
-    private HashJoinHelper hjHelper = null;
+  /* Helper class
+   * Maintains linked list of build side records with the same key
+   * Keeps information about which build records have a corresponding
+   * matching key in the probe side (for outer, right joins)
+   */
+  private HashJoinHelper hjHelper = null;
 
-    // Underlying hashtable used by the hash join
-    private HashTable hashTable = null;
+  // Underlying hashtable used by the hash join
+  private HashTable hashTable = null;
 
-    /* Hyper container to store all build side record batches.
-     * Records are retrieved from this container when there is a matching record
-     * on the probe side
-     */
-    private ExpandableHyperContainer hyperContainer;
+  /* Hyper container to store all build side record batches.
+   * Records are retrieved from this container when there is a matching record
+   * on the probe side
+   */
+  private ExpandableHyperContainer hyperContainer;
 
-    // Number of records in the output container
-    private int outputRecords;
+  // Number of records in the output container
+  private int outputRecords;
 
-    // Current batch index on the build side
-    private int buildBatchIndex = 0;
+  // Current batch index on the build side
+  private int buildBatchIndex = 0;
 
-    // Schema of the build side
-    private BatchSchema rightSchema = null;
+  // Schema of the build side
+  private BatchSchema rightSchema = null;
 
 
+  // Generator mapping for the build side
+  // Generator mapping for the build side : scalar
+  private static final GeneratorMapping PROJECT_BUILD =
+      GeneratorMapping.create("doSetup"/* setup method */, "projectBuildRecord" /* eval method */, null /* reset */,
+          null /* cleanup */);
+  // Generator mapping for the build side : constant
+  private static final GeneratorMapping PROJECT_BUILD_CONSTANT =
+      GeneratorMapping.create("doSetup"/* setup method */, "doSetup" /* eval method */, null /* reset */,
+          null /* cleanup */);
 
-    // Generator mapping for the build side
-    // Generator mapping for the build side : scalar
-    private static final GeneratorMapping PROJECT_BUILD = GeneratorMapping.create("doSetup"/* setup method */,
-                                                                                  "projectBuildRecord" /* eval method */,
-                                                                                  null /* reset */, null /* cleanup */);
-    // Generator mapping for the build side : constant
-    private static final GeneratorMapping PROJECT_BUILD_CONSTANT = GeneratorMapping.create("doSetup"/* setup method */,
-                                                                                  "doSetup" /* eval method */,
-                                                                                   null /* reset */, null /* cleanup */);
+  // Generator mapping for the probe side : scalar
+  private static final GeneratorMapping PROJECT_PROBE =
+      GeneratorMapping.create("doSetup" /* setup method */, "projectProbeRecord" /* eval method */, null /* reset */,
+          null /* cleanup */);
+  // Generator mapping for the probe side : constant
+  private static final GeneratorMapping PROJECT_PROBE_CONSTANT =
+      GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */, null /* reset */,
+          null /* cleanup */);
 
-    // Generator mapping for the probe side : scalar
-    private static final GeneratorMapping PROJECT_PROBE = GeneratorMapping.create("doSetup" /* setup method */,
-                                                                                  "projectProbeRecord" /* eval method */,
-                                                                                  null /* reset */, null /* cleanup */);
-    // Generator mapping for the probe side : constant
-    private static final GeneratorMapping PROJECT_PROBE_CONSTANT = GeneratorMapping.create("doSetup" /* setup method */,
-                                                                                  "doSetup" /* eval method */,
-                                                                                  null /* reset */, null /* cleanup */);
 
+  // Mapping set for the build side
+  private final MappingSet projectBuildMapping =
+      new MappingSet("buildIndex" /* read index */, "outIndex" /* write index */, "buildBatch" /* read container */,
+          "outgoing" /* write container */, PROJECT_BUILD_CONSTANT, PROJECT_BUILD);
 
-    // Mapping set for the build side
-    private final MappingSet projectBuildMapping = new MappingSet("buildIndex" /* read index */, "outIndex" /* write index */,
-                                                                  "buildBatch" /* read container */,
-                                                                  "outgoing" /* write container */,
-                                                                  PROJECT_BUILD_CONSTANT, PROJECT_BUILD);
+  // Mapping set for the probe side
+  private final MappingSet projectProbeMapping =
+      new MappingSet("probeIndex" /* read index */, "outIndex" /* write index */, "probeBatch" /* read container */,
+          "outgoing" /* write container */, PROJECT_PROBE_CONSTANT, PROJECT_PROBE);
 
-    // Mapping set for the probe side
-    private final MappingSet projectProbeMapping = new MappingSet("probeIndex" /* read index */, "outIndex" /* write index */,
-                                                                  "probeBatch" /* read container */,
-                                                                  "outgoing" /* write container */,
-                                                                  PROJECT_PROBE_CONSTANT, PROJECT_PROBE);
+  IterOutcome leftUpstream = IterOutcome.NONE;
+  IterOutcome rightUpstream = IterOutcome.NONE;
 
-    // indicates if we have previously returned an output batch
-    boolean firstOutputBatch = true;
-
-    IterOutcome leftUpstream = IterOutcome.NONE;
-    IterOutcome rightUpstream = IterOutcome.NONE;
-
-    private final HashTableStats htStats = new HashTableStats();
+  private final HashTableStats htStats = new HashTableStats();
 
   public enum Metric implements MetricDef {
 
-      NUM_BUCKETS,
-      NUM_ENTRIES,
-      NUM_RESIZING,
-      RESIZING_TIME;
+    NUM_BUCKETS,
+    NUM_ENTRIES,
+    NUM_RESIZING,
+    RESIZING_TIME;
 
-      // duplicate for hash ag
-
-      @Override
-      public int metricId() {
-        return ordinal();
-      }
-    }
+    // duplicate for hash ag
 
     @Override
-    public int getRecordCount() {
-        return outputRecords;
+    public int metricId() {
+      return ordinal();
     }
+  }
 
-
+  @Override
+  public int getRecordCount() {
+    return outputRecords;
+  }
 
   @Override
   protected void buildSchema() throws SchemaChangeException {
@@ -204,333 +190,332 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
     }
   }
 
-    @Override
-    public IterOutcome innerNext() {
-        try {
-            /* If we are here for the first time, execute the build phase of the
-             * hash join and setup the run time generated class for the probe side
-             */
-            if (state == BatchState.FIRST) {
-                // Build the hash table, using the build side record batches.
-                executeBuildPhase();
-//                IterOutcome next = next(HashJoinHelper.LEFT_INPUT, left);
-                hashJoinProbe.setupHashJoinProbe(context, hyperContainer, left, left.getRecordCount(), this, hashTable, hjHelper, joinType);
-
-                // Update the hash table related stats for the operator
-                updateStats(this.hashTable);
-            }
+  @Override
+  public IterOutcome innerNext() {
+    try {
+      /* If we are here for the first time, execute the build phase of the
+       * hash join and setup the run time generated class for the probe side
+       */
+      if (state == BatchState.FIRST) {
+        // Build the hash table, using the build side record batches.
+        executeBuildPhase();
+        //                IterOutcome next = next(HashJoinHelper.LEFT_INPUT, left);
+        hashJoinProbe.setupHashJoinProbe(context, hyperContainer, left, left.getRecordCount(), this, hashTable,
+            hjHelper, joinType);
+
+        // Update the hash table related stats for the operator
+        updateStats(this.hashTable);
+      }
 
-            // Store the number of records projected
-            if (hashTable != null
-                || joinType != JoinRelType.INNER) {
-
-                // Allocate the memory for the vectors in the output container
-                allocateVectors();
-
-                outputRecords = hashJoinProbe.probeAndProject();
-
-                /* We are here because of one the following
-                 * 1. Completed processing of all the records and we are done
-                 * 2. We've filled up the outgoing batch to the maximum and we need to return upstream
-                 * Either case build the output container's schema and return
-                 */
-                if (outputRecords > 0 || state == BatchState.FIRST) {
-                  if (state == BatchState.FIRST) {
-                    state = BatchState.NOT_FIRST;
-                  }
-
-
-                  for (VectorWrapper<?> v : container) {
-                    v.getValueVector().getMutator().setValueCount(outputRecords);
-                  }
-
-                  return IterOutcome.OK;
-                }
-            } else {
-                // Our build side is empty, we won't have any matches, clear the probe side
-                if (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
-                    for (VectorWrapper<?> wrapper : left) {
-                      wrapper.getValueVector().clear();
-                    }
-                    left.kill(true);
-                    leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
-                    while (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
-                      for (VectorWrapper<?> wrapper : left) {
-                        wrapper.getValueVector().clear();
-                      }
-                      leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
-                    }
-                }
-            }
+      // Store the number of records projected
+      if (hashTable != null || joinType != JoinRelType.INNER) {
 
-            // No more output records, clean up and return
-            state = BatchState.DONE;
-//            if (first) {
-//              return IterOutcome.OK_NEW_SCHEMA;
-//            }
-            return IterOutcome.NONE;
-        } catch (ClassTransformationException | SchemaChangeException | IOException e) {
-            context.fail(e);
-            killIncoming(false);
-            return IterOutcome.STOP;
-        }
-    }
+        // Allocate the memory for the vectors in the output container
+        allocateVectors();
 
-    public void setupHashTable() throws IOException, SchemaChangeException, ClassTransformationException {
+        outputRecords = hashJoinProbe.probeAndProject();
 
-        // Setup the hash table configuration object
-        int conditionsSize = conditions.size();
+        /* We are here because of one the following
+         * 1. Completed processing of all the records and we are done
+         * 2. We've filled up the outgoing batch to the maximum and we need to return upstream
+         * Either case build the output container's schema and return
+         */
+        if (outputRecords > 0 || state == BatchState.FIRST) {
+          if (state == BatchState.FIRST) {
+            state = BatchState.NOT_FIRST;
+          }
 
-        NamedExpression rightExpr[] = new NamedExpression[conditionsSize];
-        NamedExpression leftExpr[] = new NamedExpression[conditionsSize];
 
-        // Create named expressions from the conditions
-        for (int i = 0; i < conditionsSize; i++) {
-            rightExpr[i] = new NamedExpression(conditions.get(i).getRight(), new FieldReference("build_side_" + i ));
-            leftExpr[i] = new NamedExpression(conditions.get(i).getLeft(), new FieldReference("probe_side_" + i));
+          for (VectorWrapper<?> v : container) {
+            v.getValueVector().getMutator().setValueCount(outputRecords);
+          }
 
-            // Hash join only supports equality currently.
-            assert conditions.get(i).getRelationship().equals("==");
+          return IterOutcome.OK;
         }
-
-        // Set the left named expression to be null if the probe batch is empty.
-        if (leftUpstream != IterOutcome.OK_NEW_SCHEMA && leftUpstream != IterOutcome.OK) {
-            leftExpr = null;
-        } else {
-          if (left.getSchema().getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
-            throw new SchemaChangeException("Hash join does not support probe batch with selection vectors");
+      } else {
+        // Our build side is empty, we won't have any matches, clear the probe side
+        if (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
+          for (VectorWrapper<?> wrapper : left) {
+            wrapper.getValueVector().clear();
+          }
+          left.kill(true);
+          leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
+          while (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
+            for (VectorWrapper<?> wrapper : left) {
+              wrapper.getValueVector().clear();
+            }
+            leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
           }
         }
+      }
+
+      // No more output records, clean up and return
+      state = BatchState.DONE;
+      //            if (first) {
+      //              return IterOutcome.OK_NEW_SCHEMA;
+      //            }
+      return IterOutcome.NONE;
+    } catch (ClassTransformationException | SchemaChangeException | IOException e) {
+      context.fail(e);
+      killIncoming(false);
+      return IterOutcome.STOP;
+    }
+  }
+
+  public void setupHashTable() throws IOException, SchemaChangeException, ClassTransformationException {
+
+    // Setup the hash table configuration object
+    int conditionsSize = conditions.size();
 
-        HashTableConfig htConfig =
-            new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
+    NamedExpression rightExpr[] = new NamedExpression[conditionsSize];
+    NamedExpression leftExpr[] = new NamedExpression[conditionsSize];
+
+    // Create named expressions from the conditions
+    for (int i = 0; i < conditionsSize; i++) {
+      rightExpr[i] = new NamedExpression(conditions.get(i).getRight(), new FieldReference("build_side_" + i));
+      leftExpr[i] = new NamedExpression(conditions.get(i).getLeft(), new FieldReference("probe_side_" + i));
+
+      // Hash join only supports equality currently.
+      assert conditions.get(i).getRelationship().equals("==");
+    }
+
+    // Set the left named expression to be null if the probe batch is empty.
+    if (leftUpstream != IterOutcome.OK_NEW_SCHEMA && leftUpstream != IterOutcome.OK) {
+      leftExpr = null;
+    } else {
+      if (left.getSchema().getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
+        throw new SchemaChangeException("Hash join does not support probe batch with selection vectors");
+      }
+    }
+
+    HashTableConfig htConfig =
+        new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
             HashTable.DEFAULT_LOAD_FACTOR, rightExpr, leftExpr);
 
-        // Create the chained hash table
-        ChainedHashTable ht  = new ChainedHashTable(htConfig, context, oContext.getAllocator(),
-            this.right, this.left, null, false /* nulls are not equal */);
-        hashTable = ht.createAndSetupHashTable(null);
+    // Create the chained hash table
+    ChainedHashTable ht =
+        new ChainedHashTable(htConfig, context, oContext.getAllocator(), this.right, this.left, null,
+            false /* nulls are not equal */);
+    hashTable = ht.createAndSetupHashTable(null);
+  }
+
+  public void executeBuildPhase() throws SchemaChangeException, ClassTransformationException, IOException {
+
+    //Setup the underlying hash table
+
+    // skip first batch if count is zero, as it may be an empty schema batch
+    if (right.getRecordCount() == 0) {
+      for (VectorWrapper w : right) {
+        w.clear();
+      }
+      rightUpstream = next(right);
     }
 
-    public void executeBuildPhase() throws SchemaChangeException, ClassTransformationException, IOException {
+    boolean moreData = true;
+
+    while (moreData) {
+
+      switch (rightUpstream) {
+
+      case NONE:
+      case NOT_YET:
+      case STOP:
+        moreData = false;
+        continue;
 
-        //Setup the underlying hash table
+      case OK_NEW_SCHEMA:
+        if (rightSchema == null) {
+          rightSchema = right.getSchema();
 
-      // skip first batch if count is zero, as it may be an empty schema batch
-        if (right.getRecordCount() == 0) {
-          for (VectorWrapper w : right) {
-            w.clear();
+          if (rightSchema.getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
+            throw new SchemaChangeException("Hash join does not support build batch with selection vectors");
           }
-          rightUpstream = next(right);
+          setupHashTable();
+        } else {
+          if (!rightSchema.equals(right.getSchema())) {
+            throw new SchemaChangeException("Hash join does not support schema changes");
+          }
+          hashTable.updateBatches();
         }
+        // Fall through
+      case OK:
+        int currentRecordCount = right.getRecordCount();
 
-        boolean moreData = true;
-
-        while (moreData) {
-
-            switch (rightUpstream) {
-
-                case NONE:
-                case NOT_YET:
-                case STOP:
-                    moreData = false;
-                    continue;
-
-                case OK_NEW_SCHEMA:
-                    if (rightSchema == null) {
-                        rightSchema = right.getSchema();
-
-                        if (rightSchema.getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
-                          throw new SchemaChangeException("Hash join does not support build batch with selection vectors");
-                        }
-                        setupHashTable();
-                    } else {
-                      if (!rightSchema.equals(right.getSchema())) {
-                        throw new SchemaChangeException("Hash join does not support schema changes");
-                      }
-                      hashTable.updateBatches();
-                    }
-                // Fall through
-                case OK:
-                    int currentRecordCount = right.getRecordCount();
-
-                    /* For every new build batch, we store some state in the helper context
-                     * Add new state to the helper context
-                     */
-                    hjHelper.addNewBatch(currentRecordCount);
-
-                    // Holder contains the global index where the key is hashed into using the hash table
-                    IndexPointer htIndex = new IndexPointer();
-
-                    // For every record in the build batch , hash the key columns
-                    for (int i = 0; i < currentRecordCount; i++) {
-
-                        HashTable.PutStatus status = hashTable.put(i, htIndex, 1 /* retry count */);
-
-                        if (status != HashTable.PutStatus.PUT_FAILED) {
-                            /* Use the global index returned by the hash table, to store
-                             * the current record index and batch index. This will be used
-                             * later when we probe and find a match.
-                             */
-                            hjHelper.setCurrentIndex(htIndex.value, buildBatchIndex, i);
-                        }
-                    }
-
-                    /* Completed hashing all records in this batch. Transfer the batch
-                     * to the hyper vector container. Will be used when we want to retrieve
-                     * records that have matching keys on the probe side.
-                     */
-                    RecordBatchData nextBatch = new RecordBatchData(right);
-                    if (hyperContainer == null) {
-                        hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
-                    } else {
-                        hyperContainer.addBatch(nextBatch.getContainer());
-                    }
-
-                    // completed processing a batch, increment batch index
-                    buildBatchIndex++;
-                    break;
-            }
-            // Get the next record batch
-            rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
-        }
-    }
+        /* For every new build batch, we store some state in the helper context
+         * Add new state to the helper context
+         */
+        hjHelper.addNewBatch(currentRecordCount);
+
+        // Holder contains the global index where the key is hashed into using the hash table
+        IndexPointer htIndex = new IndexPointer();
 
-    public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
+        // For every record in the build batch , hash the key columns
+        for (int i = 0; i < currentRecordCount; i++) {
 
+          HashTable.PutStatus status = hashTable.put(i, htIndex, 1 /* retry count */);
 
+          if (status != HashTable.PutStatus.PUT_FAILED) {
+            /* Use the global index returned by the hash table, to store
+             * the current record index and batch index. This will be used
+             * later when we probe and find a match.
+             */
+            hjHelper.setCurrentIndex(htIndex.value, buildBatchIndex, i);
+          }
+        }
 
-        final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry());
-        ClassGenerator<HashJoinProbe> g = cg.getRoot();
+        /* Completed hashing all records in this batch. Transfer the batch
+         * to the hyper vector container. Will be used when we want to retrieve
+         * records that have matching keys on the probe side.
+         */
+        RecordBatchData nextBatch = new RecordBatchData(right);
+        if (hyperContainer == null) {
+          hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
+        } else {
+          hyperContainer.addBatch(nextBatch.getContainer());
+        }
 
-        // Generate the code to project build side records
-        g.setMappingSet(projectBuildMapping);
+        // completed processing a batch, increment batch index
+        buildBatchIndex++;
+        break;
+      }
+      // Get the next record batch
+      rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
+    }
+  }
 
+  public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
 
-        int fieldId = 0;
-        JExpression buildIndex = JExpr.direct("buildIndex");
-        JExpression outIndex = JExpr.direct("outIndex");
-        g.rotateBlock();
 
-        if (rightSchema != null) {
-            for(MaterializedField field : rightSchema) {
+    final CodeGenerator<HashJoinProbe> cg =
+        CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+    ClassGenerator<HashJoinProbe> g = cg.getRoot();
 
-                MajorType inputType = field.getType();
-                MajorType outputType;
-                if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
-                  outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
-                } else {
-                  outputType = inputType;
-                }
+    // Generate the code to project build side records
+    g.setMappingSet(projectBuildMapping);
 
-                // Add the vector to our output container
-//                ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(), outputType), context.getAllocator());
-                container.addOrGet(MaterializedField.create(field.getPath(), outputType));
 
-                JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
-                JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
-                g.getEvalBlock()._if(outVV.invoke("copyFromSafe")
-                  .arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE)))
-                  .arg(outIndex)
-                  .arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))).not())._then()._return(JExpr.FALSE);
+    int fieldId = 0;
+    JExpression buildIndex = JExpr.direct("buildIndex");
+    JExpression outIndex = JExpr.direct("outIndex");
+    g.rotateBlock();
 
-                fieldId++;
-            }
+    if (rightSchema != null) {
+      for (MaterializedField field : rightSchema) {
+
+        MajorType inputType = field.getType();
+        MajorType outputType;
+        if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
+          outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
+        } else {
+          outputType = inputType;
         }
-        g.rotateBlock();
-        g.getEvalBlock()._return(JExpr.TRUE);
 
-        // Generate the code to project probe side records
-        g.setMappingSet(projectProbeMapping);
+        // Add the vector to our output container
+        //                ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(),
+        // outputType), context.getAllocator());
+        container.addOrGet(MaterializedField.create(field.getPath(), outputType));
 
-        int outputFieldId = fieldId;
-        fieldId = 0;
-        JExpression probeIndex = JExpr.direct("probeIndex");
-        int recordCount = 0;
+        JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
+        JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
+        g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE)))
+            .arg(outIndex).arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))).not())._then()._return(JExpr.FALSE);
 
-        if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
-            for (VectorWrapper<?> vv : left) {
+        fieldId++;
+      }
+    }
+    g.rotateBlock();
+    g.getEvalBlock()._return(JExpr.TRUE);
 
-                MajorType inputType = vv.getField().getType();
-                MajorType outputType;
-                if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
-                  outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
-                } else {
-                  outputType = inputType;
-                }
+    // Generate the code to project probe side records
+    g.setMappingSet(projectProbeMapping);
 
-                ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
-                if (v instanceof AbstractContainerVector) {
-                  vv.getValueVector().makeTransferPair(v);
-                  v.clear();
-                }
+    int outputFieldId = fieldId;
+    fieldId = 0;
+    JExpression probeIndex = JExpr.direct("probeIndex");
 
-                JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
-                JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
+    if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
+      for (VectorWrapper<?> vv : left) {
 
-                g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV).not())._then()._return(JExpr.FALSE);
+        MajorType inputType = vv.getField().getType();
+        MajorType outputType;
+        if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
+          outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
+        } else {
+          outputType = inputType;
+        }
 
-                fieldId++;
-                outputFieldId++;
-            }
-            recordCount = left.getRecordCount();
+        ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
+        if (v instanceof AbstractContainerVector) {
+          vv.getValueVector().makeTransferPair(v);
+          v.clear();
         }
-        g.rotateBlock();
-        g.getEvalBlock()._return(JExpr.TRUE);
 
-        HashJoinProbe hj = context.getImplementationClass(cg);
+        JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
+        JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
 
-        return hj;
-    }
+        g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV).not())._then()
+            ._return(JExpr.FALSE);
 
-    private void allocateVectors() {
-      for(VectorWrapper<?> v : container) {
-        v.getValueVector().allocateNew();
+        fieldId++;
+        outputFieldId++;
       }
     }
+    g.rotateBlock();
+    g.getEvalBlock()._return(JExpr.TRUE);
 
-    public HashJoinBatch(HashJoinPOP popConfig, FragmentContext context, RecordBatch left, RecordBatch right) throws OutOfMemoryException {
-        super(popConfig, context, true);
-        this.left = left;
-        this.right = right;
-        this.joinType = popConfig.getJoinType();
-        this.conditions = popConfig.getConditions();
-    }
+    HashJoinProbe hj = context.getImplementationClass(cg);
 
-    private void updateStats(HashTable htable) {
-      if (htable == null) {
-        return;
-      }
-      htable.getStats(htStats);
-      this.stats.setLongStat(Metric.NUM_BUCKETS, htStats.numBuckets);
-      this.stats.setLongStat(Metric.NUM_ENTRIES, htStats.numEntries);
-      this.stats.setLongStat(Metric.NUM_RESIZING, htStats.numResizing);
-      this.stats.setLongStat(Metric.RESIZING_TIME, htStats.resizingTime);
+    return hj;
+  }
+
+  private void allocateVectors() {
+    for (VectorWrapper<?> v : container) {
+      v.getValueVector().allocateNew();
     }
+  }
 
-    @Override
-    public void killIncoming(boolean sendUpstream) {
-        this.left.kill(sendUpstream);
-        this.right.kill(sendUpstream);
+  public HashJoinBatch(HashJoinPOP popConfig, FragmentContext context, RecordBatch left,
+      RecordBatch right) throws OutOfMemoryException {
+    super(popConfig, context, true);
+    this.left = left;
+    this.right = right;
+    this.joinType = popConfig.getJoinType();
+    this.conditions = popConfig.getConditions();
+  }
+
+  private void updateStats(HashTable htable) {
+    if (htable == null) {
+      return;
     }
+    htable.getStats(htStats);
+    this.stats.setLongStat(Metric.NUM_BUCKETS, htStats.numBuckets);
+    this.stats.setLongStat(Metric.NUM_ENTRIES, htStats.numEntries);
+    this.stats.setLongStat(Metric.NUM_RESIZING, htStats.numResizing);
+    this.stats.setLongStat(Metric.RESIZING_TIME, htStats.resizingTime);
+  }
 
-    @Override
-    public void cleanup() {
-        if (hjHelper != null) {
-          hjHelper.clear();
-        }
+  @Override
+  public void killIncoming(boolean sendUpstream) {
+    this.left.kill(sendUpstream);
+    this.right.kill(sendUpstream);
+  }
 
-        // If we didn't receive any data, hyperContainer may be null, check before clearing
-        if (hyperContainer != null) {
-            hyperContainer.clear();
-        }
+  @Override
+  public void cleanup() {
+    if (hjHelper != null) {
+      hjHelper.clear();
+    }
 
-        if (hashTable != null) {
-            hashTable.clear();
-        }
-        super.cleanup();
-        right.cleanup();
-        left.cleanup();
+    // If we didn't receive any data, hyperContainer may be null, check before clearing
+    if (hyperContainer != null) {
+      hyperContainer.clear();
     }
 
+    if (hashTable != null) {
+      hashTable.clear();
+    }
+    super.cleanup();
+    right.cleanup();
+    left.cleanup();
+  }
 }