You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/01/17 00:14:55 UTC
[1/5] drill git commit: DRILL-1969: Improve the error message
displayed when querying Hive views from Drill
Repository: drill
Updated Branches:
refs/heads/master d60850bcc -> bd4d669d1
DRILL-1969: Improve the error message displayed when querying Hive views from Drill
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fe6f7efe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fe6f7efe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fe6f7efe
Branch: refs/heads/master
Commit: fe6f7efe75c97e78392231128b1a6a4971a262c5
Parents: d60850b
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 10:39:45 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:31 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java | 3 ++-
.../main/java/org/apache/drill/exec/work/foreman/Foreman.java | 4 +++-
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/fe6f7efe/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index a3d4583..f4baf3b 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -71,7 +71,8 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
try {
if (hiveReadEntry.getJdbcTableType() == TableType.VIEW) {
- throw new UnsupportedOperationException("Querying Hive views from Drill is not supported in current version.");
+ throw new UnsupportedOperationException(
+ "Querying views created in Hive from Drill is not supported in current version.");
}
return new HiveScan(hiveReadEntry, this, columns);
http://git-wip-us.apache.org/repos/asf/drill/blob/fe6f7efe/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index e10a6aa..fcf5b91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.LogicalPlan;
@@ -438,7 +439,8 @@ public class Foreman implements Runnable, Closeable, Comparable<Object> {
assert exception != null;
recordNewState(QueryState.FAILED);
cancelExecutingFragments();
- DrillPBError error = ErrorHelper.logAndConvertError(context.getCurrentEndpoint(), "Query failed: "+ exception.getMessage(), exception, logger);
+ DrillPBError error = ErrorHelper.logAndConvertError(context.getCurrentEndpoint(),
+ ExceptionUtils.getRootCauseMessage(exception), exception, logger);
QueryResult result = QueryResult //
.newBuilder() //
.addError(error) //
[4/5] drill git commit: DRILL-1993: Fix allocation issues in
HashTable and HashAgg to reduce memory waste
Posted by ve...@apache.org.
DRILL-1993: Fix allocation issues in HashTable and HashAgg to reduce memory waste
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bd4d669d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bd4d669d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bd4d669d
Branch: refs/heads/master
Commit: bd4d669d1b836d6990eb3701e81c56f3d109db18
Parents: 8cdab2e
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 01:41:53 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800
----------------------------------------------------------------------
.../impl/aggregate/HashAggTemplate.java | 40 +++++++++++++++++---
.../physical/impl/common/ChainedHashTable.java | 1 -
.../exec/physical/impl/common/HashTable.java | 3 ++
.../physical/impl/common/HashTableTemplate.java | 28 ++++++++++----
.../apache/drill/exec/vector/ObjectVector.java | 6 +++
5 files changed, 63 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index dc2b05c..2fd5ce1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -44,7 +44,10 @@ import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.FixedWidthVector;
+import org.apache.drill.exec.vector.ObjectVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import javax.inject.Named;
@@ -128,7 +131,24 @@ public abstract class HashAggTemplate implements HashAggregator {
MaterializedField outputField = materializedValueFields[i];
// Create a type-specific ValueVector for this value
vector = TypeHelper.getNewVector(outputField, allocator);
- vector.allocateNew();
+
+ // Try to allocate space to store BATCH_SIZE records. Key stored at index i in HashTable has its workspace
+ // variables (such as count, sum etc) stored at index i in HashAgg. HashTable and HashAgg both have
+ // BatchHolders. Whenever a BatchHolder in HashAgg reaches its capacity, a new BatchHolder is added to
+ // HashTable. If HashAgg can't store BATCH_SIZE records in a BatchHolder, it leaves empty slots in current
+ // BatchHolder in HashTable, causing the HashTable to be space inefficient. So it is better to allocate space
+ // to fit as close to as BATCH_SIZE records.
+ if (vector instanceof FixedWidthVector) {
+ ((FixedWidthVector) vector).allocateNew(HashTable.BATCH_SIZE);
+ } else if (vector instanceof VariableWidthVector) {
+ ((VariableWidthVector) vector).allocateNew(HashTable.VARIABLE_WIDTH_VECTOR_SIZE * HashTable.BATCH_SIZE,
+ HashTable.BATCH_SIZE);
+ } else if (vector instanceof ObjectVector) {
+ ((ObjectVector)vector).allocateNew(HashTable.BATCH_SIZE);
+ } else {
+ vector.allocateNew();
+ }
+
capacity = Math.min(capacity, vector.getValueCapacity());
aggrValuesContainer.add(vector);
@@ -149,10 +169,14 @@ public abstract class HashAggTemplate implements HashAggregator {
outStartIdxHolder.value = batchOutputCount;
outNumRecordsHolder.value = 0;
boolean status = true;
- for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
- if (outputRecordValues(i, batchOutputCount)) {
+
+ // Output records starting from 'batchOutputCount' in current batch until there are no more records
+ // or output vectors have no space left. In destination vectors, start filling records from 0th position.
+ while(batchOutputCount <= maxOccupiedIdx) {
+ if (outputRecordValues(batchOutputCount, outNumRecordsHolder.value)) {
if (EXTRA_DEBUG_2) {
- logger.debug("Outputting values to output index: {}", batchOutputCount);
+ logger.debug("Outputting values from input index {} to output index: {}",
+ batchOutputCount, outNumRecordsHolder.value);
}
batchOutputCount++;
outNumRecordsHolder.value++;
@@ -256,7 +280,7 @@ public abstract class HashAggTemplate implements HashAggregator {
numGroupByOutFields = groupByOutFieldIds.length;
batchHolders = new ArrayList<BatchHolder>();
- addBatchHolder();
+ // First BatchHolder is created when the first put request is received.
doSetup(incoming);
}
@@ -494,7 +518,11 @@ public abstract class HashAggTemplate implements HashAggregator {
logger.debug("HashAggregate: Output current batch index {} with {} records.", outBatchIndex, numOutputRecords);
lastBatchOutputCount = numOutputRecords;
- outBatchIndex++;
+ // If there are no more records to output, go to the next batch. If there are any records left refer to the
+ // same BatchHolder. Remaining records will be outputted in next outputCurrentBatch() call(s).
+ if (batchHolders.get(outBatchIndex).getNumPendingOutput() == 0) {
+ outBatchIndex++;
+ }
if (outBatchIndex == batchHolders.size()) {
allFlushed = true;
http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 322fd1f..b5cfdca 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -163,7 +163,6 @@ public class ChainedHashTable {
final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
// create a type-specific ValueVector for this key
ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
- vv.allocateNew();
htKeyFieldIds[i] = htContainerOrig.add(vv);
i++;
http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index 6966ba1..1ec74bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -51,6 +51,9 @@ public interface HashTable {
static final public int BATCH_SIZE = Character.MAX_VALUE + 1;
static final public int BATCH_MASK = 0x0000FFFF;
+ /** Variable width vector size in bytes */
+ public static final int VARIABLE_WIDTH_VECTOR_SIZE = 50 * BATCH_SIZE;
+
public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
RecordBatch incomingBuild, RecordBatch incomingProbe,
RecordBatch outgoing, VectorContainer htContainerOrig);
http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index c80e97a..ba980d7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -30,8 +30,10 @@ import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.FixedWidthVector;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
import javax.inject.Named;
import java.util.ArrayList;
@@ -116,15 +118,24 @@ public abstract class HashTableTemplate implements HashTable {
private BatchHolder(int idx) {
this.batchIndex = idx;
- if (idx == 0) { // first batch holder can use the original htContainer
- htContainer = htContainerOrig;
- } else { // otherwise create a new one using the original's fields
- htContainer = new VectorContainer();
- for (VectorWrapper<?> w : htContainerOrig) {
- ValueVector vv = TypeHelper.getNewVector(w.getField(), allocator);
+ htContainer = new VectorContainer();
+ for (VectorWrapper<?> w : htContainerOrig) {
+ ValueVector vv = TypeHelper.getNewVector(w.getField(), allocator);
+
+ // Capacity for "hashValues" and "links" vectors is BATCH_SIZE records. It is better to allocate space for
+ // "key" vectors to store as close to as BATCH_SIZE records. A new BatchHolder is created when either BATCH_SIZE
+ // records are inserted or "key" vectors ran out of space. Allocating too less space for "key" vectors will
+ // result in unused space in "hashValues" and "links" vectors in the BatchHolder. Also for each new
+ // BatchHolder we create a SV4 vector of BATCH_SIZE in HashJoinHelper.
+ if (vv instanceof FixedWidthVector) {
+ ((FixedWidthVector) vv).allocateNew(BATCH_SIZE);
+ } else if (vv instanceof VariableWidthVector) {
+ ((VariableWidthVector) vv).allocateNew(VARIABLE_WIDTH_VECTOR_SIZE, BATCH_SIZE);
+ } else {
vv.allocateNew();
- htContainer.add(vv);
}
+
+ htContainer.add(vv);
}
links = allocMetadataVector(HashTable.BATCH_SIZE, EMPTY_SLOT);
@@ -454,7 +465,7 @@ public abstract class HashTableTemplate implements HashTable {
// Create the first batch holder
batchHolders = new ArrayList<BatchHolder>();
- addBatchHolder();
+ // First BatchHolder is created when the first put request is received.
doSetup(incomingBuild, incomingProbe);
@@ -753,6 +764,7 @@ public abstract class HashTableTemplate implements HashTable {
public boolean outputKeys(int batchIdx, VectorContainer outContainer, int outStartIndex, int numRecords) {
assert batchIdx < batchHolders.size();
+
if (!batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/bd4d669d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index b68f089..3c15db3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -97,6 +97,12 @@ public class ObjectVector extends BaseValueVector{
addNewArray();
}
+ public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
+ while (maxCount < valueCount) {
+ addNewArray();
+ }
+ }
+
@Override
public boolean allocateNewSafe() {
allocateNew();
[5/5] drill git commit: DRILL-1990: Add peak memory allocation in a
operator to OperatorStats.
Posted by ve...@apache.org.
DRILL-1990: Add peak memory allocation in a operator to OperatorStats.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8cdab2ed
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8cdab2ed
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8cdab2ed
Branch: refs/heads/master
Commit: 8cdab2ed7d4c0d57ea668f6387281acbc26f4890
Parents: 10fd9e1
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Dec 30 17:13:35 2014 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800
----------------------------------------------------------------------
.../java/io/netty/buffer/FakeAllocator.java | 5 +
.../org/apache/drill/exec/memory/Accountor.java | 16 ++-
.../drill/exec/memory/BufferAllocator.java | 2 +
.../drill/exec/memory/TopLevelAllocator.java | 10 ++
.../apache/drill/exec/ops/OperatorStats.java | 2 +-
.../drill/exec/proto/SchemaUserBitShared.java | 10 +-
.../apache/drill/exec/proto/UserBitShared.java | 134 +++++++++----------
.../drill/exec/proto/beans/OperatorProfile.java | 22 +--
protocol/src/main/protobuf/UserBitShared.proto | 2 +-
9 files changed, 116 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
index df85e74..3de0a75 100644
--- a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
+++ b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -91,6 +91,11 @@ class FakeAllocator implements BufferAllocator {
return 0;
}
+ @Override
+ public long getPeakMemoryAllocation() {
+ return 0;
+ }
+
static class FakeAccountor extends Accountor {
public FakeAccountor() {
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
index 0a8c405..2b48ef0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -62,6 +62,8 @@ public class Accountor {
private final FragmentContext fragmentContext;
long fragmentLimit;
+ private long peakMemoryAllocation = 0;
+
// The top level Allocator has an accountor that keeps track of all the FragmentContexts currently executing.
// This enables the top level accountor to calculate a new fragment limit whenever necessary.
private final List<FragmentContext> fragmentContexts;
@@ -132,14 +134,22 @@ public class Accountor {
return remainder.getUsed();
}
+ public long getPeakMemoryAllocation() {
+ return peakMemoryAllocation;
+ }
+
public boolean reserve(long size) {
logger.trace("Fragment:"+fragmentStr+" Reserved "+size+" bytes. Total Allocated: "+getAllocation());
- return remainder.get(size, this.applyFragmentLimit);
+ boolean status = remainder.get(size, this.applyFragmentLimit);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
}
public boolean forceAdditionalReservation(long size) {
if (size > 0) {
- return remainder.forceGet(size);
+ boolean status = remainder.forceGet(size);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
} else {
return true;
}
@@ -156,6 +166,8 @@ public class Accountor {
if (ENABLE_ACCOUNTING) {
buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
}
+
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
index e072126..83d9d1e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -108,5 +108,7 @@ public interface BufferAllocator extends Closeable {
public abstract long getAllocatedMemory();
+ public abstract long getPeakMemoryAllocation();
+
public DrillBuf getEmpty();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 32ec37f..67e1fdb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -99,6 +99,11 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
+ public long getPeakMemoryAllocation() {
+ return acct.getPeakMemoryAllocation();
+ }
+
+ @Override
public ByteBufAllocator getUnderlyingAllocator() {
return innerAllocator;
}
@@ -290,6 +295,11 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
+ public long getPeakMemoryAllocation() {
+ return childAcct.getPeakMemoryAllocation();
+ }
+
+ @Override
public DrillBuf getEmpty() {
return empty;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 8771ab8..0e9da0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -129,7 +129,7 @@ public class OperatorStats {
.setWaitNanos(waitNanos);
if(allocator != null){
- b.setLocalMemoryAllocated(allocator.getAllocatedMemory());
+ b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
index fdd003e..68e86db 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
@@ -2175,8 +2175,8 @@ public final class SchemaUserBitShared
output.writeInt64(5, message.getSetupNanos(), false);
if(message.hasProcessNanos())
output.writeInt64(6, message.getProcessNanos(), false);
- if(message.hasLocalMemoryAllocated())
- output.writeInt64(7, message.getLocalMemoryAllocated(), false);
+ if(message.hasPeakLocalMemoryAllocated())
+ output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false);
for(org.apache.drill.exec.proto.UserBitShared.MetricValue metric : message.getMetricList())
output.writeObject(8, metric, org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true);
@@ -2238,7 +2238,7 @@ public final class SchemaUserBitShared
builder.setProcessNanos(input.readInt64());
break;
case 7:
- builder.setLocalMemoryAllocated(input.readInt64());
+ builder.setPeakLocalMemoryAllocated(input.readInt64());
break;
case 8:
builder.addMetric(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.MetricValue.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.MERGE));
@@ -2292,7 +2292,7 @@ public final class SchemaUserBitShared
case 4: return "operatorType";
case 5: return "setupNanos";
case 6: return "processNanos";
- case 7: return "localMemoryAllocated";
+ case 7: return "peakLocalMemoryAllocated";
case 8: return "metric";
case 9: return "waitNanos";
default: return null;
@@ -2311,7 +2311,7 @@ public final class SchemaUserBitShared
fieldMap.put("operatorType", 4);
fieldMap.put("setupNanos", 5);
fieldMap.put("processNanos", 6);
- fieldMap.put("localMemoryAllocated", 7);
+ fieldMap.put("peakLocalMemoryAllocated", 7);
fieldMap.put("metric", 8);
fieldMap.put("waitNanos", 9);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index e0666f2..5682a3b 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -17137,15 +17137,15 @@ public final class UserBitShared {
*/
long getProcessNanos();
- // optional int64 local_memory_allocated = 7;
+ // optional int64 peak_local_memory_allocated = 7;
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- boolean hasLocalMemoryAllocated();
+ boolean hasPeakLocalMemoryAllocated();
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- long getLocalMemoryAllocated();
+ long getPeakLocalMemoryAllocated();
// repeated .exec.shared.MetricValue metric = 8;
/**
@@ -17263,7 +17263,7 @@ public final class UserBitShared {
}
case 56: {
bitField0_ |= 0x00000010;
- localMemoryAllocated_ = input.readInt64();
+ peakLocalMemoryAllocated_ = input.readInt64();
break;
}
case 66: {
@@ -17425,20 +17425,20 @@ public final class UserBitShared {
return processNanos_;
}
- // optional int64 local_memory_allocated = 7;
- public static final int LOCAL_MEMORY_ALLOCATED_FIELD_NUMBER = 7;
- private long localMemoryAllocated_;
+ // optional int64 peak_local_memory_allocated = 7;
+ public static final int PEAK_LOCAL_MEMORY_ALLOCATED_FIELD_NUMBER = 7;
+ private long peakLocalMemoryAllocated_;
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public boolean hasLocalMemoryAllocated() {
+ public boolean hasPeakLocalMemoryAllocated() {
return ((bitField0_ & 0x00000010) == 0x00000010);
}
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public long getLocalMemoryAllocated() {
- return localMemoryAllocated_;
+ public long getPeakLocalMemoryAllocated() {
+ return peakLocalMemoryAllocated_;
}
// repeated .exec.shared.MetricValue metric = 8;
@@ -17499,7 +17499,7 @@ public final class UserBitShared {
operatorType_ = 0;
setupNanos_ = 0L;
processNanos_ = 0L;
- localMemoryAllocated_ = 0L;
+ peakLocalMemoryAllocated_ = 0L;
metric_ = java.util.Collections.emptyList();
waitNanos_ = 0L;
}
@@ -17531,7 +17531,7 @@ public final class UserBitShared {
output.writeInt64(6, processNanos_);
}
if (((bitField0_ & 0x00000010) == 0x00000010)) {
- output.writeInt64(7, localMemoryAllocated_);
+ output.writeInt64(7, peakLocalMemoryAllocated_);
}
for (int i = 0; i < metric_.size(); i++) {
output.writeMessage(8, metric_.get(i));
@@ -17570,7 +17570,7 @@ public final class UserBitShared {
}
if (((bitField0_ & 0x00000010) == 0x00000010)) {
size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(7, localMemoryAllocated_);
+ .computeInt64Size(7, peakLocalMemoryAllocated_);
}
for (int i = 0; i < metric_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
@@ -17712,7 +17712,7 @@ public final class UserBitShared {
bitField0_ = (bitField0_ & ~0x00000008);
processNanos_ = 0L;
bitField0_ = (bitField0_ & ~0x00000010);
- localMemoryAllocated_ = 0L;
+ peakLocalMemoryAllocated_ = 0L;
bitField0_ = (bitField0_ & ~0x00000020);
if (metricBuilder_ == null) {
metric_ = java.util.Collections.emptyList();
@@ -17778,7 +17778,7 @@ public final class UserBitShared {
if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
to_bitField0_ |= 0x00000010;
}
- result.localMemoryAllocated_ = localMemoryAllocated_;
+ result.peakLocalMemoryAllocated_ = peakLocalMemoryAllocated_;
if (metricBuilder_ == null) {
if (((bitField0_ & 0x00000040) == 0x00000040)) {
metric_ = java.util.Collections.unmodifiableList(metric_);
@@ -17846,8 +17846,8 @@ public final class UserBitShared {
if (other.hasProcessNanos()) {
setProcessNanos(other.getProcessNanos());
}
- if (other.hasLocalMemoryAllocated()) {
- setLocalMemoryAllocated(other.getLocalMemoryAllocated());
+ if (other.hasPeakLocalMemoryAllocated()) {
+ setPeakLocalMemoryAllocated(other.getPeakLocalMemoryAllocated());
}
if (metricBuilder_ == null) {
if (!other.metric_.isEmpty()) {
@@ -18277,35 +18277,35 @@ public final class UserBitShared {
return this;
}
- // optional int64 local_memory_allocated = 7;
- private long localMemoryAllocated_ ;
+ // optional int64 peak_local_memory_allocated = 7;
+ private long peakLocalMemoryAllocated_ ;
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public boolean hasLocalMemoryAllocated() {
+ public boolean hasPeakLocalMemoryAllocated() {
return ((bitField0_ & 0x00000020) == 0x00000020);
}
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public long getLocalMemoryAllocated() {
- return localMemoryAllocated_;
+ public long getPeakLocalMemoryAllocated() {
+ return peakLocalMemoryAllocated_;
}
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public Builder setLocalMemoryAllocated(long value) {
+ public Builder setPeakLocalMemoryAllocated(long value) {
bitField0_ |= 0x00000020;
- localMemoryAllocated_ = value;
+ peakLocalMemoryAllocated_ = value;
onChanged();
return this;
}
/**
- * <code>optional int64 local_memory_allocated = 7;</code>
+ * <code>optional int64 peak_local_memory_allocated = 7;</code>
*/
- public Builder clearLocalMemoryAllocated() {
+ public Builder clearPeakLocalMemoryAllocated() {
bitField0_ = (bitField0_ & ~0x00000020);
- localMemoryAllocated_ = 0L;
+ peakLocalMemoryAllocated_ = 0L;
onChanged();
return this;
}
@@ -19877,41 +19877,41 @@ public final class UserBitShared {
".exec.shared.OperatorProfile\022\022\n\nstart_ti" +
"me\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_use" +
"d\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010endp" +
- "oint\030\t \001(\0132\026.exec.DrillbitEndpoint\"\372\001\n\017O" +
+ "oint\030\t \001(\0132\026.exec.DrillbitEndpoint\"\377\001\n\017O" +
"peratorProfile\0221\n\rinput_profile\030\001 \003(\0132\032.",
"exec.shared.StreamProfile\022\023\n\013operator_id" +
"\030\003 \001(\005\022\025\n\roperator_type\030\004 \001(\005\022\023\n\013setup_n" +
- "anos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022\036\n\026loc" +
- "al_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(" +
- "\0132\030.exec.shared.MetricValue\022\022\n\nwait_nano" +
- "s\030\t \001(\003\"B\n\rStreamProfile\022\017\n\007records\030\001 \001(" +
- "\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013M" +
- "etricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong_va" +
- "lue\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\nRpcCh" +
- "annel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004",
- "USER\020\002*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020" +
- "\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmentState\022\013\n\007SEND" +
- "ING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNIN" +
- "G\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAIL" +
- "ED\020\005*\264\005\n\020CoreOperatorType\022\021\n\rSINGLE_SEND" +
- "ER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTER\020\002\022\022" +
- "\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMER" +
- "GE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDER\020\006\022\t\n\005" +
- "LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030ORDERED" +
- "_PARTITION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n\022UNOR",
- "DERED_RECEIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022\n\n\006SC" +
- "REEN\020\r\022\034\n\030SELECTION_VECTOR_REMOVER\020\016\022\027\n\023" +
- "STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N_SORT\020\020\022\021\n" +
- "\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNION\020\023\022\014" +
- "\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GROUP_SCAN\020\025" +
- "\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE_SCAN" +
- "\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQUET_WRITER\020" +
- "\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT_WRITER\020\033\022" +
- "\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020\035\022\030\n" +
- "\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COMPLEX_TO_JS",
- "ON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n\016HBASE_SUB" +
- "_SCAN\020!\022\n\n\006WINDOW\020\"B.\n\033org.apache.drill." +
- "exec.protoB\rUserBitSharedH\001"
+ "anos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033pea" +
+ "k_local_memory_allocated\030\007 \001(\003\022(\n\006metric" +
+ "\030\010 \003(\0132\030.exec.shared.MetricValue\022\022\n\nwait" +
+ "_nanos\030\t \001(\003\"B\n\rStreamProfile\022\017\n\007records" +
+ "\030\001 \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003" +
+ "\"J\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlo" +
+ "ng_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\n" +
+ "RpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020",
+ "\001\022\010\n\004USER\020\002*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOG" +
+ "ICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmentState\022\013\n" +
+ "\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007R" +
+ "UNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n" +
+ "\006FAILED\020\005*\264\005\n\020CoreOperatorType\022\021\n\rSINGLE" +
+ "_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTE" +
+ "R\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016" +
+ "\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDER\020" +
+ "\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030OR" +
+ "DERED_PARTITION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n",
+ "\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022" +
+ "\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_REMOVER\020" +
+ "\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N_SORT" +
+ "\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNIO" +
+ "N\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GROUP_S" +
+ "CAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE" +
+ "_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQUET_WR" +
+ "ITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT_WRIT" +
+ "ER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN" +
+ "\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COMPLEX_",
+ "TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n\016HBAS" +
+ "E_SUB_SCAN\020!\022\n\n\006WINDOW\020\"B.\n\033org.apache.d" +
+ "rill.exec.protoB\rUserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -20013,7 +20013,7 @@ public final class UserBitShared {
internal_static_exec_shared_OperatorProfile_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_exec_shared_OperatorProfile_descriptor,
- new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "LocalMemoryAllocated", "Metric", "WaitNanos", });
+ new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "PeakLocalMemoryAllocated", "Metric", "WaitNanos", });
internal_static_exec_shared_StreamProfile_descriptor =
getDescriptor().getMessageTypes().get(16);
internal_static_exec_shared_StreamProfile_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
index f1b1acc..e197f54 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
@@ -54,7 +54,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
private int operatorType;
private long setupNanos;
private long processNanos;
- private long localMemoryAllocated;
+ private long peakLocalMemoryAllocated;
private List<MetricValue> metric;
private long waitNanos;
@@ -130,16 +130,16 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
return this;
}
- // localMemoryAllocated
+ // peakLocalMemoryAllocated
- public long getLocalMemoryAllocated()
+ public long getPeakLocalMemoryAllocated()
{
- return localMemoryAllocated;
+ return peakLocalMemoryAllocated;
}
- public OperatorProfile setLocalMemoryAllocated(long localMemoryAllocated)
+ public OperatorProfile setPeakLocalMemoryAllocated(long peakLocalMemoryAllocated)
{
- this.localMemoryAllocated = localMemoryAllocated;
+ this.peakLocalMemoryAllocated = peakLocalMemoryAllocated;
return this;
}
@@ -242,7 +242,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
message.processNanos = input.readInt64();
break;
case 7:
- message.localMemoryAllocated = input.readInt64();
+ message.peakLocalMemoryAllocated = input.readInt64();
break;
case 8:
if(message.metric == null)
@@ -284,8 +284,8 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
if(message.processNanos != 0)
output.writeInt64(6, message.processNanos, false);
- if(message.localMemoryAllocated != 0)
- output.writeInt64(7, message.localMemoryAllocated, false);
+ if(message.peakLocalMemoryAllocated != 0)
+ output.writeInt64(7, message.peakLocalMemoryAllocated, false);
if(message.metric != null)
{
@@ -310,7 +310,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
case 4: return "operatorType";
case 5: return "setupNanos";
case 6: return "processNanos";
- case 7: return "localMemoryAllocated";
+ case 7: return "peakLocalMemoryAllocated";
case 8: return "metric";
case 9: return "waitNanos";
default: return null;
@@ -331,7 +331,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
__fieldMap.put("operatorType", 4);
__fieldMap.put("setupNanos", 5);
__fieldMap.put("processNanos", 6);
- __fieldMap.put("localMemoryAllocated", 7);
+ __fieldMap.put("peakLocalMemoryAllocated", 7);
__fieldMap.put("metric", 8);
__fieldMap.put("waitNanos", 9);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8cdab2ed/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 3b85e9b..8f05c45 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -164,7 +164,7 @@ message OperatorProfile {
optional int32 operator_type = 4;
optional int64 setup_nanos = 5;
optional int64 process_nanos = 6;
- optional int64 local_memory_allocated = 7;
+ optional int64 peak_local_memory_allocated = 7;
repeated MetricValue metric = 8;
optional int64 wait_nanos = 9;
}
[3/5] drill git commit: DRILL-1991: Code indendation and formatting
cleanup for few files
Posted by ve...@apache.org.
DRILL-1991: Code indendation and formatting cleanup for few files
+ Reformatted using IDE except removing two unused static variables.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/10fd9e10
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/10fd9e10
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/10fd9e10
Branch: refs/heads/master
Commit: 10fd9e101207d5d861f023e45fb46c0787bf6da0
Parents: fe6f7ef
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 9 00:02:47 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Fri Jan 16 14:03:32 2015 -0800
----------------------------------------------------------------------
.../physical/impl/aggregate/HashAggBatch.java | 77 +-
.../impl/aggregate/HashAggTemplate.java | 110 +--
.../physical/impl/aggregate/HashAggregator.java | 18 +-
.../physical/impl/common/ChainedHashTable.java | 166 +++--
.../exec/physical/impl/common/HashTable.java | 27 +-
.../physical/impl/common/HashTableTemplate.java | 160 +++--
.../exec/physical/impl/join/HashJoinBatch.java | 705 +++++++++----------
7 files changed, 654 insertions(+), 609 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index 113e883..35faf22 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.physical.impl.aggregate;
-import java.io.IOException;
-
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JVar;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
@@ -51,8 +51,7 @@ import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.ValueVector;
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JVar;
+import java.io.IOException;
public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggBatch.class);
@@ -60,21 +59,25 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
private HashAggregator aggregator;
private final RecordBatch incoming;
private LogicalExpression[] aggrExprs;
- private TypedFieldId[] groupByOutFieldIds ;
- private TypedFieldId[] aggrOutFieldIds ; // field ids for the outgoing batch
+ private TypedFieldId[] groupByOutFieldIds;
+ private TypedFieldId[] aggrOutFieldIds; // field ids for the outgoing batch
private final GeneratorMapping UPDATE_AGGR_INSIDE =
- GeneratorMapping.create("setupInterior" /* setup method */, "updateAggrValuesInternal" /* eval method */,
- "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+ GeneratorMapping.create("setupInterior" /* setup method */, "updateAggrValuesInternal" /* eval method */,
+ "resetValues" /* reset */, "cleanup" /* cleanup */);
private final GeneratorMapping UPDATE_AGGR_OUTSIDE =
- GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordValues" /* eval method */,
- "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+ GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordValues" /* eval method */,
+ "resetValues" /* reset */, "cleanup" /* cleanup */);
- private final MappingSet UpdateAggrValuesMapping = new MappingSet("incomingRowIdx" /* read index */, "outRowIdx" /* write index */, "htRowIdx" /* workspace index */, "incoming" /* read container */, "outgoing" /* write container */, "aggrValuesContainer" /* workspace container */, UPDATE_AGGR_INSIDE, UPDATE_AGGR_OUTSIDE, UPDATE_AGGR_INSIDE);
+ private final MappingSet UpdateAggrValuesMapping =
+ new MappingSet("incomingRowIdx" /* read index */, "outRowIdx" /* write index */,
+ "htRowIdx" /* workspace index */, "incoming" /* read container */, "outgoing" /* write container */,
+ "aggrValuesContainer" /* workspace container */, UPDATE_AGGR_INSIDE, UPDATE_AGGR_OUTSIDE, UPDATE_AGGR_INSIDE);
- public HashAggBatch(HashAggregate popConfig, RecordBatch incoming, FragmentContext context) throws ExecutionSetupException {
+ public HashAggBatch(HashAggregate popConfig, RecordBatch incoming, FragmentContext context) throws
+ ExecutionSetupException {
super(popConfig, context);
this.incoming = incoming;
}
@@ -122,7 +125,7 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
logger.debug("Next outcome of {}", outcome);
switch (outcome) {
case NONE:
-// throw new UnsupportedOperationException("Received NONE on first batch");
+ // throw new UnsupportedOperationException("Received NONE on first batch");
return outcome;
case NOT_YET:
case STOP:
@@ -144,13 +147,13 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
return IterOutcome.NONE;
}
- if (aggregator.buildComplete() && ! aggregator.allFlushed()) {
- // aggregation is complete and not all records have been output yet
- IterOutcome outcome = aggregator.outputCurrentBatch();
- return outcome;
- }
+ if (aggregator.buildComplete() && !aggregator.allFlushed()) {
+ // aggregation is complete and not all records have been output yet
+ IterOutcome outcome = aggregator.outputCurrentBatch();
+ return outcome;
+ }
- logger.debug("Starting aggregator doWork; incoming record count = {} ", incoming.getRecordCount());
+ logger.debug("Starting aggregator doWork; incoming record count = {} ", incoming.getRecordCount());
while (true) {
AggOutcome out = aggregator.doWork();
@@ -198,8 +201,10 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
}
}
- private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
- CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+ private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException,
+ IOException {
+ CodeGenerator<HashAggregator> top =
+ CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry());
ClassGenerator<HashAggregator> cg = top.getRoot();
ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
@@ -217,7 +222,8 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
for (i = 0; i < numGroupByExprs; i++) {
NamedExpression ne = popConfig.getGroupByExprs()[i];
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+ final LogicalExpression expr =
+ ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
if (expr == null) {
continue;
}
@@ -231,7 +237,8 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
for (i = 0; i < numAggrExprs; i++) {
NamedExpression ne = popConfig.getAggrExprs()[i];
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+ final LogicalExpression expr =
+ ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
@@ -255,17 +262,12 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
container.buildSchema(SelectionVectorMode.NONE);
HashAggregator agg = context.getImplementationClass(top);
- HashTableConfig htConfig = new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
- HashTable.DEFAULT_LOAD_FACTOR,
- popConfig.getGroupByExprs(),
- null /* no probe exprs */) ;
+ HashTableConfig htConfig =
+ new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
+ HashTable.DEFAULT_LOAD_FACTOR, popConfig.getGroupByExprs(), null /* no probe exprs */);
- agg.setup(popConfig, htConfig, context, this.stats,
- oContext.getAllocator(), incoming, this,
- aggrExprs,
- cgInner.getWorkspaceTypes(),
- groupByOutFieldIds,
- this.container);
+ agg.setup(popConfig, htConfig, context, this.stats, oContext.getAllocator(), incoming, this, aggrExprs,
+ cgInner.getWorkspaceTypes(), groupByOutFieldIds, this.container);
return agg;
}
@@ -286,25 +288,23 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
case FOUR_BYTE: {
JVar var = cg.declareClassField("sv4_", cg.getModel()._ref(SelectionVector4.class));
cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
- cg.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ cg.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));
return;
}
case NONE: {
- cg.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+ cg.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));
return;
}
case TWO_BYTE: {
JVar var = cg.declareClassField("sv2_", cg.getModel()._ref(SelectionVector2.class));
cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
- cg.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));;
+ cg.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));
return;
}
default:
throw new IllegalStateException();
-
}
-
}
@Override
@@ -320,5 +320,4 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
protected void killIncoming(boolean sendUpstream) {
incoming.kill(sendUpstream);
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index d7cf904..dc2b05c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -17,13 +17,7 @@
*/
package org.apache.drill.exec.physical.impl.aggregate;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.inject.Named;
-
+import com.google.common.collect.Lists;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
@@ -53,17 +47,19 @@ import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
-import com.google.common.collect.Lists;
+import javax.inject.Named;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
public abstract class HashAggTemplate implements HashAggregator {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggregator.class);
- private static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024;
- private static final long ALLOCATOR_MAX_RESERVATION = 20L*1000*1000*1000;
-
private static final boolean EXTRA_DEBUG_1 = false;
private static final boolean EXTRA_DEBUG_2 = false;
- private static final String TOO_BIG_ERROR = "Couldn't add value to an empty batch. This likely means that a single value is too long for a varlen field.";
+ private static final String TOO_BIG_ERROR =
+ "Couldn't add value to an empty batch. This likely means that a single value is too long for a varlen field.";
private boolean first = true;
private boolean newSchema = false;
private int underlyingIndex = 0;
@@ -93,7 +89,7 @@ public abstract class HashAggTemplate implements HashAggregator {
private MaterializedField[] materializedValueFields;
private boolean allFlushed = false;
- private boolean buildComplete = false;
+ private boolean buildComplete = false;
private OperatorStats stats = null;
private HashTableStats htStats = new HashTableStats();
@@ -113,7 +109,6 @@ public abstract class HashAggTemplate implements HashAggregator {
}
}
-
public class BatchHolder {
private VectorContainer aggrValuesContainer; // container for aggr values (workspace variables)
@@ -127,16 +122,16 @@ public abstract class HashAggTemplate implements HashAggregator {
aggrValuesContainer = new VectorContainer();
- ValueVector vector ;
+ ValueVector vector;
- for(int i = 0; i < materializedValueFields.length; i++) {
+ for (int i = 0; i < materializedValueFields.length; i++) {
MaterializedField outputField = materializedValueFields[i];
// Create a type-specific ValueVector for this value
- vector = TypeHelper.getNewVector(outputField, allocator) ;
+ vector = TypeHelper.getNewVector(outputField, allocator);
vector.allocateNew();
capacity = Math.min(capacity, vector.getValueCapacity());
- aggrValuesContainer.add(vector) ;
+ aggrValuesContainer.add(vector);
}
}
@@ -155,9 +150,9 @@ public abstract class HashAggTemplate implements HashAggregator {
outNumRecordsHolder.value = 0;
boolean status = true;
for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
- if (outputRecordValues(i, batchOutputCount) ) {
+ if (outputRecordValues(i, batchOutputCount)) {
if (EXTRA_DEBUG_2) {
- logger.debug("Outputting values to output index: {}", batchOutputCount) ;
+ logger.debug("Outputting values to output index: {}", batchOutputCount);
}
batchOutputCount++;
outNumRecordsHolder.value++;
@@ -190,26 +185,26 @@ public abstract class HashAggTemplate implements HashAggregator {
// Code-generated methods (implemented in HashAggBatch)
@RuntimeOverridden
- public void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing, @Named("aggrValuesContainer") VectorContainer aggrValuesContainer) {}
+ public void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing,
+ @Named("aggrValuesContainer") VectorContainer aggrValuesContainer) {
+ }
@RuntimeOverridden
- public void updateAggrValuesInternal(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {}
+ public void updateAggrValuesInternal(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+ }
@RuntimeOverridden
- public boolean outputRecordValues(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return true;}
+ public boolean outputRecordValues(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {
+ return true;
+ }
}
@Override
- public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig,
- FragmentContext context,
- OperatorStats stats,
- BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
- LogicalExpression[] valueExprs,
- List<TypedFieldId> valueFieldIds,
- TypedFieldId[] groupByOutFieldIds,
- VectorContainer outContainer)
- throws SchemaChangeException, ClassTransformationException, IOException {
+ public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig, FragmentContext context,
+ OperatorStats stats, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
+ LogicalExpression[] valueExprs, List<TypedFieldId> valueFieldIds, TypedFieldId[] groupByOutFieldIds,
+ VectorContainer outContainer) throws SchemaChangeException, ClassTransformationException, IOException {
if (valueExprs == null || valueFieldIds == null) {
throw new IllegalArgumentException("Invalid aggr value exprs or workspace variables.");
@@ -235,7 +230,8 @@ public abstract class HashAggTemplate implements HashAggregator {
// we need to build a hash table on the aggregation column a1.
// TODO: This functionality will be added later.
if (hashAggrConfig.getGroupByExprs().length == 0) {
- throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by expressions.");
+ throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by " +
+ "expressions.");
}
this.htIdxHolder = new IndexPointer();
@@ -246,15 +242,17 @@ public abstract class HashAggTemplate implements HashAggregator {
if (valueFieldIds.size() > 0) {
int i = 0;
- FieldReference ref = new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getIntermediateType());
+ FieldReference ref =
+ new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getIntermediateType());
for (TypedFieldId id : valueFieldIds) {
materializedValueFields[i++] = MaterializedField.create(ref, id.getIntermediateType());
}
}
- ChainedHashTable ht = new ChainedHashTable(htConfig, context, allocator, incoming,
- null /* no incoming probe */, outgoing, true /* nulls are equal */) ;
- this.htable = ht.createAndSetupHashTable(groupByOutFieldIds) ;
+ ChainedHashTable ht =
+ new ChainedHashTable(htConfig, context, allocator, incoming, null /* no incoming probe */, outgoing,
+ true /* nulls are equal */);
+ this.htable = ht.createAndSetupHashTable(groupByOutFieldIds);
numGroupByOutFields = groupByOutFieldIds.length;
batchHolders = new ArrayList<BatchHolder>();
@@ -265,19 +263,20 @@ public abstract class HashAggTemplate implements HashAggregator {
@Override
public AggOutcome doWork() {
- try{
+ try {
// Note: Keeping the outer and inner try blocks here to maintain some similarity with
// StreamingAggregate which does somethings conditionally in the outer try block.
// In the future HashAggregate may also need to perform some actions conditionally
// in the outer try block.
- outside: while(true) {
+ outside:
+ while (true) {
// loop through existing records, aggregating the values as necessary.
if (EXTRA_DEBUG_1) {
- logger.debug ("Starting outer loop of doWork()...");
+ logger.debug("Starting outer loop of doWork()...");
}
for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
- if(EXTRA_DEBUG_2) {
+ if (EXTRA_DEBUG_2) {
logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
}
boolean success = checkGroupAndAggrValues(currentIndex);
@@ -369,7 +368,7 @@ public abstract class HashAggTemplate implements HashAggregator {
// Skip the keys and only allocate for outputting the workspace values
// (keys will be output through splitAndTransfer)
Iterator<VectorWrapper<?>> outgoingIter = outContainer.iterator();
- for (int i=0; i < numGroupByOutFields; i++) {
+ for (int i = 0; i < numGroupByOutFields; i++) {
outgoingIter.next();
}
while (outgoingIter.hasNext()) {
@@ -469,16 +468,18 @@ public abstract class HashAggTemplate implements HashAggregator {
int numOutputRecords = outNumRecordsHolder.value;
if (EXTRA_DEBUG_1) {
- logger.debug("After output values: outStartIdx = {}, outNumRecords = {}", outStartIdxHolder.value, outNumRecordsHolder.value);
+ logger.debug("After output values: outStartIdx = {}, outNumRecords = {}", outStartIdxHolder.value,
+ outNumRecordsHolder.value);
}
if (outputValuesStatus) {
- outputKeysStatus = this.htable.outputKeys(outBatchIndex, this.outContainer, outStartIdxHolder.value, outNumRecordsHolder.value) ;
+ outputKeysStatus =
+ this.htable.outputKeys(outBatchIndex, this.outContainer, outStartIdxHolder.value, outNumRecordsHolder.value);
}
if (outputKeysStatus && outputValuesStatus) {
// set the value count for outgoing batch value vectors
- for(VectorWrapper<?> v : outgoing) {
+ for (VectorWrapper<?> v : outgoing) {
v.getValueVector().getMutator().setValueCount(numOutputRecords);
}
@@ -506,14 +507,16 @@ public abstract class HashAggTemplate implements HashAggregator {
if (!outputKeysStatus) {
logger.debug("Failed to output keys for current batch index: {} ", outBatchIndex);
for (VectorWrapper<?> v : outContainer) {
- logger.debug("At the time of failure, size of valuevector in outContainer = {}.", v.getValueVector().getValueCapacity());
+ logger.debug("At the time of failure, size of valuevector in outContainer = {}.",
+ v.getValueVector().getValueCapacity());
}
context.fail(new Exception("Failed to output keys for current batch !"));
}
if (!outputValuesStatus) {
logger.debug("Failed to output values for current batch index: {} ", outBatchIndex);
for (VectorWrapper<?> v : outContainer) {
- logger.debug("At the time of failure, size of valuevector in outContainer = {}.", v.getValueVector().getValueCapacity());
+ logger.debug("At the time of failure, size of valuevector in outContainer = {}.",
+ v.getValueVector().getValueCapacity());
}
context.fail(new Exception("Failed to output values for current batch !"));
}
@@ -555,7 +558,7 @@ public abstract class HashAggTemplate implements HashAggregator {
}
*/
- HashTable.PutStatus putStatus = htable.put(incomingRowIdx, htIdxHolder, 1 /* retry count */) ;
+ HashTable.PutStatus putStatus = htable.put(incomingRowIdx, htIdxHolder, 1 /* retry count */);
if (putStatus != HashTable.PutStatus.PUT_FAILED) {
int currentIdx = htIdxHolder.value;
@@ -564,11 +567,11 @@ public abstract class HashAggTemplate implements HashAggregator {
if (currentIdx >= batchHolders.size() * HashTable.BATCH_SIZE) {
addBatchHolder();
}
- BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & HashTable.BATCH_MASK);
+ BatchHolder bh = batchHolders.get((currentIdx >>> 16) & HashTable.BATCH_MASK);
int idxWithinBatch = currentIdx & HashTable.BATCH_MASK;
// Check if we have almost filled up the workspace vectors and add a batch if necessary
- if ((idxWithinBatch == (bh.capacity - 1)) && (bh.allocatedNextBatch == false)) {
+ if ((idxWithinBatch == (bh.capacity - 1)) && (bh.allocatedNextBatch == false)) {
htable.addNewKeyBatch();
addBatchHolder();
bh.allocatedNextBatch = true;
@@ -585,10 +588,9 @@ public abstract class HashAggTemplate implements HashAggregator {
// logger.debug("group-by key = {} already present at hash table index = {}", holder.value, currentIdx) ;
//}
- }
- else if (putStatus == HashTable.PutStatus.KEY_ADDED) {
+ } else if (putStatus == HashTable.PutStatus.KEY_ADDED) {
if (EXTRA_DEBUG_2) {
- logger.debug("Group-by key was added to hash table, inserting new aggregate values") ;
+ logger.debug("Group-by key was added to hash table, inserting new aggregate values");
}
// debugging
@@ -617,7 +619,9 @@ public abstract class HashAggTemplate implements HashAggregator {
// Code-generated methods (implemented in HashAggBatch)
public abstract void doSetup(@Named("incoming") RecordBatch incoming);
+
public abstract int getVectorIndex(@Named("recordIndex") int recordIndex);
+
public abstract boolean resetValues();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
index 238242b..0f7f394 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
@@ -17,9 +17,6 @@
*/
package org.apache.drill.exec.physical.impl.aggregate;
-import java.io.IOException;
-import java.util.List;
-
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.exception.ClassTransformationException;
@@ -34,21 +31,22 @@ import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
+import java.io.IOException;
+import java.util.List;
+
public interface HashAggregator {
- public static TemplateClassDefinition<HashAggregator> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashAggregator>(HashAggregator.class, HashAggTemplate.class);
+ public static TemplateClassDefinition<HashAggregator> TEMPLATE_DEFINITION =
+ new TemplateClassDefinition<HashAggregator>(HashAggregator.class, HashAggTemplate.class);
public static enum AggOutcome {
RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR
}
public abstract void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig, FragmentContext context,
- OperatorStats stats, BufferAllocator allocator, RecordBatch incoming,
- HashAggBatch outgoing, LogicalExpression[] valueExprs,
- List<TypedFieldId> valueFieldIds,
- TypedFieldId[] keyFieldIds,
- VectorContainer outContainer)
- throws SchemaChangeException, IOException, ClassTransformationException;
+ OperatorStats stats, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
+ LogicalExpression[] valueExprs, List<TypedFieldId> valueFieldIds, TypedFieldId[] keyFieldIds,
+ VectorContainer outContainer) throws SchemaChangeException, IOException, ClassTransformationException;
public abstract IterOutcome getOutcome();
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 0502f7e..322fd1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -17,19 +17,13 @@
*/
package org.apache.drill.exec.physical.impl.common;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
@@ -53,56 +47,68 @@ import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.resolver.TypeCastRules;
import org.apache.drill.exec.vector.ValueVector;
-import com.sun.codemodel.JConditional;
-import com.sun.codemodel.JExpr;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
public class ChainedHashTable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ChainedHashTable.class);
private static final GeneratorMapping KEY_MATCH_BUILD =
- GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalBuild" /* eval method */,
- null /* reset */, null /* cleanup */);
+ GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalBuild" /* eval method */,
+ null /* reset */, null /* cleanup */);
private static final GeneratorMapping KEY_MATCH_PROBE =
- GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalProbe" /* eval method */,
- null /* reset */, null /* cleanup */);
+ GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalProbe" /* eval method */,
+ null /* reset */, null /* cleanup */);
private static final GeneratorMapping GET_HASH_BUILD =
- GeneratorMapping.create("doSetup" /* setup method */, "getHashBuild" /* eval method */,
- null /* reset */, null /* cleanup */);
+ GeneratorMapping.create("doSetup" /* setup method */, "getHashBuild" /* eval method */, null /* reset */,
+ null /* cleanup */);
private static final GeneratorMapping GET_HASH_PROBE =
- GeneratorMapping.create("doSetup" /* setup method */, "getHashProbe" /* eval method */,
- null /* reset */, null /* cleanup */);
+ GeneratorMapping.create("doSetup" /* setup method */, "getHashProbe" /* eval method */, null /* reset */,
+ null /* cleanup */);
private static final GeneratorMapping SET_VALUE =
- GeneratorMapping.create("setupInterior" /* setup method */, "setValue" /* eval method */,
- null /* reset */, null /* cleanup */);
+ GeneratorMapping.create("setupInterior" /* setup method */, "setValue" /* eval method */, null /* reset */,
+ null /* cleanup */);
private static final GeneratorMapping OUTPUT_KEYS =
- GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordKeys" /* eval method */,
- null /* reset */, null /* cleanup */) ;
+ GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordKeys" /* eval method */,
+ null /* reset */, null /* cleanup */);
// GM for putting constant expression into method "setupInterior"
private static final GeneratorMapping SETUP_INTERIOR_CONSTANT =
GeneratorMapping.create("setupInterior" /* setup method */, "setupInterior" /* eval method */,
- null /* reset */, null /* cleanup */);
+ null /* reset */, null /* cleanup */);
// GM for putting constant expression into method "doSetup"
private static final GeneratorMapping DO_SETUP_CONSTANT =
- GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */,
- null /* reset */, null /* cleanup */);
-
- private final MappingSet KeyMatchIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
- private final MappingSet KeyMatchIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
- private final MappingSet KeyMatchHtableMapping = new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
- private final MappingSet KeyMatchHtableProbeMapping = new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
- private final MappingSet GetHashIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, DO_SETUP_CONSTANT, GET_HASH_BUILD);
- private final MappingSet GetHashIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, DO_SETUP_CONSTANT, GET_HASH_PROBE);
- private final MappingSet SetValueMapping = new MappingSet("incomingRowIdx" /* read index */, "htRowIdx" /* write index */, "incomingBuild" /* read container */, "htContainer" /* write container */, SETUP_INTERIOR_CONSTANT, SET_VALUE);
-
- private final MappingSet OutputRecordKeysMapping = new MappingSet("htRowIdx" /* read index */, "outRowIdx" /* write index */, "htContainer" /* read container */, "outgoing" /* write container */, SETUP_INTERIOR_CONSTANT, OUTPUT_KEYS);
+ GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */, null /* reset */,
+ null /* cleanup */);
+
+ private final MappingSet KeyMatchIncomingBuildMapping =
+ new MappingSet("incomingRowIdx", null, "incomingBuild", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
+ private final MappingSet KeyMatchIncomingProbeMapping =
+ new MappingSet("incomingRowIdx", null, "incomingProbe", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
+ private final MappingSet KeyMatchHtableMapping =
+ new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_BUILD);
+ private final MappingSet KeyMatchHtableProbeMapping =
+ new MappingSet("htRowIdx", null, "htContainer", null, SETUP_INTERIOR_CONSTANT, KEY_MATCH_PROBE);
+ private final MappingSet GetHashIncomingBuildMapping =
+ new MappingSet("incomingRowIdx", null, "incomingBuild", null, DO_SETUP_CONSTANT, GET_HASH_BUILD);
+ private final MappingSet GetHashIncomingProbeMapping =
+ new MappingSet("incomingRowIdx", null, "incomingProbe", null, DO_SETUP_CONSTANT, GET_HASH_PROBE);
+ private final MappingSet SetValueMapping =
+ new MappingSet("incomingRowIdx" /* read index */, "htRowIdx" /* write index */,
+ "incomingBuild" /* read container */, "htContainer" /* write container */, SETUP_INTERIOR_CONSTANT,
+ SET_VALUE);
+
+ private final MappingSet OutputRecordKeysMapping =
+ new MappingSet("htRowIdx" /* read index */, "outRowIdx" /* write index */, "htContainer" /* read container */,
+ "outgoing" /* write container */, SETUP_INTERIOR_CONSTANT, OUTPUT_KEYS);
private HashTableConfig htConfig;
private final FragmentContext context;
@@ -112,13 +118,8 @@ public class ChainedHashTable {
private final RecordBatch outgoing;
private final boolean areNullsEqual;
- public ChainedHashTable(HashTableConfig htConfig,
- FragmentContext context,
- BufferAllocator allocator,
- RecordBatch incomingBuild,
- RecordBatch incomingProbe,
- RecordBatch outgoing,
- boolean areNullsEqual) {
+ public ChainedHashTable(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
+ RecordBatch incomingBuild, RecordBatch incomingProbe, RecordBatch outgoing, boolean areNullsEqual) {
this.htConfig = htConfig;
this.context = context;
@@ -129,14 +130,15 @@ public class ChainedHashTable {
this.areNullsEqual = areNullsEqual;
}
- public HashTable createAndSetupHashTable (TypedFieldId[] outKeyFieldIds) throws ClassTransformationException, IOException, SchemaChangeException {
+ public HashTable createAndSetupHashTable(TypedFieldId[] outKeyFieldIds) throws ClassTransformationException,
+ IOException, SchemaChangeException {
CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getFunctionRegistry());
ClassGenerator<HashTable> cg = top.getRoot();
ClassGenerator<HashTable> cgInner = cg.getInnerGenerator("BatchHolder");
LogicalExpression[] keyExprsBuild = new LogicalExpression[htConfig.getKeyExprsBuild().length];
LogicalExpression[] keyExprsProbe = null;
- boolean isProbe = (htConfig.getKeyExprsProbe() != null) ;
+ boolean isProbe = (htConfig.getKeyExprsProbe() != null);
if (isProbe) {
keyExprsProbe = new LogicalExpression[htConfig.getKeyExprsProbe().length];
}
@@ -148,7 +150,8 @@ public class ChainedHashTable {
int i = 0;
for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
+ final LogicalExpression expr =
+ ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
}
@@ -169,7 +172,9 @@ public class ChainedHashTable {
if (isProbe) {
i = 0;
for (NamedExpression ne : htConfig.getKeyExprsProbe()) {
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector, context.getFunctionRegistry());
+ final LogicalExpression expr =
+ ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector,
+ context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
}
@@ -183,7 +188,8 @@ public class ChainedHashTable {
// generate code for isKeyMatch(), setValue(), getHash() and outputRecordKeys()
setupIsKeyMatchInternal(cgInner, KeyMatchIncomingBuildMapping, KeyMatchHtableMapping, keyExprsBuild, htKeyFieldIds);
- setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableProbeMapping, keyExprsProbe, htKeyFieldIds) ;
+ setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableProbeMapping, keyExprsProbe,
+ htKeyFieldIds);
setupSetValue(cgInner, keyExprsBuild, htKeyFieldIds);
if (outgoing != null) {
@@ -203,8 +209,8 @@ public class ChainedHashTable {
*/
addLeastRestrictiveCasts(keyExprsBuild, keyExprsProbe);
- setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, keyExprsBuild, false);
- setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, keyExprsProbe, true);
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, keyExprsBuild, false);
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, keyExprsProbe, true);
HashTable ht = context.getImplementationClass(top);
ht.setup(htConfig, context, allocator, incomingBuild, incomingProbe, outgoing, htContainerOrig);
@@ -213,9 +219,9 @@ public class ChainedHashTable {
}
- private void setupIsKeyMatchInternal(ClassGenerator<HashTable> cg, MappingSet incomingMapping, MappingSet htableMapping,
- LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
- throws SchemaChangeException {
+ private void setupIsKeyMatchInternal(ClassGenerator<HashTable> cg, MappingSet incomingMapping,
+ MappingSet htableMapping, LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds) throws
+ SchemaChangeException {
cg.setMappingSet(incomingMapping);
if (keyExprs == null || keyExprs.length == 0) {
@@ -255,24 +261,24 @@ public class ChainedHashTable {
cg.getEvalBlock()._return(JExpr.TRUE);
}
- private void setupSetValue(ClassGenerator<HashTable> cg, LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
- throws SchemaChangeException {
+ private void setupSetValue(ClassGenerator<HashTable> cg, LogicalExpression[] keyExprs,
+ TypedFieldId[] htKeyFieldIds) throws SchemaChangeException {
cg.setMappingSet(SetValueMapping);
int i = 0;
for (LogicalExpression expr : keyExprs) {
- ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(htKeyFieldIds[i++], expr, true) ;
+ ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(htKeyFieldIds[i++], expr, true);
HoldingContainer hc = cg.addExpr(vvwExpr, false); // this will write to the htContainer at htRowIdx
cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
}
cg.getEvalBlock()._return(JExpr.TRUE);
-
}
- private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds, TypedFieldId[] outKeyFieldIds) {
+ private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds,
+ TypedFieldId[] outKeyFieldIds) {
cg.setMappingSet(OutputRecordKeysMapping);
@@ -314,24 +320,34 @@ public class ChainedHashTable {
ErrorCollector errorCollector = new ErrorCollectorImpl();
if (result == null) {
- throw new DrillRuntimeException(String.format("Join conditions cannot be compared failing build expression: %s failing probe expression: %s",
- buildExpr.getMajorType().toString(), probeExpr.getMajorType().toString()));
- }
- else if (result != buildType) {
+ throw new DrillRuntimeException(String.format("Join conditions cannot be compared failing build " +
+ "expression:" + " %s failing probe expression: %s", buildExpr.getMajorType().toString(),
+ probeExpr.getMajorType().toString()));
+ } else if (result != buildType) {
// Add a cast expression on top of the build expression
- LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(buildExpr, probeExpr.getMajorType(), context.getFunctionRegistry(), errorCollector);
+ LogicalExpression castExpr =
+ ExpressionTreeMaterializer.addCastExpression(buildExpr, probeExpr.getMajorType(),
+ context.getFunctionRegistry(), errorCollector);
// Store the newly casted expression
- keyExprsBuild[i] = ExpressionTreeMaterializer.materialize(castExpr, incomingBuild, errorCollector, context.getFunctionRegistry());
+ keyExprsBuild[i] =
+ ExpressionTreeMaterializer.materialize(castExpr, incomingBuild, errorCollector,
+ context.getFunctionRegistry());
} else if (result != probeType) {
// Add a cast expression on top of the probe expression
- LogicalExpression castExpr = ExpressionTreeMaterializer.addCastExpression(probeExpr, buildExpr.getMajorType(), context.getFunctionRegistry(), errorCollector);
+ LogicalExpression castExpr =
+ ExpressionTreeMaterializer.addCastExpression(probeExpr, buildExpr.getMajorType(),
+ context.getFunctionRegistry(), errorCollector);
// store the newly casted expression
- keyExprsProbe[i] = ExpressionTreeMaterializer.materialize(castExpr, incomingProbe, errorCollector, context.getFunctionRegistry());
+ keyExprsProbe[i] =
+ ExpressionTreeMaterializer.materialize(castExpr, incomingProbe, errorCollector,
+ context.getFunctionRegistry());
}
}
}
}
- private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs, boolean isProbe) throws SchemaChangeException {
+
+ private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs,
+ boolean isProbe) throws SchemaChangeException {
cg.setMappingSet(incomingMapping);
@@ -349,24 +365,26 @@ public class ChainedHashTable {
HoldingContainer input = cg.addExpr(expr, false);
// compute the hash(expr)
- LogicalExpression hashfunc = FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT), context.getFunctionRegistry(), input);
+ LogicalExpression hashfunc =
+ FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT),
+ context.getFunctionRegistry(), input);
HoldingContainer hashValue = cg.addExpr(hashfunc, false);
if (i == 0) {
combinedHashValue = hashValue; // first expression..just use the hash value
- }
- else {
+ } else {
// compute the combined hash value using XOR
- LogicalExpression xorfunc = FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT), context.getFunctionRegistry(), hashValue, combinedHashValue);
+ LogicalExpression xorfunc =
+ FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT),
+ context.getFunctionRegistry(), hashValue, combinedHashValue);
combinedHashValue = cg.addExpr(xorfunc, false);
}
}
if (combinedHashValue != null) {
- cg.getEvalBlock()._return(combinedHashValue.getValue()) ;
- }
- else {
+ cg.getEvalBlock()._return(combinedHashValue.getValue());
+ } else {
cg.getEvalBlock()._return(JExpr.lit(0));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index e8ccd62..6966ba1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -25,26 +25,35 @@ import org.apache.drill.exec.record.VectorContainer;
public interface HashTable {
- public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
+ public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION =
+ new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
- /** The initial default capacity of the hash table (in terms of number of buckets). */
+ /**
+ * The initial default capacity of the hash table (in terms of number of buckets).
+ */
static final public int DEFAULT_INITIAL_CAPACITY = 1 << 16;
- /** The maximum capacity of the hash table (in terms of number of buckets). */
+ /**
+ * The maximum capacity of the hash table (in terms of number of buckets).
+ */
static final public int MAXIMUM_CAPACITY = 1 << 30;
- /** The default load factor of a hash table. */
+ /**
+ * The default load factor of a hash table.
+ */
static final public float DEFAULT_LOAD_FACTOR = 0.75f;
- static public enum PutStatus {KEY_PRESENT, KEY_ADDED, PUT_FAILED ;}
+ static public enum PutStatus {KEY_PRESENT, KEY_ADDED, PUT_FAILED;}
- /** The batch size used for internal batch holders */
- static final public int BATCH_SIZE = Character.MAX_VALUE+1;
+ /**
+ * The batch size used for internal batch holders
+ */
+ static final public int BATCH_SIZE = Character.MAX_VALUE + 1;
static final public int BATCH_MASK = 0x0000FFFF;
public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
- RecordBatch incomingBuild, RecordBatch incomingProbe,
- RecordBatch outgoing, VectorContainer htContainerOrig);
+ RecordBatch incomingBuild, RecordBatch incomingProbe,
+ RecordBatch outgoing, VectorContainer htContainerOrig);
public void updateBatches();
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index 5b56f8e..c80e97a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -17,11 +17,6 @@
*/
package org.apache.drill.exec.physical.impl.common;
-import java.util.ArrayList;
-import java.util.Iterator;
-
-import javax.inject.Named;
-
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -38,6 +33,10 @@ import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.ValueVector;
+import javax.inject.Named;
+import java.util.ArrayList;
+import java.util.Iterator;
+
public abstract class HashTableTemplate implements HashTable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTable.class);
@@ -115,7 +114,6 @@ public abstract class HashTableTemplate implements HashTable {
private int batchIndex = 0;
private BatchHolder(int idx) {
-
this.batchIndex = idx;
if (idx == 0) { // first batch holder can use the original htContainer
@@ -134,10 +132,10 @@ public abstract class HashTableTemplate implements HashTable {
}
private void init(IntVector links, IntVector hashValues, int size) {
- for (int i=0; i < size; i++) {
+ for (int i = 0; i < size; i++) {
links.getMutator().setSafe(i, EMPTY_SLOT);
}
- for (int i=0; i < size; i++) {
+ for (int i = 0; i < size; i++) {
hashValues.getMutator().setSafe(i, 0);
}
links.getMutator().setValueCount(size);
@@ -159,7 +157,8 @@ public abstract class HashTableTemplate implements HashTable {
boolean match = false;
if (currentIdxWithinBatch >= HashTable.BATCH_SIZE) {
- logger.debug("Batch size = {}, incomingRowIdx = {}, currentIdxWithinBatch = {}.", HashTable.BATCH_SIZE, incomingRowIdx, currentIdxWithinBatch);
+ logger.debug("Batch size = {}, incomingRowIdx = {}, currentIdxWithinBatch = {}.", HashTable.BATCH_SIZE,
+ incomingRowIdx, currentIdxWithinBatch);
}
assert (currentIdxWithinBatch < HashTable.BATCH_SIZE);
assert (incomingRowIdx < HashTable.BATCH_SIZE);
@@ -170,7 +169,7 @@ public abstract class HashTableTemplate implements HashTable {
match = isKeyMatchInternalBuild(incomingRowIdx, currentIdxWithinBatch);
}
- if (! match) {
+ if (!match) {
currentIdxHolder.value = links.getAccessor().get(currentIdxWithinBatch);
}
return match;
@@ -178,10 +177,11 @@ public abstract class HashTableTemplate implements HashTable {
// Insert a new <key1, key2...keyN> entry coming from the incoming batch into the hash table
// container at the specified index
- private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdxWithinBatch) {
+ private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch,
+ int lastEntryIdxWithinBatch) {
int currentIdxWithinBatch = currentIdx & BATCH_MASK;
- if (! setValue(incomingRowIdx, currentIdxWithinBatch)) {
+ if (!setValue(incomingRowIdx, currentIdxWithinBatch)) {
return false;
}
@@ -198,7 +198,8 @@ public abstract class HashTableTemplate implements HashTable {
maxOccupiedIdx = Math.max(maxOccupiedIdx, currentIdxWithinBatch);
if (EXTRA_DEBUG) {
- logger.debug("BatchHolder: inserted key at incomingRowIdx = {}, currentIdx = {}, hash value = {}.", incomingRowIdx, currentIdx, hashValue);
+ logger.debug("BatchHolder: inserted key at incomingRowIdx = {}, currentIdx = {}, hash value = {}.",
+ incomingRowIdx, currentIdx, hashValue);
}
return true;
@@ -210,7 +211,8 @@ public abstract class HashTableTemplate implements HashTable {
private void rehash(int numbuckets, IntVector newStartIndices, int batchStartIdx) {
- logger.debug("Rehashing entries within the batch: {}; batchStartIdx = {}, total numBuckets in hash table = {}.", batchIndex, batchStartIdx, numbuckets);
+ logger.debug("Rehashing entries within the batch: {}; batchStartIdx = {}, total numBuckets in hash table = {}" +
+ ".", batchIndex, batchStartIdx, numbuckets);
int size = links.getAccessor().getValueCount();
IntVector newLinks = allocMetadataVector(size, EMPTY_SLOT);
@@ -229,7 +231,10 @@ public abstract class HashTableTemplate implements HashTable {
newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
if (EXTRA_DEBUG) {
- logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+ logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, " +
+ "hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx),
+ entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch),
+ newHashValues.getAccessor().get(entryIdxWithinBatch));
}
} else {
@@ -251,17 +256,26 @@ public abstract class HashTableTemplate implements HashTable {
newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
if (EXTRA_DEBUG) {
- logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+ logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, " +
+ "newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch,
+ newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch,
+ newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get
+ (entryIdxWithinBatch));
}
break;
} else if (bh != this && bh.links.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
bh.links.getMutator().setSafe(idxWithinBatch, entryIdx); // update the link in the other batch
- newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this batch to mark end of the hash chain
- newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
+ newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this
+ // batch to mark end of the hash chain
+ newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
if (EXTRA_DEBUG) {
- logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+ logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, " +
+ "newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch,
+ newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch,
+ newLinks.getAccessor().get(entryIdxWithinBatch),
+ newHashValues.getAccessor().get(entryIdxWithinBatch));
}
break;
@@ -287,8 +301,8 @@ public abstract class HashTableTemplate implements HashTable {
private boolean outputKeys(VectorContainer outContainer, int outStartIndex, int numRecords) {
/** for debugging
- BigIntVector vv0 = getValueVector(0);
- BigIntHolder holder = new BigIntHolder();
+ BigIntVector vv0 = getValueVector(0);
+ BigIntHolder holder = new BigIntHolder();
*/
// set the value count for htContainer's value vectors before the transfer ..
@@ -304,7 +318,8 @@ public abstract class HashTableTemplate implements HashTable {
}
/*
- logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.", this.batchIndex, 0, maxOccupiedIdx);
+ logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.",
+ this.batchIndex, 0, maxOccupiedIdx);
for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
if (outputRecordKeys(i, batchOutputCount) ) {
if (EXTRA_DEBUG) logger.debug("Outputting keys to output index: {}", batchOutputCount) ;
@@ -312,7 +327,8 @@ public abstract class HashTableTemplate implements HashTable {
// debugging
// holder.value = vv0.getAccessor().get(i);
// if (holder.value == 100018 || holder.value == 100021) {
- // logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i, batchOutputCount);
+ // logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i,
+ // batchOutputCount);
// }
batchOutputCount++;
@@ -337,14 +353,15 @@ public abstract class HashTableTemplate implements HashTable {
if (idxWithinBatch == EMPTY_SLOT) {
break;
} else {
- logger.debug("links[ {} ] = {}, hashValues[ {} ] = {}.", idxWithinBatch, links.getAccessor().get(idxWithinBatch), idxWithinBatch, hashValues.getAccessor().get(idxWithinBatch));
+ logger.debug("links[ {} ] = {}, hashValues[ {} ] = {}.", idxWithinBatch,
+ links.getAccessor().get(idxWithinBatch), idxWithinBatch, hashValues.getAccessor().get(idxWithinBatch));
idx = links.getAccessor().get(idxWithinBatch);
}
}
}
private void clear() {
- htContainer.clear();;
+ htContainer.clear();
links.clear();
hashValues.clear();
}
@@ -363,30 +380,41 @@ public abstract class HashTableTemplate implements HashTable {
// These methods will be code-generated
@RuntimeOverridden
- protected void setupInterior(@Named("incomingBuild") RecordBatch incomingBuild,
- @Named("incomingProbe") RecordBatch incomingProbe,
- @Named("outgoing") RecordBatch outgoing,
- @Named("htContainer") VectorContainer htContainer) {}
+ protected void setupInterior(
+ @Named("incomingBuild") RecordBatch incomingBuild,
+ @Named("incomingProbe") RecordBatch incomingProbe,
+ @Named("outgoing") RecordBatch outgoing,
+ @Named("htContainer") VectorContainer htContainer) {
+ }
@RuntimeOverridden
- protected boolean isKeyMatchInternalBuild(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+ protected boolean isKeyMatchInternalBuild(
+ @Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+ return false;
+ }
@RuntimeOverridden
- protected boolean isKeyMatchInternalProbe(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+ protected boolean isKeyMatchInternalProbe(
+ @Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+ return false;
+ }
@RuntimeOverridden
- protected boolean setValue(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+ protected boolean setValue(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {
+ return false;
+ }
@RuntimeOverridden
- protected boolean outputRecordKeys(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return false;}
-
+ protected boolean outputRecordKeys(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {
+ return false;
+ }
} // class BatchHolder
@Override
public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
- RecordBatch incomingBuild, RecordBatch incomingProbe,
- RecordBatch outgoing, VectorContainer htContainerOrig) {
+ RecordBatch incomingBuild, RecordBatch incomingProbe,
+ RecordBatch outgoing, VectorContainer htContainerOrig) {
float loadf = htConfig.getLoadFactor();
int initialCap = htConfig.getInitialCapacity();
@@ -483,16 +511,16 @@ public abstract class HashTableTemplate implements HashTable {
private static int roundUpToPowerOf2(int number) {
int rounded = number >= MAXIMUM_CAPACITY
- ? MAXIMUM_CAPACITY
- : (rounded = Integer.highestOneBit(number)) != 0
- ? (Integer.bitCount(number) > 1) ? rounded << 1 : rounded
- : 1;
+ ? MAXIMUM_CAPACITY
+ : (rounded = Integer.highestOneBit(number)) != 0
+ ? (Integer.bitCount(number) > 1) ? rounded << 1 : rounded
+ : 1;
- return rounded;
+ return rounded;
}
public PutStatus put(int incomingRowIdx, IndexPointer htIdxHolder, int retryCount) {
- HashTable.PutStatus putStatus = put(incomingRowIdx, htIdxHolder) ;
+ HashTable.PutStatus putStatus = put(incomingRowIdx, htIdxHolder);
int count = retryCount;
int numBatchHolders;
while (putStatus == PutStatus.PUT_FAILED && count > 0) {
@@ -526,7 +554,8 @@ public abstract class HashTableTemplate implements HashTable {
addBatchIfNeeded(currentIdx);
if (EXTRA_DEBUG) {
- logger.debug("Empty bucket index = {}. incomingRowIdx = {}; inserting new entry at currentIdx = {}.", i, incomingRowIdx, currentIdx);
+ logger.debug("Empty bucket index = {}. incomingRowIdx = {}; inserting new entry at currentIdx = {}.", i,
+ incomingRowIdx, currentIdx);
}
if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
@@ -542,7 +571,7 @@ public abstract class HashTableTemplate implements HashTable {
currentIdx = startIdx;
boolean found = false;
- bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+ bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
currentIdxHolder.value = currentIdx;
// if startIdx is non-empty, follow the hash chain links until we find a matching
@@ -554,13 +583,12 @@ public abstract class HashTableTemplate implements HashTable {
htIdxHolder.value = currentIdxHolder.value;
found = true;
break;
- }
- else if (currentIdxHolder.value == EMPTY_SLOT) {
+ } else if (currentIdxHolder.value == EMPTY_SLOT) {
lastEntryBatch = bh;
lastEntryIdxWithinBatch = currentIdxWithinBatch;
break;
} else {
- bh = batchHolders.get( (currentIdxHolder.value >>> 16) & HashTable.BATCH_MASK);
+ bh = batchHolders.get((currentIdxHolder.value >>> 16) & HashTable.BATCH_MASK);
lastEntryBatch = bh;
}
}
@@ -571,7 +599,8 @@ public abstract class HashTableTemplate implements HashTable {
addBatchIfNeeded(currentIdx);
if (EXTRA_DEBUG) {
- logger.debug("No match was found for incomingRowIdx = {}; inserting new entry at currentIdx = {}.", incomingRowIdx, currentIdx);
+ logger.debug("No match was found for incomingRowIdx = {}; inserting new entry at currentIdx = {}.",
+ incomingRowIdx, currentIdx);
}
if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
@@ -582,17 +611,18 @@ public abstract class HashTableTemplate implements HashTable {
}
}
- return found ? PutStatus.KEY_PRESENT : PutStatus.KEY_ADDED ;
+ return found ? PutStatus.KEY_PRESENT : PutStatus.KEY_ADDED;
}
- private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdx) {
+ private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch,
+ int lastEntryIdx) {
addBatchIfNeeded(currentIdx);
- BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+ BatchHolder bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
if (bh.insertEntry(incomingRowIdx, currentIdx, hashValue, lastEntryBatch, lastEntryIdx)) {
- numEntries++ ;
+ numEntries++;
/* Resize hash table if needed and transfer the metadata
* Resize only after inserting the current entry into the hash table
@@ -617,10 +647,10 @@ public abstract class HashTableTemplate implements HashTable {
int currentIdx = startIndices.getAccessor().get(i);
if (currentIdx == EMPTY_SLOT) {
- return -1;
+ return -1;
}
- BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+ BatchHolder bh = batchHolders.get((currentIdx >>> 16) & BATCH_MASK);
currentIdxHolder.value = currentIdx;
boolean found = false;
@@ -632,7 +662,7 @@ public abstract class HashTableTemplate implements HashTable {
} else if (currentIdxHolder.value == EMPTY_SLOT) {
break;
} else {
- bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
+ bh = batchHolders.get((currentIdxHolder.value >>> 16) & BATCH_MASK);
}
}
@@ -651,8 +681,7 @@ public abstract class HashTableTemplate implements HashTable {
logger.debug("HashTable: Added new batch. Num batches = {}.", batchHolders.size());
}
return bh;
- }
- else {
+ } else {
return batchHolders.get(batchHolders.size() - 1);
}
}
@@ -700,7 +729,7 @@ public abstract class HashTableTemplate implements HashTable {
IntVector newStartIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
for (int i = 0; i < batchHolders.size(); i++) {
- BatchHolder bh = batchHolders.get(i) ;
+ BatchHolder bh = batchHolders.get(i);
int batchStartIdx = i * BATCH_SIZE;
bh.rehash(tableSize, newStartIndices, batchStartIdx);
}
@@ -714,7 +743,7 @@ public abstract class HashTableTemplate implements HashTable {
for (int i = 0; i < startIndices.getAccessor().getValueCount(); i++) {
logger.debug("Bucket: {}, startIdx[ {} ] = {}.", i, i, startIndices.getAccessor().get(i));
int idx = startIndices.getAccessor().get(i);
- BatchHolder bh = batchHolders.get( (idx >>> 16) & BATCH_MASK);
+ BatchHolder bh = batchHolders.get((idx >>> 16) & BATCH_MASK);
bh.dump(idx);
}
}
@@ -724,7 +753,7 @@ public abstract class HashTableTemplate implements HashTable {
public boolean outputKeys(int batchIdx, VectorContainer outContainer, int outStartIndex, int numRecords) {
assert batchIdx < batchHolders.size();
- if (! batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
+ if (!batchHolders.get(batchIdx).outputKeys(outContainer, outStartIndex, numRecords)) {
return false;
}
return true;
@@ -733,7 +762,7 @@ public abstract class HashTableTemplate implements HashTable {
private IntVector allocMetadataVector(int size, int initialValue) {
IntVector vector = (IntVector) TypeHelper.getNewVector(dummyIntField, allocator);
vector.allocateNew(size);
- for (int i=0; i < size; i++) {
+ for (int i = 0; i < size; i++) {
vector.getMutator().setSafe(i, initialValue);
}
vector.getMutator().setValueCount(size);
@@ -747,8 +776,11 @@ public abstract class HashTableTemplate implements HashTable {
}
// These methods will be code-generated in the context of the outer class
- protected abstract void doSetup(@Named("incomingBuild") RecordBatch incomingBuild, @Named("incomingProbe") RecordBatch incomingProbe);
- protected abstract int getHashBuild(@Named("incomingRowIdx") int incomingRowIdx) ;
- protected abstract int getHashProbe(@Named("incomingRowIdx") int incomingRowIdx) ;
+ protected abstract void doSetup(@Named("incomingBuild") RecordBatch incomingBuild,
+ @Named("incomingProbe") RecordBatch incomingProbe);
+
+ protected abstract int getHashBuild(@Named("incomingRowIdx") int incomingRowIdx);
+
+ protected abstract int getHashProbe(@Named("incomingRowIdx") int incomingRowIdx);
}
[2/5] drill git commit: DRILL-1991: Code indendation and formatting
cleanup for few files
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/10fd9e10/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 5deb67f..f370dc7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -17,9 +17,9 @@
*/
package org.apache.drill.exec.physical.impl.join;
-import java.io.IOException;
-import java.util.List;
-
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JVar;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.drill.common.logical.data.NamedExpression;
@@ -49,7 +49,6 @@ import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.ExpandableHyperContainer;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
@@ -57,118 +56,105 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
import org.eigenbase.rel.JoinRelType;
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JExpression;
-import com.sun.codemodel.JVar;
+import java.io.IOException;
+import java.util.List;
public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
+ // Probe side record batch
+ private final RecordBatch left;
- public static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024;
- public static final long ALLOCATOR_MAX_RESERVATION = 20L*1000*1000*1000;
-
- // Probe side record batch
- private final RecordBatch left;
-
- // Build side record batch
- private final RecordBatch right;
+ // Build side record batch
+ private final RecordBatch right;
- // Join type, INNER, LEFT, RIGHT or OUTER
- private final JoinRelType joinType;
+ // Join type, INNER, LEFT, RIGHT or OUTER
+ private final JoinRelType joinType;
- // Join conditions
- private final List<JoinCondition> conditions;
+ // Join conditions
+ private final List<JoinCondition> conditions;
- // Runtime generated class implementing HashJoinProbe interface
- private HashJoinProbe hashJoinProbe = null;
+ // Runtime generated class implementing HashJoinProbe interface
+ private HashJoinProbe hashJoinProbe = null;
- /* Helper class
- * Maintains linked list of build side records with the same key
- * Keeps information about which build records have a corresponding
- * matching key in the probe side (for outer, right joins)
- */
- private HashJoinHelper hjHelper = null;
+ /* Helper class
+ * Maintains linked list of build side records with the same key
+ * Keeps information about which build records have a corresponding
+ * matching key in the probe side (for outer, right joins)
+ */
+ private HashJoinHelper hjHelper = null;
- // Underlying hashtable used by the hash join
- private HashTable hashTable = null;
+ // Underlying hashtable used by the hash join
+ private HashTable hashTable = null;
- /* Hyper container to store all build side record batches.
- * Records are retrieved from this container when there is a matching record
- * on the probe side
- */
- private ExpandableHyperContainer hyperContainer;
+ /* Hyper container to store all build side record batches.
+ * Records are retrieved from this container when there is a matching record
+ * on the probe side
+ */
+ private ExpandableHyperContainer hyperContainer;
- // Number of records in the output container
- private int outputRecords;
+ // Number of records in the output container
+ private int outputRecords;
- // Current batch index on the build side
- private int buildBatchIndex = 0;
+ // Current batch index on the build side
+ private int buildBatchIndex = 0;
- // Schema of the build side
- private BatchSchema rightSchema = null;
+ // Schema of the build side
+ private BatchSchema rightSchema = null;
+ // Generator mapping for the build side
+ // Generator mapping for the build side : scalar
+ private static final GeneratorMapping PROJECT_BUILD =
+ GeneratorMapping.create("doSetup"/* setup method */, "projectBuildRecord" /* eval method */, null /* reset */,
+ null /* cleanup */);
+ // Generator mapping for the build side : constant
+ private static final GeneratorMapping PROJECT_BUILD_CONSTANT =
+ GeneratorMapping.create("doSetup"/* setup method */, "doSetup" /* eval method */, null /* reset */,
+ null /* cleanup */);
- // Generator mapping for the build side
- // Generator mapping for the build side : scalar
- private static final GeneratorMapping PROJECT_BUILD = GeneratorMapping.create("doSetup"/* setup method */,
- "projectBuildRecord" /* eval method */,
- null /* reset */, null /* cleanup */);
- // Generator mapping for the build side : constant
- private static final GeneratorMapping PROJECT_BUILD_CONSTANT = GeneratorMapping.create("doSetup"/* setup method */,
- "doSetup" /* eval method */,
- null /* reset */, null /* cleanup */);
+ // Generator mapping for the probe side : scalar
+ private static final GeneratorMapping PROJECT_PROBE =
+ GeneratorMapping.create("doSetup" /* setup method */, "projectProbeRecord" /* eval method */, null /* reset */,
+ null /* cleanup */);
+ // Generator mapping for the probe side : constant
+ private static final GeneratorMapping PROJECT_PROBE_CONSTANT =
+ GeneratorMapping.create("doSetup" /* setup method */, "doSetup" /* eval method */, null /* reset */,
+ null /* cleanup */);
- // Generator mapping for the probe side : scalar
- private static final GeneratorMapping PROJECT_PROBE = GeneratorMapping.create("doSetup" /* setup method */,
- "projectProbeRecord" /* eval method */,
- null /* reset */, null /* cleanup */);
- // Generator mapping for the probe side : constant
- private static final GeneratorMapping PROJECT_PROBE_CONSTANT = GeneratorMapping.create("doSetup" /* setup method */,
- "doSetup" /* eval method */,
- null /* reset */, null /* cleanup */);
+ // Mapping set for the build side
+ private final MappingSet projectBuildMapping =
+ new MappingSet("buildIndex" /* read index */, "outIndex" /* write index */, "buildBatch" /* read container */,
+ "outgoing" /* write container */, PROJECT_BUILD_CONSTANT, PROJECT_BUILD);
- // Mapping set for the build side
- private final MappingSet projectBuildMapping = new MappingSet("buildIndex" /* read index */, "outIndex" /* write index */,
- "buildBatch" /* read container */,
- "outgoing" /* write container */,
- PROJECT_BUILD_CONSTANT, PROJECT_BUILD);
+ // Mapping set for the probe side
+ private final MappingSet projectProbeMapping =
+ new MappingSet("probeIndex" /* read index */, "outIndex" /* write index */, "probeBatch" /* read container */,
+ "outgoing" /* write container */, PROJECT_PROBE_CONSTANT, PROJECT_PROBE);
- // Mapping set for the probe side
- private final MappingSet projectProbeMapping = new MappingSet("probeIndex" /* read index */, "outIndex" /* write index */,
- "probeBatch" /* read container */,
- "outgoing" /* write container */,
- PROJECT_PROBE_CONSTANT, PROJECT_PROBE);
+ IterOutcome leftUpstream = IterOutcome.NONE;
+ IterOutcome rightUpstream = IterOutcome.NONE;
- // indicates if we have previously returned an output batch
- boolean firstOutputBatch = true;
-
- IterOutcome leftUpstream = IterOutcome.NONE;
- IterOutcome rightUpstream = IterOutcome.NONE;
-
- private final HashTableStats htStats = new HashTableStats();
+ private final HashTableStats htStats = new HashTableStats();
public enum Metric implements MetricDef {
- NUM_BUCKETS,
- NUM_ENTRIES,
- NUM_RESIZING,
- RESIZING_TIME;
+ NUM_BUCKETS,
+ NUM_ENTRIES,
+ NUM_RESIZING,
+ RESIZING_TIME;
- // duplicate for hash ag
-
- @Override
- public int metricId() {
- return ordinal();
- }
- }
+ // duplicate for hash ag
@Override
- public int getRecordCount() {
- return outputRecords;
+ public int metricId() {
+ return ordinal();
}
+ }
-
+ @Override
+ public int getRecordCount() {
+ return outputRecords;
+ }
@Override
protected void buildSchema() throws SchemaChangeException {
@@ -204,333 +190,332 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
}
}
- @Override
- public IterOutcome innerNext() {
- try {
- /* If we are here for the first time, execute the build phase of the
- * hash join and setup the run time generated class for the probe side
- */
- if (state == BatchState.FIRST) {
- // Build the hash table, using the build side record batches.
- executeBuildPhase();
-// IterOutcome next = next(HashJoinHelper.LEFT_INPUT, left);
- hashJoinProbe.setupHashJoinProbe(context, hyperContainer, left, left.getRecordCount(), this, hashTable, hjHelper, joinType);
-
- // Update the hash table related stats for the operator
- updateStats(this.hashTable);
- }
+ @Override
+ public IterOutcome innerNext() {
+ try {
+ /* If we are here for the first time, execute the build phase of the
+ * hash join and setup the run time generated class for the probe side
+ */
+ if (state == BatchState.FIRST) {
+ // Build the hash table, using the build side record batches.
+ executeBuildPhase();
+ // IterOutcome next = next(HashJoinHelper.LEFT_INPUT, left);
+ hashJoinProbe.setupHashJoinProbe(context, hyperContainer, left, left.getRecordCount(), this, hashTable,
+ hjHelper, joinType);
+
+ // Update the hash table related stats for the operator
+ updateStats(this.hashTable);
+ }
- // Store the number of records projected
- if (hashTable != null
- || joinType != JoinRelType.INNER) {
-
- // Allocate the memory for the vectors in the output container
- allocateVectors();
-
- outputRecords = hashJoinProbe.probeAndProject();
-
- /* We are here because of one the following
- * 1. Completed processing of all the records and we are done
- * 2. We've filled up the outgoing batch to the maximum and we need to return upstream
- * Either case build the output container's schema and return
- */
- if (outputRecords > 0 || state == BatchState.FIRST) {
- if (state == BatchState.FIRST) {
- state = BatchState.NOT_FIRST;
- }
-
-
- for (VectorWrapper<?> v : container) {
- v.getValueVector().getMutator().setValueCount(outputRecords);
- }
-
- return IterOutcome.OK;
- }
- } else {
- // Our build side is empty, we won't have any matches, clear the probe side
- if (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
- for (VectorWrapper<?> wrapper : left) {
- wrapper.getValueVector().clear();
- }
- left.kill(true);
- leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
- while (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
- for (VectorWrapper<?> wrapper : left) {
- wrapper.getValueVector().clear();
- }
- leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
- }
- }
- }
+ // Store the number of records projected
+ if (hashTable != null || joinType != JoinRelType.INNER) {
- // No more output records, clean up and return
- state = BatchState.DONE;
-// if (first) {
-// return IterOutcome.OK_NEW_SCHEMA;
-// }
- return IterOutcome.NONE;
- } catch (ClassTransformationException | SchemaChangeException | IOException e) {
- context.fail(e);
- killIncoming(false);
- return IterOutcome.STOP;
- }
- }
+ // Allocate the memory for the vectors in the output container
+ allocateVectors();
- public void setupHashTable() throws IOException, SchemaChangeException, ClassTransformationException {
+ outputRecords = hashJoinProbe.probeAndProject();
- // Setup the hash table configuration object
- int conditionsSize = conditions.size();
+ /* We are here because of one the following
+ * 1. Completed processing of all the records and we are done
+ * 2. We've filled up the outgoing batch to the maximum and we need to return upstream
+ * Either case build the output container's schema and return
+ */
+ if (outputRecords > 0 || state == BatchState.FIRST) {
+ if (state == BatchState.FIRST) {
+ state = BatchState.NOT_FIRST;
+ }
- NamedExpression rightExpr[] = new NamedExpression[conditionsSize];
- NamedExpression leftExpr[] = new NamedExpression[conditionsSize];
- // Create named expressions from the conditions
- for (int i = 0; i < conditionsSize; i++) {
- rightExpr[i] = new NamedExpression(conditions.get(i).getRight(), new FieldReference("build_side_" + i ));
- leftExpr[i] = new NamedExpression(conditions.get(i).getLeft(), new FieldReference("probe_side_" + i));
+ for (VectorWrapper<?> v : container) {
+ v.getValueVector().getMutator().setValueCount(outputRecords);
+ }
- // Hash join only supports equality currently.
- assert conditions.get(i).getRelationship().equals("==");
+ return IterOutcome.OK;
}
-
- // Set the left named expression to be null if the probe batch is empty.
- if (leftUpstream != IterOutcome.OK_NEW_SCHEMA && leftUpstream != IterOutcome.OK) {
- leftExpr = null;
- } else {
- if (left.getSchema().getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
- throw new SchemaChangeException("Hash join does not support probe batch with selection vectors");
+ } else {
+ // Our build side is empty, we won't have any matches, clear the probe side
+ if (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
+ for (VectorWrapper<?> wrapper : left) {
+ wrapper.getValueVector().clear();
+ }
+ left.kill(true);
+ leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
+ while (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
+ for (VectorWrapper<?> wrapper : left) {
+ wrapper.getValueVector().clear();
+ }
+ leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
}
}
+ }
+
+ // No more output records, clean up and return
+ state = BatchState.DONE;
+ // if (first) {
+ // return IterOutcome.OK_NEW_SCHEMA;
+ // }
+ return IterOutcome.NONE;
+ } catch (ClassTransformationException | SchemaChangeException | IOException e) {
+ context.fail(e);
+ killIncoming(false);
+ return IterOutcome.STOP;
+ }
+ }
+
+ public void setupHashTable() throws IOException, SchemaChangeException, ClassTransformationException {
+
+ // Setup the hash table configuration object
+ int conditionsSize = conditions.size();
- HashTableConfig htConfig =
- new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
+ NamedExpression rightExpr[] = new NamedExpression[conditionsSize];
+ NamedExpression leftExpr[] = new NamedExpression[conditionsSize];
+
+ // Create named expressions from the conditions
+ for (int i = 0; i < conditionsSize; i++) {
+ rightExpr[i] = new NamedExpression(conditions.get(i).getRight(), new FieldReference("build_side_" + i));
+ leftExpr[i] = new NamedExpression(conditions.get(i).getLeft(), new FieldReference("probe_side_" + i));
+
+ // Hash join only supports equality currently.
+ assert conditions.get(i).getRelationship().equals("==");
+ }
+
+ // Set the left named expression to be null if the probe batch is empty.
+ if (leftUpstream != IterOutcome.OK_NEW_SCHEMA && leftUpstream != IterOutcome.OK) {
+ leftExpr = null;
+ } else {
+ if (left.getSchema().getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
+ throw new SchemaChangeException("Hash join does not support probe batch with selection vectors");
+ }
+ }
+
+ HashTableConfig htConfig =
+ new HashTableConfig(context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE_KEY).num_val.intValue(),
HashTable.DEFAULT_LOAD_FACTOR, rightExpr, leftExpr);
- // Create the chained hash table
- ChainedHashTable ht = new ChainedHashTable(htConfig, context, oContext.getAllocator(),
- this.right, this.left, null, false /* nulls are not equal */);
- hashTable = ht.createAndSetupHashTable(null);
+ // Create the chained hash table
+ ChainedHashTable ht =
+ new ChainedHashTable(htConfig, context, oContext.getAllocator(), this.right, this.left, null,
+ false /* nulls are not equal */);
+ hashTable = ht.createAndSetupHashTable(null);
+ }
+
+ public void executeBuildPhase() throws SchemaChangeException, ClassTransformationException, IOException {
+
+ //Setup the underlying hash table
+
+ // skip first batch if count is zero, as it may be an empty schema batch
+ if (right.getRecordCount() == 0) {
+ for (VectorWrapper w : right) {
+ w.clear();
+ }
+ rightUpstream = next(right);
}
- public void executeBuildPhase() throws SchemaChangeException, ClassTransformationException, IOException {
+ boolean moreData = true;
+
+ while (moreData) {
+
+ switch (rightUpstream) {
+
+ case NONE:
+ case NOT_YET:
+ case STOP:
+ moreData = false;
+ continue;
- //Setup the underlying hash table
+ case OK_NEW_SCHEMA:
+ if (rightSchema == null) {
+ rightSchema = right.getSchema();
- // skip first batch if count is zero, as it may be an empty schema batch
- if (right.getRecordCount() == 0) {
- for (VectorWrapper w : right) {
- w.clear();
+ if (rightSchema.getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
+ throw new SchemaChangeException("Hash join does not support build batch with selection vectors");
}
- rightUpstream = next(right);
+ setupHashTable();
+ } else {
+ if (!rightSchema.equals(right.getSchema())) {
+ throw new SchemaChangeException("Hash join does not support schema changes");
+ }
+ hashTable.updateBatches();
}
+ // Fall through
+ case OK:
+ int currentRecordCount = right.getRecordCount();
- boolean moreData = true;
-
- while (moreData) {
-
- switch (rightUpstream) {
-
- case NONE:
- case NOT_YET:
- case STOP:
- moreData = false;
- continue;
-
- case OK_NEW_SCHEMA:
- if (rightSchema == null) {
- rightSchema = right.getSchema();
-
- if (rightSchema.getSelectionVectorMode() != BatchSchema.SelectionVectorMode.NONE) {
- throw new SchemaChangeException("Hash join does not support build batch with selection vectors");
- }
- setupHashTable();
- } else {
- if (!rightSchema.equals(right.getSchema())) {
- throw new SchemaChangeException("Hash join does not support schema changes");
- }
- hashTable.updateBatches();
- }
- // Fall through
- case OK:
- int currentRecordCount = right.getRecordCount();
-
- /* For every new build batch, we store some state in the helper context
- * Add new state to the helper context
- */
- hjHelper.addNewBatch(currentRecordCount);
-
- // Holder contains the global index where the key is hashed into using the hash table
- IndexPointer htIndex = new IndexPointer();
-
- // For every record in the build batch , hash the key columns
- for (int i = 0; i < currentRecordCount; i++) {
-
- HashTable.PutStatus status = hashTable.put(i, htIndex, 1 /* retry count */);
-
- if (status != HashTable.PutStatus.PUT_FAILED) {
- /* Use the global index returned by the hash table, to store
- * the current record index and batch index. This will be used
- * later when we probe and find a match.
- */
- hjHelper.setCurrentIndex(htIndex.value, buildBatchIndex, i);
- }
- }
-
- /* Completed hashing all records in this batch. Transfer the batch
- * to the hyper vector container. Will be used when we want to retrieve
- * records that have matching keys on the probe side.
- */
- RecordBatchData nextBatch = new RecordBatchData(right);
- if (hyperContainer == null) {
- hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
- } else {
- hyperContainer.addBatch(nextBatch.getContainer());
- }
-
- // completed processing a batch, increment batch index
- buildBatchIndex++;
- break;
- }
- // Get the next record batch
- rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
- }
- }
+ /* For every new build batch, we store some state in the helper context
+ * Add new state to the helper context
+ */
+ hjHelper.addNewBatch(currentRecordCount);
+
+ // Holder contains the global index where the key is hashed into using the hash table
+ IndexPointer htIndex = new IndexPointer();
- public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
+ // For every record in the build batch , hash the key columns
+ for (int i = 0; i < currentRecordCount; i++) {
+ HashTable.PutStatus status = hashTable.put(i, htIndex, 1 /* retry count */);
+ if (status != HashTable.PutStatus.PUT_FAILED) {
+ /* Use the global index returned by the hash table, to store
+ * the current record index and batch index. This will be used
+ * later when we probe and find a match.
+ */
+ hjHelper.setCurrentIndex(htIndex.value, buildBatchIndex, i);
+ }
+ }
- final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry());
- ClassGenerator<HashJoinProbe> g = cg.getRoot();
+ /* Completed hashing all records in this batch. Transfer the batch
+ * to the hyper vector container. Will be used when we want to retrieve
+ * records that have matching keys on the probe side.
+ */
+ RecordBatchData nextBatch = new RecordBatchData(right);
+ if (hyperContainer == null) {
+ hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
+ } else {
+ hyperContainer.addBatch(nextBatch.getContainer());
+ }
- // Generate the code to project build side records
- g.setMappingSet(projectBuildMapping);
+ // completed processing a batch, increment batch index
+ buildBatchIndex++;
+ break;
+ }
+ // Get the next record batch
+ rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
+ }
+ }
+ public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
- int fieldId = 0;
- JExpression buildIndex = JExpr.direct("buildIndex");
- JExpression outIndex = JExpr.direct("outIndex");
- g.rotateBlock();
- if (rightSchema != null) {
- for(MaterializedField field : rightSchema) {
+ final CodeGenerator<HashJoinProbe> cg =
+ CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+ ClassGenerator<HashJoinProbe> g = cg.getRoot();
- MajorType inputType = field.getType();
- MajorType outputType;
- if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
- outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
- } else {
- outputType = inputType;
- }
+ // Generate the code to project build side records
+ g.setMappingSet(projectBuildMapping);
- // Add the vector to our output container
-// ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(), outputType), context.getAllocator());
- container.addOrGet(MaterializedField.create(field.getPath(), outputType));
- JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
- JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
- g.getEvalBlock()._if(outVV.invoke("copyFromSafe")
- .arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE)))
- .arg(outIndex)
- .arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))).not())._then()._return(JExpr.FALSE);
+ int fieldId = 0;
+ JExpression buildIndex = JExpr.direct("buildIndex");
+ JExpression outIndex = JExpr.direct("outIndex");
+ g.rotateBlock();
- fieldId++;
- }
+ if (rightSchema != null) {
+ for (MaterializedField field : rightSchema) {
+
+ MajorType inputType = field.getType();
+ MajorType outputType;
+ if (joinType == JoinRelType.LEFT && inputType.getMode() == DataMode.REQUIRED) {
+ outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
+ } else {
+ outputType = inputType;
}
- g.rotateBlock();
- g.getEvalBlock()._return(JExpr.TRUE);
- // Generate the code to project probe side records
- g.setMappingSet(projectProbeMapping);
+ // Add the vector to our output container
+ // ValueVector v = TypeHelper.getNewVector(MaterializedField.create(vv.getField().getPath(),
+ // outputType), context.getAllocator());
+ container.addOrGet(MaterializedField.create(field.getPath(), outputType));
- int outputFieldId = fieldId;
- fieldId = 0;
- JExpression probeIndex = JExpr.direct("probeIndex");
- int recordCount = 0;
+ JVar inVV = g.declareVectorValueSetupAndMember("buildBatch", new TypedFieldId(field.getType(), true, fieldId));
+ JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, fieldId));
+ g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE)))
+ .arg(outIndex).arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))).not())._then()._return(JExpr.FALSE);
- if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
- for (VectorWrapper<?> vv : left) {
+ fieldId++;
+ }
+ }
+ g.rotateBlock();
+ g.getEvalBlock()._return(JExpr.TRUE);
- MajorType inputType = vv.getField().getType();
- MajorType outputType;
- if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
- outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
- } else {
- outputType = inputType;
- }
+ // Generate the code to project probe side records
+ g.setMappingSet(projectProbeMapping);
- ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
- if (v instanceof AbstractContainerVector) {
- vv.getValueVector().makeTransferPair(v);
- v.clear();
- }
+ int outputFieldId = fieldId;
+ fieldId = 0;
+ JExpression probeIndex = JExpr.direct("probeIndex");
- JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
- JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
+ if (leftUpstream == IterOutcome.OK || leftUpstream == IterOutcome.OK_NEW_SCHEMA) {
+ for (VectorWrapper<?> vv : left) {
- g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV).not())._then()._return(JExpr.FALSE);
+ MajorType inputType = vv.getField().getType();
+ MajorType outputType;
+ if (joinType == JoinRelType.RIGHT && inputType.getMode() == DataMode.REQUIRED) {
+ outputType = Types.overrideMode(inputType, DataMode.OPTIONAL);
+ } else {
+ outputType = inputType;
+ }
- fieldId++;
- outputFieldId++;
- }
- recordCount = left.getRecordCount();
+ ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
+ if (v instanceof AbstractContainerVector) {
+ vv.getValueVector().makeTransferPair(v);
+ v.clear();
}
- g.rotateBlock();
- g.getEvalBlock()._return(JExpr.TRUE);
- HashJoinProbe hj = context.getImplementationClass(cg);
+ JVar inVV = g.declareVectorValueSetupAndMember("probeBatch", new TypedFieldId(inputType, false, fieldId));
+ JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
- return hj;
- }
+ g.getEvalBlock()._if(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV).not())._then()
+ ._return(JExpr.FALSE);
- private void allocateVectors() {
- for(VectorWrapper<?> v : container) {
- v.getValueVector().allocateNew();
+ fieldId++;
+ outputFieldId++;
}
}
+ g.rotateBlock();
+ g.getEvalBlock()._return(JExpr.TRUE);
- public HashJoinBatch(HashJoinPOP popConfig, FragmentContext context, RecordBatch left, RecordBatch right) throws OutOfMemoryException {
- super(popConfig, context, true);
- this.left = left;
- this.right = right;
- this.joinType = popConfig.getJoinType();
- this.conditions = popConfig.getConditions();
- }
+ HashJoinProbe hj = context.getImplementationClass(cg);
- private void updateStats(HashTable htable) {
- if (htable == null) {
- return;
- }
- htable.getStats(htStats);
- this.stats.setLongStat(Metric.NUM_BUCKETS, htStats.numBuckets);
- this.stats.setLongStat(Metric.NUM_ENTRIES, htStats.numEntries);
- this.stats.setLongStat(Metric.NUM_RESIZING, htStats.numResizing);
- this.stats.setLongStat(Metric.RESIZING_TIME, htStats.resizingTime);
+ return hj;
+ }
+
+ private void allocateVectors() {
+ for (VectorWrapper<?> v : container) {
+ v.getValueVector().allocateNew();
}
+ }
- @Override
- public void killIncoming(boolean sendUpstream) {
- this.left.kill(sendUpstream);
- this.right.kill(sendUpstream);
+ public HashJoinBatch(HashJoinPOP popConfig, FragmentContext context, RecordBatch left,
+ RecordBatch right) throws OutOfMemoryException {
+ super(popConfig, context, true);
+ this.left = left;
+ this.right = right;
+ this.joinType = popConfig.getJoinType();
+ this.conditions = popConfig.getConditions();
+ }
+
+ private void updateStats(HashTable htable) {
+ if (htable == null) {
+ return;
}
+ htable.getStats(htStats);
+ this.stats.setLongStat(Metric.NUM_BUCKETS, htStats.numBuckets);
+ this.stats.setLongStat(Metric.NUM_ENTRIES, htStats.numEntries);
+ this.stats.setLongStat(Metric.NUM_RESIZING, htStats.numResizing);
+ this.stats.setLongStat(Metric.RESIZING_TIME, htStats.resizingTime);
+ }
- @Override
- public void cleanup() {
- if (hjHelper != null) {
- hjHelper.clear();
- }
+ @Override
+ public void killIncoming(boolean sendUpstream) {
+ this.left.kill(sendUpstream);
+ this.right.kill(sendUpstream);
+ }
- // If we didn't receive any data, hyperContainer may be null, check before clearing
- if (hyperContainer != null) {
- hyperContainer.clear();
- }
+ @Override
+ public void cleanup() {
+ if (hjHelper != null) {
+ hjHelper.clear();
+ }
- if (hashTable != null) {
- hashTable.clear();
- }
- super.cleanup();
- right.cleanup();
- left.cleanup();
+ // If we didn't receive any data, hyperContainer may be null, check before clearing
+ if (hyperContainer != null) {
+ hyperContainer.clear();
}
+ if (hashTable != null) {
+ hashTable.clear();
+ }
+ super.cleanup();
+ right.cleanup();
+ left.cleanup();
+ }
}