You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/06/11 23:41:01 UTC
svn commit: r1491977 [1/2] - in /hive/branches/vectorization/ql/src:
java/org/apache/hadoop/hive/ql/exec/vector/
java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/
java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/ j...
Author: hashutosh
Date: Tue Jun 11 21:41:00 2013
New Revision: 1491977
URL: http://svn.apache.org/r1491977
Log:
HIVE-4606 : Implement vector group by hash spill (Remus Rusanu via Ashutosh Chauhan)
Removed:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt
Modified:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxString.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinString.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMaxString.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java Tue Jun 11 21:41:00 2013
@@ -18,38 +18,68 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
+
/**
- * This maps a batch to the aggregation buffers sets to use for each row (key)
+ * This maps a batch to the aggregation buffers sets to use for each row (key)
*
*/
public class VectorAggregationBufferBatch {
-
+
/**
- * Batch sized array of aggregation buffer sets.
+ * Batch sized array of aggregation buffer sets.
* The array is preallocated and is reused for each batch, but the individual entries
* will reference different aggregation buffer set from batch to batch.
* the array is not reset between batches, content past this.index will be stale.
*/
- private VectorAggregationBufferRow[] aggregationBuffers;
-
+ private final VectorAggregationBufferRow[] aggregationBuffers;
+
/**
- * the selection vector that maps row within a batch to the
- * specific aggregation buffer set to use.
+ * Same as aggregationBuffers but only distinct buffers
*/
- private int[] selection;
-
+ private final VectorAggregationBufferRow[] distinctAggregationBuffers;
+
/**
* versioning number gets incremented on each batch. This allows us to cache the selection
- * mapping info in the aggregation buffer set themselves while still being able to
+ * mapping info in the aggregation buffer set themselves while still being able to
* detect stale info.
*/
private int version;
-
+
/**
* Get the number of distinct aggregation buffer sets (ie. keys) used in current batch.
*/
private int distinctCount;
-
+
+ /**
+ * Memory consumed by a set of aggregation buffers
+ */
+ private int aggregatorsFixedSize;
+
+ /**
+ * Array of indexes for aggregators that have variable size
+ */
+ private int[] variableSizeAggregators;;
+
+ /**
+ * returns True if any of the aggregators has a variable size
+ * @return
+ */
+ public boolean getHasVariableSize() {
+ return variableSizeAggregators.length > 0;
+ }
+
+ /**
+ * Returns the fixed size consumed by the aggregation buffers
+ * @return
+ */
+ public int getAggregatorsFixedSize() {
+ return aggregatorsFixedSize;
+ }
+
/**
* the array of aggregation buffers for the current batch.
* content past the {@link #getDistinctBufferSetCount()} index
@@ -59,49 +89,77 @@ public class VectorAggregationBufferBatc
public VectorAggregationBufferRow[] getAggregationBuffers() {
return aggregationBuffers;
}
-
+
/**
- * number of distinct aggregation buffer sets (ie. keys) in the current batch.
+ * number of distinct aggregation buffer sets (ie. keys) in the current batch.
* @return
*/
public int getDistinctBufferSetCount () {
return distinctCount;
}
- /**
- * gets the selection vector to use for the current batch. This maps the batch rows by position
- * (row number) to an index in the {@link #getAggregationBuffers()} array.
- * @return
- */
- public int[] getSelectionVector() {
- return selection;
- }
-
+
public VectorAggregationBufferBatch() {
aggregationBuffers = new VectorAggregationBufferRow[VectorizedRowBatch.DEFAULT_SIZE];
- selection = new int [VectorizedRowBatch.DEFAULT_SIZE];
+ distinctAggregationBuffers = new VectorAggregationBufferRow[VectorizedRowBatch.DEFAULT_SIZE];
}
-
+
/**
- * resets the internal aggregation buffers sets index and increments the versioning
+ * resets the internal aggregation buffers sets index and increments the versioning
* used to optimize the selection vector population.
*/
public void startBatch() {
version++;
distinctCount = 0;
}
-
+
/**
* assigns the given aggregation buffer set to a given batch row (by row number).
* populates the selection vector appropriately. This is where the versioning numbers
- * play a role in determining if the index cached on the aggregation buffer set is stale.
+ * play a role in determining if the index cached on the aggregation buffer set is stale.
*/
public void mapAggregationBufferSet(VectorAggregationBufferRow bufferSet, int row) {
if (version != bufferSet.getVersion()) {
bufferSet.setVersionAndIndex(version, distinctCount);
+ distinctAggregationBuffers[distinctCount] = bufferSet;
++distinctCount;
}
aggregationBuffers[row] = bufferSet;
}
+ public void compileAggregationBatchInfo(VectorAggregateExpression[] aggregators) {
+ JavaDataModel model = JavaDataModel.get();
+ int[] variableSizeAggregators = new int[aggregators.length];
+ int indexVariableSizes = 0;
+
+ aggregatorsFixedSize = JavaDataModel.alignUp(
+ model.object() +
+ model.primitive1()*2 +
+ model.ref(),
+ model.memoryAlign());
+
+ aggregatorsFixedSize += model.lengthForObjectArrayOfSize(aggregators.length);
+ for(int i=0;i<aggregators.length;++i) {
+ VectorAggregateExpression aggregator = aggregators[i];
+ aggregatorsFixedSize += aggregator.getAggregationBufferFixedSize();
+ if (aggregator.hasVariableSize()) {
+ variableSizeAggregators[indexVariableSizes] = i;
+ ++indexVariableSizes;
+ }
+ }
+ this.variableSizeAggregators = Arrays.copyOfRange(
+ variableSizeAggregators, 0, indexVariableSizes);
+ }
+
+ public int getVariableSize(int batchSize) {
+ int variableSize = 0;
+ for (int i=0; i< variableSizeAggregators.length; ++i) {
+ for(int r=0; r<distinctCount; ++r) {
+ VectorAggregationBufferRow buf = distinctAggregationBuffers[r];
+ variableSize += buf.getAggregationBuffer(variableSizeAggregators[i]).getVariableSize();
+ }
+ }
+ return (variableSize * batchSize)/distinctCount;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java Tue Jun 11 21:41:00 2013
@@ -19,9 +19,12 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.io.Serializable;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -40,6 +43,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -57,18 +61,18 @@ public class VectorGroupByOperator exten
/**
* This is the vector of aggregators. They are stateless and only implement
- * the algorithm of how to compute the aggregation. state is kept in the
- * aggregation buffers and is our responsibility to match the proper state for each key.
+ * the algorithm of how to compute the aggregation. state is kept in the
+ * aggregation buffers and is our responsibility to match the proper state for each key.
*/
private transient VectorAggregateExpression[] aggregators;
-
+
/**
* Key vector expressions.
*/
private transient VectorExpression[] keyExpressions;
-
+
private VectorExpressionWriter[] keyOutputWriters;
-
+
/**
* The aggregation buffers to use for the current batch.
*/
@@ -81,6 +85,56 @@ public class VectorGroupByOperator exten
private transient VectorHashKeyWrapperBatch keyWrappersBatch;
/**
+ * Total amount of memory allowed for JVM heap.
+ */
+ private transient long maxMemory;
+
+ /**
+ * configure percent of memory threshold usable by QP.
+ */
+ private transient float memoryThreshold;
+
+ /**
+ * Max memory usable by the hashtable before it should flush.
+ */
+ private transient long maxHashTblMemory;
+
+ /**
+ * Total per hashtable entry fixed memory (does not depend on key/agg values).
+ */
+ private transient int fixedHashEntrySize;
+
+ /**
+ * Average per hashtable entry variable size memory (depends on key/agg value).
+ */
+ private transient int avgVariableSize;
+
+ /**
+ * Current number of entries in the hash table
+ */
+ private transient int numEntriesHashTable;
+
+ /**
+ * Number of entries added to the hashtable since the last check if it should flush.
+ */
+ private transient int numEntriesSinceCheck;
+
+ /**
+ * Sum of batch size processed (ie. rows).
+ */
+ private transient long sumBatchSize;
+
+ /**
+ * The number of new entries that must be added to the hashtable before a memory size check.
+ */
+ private static final int FLUSH_CHECK_THRESHOLD = 10000;
+
+ /**
+ * Percent of entries to flush when memory threshold exceeded
+ */
+ private static final float PERCENT_ENTRIES_TO_FLUSH = 0.1f;
+
+ /**
* The global key-aggregation hash map.
*/
private transient Map<KeyWrapper, VectorAggregationBufferRow> mapKeysAggregationBuffers;
@@ -103,15 +157,15 @@ public class VectorGroupByOperator exten
List<ExprNodeDesc> keysDesc = conf.getKeys();
keyExpressions = vContext.getVectorExpressions(keysDesc);
-
+
keyOutputWriters = new VectorExpressionWriter[keyExpressions.length];
-
+
for(int i = 0; i < keyExpressions.length; ++i) {
keyOutputWriters[i] = VectorExpressionWriterFactory.
genVectorExpressionWritable(keysDesc.get(i));
objectInspectors.add(keyOutputWriters[i].getObjectInspector());
}
-
+
ArrayList<AggregationDesc> aggrDesc = conf.getAggregators();
aggregators = new VectorAggregateExpression[aggrDesc.size()];
for (int i = 0; i < aggrDesc.size(); ++i) {
@@ -119,9 +173,10 @@ public class VectorGroupByOperator exten
aggregators[i] = vContext.getAggregatorExpression (desc);
objectInspectors.add(aggregators[i].getOutputObjectInspector());
}
-
+
keyWrappersBatch = VectorHashKeyWrapperBatch.compileKeyWrapperBatch(keyExpressions);
aggregationBatchInfo = new VectorAggregationBufferBatch();
+ aggregationBatchInfo.compileAggregationBatchInfo(aggregators);
mapKeysAggregationBuffers = new HashMap<KeyWrapper, VectorAggregationBufferRow>();
List<String> outputFieldNames = conf.getOutputColumnNames();
@@ -133,38 +188,178 @@ public class VectorGroupByOperator exten
} catch (Throwable e) {
throw new HiveException(e);
}
+
+ computeMemoryLimits();
+
initializeChildren(hconf);
}
+ /**
+ * Computes the memory limits for hash table flush (spill).
+ */
+ private void computeMemoryLimits() {
+ JavaDataModel model = JavaDataModel.get();
+
+ fixedHashEntrySize =
+ model.hashMapEntry() +
+ keyWrappersBatch.getKeysFixedSize() +
+ aggregationBatchInfo.getAggregatorsFixedSize();
+
+ MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+ maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
+ memoryThreshold = conf.getMemoryThreshold();
+ // Tests may leave this unitialized, so better set it to 1
+ if (memoryThreshold == 0.0f) {
+ memoryThreshold = 1.0f;
+ }
+
+ maxHashTblMemory = (int)(maxMemory * memoryThreshold);
+
+ LOG.info(String.format("maxMemory:%dMb (%d * %f) fixSize:%d (key:%d agg:%d)",
+ maxHashTblMemory/1024/1024,
+ maxMemory/1024/1024,
+ memoryThreshold,
+ fixedHashEntrySize,
+ keyWrappersBatch.getKeysFixedSize(),
+ aggregationBatchInfo.getAggregatorsFixedSize()));
+
+ }
+
@Override
public void processOp(Object row, int tag) throws HiveException {
VectorizedRowBatch batch = (VectorizedRowBatch) row;
-
+
// First we traverse the batch to evaluate and prepare the KeyWrappers
// After this the KeyWrappers are properly set and hash code is computed
keyWrappersBatch.evaluateBatch(batch);
// Next we locate the aggregation buffer set for each key
prepareBatchAggregationBufferSets(batch);
-
+
// Finally, evaluate the aggregators
processAggregators(batch);
+
+ //Flush if memory limits were reached
+ if (shouldFlush(batch)) {
+ flush(false);
+ }
+
+ if (sumBatchSize == 0 && 0 != batch.size) {
+ // Sample the first batch processed for variable sizes.
+ updateAvgVariableSize(batch);
+ }
+
+ sumBatchSize += batch.size;
+ }
+
+ /**
+ * Flushes the entries in the hash table by emiting output (forward).
+ * When parameter 'all' is true all the entries are flushed.
+ * @param all
+ * @throws HiveException
+ */
+ private void flush(boolean all) throws HiveException {
+
+ int entriesToFlush = all ? numEntriesHashTable :
+ (int)(numEntriesHashTable * PERCENT_ENTRIES_TO_FLUSH);
+ int entriesFlushed = 0;
+
+ LOG.info(String.format("Flush %d %s entries:%d fixed:%d variable:%d (used:%dMb max:%dMb)",
+ entriesToFlush, all ? "(all)" : "",
+ numEntriesHashTable, fixedHashEntrySize, avgVariableSize,
+ numEntriesHashTable * (fixedHashEntrySize + avgVariableSize)/1024/1024,
+ maxHashTblMemory/1024/1024));
+
+ Object[] forwardCache = new Object[keyExpressions.length + aggregators.length];
+ if (keyExpressions.length == 0 && mapKeysAggregationBuffers.isEmpty()) {
+
+ // if this is a global aggregation (no keys) and empty set, must still emit NULLs
+ VectorAggregationBufferRow emptyBuffers = allocateAggregationBuffer();
+ for (int i = 0; i < aggregators.length; ++i) {
+ forwardCache[i] = aggregators[i].evaluateOutput(emptyBuffers.getAggregationBuffer(i));
+ }
+ forward(forwardCache, outputObjInspector);
+ } else {
+
+ /* Iterate the global (keywrapper,aggregationbuffers) map and emit
+ a row for each key */
+ Iterator<Map.Entry<KeyWrapper, VectorAggregationBufferRow>> iter =
+ mapKeysAggregationBuffers.entrySet().iterator();
+ while(iter.hasNext()) {
+ Map.Entry<KeyWrapper, VectorAggregationBufferRow> pair = iter.next();
+ int fi = 0;
+ for (int i = 0; i < keyExpressions.length; ++i) {
+ VectorHashKeyWrapper kw = (VectorHashKeyWrapper)pair.getKey();
+ forwardCache[fi++] = keyWrappersBatch.getWritableKeyValue (
+ kw, i, keyOutputWriters[i]);
+ }
+ for (int i = 0; i < aggregators.length; ++i) {
+ forwardCache[fi++] = aggregators[i].evaluateOutput(pair.getValue()
+ .getAggregationBuffer(i));
+ }
+ LOG.debug(String.format("forwarding keys: %s: %s",
+ pair.getKey().toString(), Arrays.toString(forwardCache)));
+ forward(forwardCache, outputObjInspector);
+
+ if (!all) {
+ iter.remove();
+ --numEntriesHashTable;
+ if (++entriesFlushed >= entriesToFlush) {
+ break;
+ }
+ }
+ }
+ }
+
+ if (all) {
+ mapKeysAggregationBuffers.clear();
+ numEntriesHashTable = 0;
+ }
+ }
+
+ /**
+ * Returns true if the memory threshold for the hash table was reached.
+ */
+ private boolean shouldFlush(VectorizedRowBatch batch) {
+ if (numEntriesSinceCheck < FLUSH_CHECK_THRESHOLD ||
+ batch.size == 0) {
+ return false;
+ }
+ // Were going to update the average variable row size by sampling the current batch
+ updateAvgVariableSize(batch);
+ numEntriesSinceCheck = 0;
+ return numEntriesHashTable * (fixedHashEntrySize + avgVariableSize) > maxHashTblMemory;
+ }
+
+ /**
+ * Updates the average variable size of the hash table entries.
+ * The average is only updates by probing the batch that added the entry in the hash table
+ * that caused the check threshold to be reached.
+ */
+ private void updateAvgVariableSize(VectorizedRowBatch batch) {
+ int keyVariableSize = keyWrappersBatch.getVariableSize(batch.size);
+ int aggVariableSize = aggregationBatchInfo.getVariableSize(batch.size);
+
+ // This assumes the distribution of variable size keys/aggregates in the input
+ // is the same as the distribution of variable sizes in the hash entries
+ avgVariableSize = (int)((avgVariableSize * sumBatchSize + keyVariableSize +aggVariableSize) /
+ (sumBatchSize + batch.size));
}
-
+
/**
* Evaluates the aggregators on the current batch.
* The aggregationBatchInfo must have been prepared
- * by calling {@link #prepareBatchAggregationBufferSets} first.
+ * by calling {@link #prepareBatchAggregationBufferSets} first.
*/
private void processAggregators(VectorizedRowBatch batch) throws HiveException {
// We now have a vector of aggregation buffer sets to use for each row
// We can start computing the aggregates.
// If the number of distinct keys in the batch is 1 we can
// use the optimized code path of aggregateInput
- VectorAggregationBufferRow[] aggregationBufferSets =
+ VectorAggregationBufferRow[] aggregationBufferSets =
aggregationBatchInfo.getAggregationBuffers();
if (aggregationBatchInfo.getDistinctBufferSetCount() == 1) {
- VectorAggregateExpression.AggregationBuffer[] aggregationBuffers =
+ VectorAggregateExpression.AggregationBuffer[] aggregationBuffers =
aggregationBufferSets[0].getAggregationBuffers();
for (int i = 0; i < aggregators.length; ++i) {
aggregators[i].aggregateInput(aggregationBuffers[i], batch);
@@ -178,16 +373,16 @@ public class VectorGroupByOperator exten
}
}
}
-
+
/**
* Locates the aggregation buffer sets to use for each key in the current batch.
* The keyWrappersBatch must have evaluated the current batch first.
*/
private void prepareBatchAggregationBufferSets(VectorizedRowBatch batch) throws HiveException {
// The aggregation batch vector needs to know when we start a new batch
- // to bump its internal version.
+ // to bump its internal version.
aggregationBatchInfo.startBatch();
-
+
// We now have to probe the global hash and find-or-allocate
// the aggregation buffers to use for each key present in the batch
VectorHashKeyWrapper[] keyWrappers = keyWrappersBatch.getVectorHashKeyWrappers();
@@ -195,12 +390,14 @@ public class VectorGroupByOperator exten
VectorHashKeyWrapper kw = keyWrappers[i];
VectorAggregationBufferRow aggregationBuffer = mapKeysAggregationBuffers.get(kw);
if (null == aggregationBuffer) {
- // the probe failed, we must allocate a set of aggregation buffers
+ // the probe failed, we must allocate a set of aggregation buffers
// and push the (keywrapper,buffers) pair into the hash.
// is very important to clone the keywrapper, the one we have from our
// keyWrappersBatch is going to be reset/reused on next batch.
aggregationBuffer = allocateAggregationBuffer();
mapKeysAggregationBuffers.put(kw.copyKey(), aggregationBuffer);
+ numEntriesHashTable++;
+ numEntriesSinceCheck++;
}
aggregationBatchInfo.mapAggregationBufferSet(aggregationBuffer, i);
}
@@ -210,7 +407,7 @@ public class VectorGroupByOperator exten
* allocates a new aggregation buffer set.
*/
private VectorAggregationBufferRow allocateAggregationBuffer() throws HiveException {
- VectorAggregateExpression.AggregationBuffer[] aggregationBuffers =
+ VectorAggregateExpression.AggregationBuffer[] aggregationBuffers =
new VectorAggregateExpression.AggregationBuffer[aggregators.length];
for (int i=0; i < aggregators.length; ++i) {
aggregationBuffers[i] = aggregators[i].getNewAggregationBuffer();
@@ -223,36 +420,7 @@ public class VectorGroupByOperator exten
@Override
public void closeOp(boolean aborted) throws HiveException {
if (!aborted) {
- Object[] forwardCache = new Object[keyExpressions.length + aggregators.length];
- if (keyExpressions.length == 0 && mapKeysAggregationBuffers.isEmpty()) {
-
- // if this is a global aggregation (no keys) and empty set, must still emit NULLs
- VectorAggregationBufferRow emptyBuffers = allocateAggregationBuffer();
- for (int i = 0; i < aggregators.length; ++i) {
- forwardCache[i] = aggregators[i].evaluateOutput(emptyBuffers.getAggregationBuffer(i));
- }
- forward(forwardCache, outputObjInspector);
- } else {
-
- /* Iterate the global (keywrapper,aggregationbuffers) map and emit
- a row for each key */
- for(Map.Entry<KeyWrapper, VectorAggregationBufferRow> pair:
- mapKeysAggregationBuffers.entrySet()){
- int fi = 0;
- for (int i = 0; i < keyExpressions.length; ++i) {
- VectorHashKeyWrapper kw = (VectorHashKeyWrapper)pair.getKey();
- forwardCache[fi++] = keyWrappersBatch.getWritableKeyValue (
- kw, i, keyOutputWriters[i]);
- }
- for (int i = 0; i < aggregators.length; ++i) {
- forwardCache[fi++] = aggregators[i].evaluateOutput(pair.getValue()
- .getAggregationBuffer(i));
- }
- LOG.debug(String.format("forwarding keys: %s: %s",
- pair.getKey().toString(), Arrays.toString(forwardCache)));
- forward(forwardCache, outputObjInspector);
- }
- }
+ flush(true);
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java Tue Jun 11 21:41:00 2013
@@ -23,6 +23,7 @@ import java.util.Arrays;
import org.apache.hadoop.hive.ql.exec.KeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
@@ -233,6 +234,15 @@ public class VectorHashKeyWrapper extend
return byteLengths[i - longValues.length - doubleValues.length];
}
+ public int getVariableSize() {
+ int variableSize = 0;
+ for (int i=0; i<byteLengths.length; ++i) {
+ JavaDataModel model = JavaDataModel.get();
+ variableSize += model.lengthForByteArrayOfSize(byteLengths[i]);
+ }
+ return variableSize;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java Tue Jun 11 21:41:00 2013
@@ -20,16 +20,10 @@ package org.apache.hadoop.hive.ql.exec.v
import java.util.Arrays;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
/**
* Class for handling vectorized hash map key wrappers. It evaluates the key columns in a
@@ -83,6 +77,19 @@ public class VectorHashKeyWrapperBatch {
private KeyLookupHelper[] indexLookup;
/**
+ * The fixed size of the key wrappers.
+ */
+ private int keysFixedSize;
+
+ /**
+ * Returns the compiled fixed size for the key wrappers.
+ * @return
+ */
+ public int getKeysFixedSize() {
+ return keysFixedSize;
+ }
+
+ /**
* Accessor for the batch-sized array of key wrappers
*/
public VectorHashKeyWrapper[] getVectorHashKeyWrappers() {
@@ -428,6 +435,8 @@ public class VectorHashKeyWrapperBatch {
VectorHashKeyWrapperBatch compiledKeyWrapperBatch = new VectorHashKeyWrapperBatch();
compiledKeyWrapperBatch.keyExpressions = keyExpressions;
+ compiledKeyWrapperBatch.keysFixedSize = 0;
+
// We'll overallocate and then shrink the array for each type
int[] longIndices = new int[keyExpressions.length];
int longIndicesIndex = 0;
@@ -479,14 +488,32 @@ public class VectorHashKeyWrapperBatch {
compiledKeyWrapperBatch.vectorHashKeyWrappers[i] =
new VectorHashKeyWrapper(longIndicesIndex, doubleIndicesIndex, stringIndicesIndex);
}
+
+ JavaDataModel model = JavaDataModel.get();
+
+ // Compute the fixed size overhead for the keys
+ // start with the keywrapper itself
+ compiledKeyWrapperBatch.keysFixedSize += JavaDataModel.alignUp(
+ model.object() +
+ model.ref() * 6+
+ model.primitive1(),
+ model.memoryAlign());
+
+ // Now add the key wrapper arrays
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForLongArrayOfSize(longIndicesIndex);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(doubleIndicesIndex);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(stringIndicesIndex);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForIntArrayOfSize(longIndicesIndex) * 2;
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForBooleanArrayOfSize(keyExpressions.length);
+
return compiledKeyWrapperBatch;
}
/**
* Get the row-mode writable object value of a key from a key wrapper
- * @param keyOutputWriter
+ * @param keyOutputWriter
*/
- public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i,
+ public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i,
VectorExpressionWriter keyOutputWriter)
throws HiveException {
if (kw.getIsNull(i)) {
@@ -506,5 +533,16 @@ public class VectorHashKeyWrapperBatch {
i, klh.longIndex, klh.doubleIndex, klh.stringIndex));
}
}
+
+ public int getVariableSize(int batchSize) {
+ int variableSize = 0;
+ if ( 0 < stringIndices.length) {
+ for (int k=0; k<batchSize; ++k) {
+ VectorHashKeyWrapper hkw = vectorHashKeyWrappers[k];
+ variableSize += hkw.getVariableSize();
+ }
+ }
+ return variableSize;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java Tue Jun 11 21:41:00 2013
@@ -24,9 +24,10 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public abstract class VectorAggregateExpression {
-
+
public static interface AggregationBuffer {
- };
+ int getVariableSize();
+ };
public abstract AggregationBuffer getNewAggregationBuffer() throws HiveException;
public abstract void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
@@ -37,5 +38,9 @@ public abstract class VectorAggregateExp
public abstract Object evaluateOutput(AggregationBuffer agg) throws HiveException;
public abstract ObjectInspector getOutputObjectInspector();
+ public abstract int getAggregationBufferFixedSize();
+ public boolean hasVariableSize() {
+ return false;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java Tue Jun 11 21:41:00 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.LongWritable;
@@ -47,6 +48,11 @@ public class VectorUDAFCount extends Vec
value = 0;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private final VectorExpression inputExpression;
@@ -243,5 +249,14 @@ public class VectorUDAFCount extends Vec
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() +
+ model.primitive1(),
+ model.memoryAlign());
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java Tue Jun 11 21:41:00 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.LongWritable;
@@ -39,6 +40,11 @@ public class VectorUDAFCountStar extends
static class Aggregation implements AggregationBuffer {
long value;
boolean isNull;
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private final LongWritable result;
@@ -120,5 +126,15 @@ public class VectorUDAFCountStar extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() +
+ model.primitive1(),
+ model.memoryAlign());
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFAvgDouble extends
count++;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -435,5 +441,14 @@ public class VectorUDAFAvgDouble extends
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFAvgLong extends V
count++;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -435,5 +441,14 @@ public class VectorUDAFAvgLong extends V
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxDouble.java Tue Jun 11 21:41:00 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFMaxDouble extends
this.value = value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -411,5 +417,15 @@ public class VectorUDAFMaxDouble extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxLong.java Tue Jun 11 21:41:00 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFMaxLong extends V
this.value = value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -411,5 +417,15 @@ public class VectorUDAFMaxLong extends V
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxString.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxString.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxString.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMaxString.java Tue Jun 11 21:41:00 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -74,6 +75,12 @@ public class VectorUDAFMaxString extends
System.arraycopy(bytes, start, this.bytes, 0, length);
this.length = length;
}
+ @Override
+ public int getVariableSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return model.lengthForByteArrayOfSize(bytes.length);
+ }
+
}
private VectorExpression inputExpression;
@@ -359,5 +366,21 @@ public class VectorUDAFMaxString extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.ref()+
+ model.primitive1()*2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean hasVariableSize() {
+ return true;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinDouble.java Tue Jun 11 21:41:00 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFMinDouble extends
this.value = value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -411,5 +417,15 @@ public class VectorUDAFMinDouble extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinLong.java Tue Jun 11 21:41:00 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class VectorUDAFMinLong extends V
this.value = value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -411,5 +417,15 @@ public class VectorUDAFMinLong extends V
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinString.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinString.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinString.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFMinString.java Tue Jun 11 21:41:00 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -74,6 +75,12 @@ public class VectorUDAFMinString extends
System.arraycopy(bytes, start, this.bytes, 0, length);
this.length = length;
}
+ @Override
+ public int getVariableSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return model.lengthForByteArrayOfSize(bytes.length);
+ }
+
}
private VectorExpression inputExpression;
@@ -359,5 +366,21 @@ public class VectorUDAFMinString extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.ref()+
+ model.primitive1()*2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean hasVariableSize() {
+ return true;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFStdPopDouble exte
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFStdPopDouble exte
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFStdPopLong extend
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFStdPopLong extend
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFStdSampDouble ext
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFStdSampDouble ext
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFStdSampLong exten
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFStdSampLong exten
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFSumDouble extends
sum += value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
VectorExpression inputExpression;
@@ -404,5 +410,15 @@ public class VectorUDAFSumDouble extends
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object(),
+ model.memoryAlign());
+ }
+
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFSumLong extends V
sum += value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
VectorExpression inputExpression;
@@ -404,5 +410,15 @@ public class VectorUDAFSumLong extends V
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object(),
+ model.memoryAlign());
+ }
+
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFVarPopDouble exte
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFVarPopDouble exte
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFVarPopLong extend
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFVarPopLong extend
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFVarSampDouble ext
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFVarSampDouble ext
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class VectorUDAFVarSampLong exten
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class VectorUDAFVarSampLong exten
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class <ClassName> extends VectorA
count++;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -435,5 +441,14 @@ public class <ClassName> extends VectorA
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt Tue Jun 11 21:41:00 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -63,6 +64,11 @@ public class <ClassName> extends VectorA
this.value = value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -411,5 +417,15 @@ public class <ClassName> extends VectorA
public ObjectInspector getOutputObjectInspector() {
return <OutputTypeInspector>;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMaxString.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMaxString.txt?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMaxString.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMaxString.txt Tue Jun 11 21:41:00 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -74,6 +75,12 @@ public class <ClassName> extends VectorA
System.arraycopy(bytes, start, this.bytes, 0, length);
this.length = length;
}
+ @Override
+ public int getVariableSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return model.lengthForByteArrayOfSize(bytes.length);
+ }
+
}
private VectorExpression inputExpression;
@@ -359,5 +366,21 @@ public class <ClassName> extends VectorA
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.ref()+
+ model.primitive1()*2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean hasVariableSize() {
+ return true;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class <ClassName> extends VectorA
sum += value;
}
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
VectorExpression inputExpression;
@@ -404,5 +410,15 @@ public class <ClassName> extends VectorA
public ObjectInspector getOutputObjectInspector() {
return <OutputTypeInspector>;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object(),
+ model.memoryAlign());
+ }
+
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt Tue Jun 11 21:41:00 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -62,6 +63,11 @@ public class <ClassName> extends VectorA
count = 0;
variance = 0;
}
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
}
private VectorExpression inputExpression;
@@ -482,5 +488,16 @@ public class <ClassName> extends VectorA
public ObjectInspector getOutputObjectInspector() {
return soi;
}
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2()*3+
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java?rev=1491977&r1=1491976&r2=1491977&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java Tue Jun 11 21:41:00 2013
@@ -27,95 +27,133 @@ import org.apache.hadoop.hive.ql.udf.gen
public enum JavaDataModel {
JAVA32 {
+ @Override
public int object() {
return JAVA32_OBJECT;
}
+ @Override
public int array() {
return JAVA32_ARRAY;
}
+ @Override
public int ref() {
return JAVA32_REF;
}
+ @Override
public int hashMap(int entry) {
// base = JAVA32_OBJECT + PRIMITIVES1 * 4 + JAVA32_FIELDREF * 3 + JAVA32_ARRAY;
// entry = JAVA32_OBJECT + JAVA32_FIELDREF + PRIMITIVES1
return 64 + 24 * entry;
}
+ @Override
+ public int hashMapEntry() {
+ return 24;
+ }
+
+ @Override
public int hashSet(int entry) {
// hashMap += JAVA32_OBJECT
return 80 + 24 * entry;
}
+ @Override
public int linkedHashMap(int entry) {
// hashMap += JAVA32_FIELDREF + PRIMITIVES1
// hashMap.entry += JAVA32_FIELDREF * 2
return 72 + 32 * entry;
}
+ @Override
public int linkedList(int entry) {
// base = JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_FIELDREF;
// entry = JAVA32_OBJECT + JAVA32_FIELDREF * 2
return 28 + 24 * entry;
}
+ @Override
public int arrayList() {
// JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_ARRAY;
return 44;
}
+
+ @Override
+ public int memoryAlign() {
+ return 8;
+ }
}, JAVA64 {
+ @Override
public int object() {
return JAVA64_OBJECT;
}
+ @Override
public int array() {
return JAVA64_ARRAY;
}
+ @Override
public int ref() {
return JAVA64_REF;
}
+ @Override
public int hashMap(int entry) {
// base = JAVA64_OBJECT + PRIMITIVES1 * 4 + JAVA64_FIELDREF * 3 + JAVA64_ARRAY;
// entry = JAVA64_OBJECT + JAVA64_FIELDREF + PRIMITIVES1
return 112 + 44 * entry;
}
+ @Override
+ public int hashMapEntry() {
+ return 44;
+ }
+
+ @Override
public int hashSet(int entry) {
// hashMap += JAVA64_OBJECT
return 144 + 44 * entry;
}
+ @Override
public int linkedHashMap(int entry) {
// hashMap += JAVA64_FIELDREF + PRIMITIVES1
// hashMap.entry += JAVA64_FIELDREF * 2
return 128 + 60 * entry;
}
+ @Override
public int linkedList(int entry) {
// base = JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_FIELDREF;
// entry = JAVA64_OBJECT + JAVA64_FIELDREF * 2
return 48 + 48 * entry;
}
+ @Override
public int arrayList() {
// JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_ARRAY;
return 80;
}
+
+ @Override
+ public int memoryAlign() {
+ return 8;
+ }
};
public abstract int object();
public abstract int array();
public abstract int ref();
public abstract int hashMap(int entry);
+ public abstract int hashMapEntry();
public abstract int hashSet(int entry);
public abstract int linkedHashMap(int entry);
public abstract int linkedList(int entry);
public abstract int arrayList();
+ public abstract int memoryAlign();
// ascii string
public int lengthFor(String string) {
@@ -161,6 +199,10 @@ public enum JavaDataModel {
return PRIMITIVES2;
}
+ public static int alignUp(int value, int align) {
+ return (value + align - 1) & ~(align - 1);
+ }
+
public static final int JAVA32_META = 12;
public static final int JAVA32_ARRAY_META = 16;
public static final int JAVA32_REF = 4;
@@ -176,6 +218,8 @@ public enum JavaDataModel {
public static final int PRIMITIVES1 = 4; // void, boolean, byte, short, int, float
public static final int PRIMITIVES2 = 8; // long, double
+ public static final int PRIMITIVE_BYTE = 1; // byte
+
private static JavaDataModel current;
public static JavaDataModel get() {
@@ -200,4 +244,27 @@ public enum JavaDataModel {
}
return ((size + 8) >> 3) << 3;
}
+
+ private int lengthForPrimitiveArrayOfSize(int primitiveSize, int length) {
+ return alignUp(array() + primitiveSize*length, memoryAlign());
+ }
+
+ public int lengthForByteArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length);
+ }
+ public int lengthForObjectArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(ref(), length);
+ }
+ public int lengthForLongArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(primitive2(), length);
+ }
+ public int lengthForDoubleArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(primitive2(), length);
+ }
+ public int lengthForIntArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(primitive1(), length);
+ }
+ public int lengthForBooleanArrayOfSize(int length) {
+ return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length);
+ }
}