You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2015/11/01 05:32:55 UTC
[10/10] hive git commit: HIVE-12290 Native Vector ReduceSink (Matt
McCline, reviewed by Gopal V)
HIVE-12290 Native Vector ReduceSink (Matt McCline, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/409db57d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/409db57d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/409db57d
Branch: refs/heads/master
Commit: 409db57df5c039eb24288163f5268ea44bd1ecc7
Parents: 6d08cb1
Author: Matt McCline <mm...@hortonworks.com>
Authored: Sat Oct 31 21:26:14 2015 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Sat Oct 31 21:26:14 2015 -0700
----------------------------------------------------------------------
.../common/io/NonSyncByteArrayOutputStream.java | 4 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +-
.../apache/hadoop/hive/conf/HiveConf.java.orig | 3369 ++++++++++++++++++
.../apache/hive/common/util/HashCodeUtil.java | 132 +
.../test/resources/testconfiguration.properties | 10 +
.../apache/hadoop/hive/ql/exec/Operator.java | 8 +
.../hadoop/hive/ql/exec/OperatorUtils.java | 11 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 10 +
.../persistence/HybridHashTableContainer.java | 5 +-
.../persistence/MapJoinBytesTableContainer.java | 5 +-
.../hive/ql/exec/tez/ReduceRecordSource.java | 16 +-
.../ql/exec/vector/VectorDeserializeRow.java | 60 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 64 +-
.../exec/vector/VectorSerializeRowNoNulls.java | 412 ---
.../exec/vector/keyseries/VectorKeySeries.java | 98 +
.../VectorKeySeriesBytesSerialized.java | 271 ++
.../vector/keyseries/VectorKeySeriesImpl.java | 68 +
.../VectorKeySeriesLongSerialized.java | 249 ++
.../VectorKeySeriesMultiSerialized.java | 187 +
.../keyseries/VectorKeySeriesSerialized.java | 35 +
.../VectorKeySeriesSerializedImpl.java | 130 +
.../keyseries/VectorKeySeriesSingleImpl.java | 158 +
.../mapjoin/VectorMapJoinCommonOperator.java | 11 +-
.../VectorMapJoinGenerateResultOperator.java | 21 +-
...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 24 +-
.../VectorMapJoinInnerMultiKeyOperator.java | 24 +-
.../VectorMapJoinLeftSemiMultiKeyOperator.java | 24 +-
.../VectorMapJoinOuterMultiKeyOperator.java | 5 +-
.../fast/VectorMapJoinFastBytesHashMap.java | 3 +-
.../VectorMapJoinFastBytesHashMultiSet.java | 3 +-
.../fast/VectorMapJoinFastBytesHashSet.java | 3 +-
.../fast/VectorMapJoinFastBytesHashTable.java | 3 +-
.../fast/VectorMapJoinFastBytesHashUtil.java | 4 -
.../fast/VectorMapJoinFastIntHashUtil.java | 32 -
.../fast/VectorMapJoinFastLongHashMap.java | 3 +-
.../fast/VectorMapJoinFastLongHashMultiSet.java | 3 +-
.../fast/VectorMapJoinFastLongHashSet.java | 3 +-
.../fast/VectorMapJoinFastLongHashTable.java | 5 +-
.../fast/VectorMapJoinFastLongHashUtil.java | 11 -
.../VectorReduceSinkCommonOperator.java | 416 +++
.../VectorReduceSinkLongOperator.java | 72 +
.../VectorReduceSinkMultiKeyOperator.java | 68 +
.../VectorReduceSinkStringOperator.java | 70 +
.../hive/ql/optimizer/physical/Vectorizer.java | 218 +-
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 17 +
.../hive/ql/plan/VectorReduceSinkDesc.java | 64 +
.../hive/ql/plan/VectorReduceSinkInfo.java | 123 +
.../queries/clientpositive/vector_reduce1.q | 47 +
.../queries/clientpositive/vector_reduce2.q | 47 +
.../queries/clientpositive/vector_reduce3.q | 47 +
.../tez/vector_aggregate_without_gby.q.out | 85 +
.../clientpositive/tez/vector_bround.q.out | 66 +
.../results/clientpositive/tez/vector_nvl.q.out | 194 +
.../clientpositive/tez/vector_reduce1.q.out | 2167 +++++++++++
.../clientpositive/tez/vector_reduce2.q.out | 2167 +++++++++++
.../clientpositive/tez/vector_reduce3.q.out | 2167 +++++++++++
.../clientpositive/tez/vector_struct_in.q.out | 645 ++++
.../tez/vectorization_part_varchar.q.out | 72 +
.../tez/vectorized_parquet_types.q.out | 151 +-
.../tez/vectorized_timestamp_ints_casts.q.out | 50 +-
.../results/clientpositive/vector_reduce1.q.out | 2160 +++++++++++
.../results/clientpositive/vector_reduce2.q.out | 2160 +++++++++++
.../results/clientpositive/vector_reduce3.q.out | 2160 +++++++++++
.../apache/hadoop/hive/serde2/WriteBuffers.java | 58 +-
.../fast/BinarySortableDeserializeRead.java | 25 +-
.../fast/BinarySortableSerializeWrite.java | 7 +-
.../hive/serde2/fast/DeserializeRead.java | 6 +-
.../lazy/fast/LazySimpleDeserializeRead.java | 29 +-
.../lazy/fast/LazySimpleSerializeWrite.java | 2 +-
.../fast/LazyBinaryDeserializeRead.java | 25 +-
.../hive/ql/exec/vector/ColumnVector.java | 3 +-
71 files changed, 20271 insertions(+), 806 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/409db57d/common/src/java/org/apache/hadoop/hive/common/io/NonSyncByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/NonSyncByteArrayOutputStream.java b/common/src/java/org/apache/hadoop/hive/common/io/NonSyncByteArrayOutputStream.java
index a9f5ed6..4ed3a02 100644
--- a/common/src/java/org/apache/hadoop/hive/common/io/NonSyncByteArrayOutputStream.java
+++ b/common/src/java/org/apache/hadoop/hive/common/io/NonSyncByteArrayOutputStream.java
@@ -43,6 +43,10 @@ public class NonSyncByteArrayOutputStream extends ByteArrayOutputStream {
return count;
}
+ public void setWritePosition(int writePosition) {
+ count = writePosition;
+ }
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/409db57d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index f05f224..b214344 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2206,7 +2206,10 @@ public class HiveConf extends Configuration {
"Exceeding this will trigger a flush irrelevant of memory pressure condition."),
HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", (float) 0.1,
"Percent of entries in the group by aggregation hash flushed when the memory threshold is exceeded."),
-
+ HIVE_VECTORIZATION_REDUCESINK_NEW_ENABLED("hive.vectorized.execution.reducesink.new.enabled", true,
+ "This flag should be set to true to enable the new vectorization\n" +
+ "of queries using ReduceSink.\ni" +
+ "The default value is true."),
HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, "This property has been extended to control "
+ "whether to check, convert, and normalize partition value to conform to its column type in "
+ "partition operations including but not limited to insert, such as alter, describe etc."),