You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/04/10 08:02:39 UTC
[16/16] hive git commit: HIVE-9862 Vectorized execution corrupts
timestamp values (Matt McCline, reviewed by Jason Dere)
HIVE-9862 Vectorized execution corrupts timestamp values (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a479d0c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a479d0c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a479d0c
Branch: refs/heads/branch-2.0
Commit: 4a479d0c5f0f6e6feb43daacaacc9c55aec402d1
Parents: 226f4d6
Author: Matt McCline <mm...@hortonworks.com>
Authored: Sat Apr 9 23:02:07 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Sat Apr 9 23:02:07 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 1547 +++++++++++-------
.../hive/common/type/HiveIntervalDayTime.java | 11 +
.../java/org/apache/orc/TypeDescription.java | 4 +-
.../java/org/apache/orc/impl/WriterImpl.java | 37 +-
.../DTIColumnArithmeticDTIColumnNoConvert.txt | 2 +
.../DTIColumnArithmeticDTIScalarNoConvert.txt | 2 +
.../DTIColumnCompareScalar.txt | 2 +
.../DTIScalarArithmeticDTIColumnNoConvert.txt | 2 +
.../DTIScalarCompareColumn.txt | 2 +
...eColumnArithmeticIntervalYearMonthColumn.txt | 161 ++
...eColumnArithmeticIntervalYearMonthScalar.txt | 135 ++
.../DateColumnArithmeticTimestampColumn.txt | 59 +
.../DateColumnArithmeticTimestampColumnBase.txt | 171 ++
.../DateColumnArithmeticTimestampScalar.txt | 63 +
.../DateColumnArithmeticTimestampScalarBase.txt | 137 ++
...eScalarArithmeticIntervalYearMonthColumn.txt | 149 ++
.../DateScalarArithmeticTimestampColumn.txt | 65 +
.../DateScalarArithmeticTimestampColumnBase.txt | 147 ++
...olumnArithmeticIntervalColumnWithConvert.txt | 175 --
...olumnArithmeticIntervalScalarWithConvert.txt | 152 --
...calarArithmeticIntervalColumnWithConvert.txt | 165 --
.../FilterDTIColumnCompareScalar.txt | 2 -
.../FilterDecimalColumnCompareColumn.txt | 445 -----
.../FilterDecimalColumnCompareDecimalColumn.txt | 445 +++++
.../FilterDecimalColumnCompareDecimalScalar.txt | 160 ++
.../FilterDecimalColumnCompareScalar.txt | 160 --
.../FilterDecimalScalarCompareColumn.txt | 160 --
.../FilterDecimalScalarCompareDecimalColumn.txt | 160 ++
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 +
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 +
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 +
...erLongDoubleColumnCompareTimestampColumn.txt | 185 +++
...erLongDoubleColumnCompareTimestampScalar.txt | 59 +
...erLongDoubleScalarCompareTimestampColumn.txt | 165 ++
.../FilterScalarCompareTimestampColumn.txt | 55 -
.../FilterTimestampColumnBetween.txt | 172 ++
...erTimestampColumnCompareLongDoubleColumn.txt | 182 +++
...erTimestampColumnCompareLongDoubleScalar.txt | 165 ++
.../FilterTimestampColumnCompareScalar.txt | 56 -
...terTimestampColumnCompareTimestampColumn.txt | 52 +
...imestampColumnCompareTimestampColumnBase.txt | 429 +++++
...terTimestampColumnCompareTimestampScalar.txt | 18 +-
...imestampColumnCompareTimestampScalarBase.txt | 145 ++
...erTimestampScalarCompareLongDoubleColumn.txt | 59 +
...terTimestampScalarCompareTimestampColumn.txt | 15 +-
...imestampScalarCompareTimestampColumnBase.txt | 147 ++
...olumnArithmeticDateTimeColumnWithConvert.txt | 177 --
...olumnArithmeticDateTimeScalarWithConvert.txt | 154 --
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 +
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 +
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 +
...calarArithmeticDateTimeColumnWithConvert.txt | 167 --
...ervalYearMonthColumnArithmeticDateColumn.txt | 163 ++
...ervalYearMonthColumnArithmeticDateScalar.txt | 135 ++
...YearMonthColumnArithmeticTimestampColumn.txt | 179 ++
...YearMonthColumnArithmeticTimestampScalar.txt | 150 ++
...ervalYearMonthScalarArithmeticDateColumn.txt | 149 ++
...YearMonthScalarArithmeticTimestampColumn.txt | 159 ++
.../LongDoubleColumnCompareTimestampColumn.txt | 155 ++
.../LongDoubleColumnCompareTimestampScalar.txt | 147 ++
.../LongDoubleScalarCompareTimestampColumn.txt | 143 ++
.../ScalarCompareTimestampColumn.txt | 63 -
.../TimestampColumnArithmeticDateColumn.txt | 59 +
.../TimestampColumnArithmeticDateColumnBase.txt | 172 ++
.../TimestampColumnArithmeticDateScalar.txt | 59 +
.../TimestampColumnArithmeticDateScalarBase.txt | 126 ++
...pColumnArithmeticIntervalYearMonthColumn.txt | 177 ++
...pColumnArithmeticIntervalYearMonthScalar.txt | 146 ++
...TimestampColumnArithmeticTimestampColumn.txt | 59 +
...stampColumnArithmeticTimestampColumnBase.txt | 152 ++
...TimestampColumnArithmeticTimestampScalar.txt | 63 +
...stampColumnArithmeticTimestampScalarBase.txt | 125 ++
.../TimestampColumnCompareLongDoubleColumn.txt | 153 ++
.../TimestampColumnCompareLongDoubleScalar.txt | 143 ++
.../TimestampColumnCompareScalar.txt | 56 -
.../TimestampColumnCompareTimestampColumn.txt | 54 +
...imestampColumnCompareTimestampColumnBase.txt | 140 ++
.../TimestampColumnCompareTimestampScalar.txt | 16 +-
...imestampColumnCompareTimestampScalarBase.txt | 131 ++
.../TimestampScalarArithmeticDateColumn.txt | 69 +
.../TimestampScalarArithmeticDateColumnBase.txt | 151 ++
...pScalarArithmeticIntervalYearMonthColumn.txt | 161 ++
...TimestampScalarArithmeticTimestampColumn.txt | 69 +
...stampScalarArithmeticTimestampColumnBase.txt | 139 ++
.../TimestampScalarCompareLongDoubleColumn.txt | 58 +
.../TimestampScalarCompareTimestampColumn.txt | 24 +-
...imestampScalarCompareTimestampColumnBase.txt | 132 ++
...runcStringScalarCompareStringGroupColumn.txt | 2 +-
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 455 ++++++
.../hive/ql/exec/vector/TimestampUtils.java | 48 +-
.../hive/ql/exec/vector/VectorAssignRow.java | 30 +-
.../exec/vector/VectorColumnAssignFactory.java | 19 +-
.../ql/exec/vector/VectorColumnSetInfo.java | 26 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 32 +-
.../ql/exec/vector/VectorDeserializeRow.java | 24 +-
.../exec/vector/VectorExpressionDescriptor.java | 19 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 39 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 14 +-
.../ql/exec/vector/VectorHashKeyWrapper.java | 61 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 153 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 23 +-
.../ql/exec/vector/VectorizationContext.java | 102 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 18 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 22 +-
.../expressions/CastDecimalToTimestamp.java | 20 +-
.../expressions/CastDoubleToTimestamp.java | 123 ++
.../exec/vector/expressions/CastLongToDate.java | 58 +-
.../vector/expressions/CastLongToTimestamp.java | 124 ++
.../CastMillisecondsLongToTimestamp.java | 120 ++
.../CastStringToIntervalDayTime.java | 9 +-
.../expressions/CastTimestampToBoolean.java | 138 ++
.../vector/expressions/CastTimestampToDate.java | 49 +
.../expressions/CastTimestampToDecimal.java | 14 +-
.../expressions/CastTimestampToDouble.java | 132 ++
.../vector/expressions/CastTimestampToLong.java | 133 ++
.../expressions/ConstantVectorExpression.java | 92 +-
.../expressions/DateColSubtractDateColumn.java | 187 +++
.../expressions/DateColSubtractDateScalar.java | 150 ++
.../DateScalarSubtractDateColumn.java | 154 ++
.../FilterTimestampColumnInList.java | 181 ++
.../expressions/FuncDecimalToTimestamp.java | 135 ++
.../expressions/FuncTimestampToDecimal.java | 144 ++
.../vector/expressions/FuncTimestampToLong.java | 145 ++
.../vector/expressions/ITimestampInExpr.java | 25 +
.../IfExprIntervalDayTimeColumnColumn.java | 54 +
.../IfExprIntervalDayTimeColumnScalar.java | 59 +
.../IfExprIntervalDayTimeScalarColumn.java | 59 +
.../IfExprIntervalDayTimeScalarScalar.java | 59 +
.../expressions/IfExprLongColumnLongColumn.java | 4 +-
.../IfExprTimestampColumnColumn.java | 54 +
.../IfExprTimestampColumnColumnBase.java | 136 ++
.../IfExprTimestampColumnScalar.java | 60 +
.../IfExprTimestampColumnScalarBase.java | 133 ++
.../IfExprTimestampScalarColumn.java | 60 +
.../IfExprTimestampScalarColumnBase.java | 133 ++
.../IfExprTimestampScalarScalar.java | 60 +
.../IfExprTimestampScalarScalarBase.java | 123 ++
.../expressions/LongColEqualLongColumn.java | 4 +-
.../LongColGreaterEqualLongColumn.java | 4 +-
.../expressions/LongColGreaterLongColumn.java | 4 +-
.../expressions/LongColLessEqualLongColumn.java | 4 +-
.../expressions/LongColLessLongColumn.java | 4 +-
.../expressions/LongColNotEqualLongColumn.java | 4 +-
.../ql/exec/vector/expressions/NullUtil.java | 31 +-
.../expressions/TimestampColumnInList.java | 164 ++
.../expressions/TimestampToStringUnaryUDF.java | 147 ++
.../vector/expressions/VectorExpression.java | 2 +
.../expressions/VectorExpressionWriter.java | 8 +-
.../VectorExpressionWriterFactory.java | 197 ++-
.../expressions/VectorUDFDateAddColCol.java | 5 +-
.../expressions/VectorUDFDateAddColScalar.java | 5 +-
.../expressions/VectorUDFDateAddScalarCol.java | 6 +-
.../expressions/VectorUDFDateDiffColCol.java | 93 +-
.../expressions/VectorUDFDateDiffColScalar.java | 11 +-
.../expressions/VectorUDFDateDiffScalarCol.java | 11 +-
.../vector/expressions/VectorUDFDateLong.java | 5 +-
.../expressions/VectorUDFDateTimestamp.java | 74 +
.../expressions/VectorUDFDayOfMonthDate.java | 38 +
.../expressions/VectorUDFDayOfMonthLong.java | 38 -
.../VectorUDFDayOfMonthTimestamp.java | 38 +
.../vector/expressions/VectorUDFHourDate.java | 39 +
.../vector/expressions/VectorUDFHourLong.java | 39 -
.../expressions/VectorUDFHourTimestamp.java | 39 +
.../vector/expressions/VectorUDFMinuteDate.java | 38 +
.../vector/expressions/VectorUDFMinuteLong.java | 38 -
.../expressions/VectorUDFMinuteTimestamp.java | 38 +
.../vector/expressions/VectorUDFMonthDate.java | 44 +
.../vector/expressions/VectorUDFMonthLong.java | 50 -
.../expressions/VectorUDFMonthTimestamp.java | 46 +
.../vector/expressions/VectorUDFSecondDate.java | 38 +
.../vector/expressions/VectorUDFSecondLong.java | 38 -
.../expressions/VectorUDFSecondTimestamp.java | 38 +
.../VectorUDFTimestampFieldDate.java | 163 ++
.../VectorUDFTimestampFieldLong.java | 229 ---
.../VectorUDFTimestampFieldTimestamp.java | 162 ++
.../expressions/VectorUDFUnixTimeStampDate.java | 46 +
.../expressions/VectorUDFUnixTimeStampLong.java | 57 -
.../VectorUDFUnixTimeStampTimestamp.java | 48 +
.../expressions/VectorUDFWeekOfYearDate.java | 46 +
.../expressions/VectorUDFWeekOfYearLong.java | 46 -
.../VectorUDFWeekOfYearTimestamp.java | 46 +
.../vector/expressions/VectorUDFYearDate.java | 38 +
.../vector/expressions/VectorUDFYearLong.java | 67 -
.../expressions/VectorUDFYearTimestamp.java | 38 +
.../aggregates/VectorUDAFAvgTimestamp.java | 482 ++++++
.../aggregates/VectorUDAFStdPopTimestamp.java | 527 ++++++
.../aggregates/VectorUDAFStdSampTimestamp.java | 528 ++++++
.../aggregates/VectorUDAFVarPopTimestamp.java | 527 ++++++
.../aggregates/VectorUDAFVarSampTimestamp.java | 527 ++++++
.../hive/ql/io/orc/TreeReaderFactory.java | 18 +-
.../hadoop/hive/ql/io/orc/WriterImpl.java | 6 +-
.../hadoop/hive/ql/udf/UDFDayOfMonth.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFHour.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFMinute.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFMonth.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFSecond.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFToBoolean.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToByte.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToDouble.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToFloat.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToInteger.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToLong.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToShort.java | 4 +-
.../hadoop/hive/ql/udf/UDFWeekOfYear.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFYear.java | 5 +-
.../hive/ql/udf/generic/GenericUDFDate.java | 3 +-
.../hive/ql/udf/generic/GenericUDFIf.java | 16 +-
.../hive/ql/udf/generic/GenericUDFOPEqual.java | 32 +-
.../generic/GenericUDFOPEqualOrGreaterThan.java | 32 +-
.../generic/GenericUDFOPEqualOrLessThan.java | 31 +-
.../ql/udf/generic/GenericUDFOPGreaterThan.java | 31 +-
.../ql/udf/generic/GenericUDFOPLessThan.java | 31 +-
.../hive/ql/udf/generic/GenericUDFOPMinus.java | 1 +
.../ql/udf/generic/GenericUDFOPNotEqual.java | 32 +-
.../hive/ql/udf/generic/GenericUDFOPPlus.java | 1 +
.../ql/udf/generic/GenericUDFTimestamp.java | 8 +-
.../hive/ql/udf/generic/GenericUDFToDate.java | 3 +-
.../udf/generic/GenericUDFToUnixTimeStamp.java | 5 +-
.../hadoop/hive/ql/util/DateTimeMath.java | 21 +
.../ql/exec/vector/RandomRowObjectSource.java | 21 +-
.../exec/vector/TestVectorizationContext.java | 32 +-
.../TestConstantVectorExpression.java | 4 +-
.../expressions/TestVectorDateExpressions.java | 10 +-
.../TestVectorExpressionWriters.java | 135 +-
.../TestVectorFilterExpressions.java | 49 +-
.../TestVectorGenericDateExpressions.java | 22 +-
.../expressions/TestVectorMathFunctions.java | 74 +
.../TestVectorTimestampExpressions.java | 292 ++--
.../vector/expressions/TestVectorTypeCasts.java | 99 +-
.../FakeVectorRowBatchFromObjectIterables.java | 9 +-
.../vector/util/VectorizedRowGroupGenUtil.java | 29 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 7 +-
.../hive/ql/io/orc/TestVectorOrcFile.java | 32 +-
.../hive/ql/io/orc/TestVectorizedORCReader.java | 10 +-
.../clientpositive/vectorized_timestamp.q | 27 +
.../tez/vector_groupby_mapjoin.q.out | 44 +-
.../clientpositive/vector_groupby_mapjoin.q.out | 4 +-
.../clientpositive/vectorized_timestamp.q.out | 239 +++
.../fast/BinarySortableSerializeWrite.java | 13 -
.../hadoop/hive/serde2/fast/SerializeWrite.java | 3 -
.../hive/serde2/io/TimestampWritable.java | 19 +-
.../lazy/fast/LazySimpleSerializeWrite.java | 20 -
.../fast/LazyBinarySerializeWrite.java | 36 -
.../hive/serde2/binarysortable/MyTestClass.java | 3 +-
.../binarysortable/MyTestPrimitiveClass.java | 21 +-
.../serde2/lazybinary/MyTestClassBigger.java | 3 +-
.../serde2/lazybinary/MyTestClassSmaller.java | 3 +-
.../hadoop/hive/common/type/PisaTimestamp.java | 609 +++++++
.../hadoop/hive/common/type/RandomTypeUtil.java | 51 +
.../hive/ql/exec/vector/ColumnVector.java | 1 +
.../ql/exec/vector/TimestampColumnVector.java | 498 ++++++
.../hive/common/type/TestPisaTimestamp.java | 118 ++
252 files changed, 19460 insertions(+), 4113 deletions(-)
----------------------------------------------------------------------