You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/04/19 12:13:14 UTC
[20/20] hive git commit: HIVE-9862 Vectorized execution corrupts
timestamp values (Matt McCline,
reviewed by Jason Dere) HIVE-13111: Fix timestamp / interval_day_time wrong
results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
HIVE-9862 Vectorized execution corrupts timestamp values (Matt McCline, reviewed by Jason Dere)
HIVE-13111: Fix timestamp / interval_day_time wrong results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/130293e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/130293e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/130293e5
Branch: refs/heads/branch-1
Commit: 130293e5645b9029578d96dd2cc7e07bb1f76015
Parents: f42b984
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Apr 19 03:12:15 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Apr 19 03:12:15 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 1445 ++++++++-----
.../hive/common/type/HiveIntervalDayTime.java | 39 +-
.../hadoop/hive/common/type/RandomTypeUtil.java | 115 +
.../org/apache/hive/common/util/DateUtils.java | 17 -
.../hive/common/util/IntervalDayTimeUtils.java | 77 +
data/files/timestamps.txt | 50 +
.../test/resources/testconfiguration.properties | 4 +
.../DTIColumnArithmeticDTIColumnNoConvert.txt | 2 +
.../DTIColumnArithmeticDTIScalarNoConvert.txt | 2 +
.../DTIColumnCompareScalar.txt | 2 +
.../DTIScalarArithmeticDTIColumnNoConvert.txt | 2 +
.../DTIScalarCompareColumn.txt | 2 +
...eColumnArithmeticIntervalYearMonthColumn.txt | 197 ++
...eColumnArithmeticIntervalYearMonthScalar.txt | 156 ++
.../DateColumnArithmeticTimestampColumn.txt | 186 ++
.../DateColumnArithmeticTimestampScalar.txt | 154 ++
...eScalarArithmeticIntervalYearMonthColumn.txt | 170 ++
.../DateScalarArithmeticTimestampColumn.txt | 161 ++
...olumnArithmeticIntervalColumnWithConvert.txt | 175 --
...olumnArithmeticIntervalScalarWithConvert.txt | 152 --
...calarArithmeticIntervalColumnWithConvert.txt | 165 --
.../FilterDTIColumnCompareScalar.txt | 2 -
.../FilterDecimalColumnCompareColumn.txt | 445 ----
.../FilterDecimalColumnCompareDecimalColumn.txt | 445 ++++
.../FilterDecimalColumnCompareDecimalScalar.txt | 160 ++
.../FilterDecimalColumnCompareScalar.txt | 160 --
.../FilterDecimalScalarCompareColumn.txt | 160 --
.../FilterDecimalScalarCompareDecimalColumn.txt | 160 ++
...erLongDoubleColumnCompareTimestampColumn.txt | 185 ++
...erLongDoubleColumnCompareTimestampScalar.txt | 59 +
...erLongDoubleScalarCompareTimestampColumn.txt | 169 ++
.../FilterScalarCompareTimestampColumn.txt | 55 -
.../FilterTimestampColumnBetween.txt | 171 ++
...erTimestampColumnCompareLongDoubleColumn.txt | 182 ++
...erTimestampColumnCompareLongDoubleScalar.txt | 165 ++
.../FilterTimestampColumnCompareScalar.txt | 56 -
...terTimestampColumnCompareTimestampColumn.txt | 451 ++++
...terTimestampColumnCompareTimestampScalar.txt | 132 +-
...erTimestampScalarCompareLongDoubleColumn.txt | 58 +
...terTimestampScalarCompareTimestampColumn.txt | 133 +-
...olumnArithmeticDateTimeColumnWithConvert.txt | 177 --
...olumnArithmeticDateTimeScalarWithConvert.txt | 154 --
...calarArithmeticDateTimeColumnWithConvert.txt | 167 --
...ervalYearMonthColumnArithmeticDateColumn.txt | 196 ++
...ervalYearMonthColumnArithmeticDateScalar.txt | 156 ++
...YearMonthColumnArithmeticTimestampColumn.txt | 186 ++
...YearMonthColumnArithmeticTimestampScalar.txt | 154 ++
...ervalYearMonthScalarArithmeticDateColumn.txt | 170 ++
...YearMonthScalarArithmeticTimestampColumn.txt | 158 ++
.../LongDoubleColumnCompareTimestampColumn.txt | 154 ++
.../LongDoubleColumnCompareTimestampScalar.txt | 146 ++
.../LongDoubleScalarCompareTimestampColumn.txt | 144 ++
.../ScalarCompareTimestampColumn.txt | 63 -
.../TimestampColumnArithmeticDateColumn.txt | 187 ++
.../TimestampColumnArithmeticDateScalar.txt | 147 ++
...pColumnArithmeticIntervalYearMonthColumn.txt | 186 ++
...pColumnArithmeticIntervalYearMonthScalar.txt | 143 ++
...TimestampColumnArithmeticTimestampColumn.txt | 177 ++
...TimestampColumnArithmeticTimestampScalar.txt | 145 ++
.../TimestampColumnCompareLongDoubleColumn.txt | 153 ++
.../TimestampColumnCompareLongDoubleScalar.txt | 144 ++
.../TimestampColumnCompareScalar.txt | 56 -
.../TimestampColumnCompareTimestampColumn.txt | 158 ++
.../TimestampColumnCompareTimestampScalar.txt | 118 +-
.../TimestampScalarArithmeticDateColumn.txt | 168 ++
...pScalarArithmeticIntervalYearMonthColumn.txt | 167 ++
...TimestampScalarArithmeticTimestampColumn.txt | 158 ++
.../TimestampScalarCompareLongDoubleColumn.txt | 58 +
.../TimestampScalarCompareTimestampColumn.txt | 119 +-
...runcStringScalarCompareStringGroupColumn.txt | 2 +-
.../VectorUDAFMinMaxIntervalDayTime.txt | 454 ++++
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 456 ++++
.../hive/ql/exec/vector/ColumnVector.java | 3 +-
.../vector/IntervalDayTimeColumnVector.java | 348 +++
.../ql/exec/vector/TimestampColumnVector.java | 395 ++++
.../hive/ql/exec/vector/TimestampUtils.java | 52 +-
.../hive/ql/exec/vector/VectorAssignRow.java | 53 +-
.../exec/vector/VectorColumnAssignFactory.java | 38 +-
.../ql/exec/vector/VectorColumnSetInfo.java | 85 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 113 +-
.../ql/exec/vector/VectorDeserializeRow.java | 33 +-
.../exec/vector/VectorExpressionDescriptor.java | 24 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 56 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 26 +-
.../ql/exec/vector/VectorHashKeyWrapper.java | 109 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 259 ++-
.../hive/ql/exec/vector/VectorSerializeRow.java | 22 +-
.../exec/vector/VectorSerializeRowNoNulls.java | 13 +-
.../ql/exec/vector/VectorizationContext.java | 104 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 39 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 18 +-
.../expressions/CastDecimalToTimestamp.java | 19 +-
.../expressions/CastDoubleToTimestamp.java | 126 ++
.../exec/vector/expressions/CastLongToDate.java | 58 +-
.../vector/expressions/CastLongToTimestamp.java | 126 ++
.../CastMillisecondsLongToTimestamp.java | 126 ++
.../CastStringToIntervalDayTime.java | 9 +-
.../expressions/CastTimestampToBoolean.java | 138 ++
.../vector/expressions/CastTimestampToDate.java | 49 +
.../expressions/CastTimestampToDecimal.java | 15 +-
.../expressions/CastTimestampToDouble.java | 131 ++
.../vector/expressions/CastTimestampToLong.java | 133 ++
.../expressions/ConstantVectorExpression.java | 106 +-
.../expressions/DateColSubtractDateColumn.java | 185 ++
.../expressions/DateColSubtractDateScalar.java | 147 ++
.../DateScalarSubtractDateColumn.java | 150 ++
.../FilterTimestampColumnInList.java | 172 ++
.../expressions/FuncDecimalToTimestamp.java | 135 ++
.../expressions/FuncTimestampToDecimal.java | 144 ++
.../vector/expressions/FuncTimestampToLong.java | 145 ++
.../vector/expressions/ITimestampInExpr.java | 25 +
.../IfExprIntervalDayTimeColumnColumn.java | 153 ++
.../IfExprIntervalDayTimeColumnScalar.java | 147 ++
.../IfExprIntervalDayTimeScalarColumn.java | 149 ++
.../IfExprIntervalDayTimeScalarScalar.java | 137 ++
.../IfExprTimestampColumnColumn.java | 54 +
.../IfExprTimestampColumnColumnBase.java | 136 ++
.../IfExprTimestampColumnScalar.java | 59 +
.../IfExprTimestampColumnScalarBase.java | 133 ++
.../IfExprTimestampScalarColumn.java | 59 +
.../IfExprTimestampScalarColumnBase.java | 134 ++
.../IfExprTimestampScalarScalar.java | 59 +
.../IfExprTimestampScalarScalarBase.java | 120 ++
.../ql/exec/vector/expressions/NullUtil.java | 84 +-
.../expressions/TimestampColumnInList.java | 153 ++
.../expressions/TimestampToStringUnaryUDF.java | 147 ++
.../vector/expressions/VectorExpression.java | 2 +
.../expressions/VectorExpressionWriter.java | 10 +-
.../VectorExpressionWriterFactory.java | 275 ++-
.../expressions/VectorUDFDateAddColCol.java | 5 +-
.../expressions/VectorUDFDateAddColScalar.java | 10 +-
.../expressions/VectorUDFDateAddScalarCol.java | 13 +-
.../expressions/VectorUDFDateDiffColCol.java | 93 +-
.../expressions/VectorUDFDateDiffColScalar.java | 26 +-
.../expressions/VectorUDFDateDiffScalarCol.java | 28 +-
.../vector/expressions/VectorUDFDateLong.java | 5 +-
.../expressions/VectorUDFDateTimestamp.java | 74 +
.../expressions/VectorUDFDayOfMonthDate.java | 38 +
.../expressions/VectorUDFDayOfMonthLong.java | 38 -
.../VectorUDFDayOfMonthTimestamp.java | 38 +
.../vector/expressions/VectorUDFHourDate.java | 39 +
.../vector/expressions/VectorUDFHourLong.java | 39 -
.../expressions/VectorUDFHourTimestamp.java | 39 +
.../vector/expressions/VectorUDFMinuteDate.java | 38 +
.../vector/expressions/VectorUDFMinuteLong.java | 38 -
.../expressions/VectorUDFMinuteTimestamp.java | 38 +
.../vector/expressions/VectorUDFMonthDate.java | 44 +
.../vector/expressions/VectorUDFMonthLong.java | 50 -
.../expressions/VectorUDFMonthTimestamp.java | 46 +
.../vector/expressions/VectorUDFSecondDate.java | 38 +
.../vector/expressions/VectorUDFSecondLong.java | 38 -
.../expressions/VectorUDFSecondTimestamp.java | 38 +
.../VectorUDFTimestampFieldDate.java | 163 ++
.../VectorUDFTimestampFieldLong.java | 228 --
.../VectorUDFTimestampFieldString.java | 5 +-
.../VectorUDFTimestampFieldTimestamp.java | 162 ++
.../expressions/VectorUDFUnixTimeStampDate.java | 49 +
.../expressions/VectorUDFUnixTimeStampLong.java | 57 -
.../VectorUDFUnixTimeStampTimestamp.java | 45 +
.../expressions/VectorUDFWeekOfYearDate.java | 46 +
.../expressions/VectorUDFWeekOfYearLong.java | 46 -
.../VectorUDFWeekOfYearTimestamp.java | 46 +
.../vector/expressions/VectorUDFYearDate.java | 38 +
.../vector/expressions/VectorUDFYearLong.java | 67 -
.../expressions/VectorUDFYearTimestamp.java | 38 +
.../aggregates/VectorUDAFAvgTimestamp.java | 482 +++++
.../aggregates/VectorUDAFStdPopTimestamp.java | 527 +++++
.../aggregates/VectorUDAFStdSampTimestamp.java | 527 +++++
.../aggregates/VectorUDAFVarPopTimestamp.java | 527 +++++
.../aggregates/VectorUDAFVarSampTimestamp.java | 527 +++++
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +-
.../hive/ql/io/orc/TreeReaderFactory.java | 18 +-
.../hadoop/hive/ql/io/orc/TypeDescription.java | 4 +-
.../hadoop/hive/ql/udf/UDFDayOfMonth.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFHour.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFMinute.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFMonth.java | 7 +-
.../apache/hadoop/hive/ql/udf/UDFSecond.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFToBoolean.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToByte.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToDouble.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToFloat.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToInteger.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToLong.java | 4 +-
.../apache/hadoop/hive/ql/udf/UDFToShort.java | 4 +-
.../hadoop/hive/ql/udf/UDFWeekOfYear.java | 5 +-
.../org/apache/hadoop/hive/ql/udf/UDFYear.java | 5 +-
.../hive/ql/udf/generic/GenericUDFDate.java | 3 +-
.../hive/ql/udf/generic/GenericUDFIf.java | 18 +-
.../hive/ql/udf/generic/GenericUDFOPEqual.java | 32 +-
.../generic/GenericUDFOPEqualOrGreaterThan.java | 32 +-
.../generic/GenericUDFOPEqualOrLessThan.java | 31 +-
.../ql/udf/generic/GenericUDFOPGreaterThan.java | 31 +-
.../ql/udf/generic/GenericUDFOPLessThan.java | 31 +-
.../hive/ql/udf/generic/GenericUDFOPMinus.java | 3 +
.../ql/udf/generic/GenericUDFOPNotEqual.java | 34 +-
.../ql/udf/generic/GenericUDFTimestamp.java | 8 +-
.../hive/ql/udf/generic/GenericUDFToDate.java | 3 +-
.../udf/generic/GenericUDFToUnixTimeStamp.java | 11 +-
.../hadoop/hive/ql/util/DateTimeMath.java | 193 +-
.../ql/exec/vector/RandomRowObjectSource.java | 21 +-
.../TestTimestampWritableAndColumnVector.java | 68 +
.../exec/vector/TestVectorizationContext.java | 38 +-
.../TestConstantVectorExpression.java | 4 +-
.../expressions/TestVectorDateExpressions.java | 10 +-
.../TestVectorExpressionWriters.java | 136 +-
.../TestVectorFilterExpressions.java | 48 +-
.../TestVectorGenericDateExpressions.java | 22 +-
.../expressions/TestVectorMathFunctions.java | 87 +
.../TestVectorTimestampExpressions.java | 364 ++--
.../vector/expressions/TestVectorTypeCasts.java | 221 +-
.../FakeVectorRowBatchFromObjectIterables.java | 8 +-
.../vector/util/VectorizedRowGroupGenUtil.java | 35 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 7 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java.orig | 2034 ++++++++++++++++++
.../hive/ql/io/orc/TestVectorizedORCReader.java | 10 +-
.../clientpositive/vector_interval_arithmetic.q | 174 ++
.../clientpositive/vectorized_timestamp.q | 27 +
.../tez/vector_interval_arithmetic.q.out | 1078 ++++++++++
.../tez/vector_join_part_col_char.q.out | 172 +-
.../tez/vectorized_timestamp.q.out | 258 +++
.../vector_interval_arithmetic.q.out | 1026 +++++++++
.../results/clientpositive/vector_udf1.q.out | 113 +-
.../clientpositive/vectorized_casts.q.out | 18 +-
.../clientpositive/vectorized_timestamp.q.out | 239 ++
.../fast/BinarySortableSerializeWrite.java | 13 -
.../hadoop/hive/serde2/fast/SerializeWrite.java | 3 -
.../hive/serde2/io/TimestampWritable.java | 88 +-
.../lazy/fast/LazySimpleSerializeWrite.java | 20 -
.../fast/LazyBinarySerializeWrite.java | 36 -
231 files changed, 24767 insertions(+), 4361 deletions(-)
----------------------------------------------------------------------