You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/14 00:20:16 UTC
[55/57] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Sergey Shelukhin)
HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Conflicts:
ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bd78d660
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bd78d660
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bd78d660
Branch: refs/heads/hive-14535
Commit: bd78d6605c6c57f723045b47bab06fccc5053047
Parents: b9e8157 5353161
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 13 17:12:24 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 13 17:12:24 2016 -0700
----------------------------------------------------------------------
.gitignore | 1 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 12 +-
.../org/apache/hive/common/util/DateUtils.java | 20 +
data/files/identity_udf.jar | Bin 710 -> 0 bytes
itests/hive-blobstore/README | 25 +
itests/hive-blobstore/pom.xml | 355 +
.../hadoop/hive/cli/TestBlobstoreCliDriver.java | 64 +
.../cli/TestBlobstoreNegativeCliDriver.java | 64 +
.../clientnegative/select_dropped_table.q | 4 +
.../test/queries/clientpositive/insert_into.q | 4 +
.../test/resources/blobstore-conf.xml.template | 22 +
.../src/test/resources/hive-site.xml | 271 +
.../test/resources/testconfiguration.properties | 2 +
.../src/test/resources/tez-site.xml | 6 +
.../clientnegative/select_dropped_table.q.out | 21 +
.../results/clientpositive/insert_into.q.out | 35 +
.../minikdc/JdbcWithMiniKdcSQLAuthTest.java | 2 +
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 32 +-
.../org/apache/hive/jdbc/TestJdbcDriver2.java | 171 +-
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 1005 +-
.../apache/hive/jdbc/TestJdbcWithMiniMr.java | 344 -
itests/pom.xml | 1 +
.../test/resources/testconfiguration.properties | 80 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 167 +
.../hadoop/hive/cli/control/CliConfigs.java | 40 +
.../cli/control/CoreBlobstoreCliDriver.java | 29 +
.../control/CoreBlobstoreNegativeCliDriver.java | 29 +
.../org/apache/hadoop/hive/ql/QTestUtil.java | 18 +-
.../ColumnArithmeticColumn.txt | 7 +-
.../ColumnArithmeticColumnDecimal.txt | 5 +
.../ColumnArithmeticColumnWithConvert.txt | 173 -
.../ColumnArithmeticScalar.txt | 5 +
.../ColumnArithmeticScalarDecimal.txt | 5 +
.../ColumnArithmeticScalarWithConvert.txt | 150 -
.../ExpressionTemplates/ColumnCompareColumn.txt | 5 +
.../ExpressionTemplates/ColumnCompareScalar.txt | 5 +
.../ExpressionTemplates/ColumnDivideColumn.txt | 5 +
.../ColumnDivideColumnDecimal.txt | 5 +
.../ExpressionTemplates/ColumnDivideScalar.txt | 5 +
.../ColumnDivideScalarDecimal.txt | 5 +
.../ExpressionTemplates/ColumnUnaryFunc.txt | 5 +
.../ExpressionTemplates/ColumnUnaryMinus.txt | 5 +
...eColumnArithmeticIntervalYearMonthColumn.txt | 5 +
...eColumnArithmeticIntervalYearMonthScalar.txt | 5 +
.../DateColumnArithmeticTimestampColumn.txt | 5 +
.../DateColumnArithmeticTimestampScalar.txt | 5 +
...eScalarArithmeticIntervalYearMonthColumn.txt | 5 +
.../DateScalarArithmeticTimestampColumn.txt | 5 +
.../DecimalColumnUnaryFunc.txt | 5 +
.../ExpressionTemplates/FilterColumnBetween.txt | 7 +-
.../FilterColumnCompareColumn.txt | 9 +-
.../FilterColumnCompareScalar.txt | 9 +-
.../FilterDecimalColumnBetween.txt | 5 +
.../FilterDecimalColumnCompareDecimalColumn.txt | 5 +
.../FilterDecimalColumnCompareDecimalScalar.txt | 5 +
.../FilterDecimalScalarCompareDecimalColumn.txt | 5 +
...erLongDoubleColumnCompareTimestampColumn.txt | 5 +
...erLongDoubleScalarCompareTimestampColumn.txt | 5 +
.../FilterScalarCompareColumn.txt | 9 +-
.../FilterStringColumnBetween.txt | 9 +-
...tringGroupColumnCompareStringGroupColumn.txt | 5 +
...gGroupColumnCompareStringGroupScalarBase.txt | 7 +
...gGroupScalarCompareStringGroupColumnBase.txt | 8 +
.../FilterTimestampColumnBetween.txt | 5 +
...erTimestampColumnCompareLongDoubleColumn.txt | 5 +
...erTimestampColumnCompareLongDoubleScalar.txt | 5 +
...terTimestampColumnCompareTimestampColumn.txt | 5 +
...terTimestampColumnCompareTimestampScalar.txt | 5 +
...erTimestampScalarCompareLongDoubleColumn.txt | 5 +
...terTimestampScalarCompareTimestampColumn.txt | 5 +
.../FilterTruncStringColumnBetween.txt | 10 +-
.../ExpressionTemplates/IfExprColumnScalar.txt | 5 +
.../ExpressionTemplates/IfExprScalarColumn.txt | 5 +
.../ExpressionTemplates/IfExprScalarScalar.txt | 5 +
...ervalYearMonthColumnArithmeticDateColumn.txt | 5 +
...ervalYearMonthColumnArithmeticDateScalar.txt | 5 +
...YearMonthColumnArithmeticTimestampColumn.txt | 5 +
...YearMonthColumnArithmeticTimestampScalar.txt | 5 +
...ervalYearMonthScalarArithmeticDateColumn.txt | 5 +
...YearMonthScalarArithmeticTimestampColumn.txt | 5 +
.../LongDoubleColumnCompareTimestampColumn.txt | 5 +
.../LongDoubleColumnCompareTimestampScalar.txt | 4 +
.../LongDoubleScalarCompareTimestampColumn.txt | 5 +
.../ScalarArithmeticColumn.txt | 5 +
.../ScalarArithmeticColumnDecimal.txt | 5 +
.../ScalarArithmeticColumnWithConvert.txt | 163 -
.../ExpressionTemplates/ScalarCompareColumn.txt | 5 +
.../ExpressionTemplates/ScalarDivideColumn.txt | 5 +
.../ScalarDivideColumnDecimal.txt | 5 +
...tringGroupColumnCompareStringGroupColumn.txt | 5 +
...gGroupColumnCompareStringGroupScalarBase.txt | 6 +
...tringGroupColumnCompareTruncStringScalar.txt | 7 +
...gGroupScalarCompareStringGroupColumnBase.txt | 7 +
.../TimestampColumnArithmeticDateColumn.txt | 5 +
.../TimestampColumnArithmeticDateScalar.txt | 5 +
...pColumnArithmeticIntervalYearMonthColumn.txt | 5 +
...pColumnArithmeticIntervalYearMonthScalar.txt | 5 +
...TimestampColumnArithmeticTimestampColumn.txt | 5 +
...TimestampColumnArithmeticTimestampScalar.txt | 5 +
.../TimestampColumnCompareLongDoubleColumn.txt | 5 +
.../TimestampColumnCompareLongDoubleScalar.txt | 5 +
.../TimestampColumnCompareTimestampColumn.txt | 5 +
.../TimestampColumnCompareTimestampScalar.txt | 5 +
.../TimestampScalarArithmeticDateColumn.txt | 5 +
...pScalarArithmeticIntervalYearMonthColumn.txt | 5 +
...TimestampScalarArithmeticTimestampColumn.txt | 5 +
.../TimestampScalarCompareTimestampColumn.txt | 5 +
...runcStringScalarCompareStringGroupColumn.txt | 2 +
.../UDAFTemplates/VectorUDAFAvg.txt | 6 +
.../UDAFTemplates/VectorUDAFMinMax.txt | 6 +
.../UDAFTemplates/VectorUDAFMinMaxDecimal.txt | 6 +
.../VectorUDAFMinMaxIntervalDayTime.txt | 6 +
.../UDAFTemplates/VectorUDAFMinMaxString.txt | 6 +
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 6 +
.../UDAFTemplates/VectorUDAFSum.txt | 6 +
.../UDAFTemplates/VectorUDAFVar.txt | 6 +
.../UDAFTemplates/VectorUDAFVarDecimal.txt | 6 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 252 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 2 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 6 +-
.../hadoop/hive/ql/exec/OperatorFactory.java | 5 +
.../hadoop/hive/ql/exec/SelectOperator.java | 1 -
.../hadoop/hive/ql/exec/StatsNoJobTask.java | 8 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 7 +
.../hive/ql/exec/persistence/MapJoinKey.java | 9 +-
.../hive/ql/exec/spark/HashTableLoader.java | 2 +-
.../ql/exec/vector/VectorColumnMapping.java | 12 +-
.../ql/exec/vector/VectorColumnOrderedMap.java | 33 +-
.../exec/vector/VectorColumnOutputMapping.java | 7 +-
.../exec/vector/VectorColumnSourceMapping.java | 7 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 3 +-
.../ql/exec/vector/VectorFilterOperator.java | 4 +-
.../ql/exec/vector/VectorGroupByOperator.java | 21 +-
.../ql/exec/vector/VectorSelectOperator.java | 33 +-
.../ql/exec/vector/VectorizationContext.java | 38 +-
.../exec/vector/VectorizationContextRegion.java | 4 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 3 +
...AbstractFilterStringColLikeStringScalar.java | 5 +
.../CastBooleanToCharViaLongToChar.java | 5 +
.../CastBooleanToVarCharViaLongToVarChar.java | 5 +
.../exec/vector/expressions/CastDateToChar.java | 4 +
.../vector/expressions/CastDateToVarChar.java | 5 +
.../vector/expressions/CastDecimalToChar.java | 5 +
.../expressions/CastDecimalToDecimal.java | 5 +
.../expressions/CastDecimalToVarChar.java | 5 +
.../expressions/CastDoubleToTimestamp.java | 5 +
.../exec/vector/expressions/CastLongToChar.java | 5 +
.../exec/vector/expressions/CastLongToDate.java | 5 +
.../vector/expressions/CastLongToTimestamp.java | 5 +
.../vector/expressions/CastLongToVarChar.java | 5 +
.../CastMillisecondsLongToTimestamp.java | 5 +
.../expressions/CastStringGroupToChar.java | 4 +
.../expressions/CastStringGroupToVarChar.java | 5 +
.../vector/expressions/CastStringToDate.java | 5 +
.../vector/expressions/CastStringToDecimal.java | 5 +
.../CastStringToIntervalDayTime.java | 5 +
.../CastStringToIntervalYearMonth.java | 5 +
.../expressions/CastTimestampToBoolean.java | 7 +-
.../expressions/CastTimestampToDouble.java | 5 +
.../vector/expressions/CastTimestampToLong.java | 7 +-
.../ql/exec/vector/expressions/ColAndCol.java | 5 +
.../ql/exec/vector/expressions/ColOrCol.java | 5 +
.../expressions/ConstantVectorExpression.java | 33 +
.../expressions/DateColSubtractDateColumn.java | 4 +
.../expressions/DateColSubtractDateScalar.java | 5 +
.../DateScalarSubtractDateColumn.java | 5 +
.../vector/expressions/DecimalColumnInList.java | 7 +
.../expressions/DecimalToStringUnaryUDF.java | 5 +
.../vector/expressions/DoubleColumnInList.java | 7 +
.../vector/expressions/FilterColAndScalar.java | 5 +
.../vector/expressions/FilterColOrScalar.java | 5 +
.../expressions/FilterDecimalColumnInList.java | 7 +
.../expressions/FilterDoubleColumnInList.java | 6 +
.../vector/expressions/FilterExprAndExpr.java | 8 +
.../vector/expressions/FilterExprOrExpr.java | 6 +
.../expressions/FilterLongColumnInList.java | 7 +
.../expressions/FilterScalarAndColumn.java | 5 +
.../expressions/FilterScalarOrColumn.java | 5 +
.../expressions/FilterStringColumnInList.java | 7 +
.../expressions/FilterStructColumnInList.java | 9 +
.../FilterTimestampColumnInList.java | 7 +
.../vector/expressions/FuncDecimalToDouble.java | 5 +
.../vector/expressions/FuncDecimalToLong.java | 5 +
.../expressions/FuncDecimalToTimestamp.java | 5 +
.../vector/expressions/FuncDoubleToDecimal.java | 4 +
.../vector/expressions/FuncLongToDecimal.java | 4 +
.../vector/expressions/FuncLongToString.java | 4 +
.../FuncRoundWithNumDigitsDecimalToDecimal.java | 4 +
.../expressions/FuncTimestampToDecimal.java | 5 +
.../vector/expressions/FuncTimestampToLong.java | 5 +
.../vector/expressions/IdentityExpression.java | 5 +
.../IfExprDoubleColumnDoubleColumn.java | 5 +
.../IfExprIntervalDayTimeColumnColumn.java | 5 +
.../IfExprIntervalDayTimeColumnScalar.java | 5 +
.../IfExprIntervalDayTimeScalarColumn.java | 5 +
.../IfExprIntervalDayTimeScalarScalar.java | 5 +
.../expressions/IfExprLongColumnLongColumn.java | 5 +
...fExprStringGroupColumnStringGroupColumn.java | 5 +
.../IfExprStringGroupColumnStringScalar.java | 7 +
.../IfExprStringScalarStringGroupColumn.java | 7 +
.../IfExprStringScalarStringScalar.java | 7 +
.../IfExprTimestampColumnColumnBase.java | 5 +
.../IfExprTimestampColumnScalarBase.java | 6 +
.../IfExprTimestampScalarColumnBase.java | 6 +
.../IfExprTimestampScalarScalarBase.java | 6 +
.../ql/exec/vector/expressions/IsNotNull.java | 5 +
.../hive/ql/exec/vector/expressions/IsNull.java | 5 +
.../expressions/LongColDivideLongColumn.java | 5 +
.../expressions/LongColDivideLongScalar.java | 5 +
.../expressions/LongColEqualLongColumn.java | 5 +
.../expressions/LongColEqualLongScalar.java | 4 +
.../LongColGreaterEqualLongColumn.java | 5 +
.../LongColGreaterEqualLongScalar.java | 5 +
.../expressions/LongColGreaterLongColumn.java | 5 +
.../expressions/LongColGreaterLongScalar.java | 5 +
.../expressions/LongColLessEqualLongColumn.java | 5 +
.../expressions/LongColLessEqualLongScalar.java | 5 +
.../expressions/LongColLessLongColumn.java | 5 +
.../expressions/LongColLessLongScalar.java | 5 +
.../expressions/LongColNotEqualLongColumn.java | 5 +
.../expressions/LongColNotEqualLongScalar.java | 5 +
.../vector/expressions/LongColumnInList.java | 6 +
.../expressions/LongScalarDivideLongColumn.java | 5 +
.../expressions/LongScalarEqualLongColumn.java | 4 +
.../LongScalarGreaterEqualLongColumn.java | 5 +
.../LongScalarGreaterLongColumn.java | 5 +
.../LongScalarLessEqualLongColumn.java | 4 +
.../expressions/LongScalarLessLongColumn.java | 5 +
.../LongScalarNotEqualLongColumn.java | 5 +
.../expressions/LongToStringUnaryUDF.java | 5 +
.../expressions/MathFuncDoubleToDouble.java | 7 +-
.../expressions/MathFuncLongToDouble.java | 5 +
.../vector/expressions/MathFuncLongToLong.java | 7 +-
.../hive/ql/exec/vector/expressions/NotCol.java | 5 +
.../expressions/PosModDoubleToDouble.java | 5 +
.../vector/expressions/PosModLongToLong.java | 5 +
.../RoundWithNumDigitsDoubleToDouble.java | 5 +
.../vector/expressions/SelectColumnIsFalse.java | 4 +
.../expressions/SelectColumnIsNotNull.java | 5 +
.../vector/expressions/SelectColumnIsNull.java | 5 +
.../vector/expressions/SelectColumnIsTrue.java | 5 +
.../SelectStringColLikeStringScalar.java | 10 +-
.../vector/expressions/StringColumnInList.java | 5 +
.../StringGroupColConcatStringScalar.java | 7 +
.../expressions/StringGroupConcatColCol.java | 5 +
.../exec/vector/expressions/StringLength.java | 4 +
.../StringScalarConcatStringGroupCol.java | 7 +
.../expressions/StringSubstrColStart.java | 5 +
.../expressions/StringSubstrColStartLen.java | 5 +
.../exec/vector/expressions/StringUnaryUDF.java | 5 +
.../expressions/StringUnaryUDFDirect.java | 4 +
.../vector/expressions/StructColumnInList.java | 8 +
.../expressions/TimestampColumnInList.java | 6 +
.../expressions/TimestampToStringUnaryUDF.java | 4 +
.../exec/vector/expressions/VectorCoalesce.java | 5 +
.../ql/exec/vector/expressions/VectorElt.java | 7 +
.../vector/expressions/VectorExpression.java | 44 +-
.../expressions/VectorUDFDateAddColCol.java | 5 +
.../expressions/VectorUDFDateAddColScalar.java | 5 +
.../expressions/VectorUDFDateAddScalarCol.java | 5 +
.../expressions/VectorUDFDateDiffColCol.java | 5 +
.../expressions/VectorUDFDateDiffColScalar.java | 6 +
.../expressions/VectorUDFDateDiffScalarCol.java | 5 +
.../VectorUDFTimestampFieldDate.java | 10 +
.../VectorUDFTimestampFieldString.java | 9 +
.../VectorUDFTimestampFieldTimestamp.java | 9 +
.../aggregates/VectorAggregateExpression.java | 19 +
.../aggregates/VectorUDAFAvgDecimal.java | 6 +
.../aggregates/VectorUDAFAvgTimestamp.java | 6 +
.../expressions/aggregates/VectorUDAFCount.java | 6 +
.../aggregates/VectorUDAFCountMerge.java | 6 +
.../aggregates/VectorUDAFCountStar.java | 7 +
.../aggregates/VectorUDAFStdPopTimestamp.java | 6 +
.../aggregates/VectorUDAFStdSampTimestamp.java | 6 +
.../aggregates/VectorUDAFSumDecimal.java | 6 +
.../aggregates/VectorUDAFVarPopTimestamp.java | 6 +
.../aggregates/VectorUDAFVarSampTimestamp.java | 6 +
.../mapjoin/VectorMapJoinCommonOperator.java | 363 +-
.../VectorMapJoinInnerBigOnlyLongOperator.java | 11 +-
...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 15 +-
...VectorMapJoinInnerBigOnlyStringOperator.java | 11 +-
.../mapjoin/VectorMapJoinInnerLongOperator.java | 11 +-
.../VectorMapJoinInnerMultiKeyOperator.java | 15 +-
.../VectorMapJoinInnerStringOperator.java | 11 +-
.../VectorMapJoinLeftSemiLongOperator.java | 11 +-
.../VectorMapJoinLeftSemiMultiKeyOperator.java | 15 +-
.../VectorMapJoinLeftSemiStringOperator.java | 11 +-
.../mapjoin/VectorMapJoinOuterLongOperator.java | 11 +-
.../VectorMapJoinOuterMultiKeyOperator.java | 15 +-
.../VectorMapJoinOuterStringOperator.java | 11 +-
.../fast/VectorMapJoinFastTableContainer.java | 2 +-
.../VectorMapJoinOptimizedCreateHashTable.java | 2 +-
.../VectorReduceSinkCommonOperator.java | 2 +-
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 5 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 4 +
.../hive/ql/optimizer/physical/Vectorizer.java | 1458 +-
.../ql/optimizer/physical/VectorizerReason.java | 123 +
.../hive/ql/parse/ExplainConfiguration.java | 39 +
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 45 +-
.../hadoop/hive/ql/parse/FromClauseParser.g | 3 -
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 5 +
.../apache/hadoop/hive/ql/parse/HiveParser.g | 38 +-
.../hadoop/hive/ql/parse/IdentifiersParser.g | 54 +-
.../hadoop/hive/ql/parse/SelectClauseParser.g | 3 -
.../hive/ql/plan/AbstractOperatorDesc.java | 12 +
.../hadoop/hive/ql/plan/AbstractVectorDesc.java | 14 +
.../hadoop/hive/ql/plan/AppMasterEventDesc.java | 24 +
.../apache/hadoop/hive/ql/plan/BaseWork.java | 197 +-
.../org/apache/hadoop/hive/ql/plan/Explain.java | 29 +
.../apache/hadoop/hive/ql/plan/ExplainWork.java | 13 +
.../apache/hadoop/hive/ql/plan/FetchWork.java | 45 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 18 +-
.../apache/hadoop/hive/ql/plan/FilterDesc.java | 28 +
.../apache/hadoop/hive/ql/plan/GroupByDesc.java | 84 +-
.../hadoop/hive/ql/plan/HashTableSinkDesc.java | 1 +
.../apache/hadoop/hive/ql/plan/LimitDesc.java | 18 +
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 218 +-
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 99 +-
.../hadoop/hive/ql/plan/MapredLocalWork.java | 4 +-
.../apache/hadoop/hive/ql/plan/MapredWork.java | 11 +-
.../ql/plan/OperatorExplainVectorization.java | 85 +
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 119 +-
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 92 +-
.../apache/hadoop/hive/ql/plan/SelectDesc.java | 35 +
.../hive/ql/plan/SparkHashTableSinkDesc.java | 25 +
.../apache/hadoop/hive/ql/plan/SparkWork.java | 6 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 26 +
.../org/apache/hadoop/hive/ql/plan/TezWork.java | 11 +-
.../hive/ql/plan/VectorAppMasterEventDesc.java | 35 +
.../apache/hadoop/hive/ql/plan/VectorDesc.java | 5 +
.../hadoop/hive/ql/plan/VectorFileSinkDesc.java | 35 +
.../hadoop/hive/ql/plan/VectorFilterDesc.java | 48 +
.../hadoop/hive/ql/plan/VectorGroupByDesc.java | 31 +
.../hadoop/hive/ql/plan/VectorLimitDesc.java | 35 +
.../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 110 +
.../hadoop/hive/ql/plan/VectorMapJoinInfo.java | 169 +
.../hive/ql/plan/VectorReduceSinkDesc.java | 68 +
.../hadoop/hive/ql/plan/VectorSMBJoinDesc.java | 35 +
.../hadoop/hive/ql/plan/VectorSelectDesc.java | 56 +
.../ql/plan/VectorSparkHashTableSinkDesc.java | 35 +
.../hive/ql/plan/VectorTableScanDesc.java | 45 +
.../hive/ql/plan/VectorizationCondition.java | 76 +
.../exec/vector/TestVectorFilterOperator.java | 15 +-
.../exec/vector/TestVectorGroupByOperator.java | 90 +-
.../exec/vector/TestVectorSelectOperator.java | 16 +
.../ql/optimizer/physical/TestVectorizer.java | 16 +-
.../TestSQL11ReservedKeyWordsNegative.java | 2158 +-
.../TestSQL11ReservedKeyWordsPositive.java | 822 -
.../dynpart_sort_optimization_acid.q | 6 +
.../queries/clientpositive/explainanalyze_3.q | 8 +-
.../queries/clientpositive/explainanalyze_5.q | 8 +-
.../clientpositive/insert_into_with_schema.q | 10 +-
ql/src/test/queries/clientpositive/keyword_2.q | 7 +-
.../queries/clientpositive/ppd_field_garbage.q | 6 +-
.../clientpositive/schema_evol_orc_vec_part.q | 20 +-
.../schema_evol_orc_vec_part_all_complex.q | 8 +-
.../schema_evol_orc_vec_part_all_primitive.q | 12 +-
.../clientpositive/schema_evol_orc_vec_table.q | 12 +-
.../clientpositive/schema_evol_text_vec_part.q | 20 +-
.../schema_evol_text_vec_part_all_complex.q | 8 +-
.../schema_evol_text_vec_part_all_primitive.q | 12 +-
.../clientpositive/schema_evol_text_vec_table.q | 12 +-
.../schema_evol_text_vecrow_part.q | 20 +-
.../schema_evol_text_vecrow_part_all_complex.q | 8 +-
...schema_evol_text_vecrow_part_all_primitive.q | 12 +-
.../schema_evol_text_vecrow_table.q | 12 +-
.../queries/clientpositive/udaf_collect_set_2.q | 34 +-
.../clientpositive/vector_adaptor_usage_mode.q | 24 +-
.../queries/clientpositive/vector_aggregate_9.q | 3 +-
.../vector_aggregate_without_gby.q | 4 +-
.../clientpositive/vector_auto_smb_mapjoin_14.q | 30 +-
.../clientpositive/vector_between_columns.q | 4 +-
.../queries/clientpositive/vector_between_in.q | 25 +-
.../clientpositive/vector_binary_join_groupby.q | 7 +-
.../test/queries/clientpositive/vector_bround.q | 4 +-
.../test/queries/clientpositive/vector_bucket.q | 3 +-
.../clientpositive/vector_cast_constant.q | 4 +-
.../test/queries/clientpositive/vector_char_2.q | 6 +-
.../test/queries/clientpositive/vector_char_4.q | 3 +-
.../queries/clientpositive/vector_char_cast.q | 2 +
.../clientpositive/vector_char_mapjoin1.q | 6 +-
.../queries/clientpositive/vector_char_simple.q | 8 +-
.../queries/clientpositive/vector_coalesce.q | 13 +-
.../queries/clientpositive/vector_coalesce_2.q | 8 +-
.../queries/clientpositive/vector_complex_all.q | 6 +-
.../clientpositive/vector_complex_join.q | 4 +-
.../test/queries/clientpositive/vector_count.q | 8 +-
.../clientpositive/vector_count_distinct.q | 3 +-
.../queries/clientpositive/vector_data_types.q | 5 +-
.../test/queries/clientpositive/vector_date_1.q | 3 +-
.../queries/clientpositive/vector_decimal_1.q | 2 +-
.../clientpositive/vector_decimal_10_0.q | 2 +-
.../queries/clientpositive/vector_decimal_2.q | 2 +-
.../queries/clientpositive/vector_decimal_3.q | 2 +-
.../queries/clientpositive/vector_decimal_4.q | 2 +-
.../queries/clientpositive/vector_decimal_5.q | 2 +-
.../queries/clientpositive/vector_decimal_6.q | 2 +-
.../clientpositive/vector_decimal_aggregate.q | 6 +-
.../clientpositive/vector_decimal_cast.q | 3 +-
.../clientpositive/vector_decimal_expressions.q | 3 +-
.../clientpositive/vector_decimal_mapjoin.q | 3 +-
.../clientpositive/vector_decimal_math_funcs.q | 4 +-
.../clientpositive/vector_decimal_precision.q | 4 +-
.../clientpositive/vector_decimal_round.q | 14 +-
.../clientpositive/vector_decimal_round_2.q | 10 +-
.../clientpositive/vector_decimal_trailing.q | 2 +-
.../clientpositive/vector_decimal_udf2.q | 6 +-
.../queries/clientpositive/vector_distinct_2.q | 3 +-
ql/src/test/queries/clientpositive/vector_elt.q | 5 +-
.../queries/clientpositive/vector_empty_where.q | 8 +-
.../queries/clientpositive/vector_groupby4.q | 2 +-
.../queries/clientpositive/vector_groupby6.q | 2 +-
.../queries/clientpositive/vector_groupby_3.q | 3 +-
.../clientpositive/vector_groupby_mapjoin.q | 4 +-
.../clientpositive/vector_groupby_reduce.q | 9 +-
.../clientpositive/vector_grouping_sets.q | 4 +-
.../queries/clientpositive/vector_if_expr.q | 4 +-
.../clientpositive/vector_include_no_sel.q | 3 +-
.../queries/clientpositive/vector_inner_join.q | 19 +-
.../queries/clientpositive/vector_interval_1.q | 19 +-
.../queries/clientpositive/vector_interval_2.q | 22 +-
.../clientpositive/vector_interval_arithmetic.q | 16 +-
.../clientpositive/vector_interval_mapjoin.q | 3 +-
.../test/queries/clientpositive/vector_join.q | 1 +
.../test/queries/clientpositive/vector_join30.q | 16 +-
.../clientpositive/vector_join_part_col_char.q | 3 +-
.../clientpositive/vector_left_outer_join.q | 4 +-
.../clientpositive/vector_left_outer_join2.q | 12 +-
.../clientpositive/vector_leftsemi_mapjoin.q | 361 +-
.../clientpositive/vector_mapjoin_reduce.q | 5 +-
.../vector_mr_diff_schema_alias.q | 3 +-
.../clientpositive/vector_multi_insert.q | 4 +-
.../vector_non_constant_in_expr.q | 2 +-
.../vector_non_string_partition.q | 5 +-
.../clientpositive/vector_null_projection.q | 4 +-
.../clientpositive/vector_nullsafe_join.q | 21 +-
.../vector_number_compare_projection.q | 4 +-
ql/src/test/queries/clientpositive/vector_nvl.q | 9 +-
.../queries/clientpositive/vector_orderby_5.q | 3 +-
.../queries/clientpositive/vector_outer_join0.q | 5 +-
.../queries/clientpositive/vector_outer_join1.q | 7 +-
.../queries/clientpositive/vector_outer_join2.q | 3 +-
.../queries/clientpositive/vector_outer_join3.q | 7 +-
.../queries/clientpositive/vector_outer_join4.q | 7 +-
.../queries/clientpositive/vector_outer_join5.q | 21 +-
.../queries/clientpositive/vector_outer_join6.q | 5 +-
.../vector_partition_diff_num_cols.q | 12 +-
.../vector_partitioned_date_time.q | 18 +-
.../vector_partitioned_date_time_win.q | 16 +-
.../queries/clientpositive/vector_reduce1.q | 3 +-
.../queries/clientpositive/vector_reduce2.q | 3 +-
.../queries/clientpositive/vector_reduce3.q | 3 +-
.../vector_reduce_groupby_decimal.q | 4 +-
.../clientpositive/vector_string_concat.q | 5 +-
.../clientpositive/vector_string_decimal.q | 2 +-
.../queries/clientpositive/vector_struct_in.q | 19 +-
.../clientpositive/vector_tablesample_rows.q | 8 +-
.../test/queries/clientpositive/vector_udf2.q | 2 +-
.../test/queries/clientpositive/vector_udf3.q | 3 +-
.../queries/clientpositive/vector_varchar_4.q | 3 +-
.../clientpositive/vector_varchar_mapjoin1.q | 6 +-
.../clientpositive/vector_varchar_simple.q | 8 +-
.../clientpositive/vector_when_case_null.q | 2 +-
.../queries/clientpositive/vectorization_0.q | 21 +-
.../queries/clientpositive/vectorization_1.q | 1 +
.../queries/clientpositive/vectorization_10.q | 1 +
.../queries/clientpositive/vectorization_11.q | 1 +
.../queries/clientpositive/vectorization_12.q | 1 +
.../queries/clientpositive/vectorization_13.q | 6 +-
.../queries/clientpositive/vectorization_14.q | 3 +-
.../queries/clientpositive/vectorization_15.q | 3 +-
.../queries/clientpositive/vectorization_16.q | 3 +-
.../queries/clientpositive/vectorization_17.q | 3 +-
.../queries/clientpositive/vectorization_2.q | 1 +
.../queries/clientpositive/vectorization_3.q | 1 +
.../queries/clientpositive/vectorization_4.q | 1 +
.../queries/clientpositive/vectorization_5.q | 1 +
.../queries/clientpositive/vectorization_6.q | 1 +
.../queries/clientpositive/vectorization_7.q | 6 +-
.../queries/clientpositive/vectorization_8.q | 6 +-
.../queries/clientpositive/vectorization_9.q | 3 +-
.../clientpositive/vectorization_decimal_date.q | 4 +-
.../queries/clientpositive/vectorization_div0.q | 7 +-
.../clientpositive/vectorization_limit.q | 16 +-
.../clientpositive/vectorization_nested_udf.q | 2 +
.../queries/clientpositive/vectorization_not.q | 2 +
.../clientpositive/vectorization_offset_limit.q | 5 +-
.../queries/clientpositive/vectorization_part.q | 2 +
.../clientpositive/vectorization_part_project.q | 4 +-
.../clientpositive/vectorization_part_varchar.q | 2 +
.../clientpositive/vectorization_pushdown.q | 4 +-
.../vectorization_short_regress.q | 54 +-
.../clientpositive/vectorized_bucketmapjoin1.q | 8 +-
.../queries/clientpositive/vectorized_case.q | 4 +-
.../queries/clientpositive/vectorized_casts.q | 2 +-
.../queries/clientpositive/vectorized_context.q | 4 +-
.../clientpositive/vectorized_date_funcs.q | 11 +-
.../clientpositive/vectorized_distinct_gby.q | 5 +-
.../vectorized_dynamic_partition_pruning.q | 79 +-
.../queries/clientpositive/vectorized_mapjoin.q | 3 +-
.../clientpositive/vectorized_mapjoin2.q | 2 +-
.../clientpositive/vectorized_math_funcs.q | 3 +-
.../clientpositive/vectorized_nested_mapjoin.q | 3 +-
.../queries/clientpositive/vectorized_parquet.q | 4 +-
.../clientpositive/vectorized_parquet_types.q | 6 +-
.../queries/clientpositive/vectorized_ptf.q | 47 +-
.../clientpositive/vectorized_shufflejoin.q | 3 +-
.../clientpositive/vectorized_string_funcs.q | 3 +-
.../clientpositive/vectorized_timestamp.q | 8 +-
.../clientpositive/vectorized_timestamp_funcs.q | 15 +-
.../vectorized_timestamp_ints_casts.q | 5 +-
.../authorization_cannot_create_all_role.q.out | 2 +-
.../authorization_cannot_create_none_role.q.out | 2 +-
.../clientnegative/cte_with_in_subquery.q.out | 2 +-
.../clientnegative/lateral_view_join.q.out | 2 +-
.../results/clientnegative/subq_insert.q.out | 2 +-
.../test/results/clientpositive/keyword_2.q.out | 12 +-
.../clientpositive/llap/autoColumnStats_1.q.out | 1361 ++
.../clientpositive/llap/autoColumnStats_2.q.out | 1480 ++
.../clientpositive/llap/bucket_groupby.q.out | 1666 ++
.../llap/bucketsortoptimize_insert_2.q.out | 1256 ++
.../clientpositive/llap/cbo_rp_gby.q.out | 124 +
.../clientpositive/llap/cbo_rp_join.q.out | 15028 +++++++++++++
.../clientpositive/llap/cbo_rp_lineage2.q.out | 677 +
.../clientpositive/llap/cbo_rp_semijoin.q.out | 440 +
.../llap/cbo_rp_subq_not_in.q.out | 365 +
.../llap/cbo_rp_unionDistinct_2.q.out | 551 +
.../llap/cbo_rp_windowing_2.q.out | 2338 ++
.../llap/correlationoptimizer2.q.out | 2130 ++
.../llap/correlationoptimizer4.q.out | 1922 ++
.../llap/correlationoptimizer6.q.out | 3994 ++++
.../llap/dynpart_sort_optimization_acid.q.out | 1709 ++
.../results/clientpositive/llap/escape1.q.out | Bin 0 -> 54831 bytes
.../results/clientpositive/llap/escape2.q.out | Bin 0 -> 78516 bytes
.../clientpositive/llap/global_limit.q.out | 1677 ++
.../clientpositive/llap/groupby_sort_1_23.q.out | 4310 ++++
.../llap/groupby_sort_skew_1_23.q.out | 4482 ++++
.../llap/insert_into_with_schema.q.out | 364 +
.../results/clientpositive/llap/join43.q.out | 648 +
.../clientpositive/llap/join_filters.q.out | 1484 ++
.../clientpositive/llap/join_nulls.q.out | 652 +
.../llap/limit_join_transpose.q.out | 1825 ++
.../results/clientpositive/llap/lineage2.q.out | 707 +
.../results/clientpositive/llap/lineage3.q.out | 388 +
.../clientpositive/llap/llap_partitioned.q.out | 2019 ++
.../clientpositive/llap/load_dyn_part5.q.out | 1042 +
.../clientpositive/llap/multiMapJoin1.q.out | 2145 ++
.../clientpositive/llap/multiMapJoin2.q.out | 3187 +++
...i_insert_move_tasks_share_dependencies.q.out | 3765 ++++
.../clientpositive/llap/orc_ppd_date.q.out | 301 +
.../clientpositive/llap/orc_ppd_decimal.q.out | 490 +
.../clientpositive/llap/orc_ppd_timestamp.q.out | 292 +
.../llap/parquet_ppd_decimal.q.out | 858 +
.../llap/partition_multilevels.q.out | 1644 ++
.../clientpositive/llap/rcfile_createas1.q.out | 180 +
.../clientpositive/llap/rcfile_merge2.q.out | 230 +
.../clientpositive/llap/rcfile_merge3.q.out | 169 +
.../clientpositive/llap/rcfile_merge4.q.out | 169 +
.../results/clientpositive/llap/sample10.q.out | 437 +
.../llap/schema_evol_orc_vec_part.q.out | 666 +-
.../schema_evol_orc_vec_part_all_complex.q.out | 168 +-
...schema_evol_orc_vec_part_all_primitive.q.out | 370 +-
.../llap/schema_evol_orc_vec_table.q.out | 365 +-
.../llap/schema_evol_text_vec_part.q.out | 666 +-
.../schema_evol_text_vec_part_all_complex.q.out | 168 +-
...chema_evol_text_vec_part_all_primitive.q.out | 370 +-
.../llap/schema_evol_text_vec_table.q.out | 365 +-
.../llap/schema_evol_text_vecrow_part.q.out | 666 +-
...hema_evol_text_vecrow_part_all_complex.q.out | 168 +-
...ma_evol_text_vecrow_part_all_primitive.q.out | 370 +-
.../llap/schema_evol_text_vecrow_table.q.out | 365 +-
.../results/clientpositive/llap/semijoin.q.out | 2751 +++
.../llap/special_character_in_tabnames_1.q.out | 19609 +++++++++++++++++
.../clientpositive/llap/subquery_notin.q.out | 1539 ++
.../llap/table_access_keys_stats.q.out | 606 +
.../llap/udaf_collect_set_2.q.out | 742 +
.../clientpositive/llap/union_remove_26.q.out | 849 +
.../clientpositive/llap/union_top_level.q.out | 1193 +
.../llap/vector_aggregate_9.q.out | 35 +-
.../llap/vector_aggregate_without_gby.q.out | 4 +-
.../llap/vector_auto_smb_mapjoin_14.q.out | 1973 +-
.../llap/vector_between_columns.q.out | 115 +-
.../clientpositive/llap/vector_between_in.q.out | 600 +-
.../llap/vector_binary_join_groupby.q.out | 152 +-
.../clientpositive/llap/vector_bround.q.out | 15 +-
.../clientpositive/llap/vector_bucket.q.out | 27 +-
.../llap/vector_cast_constant.q.out | 53 +-
.../clientpositive/llap/vector_char_2.q.out | 144 +-
.../clientpositive/llap/vector_char_4.q.out | 27 +-
.../llap/vector_char_mapjoin1.q.out | 220 +-
.../llap/vector_char_simple.q.out | 209 +-
.../clientpositive/llap/vector_coalesce.q.out | 473 +-
.../clientpositive/llap/vector_coalesce_2.q.out | 100 +-
.../llap/vector_complex_all.q.out | 106 +-
.../llap/vector_complex_join.q.out | 40 +-
.../clientpositive/llap/vector_count.q.out | 146 +-
.../llap/vector_count_distinct.q.out | 73 +-
.../clientpositive/llap/vector_data_types.q.out | 53 +-
.../llap/vector_decimal_aggregate.q.out | 95 +-
.../llap/vector_decimal_cast.q.out | 77 +-
.../llap/vector_decimal_expressions.q.out | 50 +-
.../llap/vector_decimal_mapjoin.q.out | 59 +-
.../llap/vector_decimal_math_funcs.q.out | 69 +-
.../llap/vector_decimal_precision.q.out | 35 +-
.../llap/vector_decimal_round.q.out | 189 +-
.../llap/vector_decimal_round_2.q.out | 173 +-
.../llap/vector_decimal_udf2.q.out | 62 +-
.../clientpositive/llap/vector_distinct_2.q.out | 53 +-
.../clientpositive/llap/vector_elt.q.out | 145 +-
.../clientpositive/llap/vector_groupby4.q.out | 62 +-
.../clientpositive/llap/vector_groupby6.q.out | 62 +-
.../clientpositive/llap/vector_groupby_3.q.out | 55 +-
.../llap/vector_groupby_mapjoin.q.out | 238 +-
.../llap/vector_groupby_reduce.q.out | 286 +-
.../llap/vector_grouping_sets.q.out | 53 +-
.../clientpositive/llap/vector_if_expr.q.out | 47 +-
.../llap/vector_include_no_sel.q.out | 75 +-
.../clientpositive/llap/vector_inner_join.q.out | 686 +-
.../clientpositive/llap/vector_interval_1.q.out | 347 +-
.../clientpositive/llap/vector_interval_2.q.out | 448 +-
.../llap/vector_interval_arithmetic.q.out | 334 +-
.../llap/vector_interval_mapjoin.q.out | 63 +-
.../clientpositive/llap/vector_join30.q.out | 907 +-
.../llap/vector_join_part_col_char.q.out | 4 +-
.../llap/vector_left_outer_join.q.out | 39 +-
.../llap/vector_left_outer_join2.q.out | 230 +-
.../llap/vector_leftsemi_mapjoin.q.out | 11428 +++++++---
.../llap/vector_mapjoin_reduce.q.out | 88 +-
.../llap/vector_mr_diff_schema_alias.q.out | 44 +-
.../llap/vector_multi_insert.q.out | 16 +-
.../llap/vector_null_projection.q.out | 39 +-
.../llap/vector_nullsafe_join.q.out | 628 +-
.../llap/vector_number_compare_projection.q.out | 796 +
.../clientpositive/llap/vector_nvl.q.out | 265 +-
.../clientpositive/llap/vector_orderby_5.q.out | 67 +-
.../llap/vector_outer_join0.q.out | 132 +-
.../llap/vector_outer_join1.q.out | 252 +-
.../llap/vector_outer_join2.q.out | 123 +-
.../llap/vector_outer_join3.q.out | 327 +-
.../llap/vector_outer_join4.q.out | 242 +-
.../llap/vector_outer_join5.q.out | 914 +-
.../llap/vector_outer_join6.q.out | 199 +-
.../llap/vector_partition_diff_num_cols.q.out | 250 +-
.../llap/vector_partitioned_date_time.q.out | 1053 +-
.../clientpositive/llap/vector_reduce1.q.out | 42 +-
.../clientpositive/llap/vector_reduce2.q.out | 42 +-
.../clientpositive/llap/vector_reduce3.q.out | 42 +-
.../llap/vector_reduce_groupby_decimal.q.out | 71 +-
.../llap/vector_string_concat.q.out | 138 +-
.../clientpositive/llap/vector_struct_in.q.out | 568 +-
.../clientpositive/llap/vector_udf1.q.out | 1767 ++
.../clientpositive/llap/vector_varchar_4.q.out | 27 +-
.../llap/vector_varchar_mapjoin1.q.out | 93 +-
.../llap/vector_varchar_simple.q.out | 99 +-
.../llap/vector_when_case_null.q.out | 52 +-
.../clientpositive/llap/vectorization_0.q.out | 519 +-
.../clientpositive/llap/vectorization_13.q.out | 114 +-
.../clientpositive/llap/vectorization_14.q.out | 28 +-
.../clientpositive/llap/vectorization_15.q.out | 28 +-
.../clientpositive/llap/vectorization_16.q.out | 21 +-
.../clientpositive/llap/vectorization_17.q.out | 23 +-
.../clientpositive/llap/vectorization_7.q.out | 100 +-
.../clientpositive/llap/vectorization_8.q.out | 100 +-
.../clientpositive/llap/vectorization_9.q.out | 21 +-
.../llap/vectorization_decimal_date.q.out | 76 +-
.../llap/vectorization_part_project.q.out | 23 +-
.../llap/vectorization_pushdown.q.out | 21 +-
.../llap/vectorization_short_regress.q.out | 1002 +-
.../llap/vectorized_bucketmapjoin1.q.out | 108 +-
.../clientpositive/llap/vectorized_case.q.out | 62 +-
.../clientpositive/llap/vectorized_casts.q.out | 16 +-
.../llap/vectorized_context.q.out | 32 +-
.../llap/vectorized_date_funcs.q.out | 314 +-
.../llap/vectorized_distinct_gby.q.out | 51 +-
.../vectorized_dynamic_partition_pruning.q.out | 1159 +-
.../llap/vectorized_mapjoin.q.out | 71 +-
.../llap/vectorized_math_funcs.q.out | 69 +-
.../llap/vectorized_nested_mapjoin.q.out | 39 +-
.../llap/vectorized_parquet.q.out | 25 +-
.../llap/vectorized_parquet_types.q.out | 12 +-
.../clientpositive/llap/vectorized_ptf.q.out | 668 +-
.../llap/vectorized_shufflejoin.q.out | 73 +-
.../llap/vectorized_string_funcs.q.out | 54 +-
.../llap/vectorized_timestamp.q.out | 16 +-
.../llap/vectorized_timestamp_funcs.q.out | 292 +-
.../llap/vectorized_timestamp_ints_casts.q.out | 138 +-
.../results/clientpositive/llap/windowing.q.out | 2338 ++
.../llap/windowing_windowspec2.q.out | 3084 +++
.../clientpositive/ppd_field_garbage.q.out | 14 +-
.../spark/vector_between_in.q.out | 604 +-
.../spark/vector_cast_constant.q.out | 53 +-
.../clientpositive/spark/vector_char_4.q.out | 27 +-
.../spark/vector_count_distinct.q.out | 74 +-
.../spark/vector_data_types.q.out | 53 +-
.../spark/vector_decimal_aggregate.q.out | 96 +-
.../spark/vector_decimal_mapjoin.q.out | 58 +-
.../spark/vector_distinct_2.q.out | 54 +-
.../clientpositive/spark/vector_elt.q.out | 143 +-
.../clientpositive/spark/vector_groupby_3.q.out | 56 +-
.../spark/vector_inner_join.q.out | 678 +-
.../spark/vector_left_outer_join.q.out | 39 +-
.../spark/vector_mapjoin_reduce.q.out | 72 +-
.../clientpositive/spark/vector_orderby_5.q.out | 68 +-
.../spark/vector_outer_join0.q.out | 130 +-
.../spark/vector_outer_join1.q.out | 248 +-
.../spark/vector_outer_join2.q.out | 121 +-
.../spark/vector_outer_join3.q.out | 342 +-
.../spark/vector_outer_join4.q.out | 254 +-
.../spark/vector_outer_join5.q.out | 968 +-
.../spark/vector_string_concat.q.out | 137 +-
.../clientpositive/spark/vector_varchar_4.q.out | 27 +-
.../clientpositive/spark/vectorization_0.q.out | 519 +-
.../clientpositive/spark/vectorization_13.q.out | 114 +-
.../clientpositive/spark/vectorization_14.q.out | 28 +-
.../clientpositive/spark/vectorization_15.q.out | 28 +-
.../clientpositive/spark/vectorization_16.q.out | 21 +-
.../clientpositive/spark/vectorization_17.q.out | 23 +-
.../clientpositive/spark/vectorization_7.q.out | 118 +-
.../clientpositive/spark/vectorization_8.q.out | 146 +-
.../clientpositive/spark/vectorization_9.q.out | 21 +-
.../spark/vectorization_decimal_date.q.out | 75 +-
.../spark/vectorization_div0.q.out | 167 +-
.../spark/vectorization_part_project.q.out | 23 +-
.../spark/vectorization_pushdown.q.out | 21 +-
.../spark/vectorization_short_regress.q.out | 1002 +-
.../spark/vectorized_bucketmapjoin1.q.out | 135 +-
.../clientpositive/spark/vectorized_case.q.out | 62 +-
.../spark/vectorized_mapjoin.q.out | 70 +-
.../spark/vectorized_math_funcs.q.out | 68 +-
.../spark/vectorized_nested_mapjoin.q.out | 39 +-
.../clientpositive/spark/vectorized_ptf.q.out | 693 +-
.../spark/vectorized_shufflejoin.q.out | 80 +-
.../spark/vectorized_string_funcs.q.out | 53 +-
.../spark/vectorized_timestamp_funcs.q.out | 292 +-
.../clientpositive/tez/explainanalyze_3.q.out | 58 +-
.../clientpositive/tez/explainanalyze_5.q.out | 58 +-
.../tez/vector_join_part_col_char.q.out | 4 +-
.../tez/vector_non_string_partition.q.out | 98 +-
.../clientpositive/tez/vectorization_div0.q.out | 167 +-
.../tez/vectorization_limit.q.out | 339 +-
.../clientpositive/udaf_collect_set_2.q.out | 64 +-
.../vector_adaptor_usage_mode.q.out | 230 +-
.../clientpositive/vector_aggregate_9.q.out | 34 +-
.../vector_aggregate_without_gby.q.out | 20 +-
.../vector_auto_smb_mapjoin_14.q.out | 604 +-
.../clientpositive/vector_between_columns.q.out | 75 +-
.../clientpositive/vector_between_in.q.out | 466 +-
.../vector_binary_join_groupby.q.out | 96 +-
.../results/clientpositive/vector_bround.q.out | 16 +-
.../results/clientpositive/vector_bucket.q.out | 16 +-
.../clientpositive/vector_cast_constant.q.out | 43 +-
.../results/clientpositive/vector_char_2.q.out | 96 +-
.../results/clientpositive/vector_char_4.q.out | 27 +-
.../clientpositive/vector_char_mapjoin1.q.out | 123 +-
.../clientpositive/vector_char_simple.q.out | 180 +-
.../clientpositive/vector_coalesce.q.out | 394 +-
.../clientpositive/vector_coalesce_2.q.out | 83 +-
.../clientpositive/vector_complex_all.q.out | 84 +-
.../clientpositive/vector_complex_join.q.out | 28 +-
.../results/clientpositive/vector_count.q.out | 142 +-
.../clientpositive/vector_count_distinct.q.out | 39 +-
.../clientpositive/vector_data_types.q.out | 40 +-
.../vector_decimal_aggregate.q.out | 74 +-
.../clientpositive/vector_decimal_cast.q.out | 34 +-
.../vector_decimal_expressions.q.out | 37 +-
.../clientpositive/vector_decimal_mapjoin.q.out | 36 +-
.../vector_decimal_math_funcs.q.out | 31 +-
.../vector_decimal_precision.q.out | 34 +-
.../clientpositive/vector_decimal_round.q.out | 129 +-
.../clientpositive/vector_decimal_round_2.q.out | 132 +-
.../clientpositive/vector_decimal_udf2.q.out | 62 +-
.../clientpositive/vector_distinct_2.q.out | 38 +-
.../results/clientpositive/vector_elt.q.out | 101 +-
.../clientpositive/vector_empty_where.q.out | 168 +-
.../clientpositive/vector_groupby4.q.out | 41 +-
.../clientpositive/vector_groupby6.q.out | 41 +-
.../clientpositive/vector_groupby_3.q.out | 39 +-
.../clientpositive/vector_groupby_mapjoin.q.out | 24 +-
.../clientpositive/vector_groupby_reduce.q.out | 178 +-
.../clientpositive/vector_grouping_sets.q.out | 36 +-
.../results/clientpositive/vector_if_expr.q.out | 37 +-
.../clientpositive/vector_include_no_sel.q.out | 47 +-
.../clientpositive/vector_inner_join.q.out | 396 +-
.../clientpositive/vector_interval_1.q.out | 264 +-
.../clientpositive/vector_interval_2.q.out | 348 +-
.../vector_interval_arithmetic.q.out | 260 +-
.../vector_interval_mapjoin.q.out | 40 +-
.../results/clientpositive/vector_join30.q.out | 700 +-
.../vector_join_part_col_char.q.out | 15 +-
.../clientpositive/vector_left_outer_join.q.out | 20 +-
.../vector_left_outer_join2.q.out | 156 +-
.../vector_leftsemi_mapjoin.q.out | 7103 +++---
.../clientpositive/vector_mapjoin_reduce.q.out | 56 +-
.../vector_mr_diff_schema_alias.q.out | 38 +-
.../clientpositive/vector_multi_insert.q.out | 16 +-
.../vector_non_constant_in_expr.q.out | 14 +-
.../vector_non_string_partition.q.out | 72 +-
.../clientpositive/vector_null_projection.q.out | 29 +-
.../clientpositive/vector_nullsafe_join.q.out | 326 +-
.../vector_number_compare_projection.q.out | 66 +-
.../results/clientpositive/vector_nvl.q.out | 181 +-
.../clientpositive/vector_orderby_5.q.out | 47 +-
.../clientpositive/vector_outer_join0.q.out | 74 +-
.../clientpositive/vector_outer_join1.q.out | 126 +-
.../clientpositive/vector_outer_join2.q.out | 53 +-
.../clientpositive/vector_outer_join3.q.out | 309 +-
.../clientpositive/vector_outer_join4.q.out | 235 +-
.../clientpositive/vector_outer_join5.q.out | 882 +-
.../clientpositive/vector_outer_join6.q.out | 190 +-
.../vector_partition_diff_num_cols.q.out | 190 +-
.../vector_partitioned_date_time.q.out | 348 +-
.../results/clientpositive/vector_reduce1.q.out | 32 +-
.../results/clientpositive/vector_reduce2.q.out | 32 +-
.../results/clientpositive/vector_reduce3.q.out | 32 +-
.../vector_reduce_groupby_decimal.q.out | 47 +-
.../clientpositive/vector_string_concat.q.out | 77 +-
.../clientpositive/vector_string_decimal.q.out | 14 +-
.../clientpositive/vector_struct_in.q.out | 228 +-
.../vector_tablesample_rows.q.out | 88 +-
.../results/clientpositive/vector_udf2.q.out | 30 +-
.../results/clientpositive/vector_udf3.q.out | 27 +-
.../clientpositive/vector_varchar_4.q.out | 27 +-
.../vector_varchar_mapjoin1.q.out | 60 +-
.../clientpositive/vector_varchar_simple.q.out | 75 +-
.../clientpositive/vector_when_case_null.q.out | 40 +-
.../clientpositive/vectorization_0.q.out | 398 +-
.../clientpositive/vectorization_13.q.out | 94 +-
.../clientpositive/vectorization_14.q.out | 28 +-
.../clientpositive/vectorization_15.q.out | 28 +-
.../clientpositive/vectorization_16.q.out | 20 +-
.../clientpositive/vectorization_17.q.out | 20 +-
.../clientpositive/vectorization_7.q.out | 74 +-
.../clientpositive/vectorization_8.q.out | 74 +-
.../clientpositive/vectorization_9.q.out | 20 +-
.../vectorization_decimal_date.q.out | 33 +-
.../clientpositive/vectorization_div0.q.out | 104 +-
.../clientpositive/vectorization_limit.q.out | 224 +-
.../vectorization_offset_limit.q.out | 52 +-
.../vectorization_part_project.q.out | 20 +-
.../clientpositive/vectorization_pushdown.q.out | 20 +-
.../vectorization_short_regress.q.out | 800 +-
.../vectorized_bucketmapjoin1.q.out | 99 +-
.../clientpositive/vectorized_case.q.out | 62 +-
.../clientpositive/vectorized_casts.q.out | 16 +-
.../clientpositive/vectorized_context.q.out | 16 +-
.../clientpositive/vectorized_date_funcs.q.out | 157 +-
.../vectorized_distinct_gby.q.out | 40 +-
.../clientpositive/vectorized_mapjoin.q.out | 48 +-
.../clientpositive/vectorized_mapjoin2.q.out | 47 +-
.../clientpositive/vectorized_math_funcs.q.out | 31 +-
.../vectorized_nested_mapjoin.q.out | 20 +-
.../clientpositive/vectorized_parquet.q.out | 28 +-
.../vectorized_parquet_types.q.out | 48 +-
.../results/clientpositive/vectorized_ptf.q.out | 728 +-
.../clientpositive/vectorized_shufflejoin.q.out | 31 +-
.../vectorized_string_funcs.q.out | 16 +-
.../clientpositive/vectorized_timestamp.q.out | 80 +-
.../vectorized_timestamp_funcs.q.out | 222 +-
.../vectorized_timestamp_ints_casts.q.out | 62 +-
.../compiler/errors/missing_overwrite.q.out | 2 +-
.../org/apache/hive/service/cli/CLIService.java | 4 +-
.../hive/service/cli/CLIServiceClient.java | 13 +-
.../service/cli/EmbeddedCLIServiceClient.java | 5 +-
.../hive/service/cli/operation/Operation.java | 6 +-
.../service/cli/operation/OperationManager.java | 8 +-
.../thrift/RetryingThriftCLIServiceClient.java | 8 +-
.../cli/thrift/ThriftCLIServiceClient.java | 15 +-
.../cli/TestRetryingThriftCLIServiceClient.java | 2 +-
.../hive/ptest/api/server/TestExecutor.java | 1 +
.../hive/ptest/execution/ExecutionPhase.java | 12 +-
.../hive/ptest/execution/HostExecutor.java | 11 +-
.../hive/ptest/execution/JUnitReportParser.java | 74 +-
.../org/apache/hive/ptest/execution/PTest.java | 5 +-
.../ptest/execution/conf/QFileTestBatch.java | 13 +-
.../hive/ptest/execution/conf/TestBatch.java | 13 +-
.../ptest/execution/conf/TestConfiguration.java | 4 +
.../ptest/execution/conf/UnitTestBatch.java | 11 +-
.../hive/ptest/execution/ssh/RSyncCommand.java | 3 +-
.../execution/ssh/RSyncCommandExecutor.java | 5 +
.../ptest2/src/main/resources/batch-exec.vm | 13 +-
.../ptest2/src/main/resources/source-prep.vm | 4 +-
.../hive/ptest/execution/AbstractTestPhase.java | 3 +-
.../ptest/execution/TestExecutionPhase.java | 30 +-
...ExecutionPhase.testFailingQFile.approved.txt | 8 +-
...utionPhase.testPassingQFileTest.approved.txt | 8 +-
...utionPhase.testTimedOutUnitTest.approved.txt | 14 +
.../hive/ptest/execution/TestHostExecutor.java | 3 +-
.../hive/ptest/execution/TestReportParser.java | 6 +-
...tScripts.testAlternativeTestJVM.approved.txt | 13 +-
.../TestScripts.testBatch.approved.txt | 13 +-
.../execution/conf/TestQFileTestBatch.java | 4 +-
891 files changed, 164006 insertions(+), 21154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/itests/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 538bf79,ec21cd6..9bc4836
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@@ -378,223 -553,6 +378,225 @@@ public class MoveTask extends Task<Move
}
}
+ private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd,
+ TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
+ List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec());
+ db.validatePartitionNameCharacters(partVals);
+ Utilities.LOG14535.info("loadPartition called from " + tbd.getSourcePath()
+ + " into " + tbd.getTable().getTableName());
+ db.loadSinglePartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
+ tbd.getPartitionSpec(), tbd.getReplace(),
+ tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID,
+ hasFollowingStatsTask(), tbd.getMmWriteId());
+ Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
+
+ if (ti.bucketCols != null || ti.sortCols != null) {
+ updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols,
+ ti.numBuckets, ti.sortCols);
+ }
+
+ DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
+ // add this partition to post-execution hook
+ if (work.getOutputs() != null) {
+ work.getOutputs().add(new WriteEntity(partn,
+ (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE
+ : WriteEntity.WriteType.INSERT)));
+ }
+ return dc;
+ }
+
+ private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
+ TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException,
+ IOException, InvalidOperationException {
+ DataContainer dc;
+ List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
+
+ // publish DP columns to its subscribers
+ if (dps != null && dps.size() > 0) {
+ pushFeed(FeedType.DYNAMIC_PARTITIONS, dps);
+ }
+ console.printInfo(System.getProperty("line.separator"));
+ long startTime = System.currentTimeMillis();
+ // load the list of DP partitions and return the list of partition specs
+ // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
+ // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
+ // After that check the number of DPs created to not exceed the limit and
+ // iterate over it and call loadPartition() here.
+ // The reason we don't do inside HIVE-1361 is the latter is large and we
+ // want to isolate any potential issue it may introduce.
+ Map<Map<String, String>, Partition> dp =
+ db.loadDynamicPartitions(
+ tbd.getSourcePath(),
+ tbd.getTable().getTableName(),
+ tbd.getPartitionSpec(),
+ tbd.getReplace(),
+ dpCtx.getNumDPCols(),
+ isSkewedStoredAsDirs(tbd),
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID,
+ SessionState.get().getTxnMgr().getCurrentTxnId(), hasFollowingStatsTask(),
+ work.getLoadTableWork().getWriteType(),
+ tbd.getMmWriteId());
+
- console.printInfo("\t Time taken to load dynamic partitions: " +
- (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
++ String loadTime = "\t Time taken to load dynamic partitions: " +
++ (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
++ console.printInfo(loadTime);
++ LOG.info(loadTime);
+
+ if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
+ throw new HiveException("This query creates no partitions." +
+ " To turn off this error, set hive.error.on.empty.partition=false.");
+ }
+
+ startTime = System.currentTimeMillis();
+ // for each partition spec, get the partition
+ // and put it to WriteEntity for post-exec hook
+ for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
+ Partition partn = entry.getValue();
+
+ if (ti.bucketCols != null || ti.sortCols != null) {
+ updatePartitionBucketSortColumns(
+ db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
+ }
+
+ WriteEntity enty = new WriteEntity(partn,
+ (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE :
+ WriteEntity.WriteType.INSERT));
+ if (work.getOutputs() != null) {
+ work.getOutputs().add(enty);
+ }
+ // Need to update the queryPlan's output as well so that post-exec hook get executed.
+ // This is only needed for dynamic partitioning since for SP the the WriteEntity is
+ // constructed at compile time and the queryPlan already contains that.
+ // For DP, WriteEntity creation is deferred at this stage so we need to update
+ // queryPlan here.
+ if (queryPlan.getOutputs() == null) {
+ queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
+ }
+ queryPlan.getOutputs().add(enty);
+
+ // update columnar lineage for each partition
+ dc = new DataContainer(table.getTTable(), partn.getTPartition());
+
+ // Don't set lineage on delete as we don't have all the columns
+ if (SessionState.get() != null &&
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
+ SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc,
+ table.getCols());
+ }
+ LOG.info("\tLoading partition " + entry.getKey());
+ }
+ console.printInfo("\t Time taken for adding to write entity : " +
+ (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
+ dc = null; // reset data container to prevent it being added again.
+ return dc;
+ }
+
+ private void inferTaskInformation(TaskInformation ti) {
+ // Find the first ancestor of this MoveTask which is some form of map reduce task
+ // (Either standard, local, or a merge)
+ while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) {
+ ti.task = (Task)ti.task.getParentTasks().get(0);
+ // If it was a merge task or a local map reduce task, nothing can be inferred
+ if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) {
+ break;
+ }
+
+ // If it's a standard map reduce task, check what, if anything, it inferred about
+ // the directory this move task is moving
+ if (ti.task instanceof MapRedTask) {
+ MapredWork work = (MapredWork)ti.task.getWork();
+ MapWork mapWork = work.getMapWork();
+ ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path);
+ ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path);
+ if (work.getReduceWork() != null) {
+ ti.numBuckets = work.getReduceWork().getNumReduceTasks();
+ }
+
+ if (ti.bucketCols != null || ti.sortCols != null) {
+ // This must be a final map reduce task (the task containing the file sink
+ // operator that writes the final output)
+ assert work.isFinalMapRed();
+ }
+ break;
+ }
+
+ // If it's a move task, get the path the files were moved from, this is what any
+ // preceding map reduce task inferred information about, and moving does not invalidate
+ // those assumptions
+ // This can happen when a conditional merge is added before the final MoveTask, but the
+ // condition for merging is not met, see GenMRFileSink1.
+ if (ti.task instanceof MoveTask) {
+ MoveTask mt = (MoveTask)ti.task;
+ if (mt.getWork().getLoadFileWork() != null) {
+ ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString();
+ }
+ }
+ }
+ }
+
+ private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table)
+ throws HiveException {
+ if (work.getCheckFileFormat()) {
+ // Get all files from the src directory
+ FileStatus[] dirs;
+ ArrayList<FileStatus> files;
+ FileSystem srcFs; // source filesystem
+ try {
+ srcFs = tbd.getSourcePath().getFileSystem(conf);
+ dirs = srcFs.globStatus(tbd.getSourcePath());
+ files = new ArrayList<FileStatus>();
+ for (int i = 0; (dirs != null && i < dirs.length); i++) {
+ files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
+ // We only check one file, so exit the loop when we have at least
+ // one.
+ if (files.size() > 0) {
+ break;
+ }
+ }
+ } catch (IOException e) {
+ throw new HiveException(
+ "addFiles: filesystem error in check phase", e);
+ }
+
+ // handle file format check for table level
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
+ boolean flag = true;
+ // work.checkFileFormat is set to true only for Load Task, so assumption here is
+ // dynamic partition context is null
+ if (tbd.getDPCtx() == null) {
+ if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
+ // Check if the file format of the file matches that of the table.
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ } else {
+ // Check if the file format of the file matches that of the partition
+ Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
+ if (oldPart == null) {
+ // this means we have just created a table and are specifying partition in the
+ // load statement (without pre-creating the partition), in which case lets use
+ // table input format class. inheritTableSpecs defaults to true so when a new
+ // partition is created later it will automatically inherit input format
+ // from table object
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ } else {
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, oldPart.getInputFormatClass(), files);
+ }
+ }
+ if (!flag) {
+ throw new HiveException(
+ "Wrong file format. Please check the file's format.");
+ }
+ } else {
+ LOG.warn("Skipping file format check as dpCtx is not null");
+ }
+ }
+ }
+ }
+
private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) {
return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx()
.isSkewedStoredAsDir();
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index a2eff8b,a9dbc3e..d343e32
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@@ -1419,21 -1402,27 +1419,28 @@@ public final class Utilities
FileStatus[] statuses = HiveStatsUtils.getFileStatusRecurse(
tmpPath, ((dpCtx == null) ? 1 : dpCtx.getNumDPCols()), fs);
if(statuses != null && statuses.length > 0) {
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.PerfLogBegin("FileSinkOperator", "RemoveTempOrDuplicateFiles");
// remove any tmp file or double-committed output files
List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, statuses, dpCtx, conf, hconf);
+ perfLogger.PerfLogEnd("FileSinkOperator", "RemoveTempOrDuplicateFiles");
// create empty buckets if necessary
if (emptyBuckets.size() > 0) {
+ perfLogger.PerfLogBegin("FileSinkOperator", "CreateEmptyBuckets");
createEmptyBuckets(hconf, emptyBuckets, conf, reporter);
+ perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets");
}
-
// move to the file destination
- log.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
+ Utilities.LOG14535.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
+ perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles");
Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+ perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");
}
} else {
+ Utilities.LOG14535.info("deleting tmpPath " + tmpPath);
fs.delete(tmpPath, true);
}
+ Utilities.LOG14535.info("deleting taskTmpPath " + taskTmpPath);
fs.delete(taskTmpPath, true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 84c1e7b,dab4c6a..e66948f
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -1585,31 -1568,20 +1586,34 @@@ public class Hive
newPartPath = oldPartPath;
}
List<Path> newFiles = null;
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.PerfLogBegin("MoveTask", "FileMoves");
- if (replace || (oldPart == null && !isAcid)) {
- replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
- isSrcLocal);
- } else {
- if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) {
- newFiles = Collections.synchronizedList(new ArrayList<Path>());
+ if (mmWriteId != null) {
+ Utilities.LOG14535.info("not moving " + loadPath + " to " + newPartPath);
+ assert !isAcid;
+ if (areEventsForDmlNeeded(tbl, oldPart)) {
+ newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
+ }
+ Utilities.LOG14535.info("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
+ if (replace && oldPartPath != null) { // TODO# is this correct? ignore until iow jira
+ deleteOldPathForReplace(newPartPath, oldPartPath,
+ getConf(), new ValidWriteIds.IdPathFilter(mmWriteId, false));
}
+ } else {
+ Utilities.LOG14535.info("moving " + loadPath + " to " + newPartPath);
+ if (replace || (oldPart == null && !isAcid)) {
+ replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
+ isSrcLocal);
+ } else {
+ if (areEventsForDmlNeeded(tbl, oldPart)) {
+ newFiles = Collections.synchronizedList(new ArrayList<Path>());
+ }
- FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
- Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);
+ FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
+ Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);
+ }
}
+ perfLogger.PerfLogEnd("MoveTask", "FileMoves");
Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
validatePartition(newTPart);
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd78d660/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------