You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/08/30 22:21:24 UTC
[17/17] hive git commit: HIVE-14671 : merge master into hive-14535
(Sergey Shelukhin)
HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Conflicts:
ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/67f1b92b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/67f1b92b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/67f1b92b
Branch: refs/heads/hive-14535
Commit: 67f1b92ba865926173cbbe56595ed8b3aa0b595c
Parents: d0f5b89 20824f2
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Aug 30 15:20:04 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Aug 30 15:20:04 2016 -0700
----------------------------------------------------------------------
beeline/pom.xml | 5 +
.../java/org/apache/hive/beeline/BeeLine.java | 10 +-
.../org/apache/hive/beeline/BeeLineOpts.java | 10 +
.../org/apache/hive/beeline/BufferedRows.java | 23 +-
.../apache/hive/beeline/IncrementalRows.java | 10 +-
.../IncrementalRowsWithNormalization.java | 86 +
.../src/java/org/apache/hive/beeline/Rows.java | 2 +-
beeline/src/main/resources/BeeLine.properties | 4 +
.../TestIncrementalRowsWithNormalization.java | 90 +
.../hadoop/hive/common/StatsSetupConst.java | 2 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
data/scripts/q_test_init_for_minimr.sql | 48 +
itests/hive-unit-hadoop2/pom.xml | 6 +
itests/hive-unit/pom.xml | 18 +
itests/qtest/pom.xml | 12 +
.../test/resources/testconfiguration.properties | 14 +-
.../resources/testconfiguration.properties.orig | 1373 ++
.../hadoop/hive/cli/control/CliConfigs.java | 2 +-
.../hive/llap/cache/EvictionDispatcher.java | 4 +-
.../hadoop/hive/llap/cache/LowLevelCache.java | 3 +
.../hive/llap/cache/LowLevelCacheImpl.java | 5 +-
.../hadoop/hive/llap/cache/SimpleAllocator.java | 1 +
.../hive/llap/cache/SimpleBufferManager.java | 33 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 36 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 3 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 11 +-
.../resources/llap-daemon-log4j2.properties | 7 +-
pom.xml | 6 +-
.../java/org/apache/hadoop/hive/ql/Context.java | 80 +-
.../org/apache/hadoop/hive/ql/Context.java.orig | 829 ++
.../java/org/apache/hadoop/hive/ql/Driver.java | 13 +-
.../org/apache/hadoop/hive/ql/debug/Utils.java | 56 +-
.../hadoop/hive/ql/exec/ColumnStatsTask.java | 4 +
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 1 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 4 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 6 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 4 +-
.../hadoop/hive/ql/exec/GroupByOperator.java | 2 +-
.../hadoop/hive/ql/exec/JoinOperator.java | 2 -
.../hadoop/hive/ql/exec/LimitOperator.java | 1 +
.../hadoop/hive/ql/exec/ListSinkOperator.java | 1 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 4 +
.../apache/hadoop/hive/ql/exec/Operator.java | 50 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 1 +
.../hive/ql/exec/SerializationUtilities.java | 22 +
.../apache/hadoop/hive/ql/exec/StatsTask.java | 9 +-
.../hadoop/hive/ql/exec/TableScanOperator.java | 1 +
.../hadoop/hive/ql/exec/UDTFOperator.java | 1 +
.../apache/hadoop/hive/ql/exec/Utilities.java | 1 +
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 10 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 3 +-
.../physical/AnnotateRunTimeStatsOptimizer.java | 174 +
.../optimizer/physical/PhysicalOptimizer.java | 7 +-
.../ql/parse/ColumnStatsAutoGatherContext.java | 9 +-
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 1 +
.../hive/ql/parse/ExplainConfiguration.java | 117 +
.../parse/ExplainSQRewriteSemanticAnalyzer.java | 3 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 137 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 7 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 2 +-
.../hadoop/hive/ql/parse/MapReduceCompiler.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 18 +-
.../hive/ql/parse/SemanticAnalyzer.java.orig | 13038 +++++++++++++++++
.../hive/ql/parse/SubQueryDiagnostic.java | 2 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 5 +
.../hadoop/hive/ql/parse/TezCompiler.java | 12 +-
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 1 +
.../hive/ql/parse/spark/SparkCompiler.java | 8 +-
.../hive/ql/plan/AbstractOperatorDesc.java | 12 +
.../apache/hadoop/hive/ql/plan/ExplainWork.java | 66 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 9 -
.../hadoop/hive/ql/plan/MergeJoinWork.java | 8 +-
.../hadoop/hive/ql/plan/OperatorDesc.java | 2 +
.../apache/hadoop/hive/ql/plan/Statistics.java | 22 +-
.../hive/ql/processors/ResetProcessor.java | 109 +-
.../hadoop/hive/ql/processors/SetProcessor.java | 3 +-
.../apache/hadoop/hive/ql/exec/TestContext.java | 5 +
.../hadoop/hive/ql/exec/TestExplainTask.java | 2 +
.../parse/TestUpdateDeleteSemanticAnalyzer.java | 4 +-
.../clientpositive/encryption_move_tbl.q | 15 +-
.../queries/clientpositive/explainanalyze_1.q | 38 +
.../queries/clientpositive/explainanalyze_2.q | 329 +
.../queries/clientpositive/explainanalyze_3.q | 158 +
.../queries/clientpositive/explainanalyze_4.q | 103 +
.../queries/clientpositive/explainanalyze_5.q | 81 +
ql/src/test/queries/clientpositive/reset_conf.q | 18 +
.../test/queries/clientpositive/set_metaconf.q | 4 +
.../test/results/clientpositive/bucket4.q.out | 20 +-
.../test/results/clientpositive/bucket5.q.out | 28 +-
.../test/results/clientpositive/bucket6.q.out | 10 +-
.../results/clientpositive/bucket_many.q.out | 20 +-
.../clientpositive/columnstats_partlvl.q.out | 48 +
.../clientpositive/columnstats_partlvl_dp.q.out | 28 +
.../clientpositive/columnstats_quoting.q.out | 12 +
.../clientpositive/columnstats_tbllvl.q.out | 42 +
.../clientpositive/compute_stats_date.q.out | 6 +
.../clientpositive/constant_prop_2.q.out | 7 +
.../disable_merge_for_bucketing.q.out | 20 +-
.../display_colstats_tbllvl.q.out | 18 +
.../dynpart_sort_optimization_acid.q.out | 60 +
.../encrypted/encryption_move_tbl.q.out | 57 +-
.../exec_parallel_column_stats.q.out | 6 +
.../test/results/clientpositive/groupby2.q.out | 16 +-
.../clientpositive/index_bitmap_auto.q.out | 4 -
.../infer_bucket_sort_dyn_part.q.out | 14 +-
.../infer_bucket_sort_map_operators.q.out | 12 +-
.../infer_bucket_sort_num_buckets.q.out | 10 +-
.../infer_bucket_sort_reducers_power_two.q.out | 4 +-
ql/src/test/results/clientpositive/join1.q.out | 22 +-
.../clientpositive/list_bucket_dml_10.q.out | 16 +-
.../clientpositive/orc_merge_diff_fs.q.out | 18 +-
.../clientpositive/reduce_deduplicate.q.out | 20 +-
.../results/clientpositive/reset_conf.q.out | 32 +
.../results/clientpositive/set_metaconf.q.out | 2 +
.../temp_table_display_colstats_tbllvl.q.out | 18 +
.../clientpositive/tez/explainanalyze_1.q.out | 471 +
.../clientpositive/tez/explainanalyze_2.q.out | 4827 ++++++
.../clientpositive/tez/explainanalyze_3.q.out | 839 ++
.../clientpositive/tez/explainanalyze_4.q.out | 590 +
.../clientpositive/tez/explainanalyze_5.q.out | 445 +
shims/0.23/pom.xml | 16 +
.../apache/hadoop/hive/shims/ShimLoader.java | 10 +-
.../hive/spark/client/TestSparkClient.java | 47 +-
testutils/ptest2/pom.xml | 28 +-
.../hive/ptest/execution/conf/TestParser.java | 3 -
...tScripts.testAlternativeTestJVM.approved.txt | 8 +-
.../TestScripts.testBatch.approved.txt | 8 +-
127 files changed, 24931 insertions(+), 397 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index f4b23e6,fd80e6c..73cc95a
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@@ -304,10 -304,7 +304,9 @@@ public class GenTezUtils
linked = context.linkedFileSinks.get(path);
linked.add(desc);
- desc.setIndexInTezUnion(linked.size());
+ // TODO# special case #N - unions (tez)
- desc.setDirName(new Path(path, "" + desc.getIndexInTezUnion()));
+ desc.setDirName(new Path(path, "" + linked.size()));
+ Utilities.LOG14535.info("removing union - new desc with " + desc.getDirName() + "; parent " + path);
desc.setLinkedFileSink(true);
desc.setParentDir(path);
desc.setLinkedFileSinkDesc(linked);
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/67f1b92b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------