You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/11 02:05:04 UTC
[32/32] hive git commit: HIVE-19416 : merge master into branch
(Sergey Shelukhin) 0710
HIVE-19416 : merge master into branch (Sergey Shelukhin) 0710
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/04ea1455
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/04ea1455
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/04ea1455
Branch: refs/heads/master-txnstats
Commit: 04ea1455febfbddeab505a71232465f0f24fa164
Parents: f384d4c b574bcd
Author: sergey <se...@apache.org>
Authored: Tue Jul 10 18:59:34 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Jul 10 18:59:34 2018 -0700
----------------------------------------------------------------------
RELEASE_NOTES.txt | 303 +
.../hive/accumulo/AccumuloStorageHandler.java | 19 +-
.../accumulo/TestAccumuloStorageHandler.java | 44 +-
.../test/queries/positive/accumulo_custom_key.q | 5 +-
.../queries/positive/accumulo_custom_key2.q | 5 +-
.../src/test/queries/positive/accumulo_index.q | 5 +-
.../src/test/queries/positive/accumulo_joins.q | 25 +-
.../positive/accumulo_predicate_pushdown.q | 5 +-
.../test/queries/positive/accumulo_queries.q | 35 +-
.../accumulo_single_sourced_multi_insert.q | 5 +-
.../results/positive/accumulo_custom_key.q.out | 6 +-
.../results/positive/accumulo_custom_key2.q.out | 6 +-
.../test/results/positive/accumulo_index.q.out | 6 +-
.../test/results/positive/accumulo_joins.q.out | 30 +-
.../positive/accumulo_predicate_pushdown.q.out | 6 +-
.../results/positive/accumulo_queries.q.out | 46 +-
.../accumulo_single_sourced_multi_insert.q.out | 6 +-
.../hive/beeline/schematool/HiveSchemaTool.java | 338 +-
.../schematool/HiveSchemaToolCommandLine.java | 286 -
.../beeline/schematool/HiveSchemaToolTask.java | 32 -
.../HiveSchemaToolTaskAlterCatalog.java | 90 -
.../HiveSchemaToolTaskCreateCatalog.java | 132 -
.../schematool/HiveSchemaToolTaskInfo.java | 43 -
.../schematool/HiveSchemaToolTaskInit.java | 73 -
.../HiveSchemaToolTaskMoveDatabase.java | 96 -
.../schematool/HiveSchemaToolTaskMoveTable.java | 142 -
.../schematool/HiveSchemaToolTaskUpgrade.java | 116 -
.../schematool/HiveSchemaToolTaskValidate.java | 631 --
.../beeline/schematool/TestHiveSchemaTool.java | 6 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hive/common/util/HiveStringUtils.java | 21 +-
data/files/part.parquet | Bin 0 -> 4758 bytes
data/files/studenttab10k | 10000 +++++++++++++++++
.../hadoop/hive/druid/DruidStorageHandler.java | 71 +-
.../hadoop/hive/druid/serde/DruidSerDe.java | 3 +-
.../hive/druid/TestDruidStorageHandler.java | 4 +-
hcatalog/bin/hcat | 41 +-
.../apache/hive/hcatalog/data/JsonSerDe.java | 612 +-
.../hive/hcatalog/data/TestJsonSerDe.java | 12 +-
.../insert_into_dynamic_partitions.q | 3 +-
.../clientpositive/insert_overwrite_directory.q | 3 +-
.../insert_overwrite_dynamic_partitions.q | 3 +-
.../queries/clientpositive/parquet_buckets.q | 31 +
.../clientpositive/parquet_format_nonpart.q | 30 +
.../clientpositive/parquet_format_part.q | 67 +
.../parquet_nonstd_partitions_loc.q | 100 +
.../queries/clientpositive/ptf_parquetfile.q | 32 +
.../insert_into_dynamic_partitions.q.out | 8 +-
.../insert_overwrite_dynamic_partitions.q.out | 12 +-
.../clientpositive/parquet_buckets.q.out | 183 +
.../clientpositive/parquet_format_nonpart.q.out | 201 +
.../clientpositive/parquet_format_part.q.out | 290 +
.../parquet_nonstd_partitions_loc.q.out | 537 +
.../clientpositive/ptf_parquetfile.q.out | 104 +
.../benchmark/udf/json_read/JsonReadBench.java | 83 +
.../hive/benchmark/udf/json_read/val1.json | 86 +
.../hive/benchmark/udf/json_read/val1.type | 1 +
.../hive/metastore/TestMetaStoreMetrics.java | 5 +-
.../tools/TestSchemaToolCatalogOps.java | 479 +
.../TestReplicationScenariosAcidTables.java | 2 +
.../hive/beeline/schematool/TestSchemaTool.java | 550 +-
.../schematool/TestSchemaToolCatalogOps.java | 478 -
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 2 +-
.../jdbc/TestJdbcWithMiniHS2ErasureCoding.java | 215 +
.../test/resources/testconfiguration.properties | 16 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 4 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 23 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 44 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../ql/exec/tez/monitoring/TezJobMonitor.java | 19 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 6 +-
.../ql/io/orc/VectorizedOrcInputFormat.java | 2 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../formatting/MetaDataFormatUtils.java | 11 +-
.../formatting/TextMetaDataFormatter.java | 10 +
.../annotation/OpTraitsRulesProcFactory.java | 59 +-
.../optimizer/spark/SparkMapJoinOptimizer.java | 2 +-
.../stats/annotation/StatsRulesProcFactory.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/plan/PartitionDesc.java | 4 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 31 +
.../apache/hadoop/hive/ql/plan/Statistics.java | 12 +-
.../apache/hadoop/hive/ql/plan/TableDesc.java | 3 +-
.../hadoop/hive/ql/session/SessionState.java | 6 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 7 +-
.../hadoop/hive/ql/stats/BasicStatsTask.java | 4 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 20 +-
.../hive/ql/txn/compactor/CompactorMR.java | 4 +-
.../hive/ql/udf/generic/GenericUDFJsonRead.java | 92 +
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 4 +-
.../ql/udf/generic/TestGenericUDFJsonRead.java | 204 +
ql/src/test/queries/clientnegative/ctas.q | 2 +-
.../queries/clientnegative/druid_datasource2.q | 1 +
.../queries/clientpositive/druid_timestamptz.q | 5 +-
.../queries/clientpositive/druid_timestamptz2.q | 14 +-
.../clientpositive/druidkafkamini_basic.q | 2 +-
.../druidmini_dynamic_partition.q | 10 +-
.../clientpositive/druidmini_expressions.q | 3 +-
.../clientpositive/druidmini_extractTime.q | 6 +-
.../clientpositive/druidmini_floorTime.q | 4 +-
.../queries/clientpositive/druidmini_joins.q | 5 +-
.../queries/clientpositive/druidmini_masking.q | 4 +-
.../queries/clientpositive/druidmini_test1.q | 5 +-
.../clientpositive/druidmini_test_alter.q | 5 +-
.../clientpositive/druidmini_test_insert.q | 8 +-
ql/src/test/queries/clientpositive/dst.q | 12 +
.../queries/clientpositive/erasure_explain.q | 24 +
.../queries/clientpositive/erasure_simple.q | 13 +
.../test/queries/clientpositive/explainuser_4.q | 1 +
.../test/queries/clientpositive/json_serde2.q | 37 +
ql/src/test/queries/clientpositive/mm_all.q | 1 +
.../clientpositive/orc_schema_evol_quoted.q | 42 +
.../clientpositive/orc_schema_evolution_float.q | 1 +
.../clientpositive/parquet_ppd_multifiles.q | 1 +
.../test/queries/clientpositive/selectindate.q | 1 +
.../test/queries/clientpositive/udf_json_read.q | 44 +
ql/src/test/queries/clientpositive/union38.q | 1 +
.../test/queries/clientpositive/vector_bround.q | 1 +
.../queries/clientpositive/vector_case_when_1.q | 3 +-
.../queries/clientpositive/vector_coalesce_2.q | 1 +
.../queries/clientpositive/vector_coalesce_3.q | 1 +
.../queries/clientpositive/vector_interval_1.q | 1 +
.../clientpositive/vectorized_parquet_types.q | 1 +
.../clientnegative/druid_datasource2.q.out | 2 +-
.../clientnegative/unset_table_property.q.out | 1 +
.../materialized_view_create_rewrite.q.out | 2 +
.../druid/druid_timestamptz.q.out | 4 +-
.../druid/druid_timestamptz2.q.out | 20 +-
.../druid/druidkafkamini_basic.q.out | 6 +-
.../druid/druidmini_dynamic_partition.q.out | 18 +-
.../druid/druidmini_expressions.q.out | 6 +-
.../druid/druidmini_extractTime.q.out | 8 +-
.../druid/druidmini_floorTime.q.out | 4 +-
.../clientpositive/druid/druidmini_joins.q.out | 4 +-
.../druid/druidmini_masking.q.out | 4 +-
.../clientpositive/druid/druidmini_mv.q.out | 3 +
.../clientpositive/druid/druidmini_test1.q.out | 4 +-
.../druid/druidmini_test_alter.q.out | 4 +-
.../druid/druidmini_test_insert.q.out | 12 +-
.../erasurecoding/erasure_explain.q.out | 409 +
.../erasurecoding/erasure_simple.q.out | 20 +-
.../results/clientpositive/json_serde2.q.out | 113 +
.../test/results/clientpositive/llap/dst.q.out | 108 +
.../clientpositive/llap/explainuser_4.q.out | 28 +-
.../llap/materialized_view_create.q.out | 1 +
.../llap/materialized_view_create_rewrite.q.out | 2 +
...materialized_view_create_rewrite_dummy.q.out | 2 +
...erialized_view_create_rewrite_multi_db.q.out | 2 +
.../llap/materialized_view_describe.q.out | 1 +
.../llap/orc_schema_evol_quoted.q.out | 258 +
.../clientpositive/llap/subquery_notin.q.out | 118 +-
.../results/clientpositive/llap/tez_join.q.out | 51 +-
.../clientpositive/llap/vector_bround.q.out | 4 +-
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +-
.../clientpositive/llap/vector_interval_1.q.out | 16 +-
.../llap/vectorized_parquet_types.q.out | 8 +-
.../clientpositive/orc_schema_evol_quoted.q.out | 258 +
.../orc_schema_evolution_float.q.out | 12 +-
.../results/clientpositive/show_functions.q.out | 1 +
.../clientpositive/show_tblproperties.q.out | 5 +
.../results/clientpositive/udf_json_read.q.out | 107 +
.../test/results/clientpositive/union38.q.out | 14 +-
.../unset_table_view_property.q.out | 9 +
.../results/clientpositive/vector_bround.q.out | 4 +-
.../clientpositive/vector_coalesce_2.q.out | 4 +-
.../clientpositive/vector_interval_1.q.out | 16 +-
.../vectorized_parquet_types.q.out | 8 +-
.../apache/hadoop/hive/serde2/JsonSerDe.java | 645 +-
.../hive/serde2/json/HiveJsonStructReader.java | 402 +
.../java/org/apache/hive/http/LlapServlet.java | 2 +-
.../hadoop/hive/common/StatsSetupConst.java | 21 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 5 +-
.../hive/metastore/StatObjectConverter.java | 10 +-
.../hive/metastore/events/InsertEvent.java | 9 +-
.../metastore/messaging/PartitionFiles.java | 2 +-
.../messaging/json/JSONAbortTxnMessage.java | 3 +-
.../messaging/json/JSONAcidWriteMessage.java | 2 +-
.../json/JSONAddForeignKeyMessage.java | 3 +-
.../json/JSONAddNotNullConstraintMessage.java | 3 +-
.../messaging/json/JSONAddPartitionMessage.java | 26 +-
.../json/JSONAddPrimaryKeyMessage.java | 3 +-
.../json/JSONAddUniqueConstraintMessage.java | 3 +-
.../messaging/json/JSONAllocWriteIdMessage.java | 8 +-
.../json/JSONAlterDatabaseMessage.java | 3 +-
.../json/JSONAlterPartitionMessage.java | 11 +-
.../messaging/json/JSONAlterTableMessage.java | 9 +-
.../messaging/json/JSONCommitTxnMessage.java | 3 +-
.../json/JSONCreateCatalogMessage.java | 3 +-
.../json/JSONCreateDatabaseMessage.java | 4 +-
.../json/JSONCreateFunctionMessage.java | 3 +-
.../messaging/json/JSONCreateTableMessage.java | 8 +-
.../messaging/json/JSONDropCatalogMessage.java | 3 +-
.../json/JSONDropConstraintMessage.java | 3 +-
.../messaging/json/JSONDropDatabaseMessage.java | 3 +-
.../messaging/json/JSONDropFunctionMessage.java | 3 +-
.../json/JSONDropPartitionMessage.java | 13 +-
.../messaging/json/JSONDropTableMessage.java | 9 +-
.../messaging/json/JSONInsertMessage.java | 14 +-
.../messaging/json/JSONMessageDeserializer.java | 22 +-
.../messaging/json/JSONMessageFactory.java | 21 +-
.../messaging/json/JSONOpenTxnMessage.java | 6 +-
.../metastore/tools/MetastoreSchemaTool.java | 1282 +--
.../metastore/tools/SchemaToolCommandLine.java | 308 +
.../hive/metastore/tools/SchemaToolTask.java | 32 +
.../tools/SchemaToolTaskAlterCatalog.java | 90 +
.../tools/SchemaToolTaskCreateCatalog.java | 132 +
.../tools/SchemaToolTaskCreateUser.java | 115 +
.../metastore/tools/SchemaToolTaskInfo.java | 43 +
.../metastore/tools/SchemaToolTaskInit.java | 73 +
.../tools/SchemaToolTaskMoveDatabase.java | 96 +
.../tools/SchemaToolTaskMoveTable.java | 142 +
.../metastore/tools/SchemaToolTaskUpgrade.java | 116 +
.../metastore/tools/SchemaToolTaskValidate.java | 630 ++
.../hive/metastore/utils/MetaStoreUtils.java | 30 +-
.../hive/metastore/dbinstall/DbInstallBase.java | 25 +-
.../json/TestJSONMessageDeserializer.java | 28 +-
.../tools/TestSchemaToolForMetastore.java | 277 +-
.../metastore/utils/TestMetaStoreUtils.java | 37 +-
.../apache/hive/streaming/StrictJsonWriter.java | 2 +-
219 files changed, 18301 insertions(+), 5801 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index cb6913e,95a4440..ae51b4d
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@@ -263,11 -261,10 +263,13 @@@ public class StatsUtils
long nr = basicStats.getNumRows();
List<ColStatistics> colStats = Lists.newArrayList();
+ long numErasureCodedFiles = getErasureCodedFiles(table);
+
if (fetchColStats) {
- colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache);
+ // Note: this is currently called from two notable places (w/false for checkTxn)
+ // 1) StatsRulesProcFactory.TableScanStatsRule via collectStatistics
+ // 2) RelOptHiveTable via getColStats and updateColStats.
+ colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache, false);
if(colStats == null) {
colStats = Lists.newArrayList();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------