You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/08/28 05:15:19 UTC
svn commit: r1621031 [1/10] - in /hive/branches/cbo: ./
common/src/java/org/apache/hadoop/hive/conf/
contrib/src/test/results/clientnegative/
contrib/src/test/results/clientpositive/ data/files/
hbase-handler/src/test/results/negative/ hcatalog/core/sr...
Author: gunther
Date: Thu Aug 28 03:15:13 2014
New Revision: 1621031
URL: http://svn.apache.org/r1621031
Log:
Merge latest trunk into cbo branch. (Gunther Hagleitner)
Added:
hive/branches/cbo/data/files/extrapolate_stats_full.txt
- copied unchanged from r1621023, hive/trunk/data/files/extrapolate_stats_full.txt
hive/branches/cbo/data/files/extrapolate_stats_partial.txt
- copied unchanged from r1621023, hive/trunk/data/files/extrapolate_stats_partial.txt
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
- copied unchanged from r1621023, hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
- copied unchanged from r1621023, hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java
- copied unchanged from r1621023, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java
hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_create_tbl.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_create_tbl.q
hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_create_view.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_create_view.q
hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_ctas2.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_ctas2.q
hive/branches/cbo/ql/src/test/queries/clientpositive/extrapolate_part_stats_full.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/extrapolate_part_stats_full.q
hive/branches/cbo/ql/src/test/queries/clientpositive/extrapolate_part_stats_partial.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/extrapolate_part_stats_partial.q
hive/branches/cbo/ql/src/test/queries/clientpositive/orc_merge_incompat1.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_merge_incompat1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/orc_merge_incompat2.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_merge_incompat2.q
hive/branches/cbo/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q
- copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_create_tbl.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_create_tbl.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_create_view.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_create_view.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_ctas2.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_ctas2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_ppd_timestamp.q.out
- copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_ppd_timestamp.q.out
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/FetchType.java
- copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/FetchType.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
- copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationLog.java
- copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/operation/OperationLog.java
hive/branches/cbo/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java
- copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java
hive/branches/cbo/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
- copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
hive/branches/cbo/service/src/test/org/apache/hive/service/cli/operation/
- copied from r1621023, hive/trunk/service/src/test/org/apache/hive/service/cli/operation/
Removed:
hive/branches/cbo/ql/src/test/queries/clientnegative/authorize_create_tbl.q
hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge1.q
hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge2.q
hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge3.q
hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge4.q
hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge5.q
hive/branches/cbo/ql/src/test/results/clientnegative/authorize_create_tbl.q.out
Modified:
hive/branches/cbo/ (props changed)
hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out
hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out
hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out
hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out
hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java
hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java
hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java
hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
hive/branches/cbo/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeMapper.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileMergeMapper.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
hive/branches/cbo/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
hive/branches/cbo/ql/src/test/queries/clientpositive/add_part_exist.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_char1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_index.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_partition_coltype.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_skewed_table.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_varchar1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_view_as_select.q
hive/branches/cbo/ql/src/test/queries/clientpositive/alter_view_rename.q
hive/branches/cbo/ql/src/test/queries/clientpositive/archive_multi.q
hive/branches/cbo/ql/src/test/queries/clientpositive/create_or_replace_view.q
hive/branches/cbo/ql/src/test/queries/clientpositive/drop_multi_partitions.q
hive/branches/cbo/ql/src/test/queries/clientpositive/exchange_partition.q
hive/branches/cbo/ql/src/test/queries/clientpositive/index_auto_empty.q
hive/branches/cbo/ql/src/test/queries/clientpositive/input46.q
hive/branches/cbo/ql/src/test/queries/clientpositive/optimize_nullscan.q
hive/branches/cbo/ql/src/test/queries/clientpositive/touch.q
hive/branches/cbo/ql/src/test/queries/clientpositive/unset_table_view_property.q
hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
hive/branches/cbo/ql/src/test/resources/orc-file-dump.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_as_select_not_exist.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure2.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure4.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure5.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure6.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure7.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure9.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/analyze_view.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_drop_db_cascade.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_not_owner_drop_view.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_part.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_select_view.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view1.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view2.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view3.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view4.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view5.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view6.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view7.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view8.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure1.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure2.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure4.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/desc_failure3.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/drop_table_failure2.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/insert_view_failure.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/invalidate_view1.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/load_view_failure.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/recursive_view.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/unset_view_property.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/add_part_exist.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/allcolref_in_udf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_char1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_index.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_rename_partition_authorization.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_rename_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_skewed_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_table_update_status.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_varchar1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_view_as_select.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/alter_view_rename.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_part.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/archive_multi.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_8.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_cli_createtab.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_create_temp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_owner_actions.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/autogen_colalias.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/char_nested_types.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/colstats_all_nulls.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_big_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_default_prop.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_like_tbl_props.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_like_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_or_replace_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_view_partitioned.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/create_view_translate.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/cross_product_check_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/cross_product_check_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas_char.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas_date.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ctas_varchar.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/cte_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/database.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/database_drop.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/dbtxnmgr_ddl1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/decimal_6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/decimal_serde.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/describe_formatted_view_partitioned_json.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/describe_syntax.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/drop_multi_partitions.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/exchange_partition.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/explain_dependency.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/explain_logical.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/fileformat_sequencefile.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/fileformat_text.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/global_limit.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_empty.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/input46.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/insert1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/join41.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/join_filters_overlap.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/join_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/lateral_view_outer.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/multi_join_union.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nestedvirtual.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/optimize_nullscan.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_createas1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/parallel_orderby.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/parquet_ctas.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/parquet_join.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/partition_decode_name.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/partition_special_char.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ppd_field_garbage.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ppd_union_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ptf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/query_result_fileformat.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/quotedid_basic.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/rcfile_createas1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/rcfile_default_format.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/sample_islocalmode_hook.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/semijoin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/serde_regex.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_temp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/skewjoin_noskew.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/stats5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/stats_counter.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/str_to_map.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_exists.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_exists_having.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_notin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_views.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/symlink_text_input_format.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_gb1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_join1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_names.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_precedence.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_subquery1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/ctas.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/insert1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/ptf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/stats_counter.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/subquery_exists.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/temp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_dml.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_union.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorization_part_project.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/touch.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union24.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union25.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union27.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union31.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union32.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_top_level.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/unset_table_view_property.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/updateAccessTime.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/varchar_nested_types.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_between_in.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorization_decimal_date.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorized_ptf.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/view_cast.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/view_inputs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/windowing.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/windowing_navfn.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/windowing_streaming.q.out
hive/branches/cbo/service/if/TCLIService.thrift
hive/branches/cbo/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp
hive/branches/cbo/service/src/gen/thrift/gen-cpp/TCLIService_types.h
hive/branches/cbo/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TFetchResultsReq.java
hive/branches/cbo/service/src/gen/thrift/gen-py/TCLIService/ttypes.py
hive/branches/cbo/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/ICLIService.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/Operation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSession.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
Merged /hive/trunk:r1619930-1621023
Modified: hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Aug 28 03:15:13 2014
@@ -54,7 +54,6 @@ import org.apache.hive.common.HiveCompat
* Hive Configuration.
*/
public class HiveConf extends Configuration {
-
protected String hiveJar;
protected Properties origProp;
protected String auxJars;
@@ -1408,6 +1407,14 @@ public class HiveConf extends Configurat
"to construct a list exception handlers to handle exceptions thrown\n" +
"by record readers"),
+ // operation log configuration
+ HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true,
+ "When true, HS2 will save operation logs"),
+ HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location",
+ "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator +
+ "operation_logs",
+ "Top level directory where operation logs are stored if logging functionality is enabled"),
+
// logging configuration
HIVE_LOG4J_FILE("hive.log4j.file", "",
"Hive log4j configuration file.\n" +
@@ -1479,8 +1486,11 @@ public class HiveConf extends Configurat
"Minimum number of worker threads when in HTTP mode."),
HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS("hive.server2.thrift.http.max.worker.threads", 500,
"Maximum number of worker threads when in HTTP mode."),
- HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME("hive.server2.thrift.http.max.idle.time", 1800000,
+ HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME("hive.server2.thrift.http.max.idle.time", 1800000,
"Maximum idle time in milliseconds for a connection on the server when in HTTP mode."),
+ HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME("hive.server2.thrift.http.worker.keepalive.time", 60,
+ "Keepalive time (in seconds) for an idle http worker thread. When number of workers > min workers, " +
+ "excess threads are killed after this time interval."),
// binary transport settings
HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000,
@@ -1503,7 +1513,9 @@ public class HiveConf extends Configurat
"Minimum number of Thrift worker threads"),
HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 500,
"Maximum number of Thrift worker threads"),
-
+ HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME("hive.server2.thrift.worker.keepalive.time", 60,
+ "Keepalive time (in seconds) for an idle worker thread. When number of workers > min workers, " +
+ "excess threads are killed after this time interval."),
// Configuration for async thread pool in SessionManager
HIVE_SERVER2_ASYNC_EXEC_THREADS("hive.server2.async.exec.threads", 100,
"Number of threads in the async thread pool for HiveServer2"),
Modified: hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out (original)
+++ hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out Thu Aug 28 03:15:13 2014
@@ -56,7 +56,7 @@ STAGE PLANS:
serde properties:
input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
- name: serde_regex
+ name: default.serde_regex
PREHOOK: query: CREATE TABLE serde_regex(
host STRING,
Modified: hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out (original)
+++ hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out Thu Aug 28 03:15:13 2014
@@ -22,7 +22,7 @@ STAGE PLANS:
columns: key int, value string
input format: org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextInputFormat
output format: org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextOutputFormat
- name: base64_test
+ name: default.base64_test
PREHOOK: query: CREATE TABLE base64_test(key INT, value STRING) STORED AS
INPUTFORMAT 'org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextInputFormat'
Modified: hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out (original)
+++ hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out Thu Aug 28 03:15:13 2014
@@ -48,7 +48,7 @@ STAGE PLANS:
serde properties:
input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
- name: serde_regex
+ name: default.serde_regex
PREHOOK: query: CREATE TABLE serde_regex(
host STRING,
Modified: hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out (original)
+++ hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out Thu Aug 28 03:15:13 2014
@@ -20,7 +20,7 @@ WITH SERDEPROPERTIES ("hbase.columns.map
TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:hbasedb
-PREHOOK: Output: hbaseDB@hbaseDB.hbase_table_0
+PREHOOK: Output: hbaseDB@hbase_table_0
POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S)
-- Hadoop 0.23 changes the behavior FsShell on Exit Codes
-- In Hadoop 0.20
@@ -37,7 +37,6 @@ WITH SERDEPROPERTIES ("hbase.columns.map
TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:hbasedb
-POSTHOOK: Output: hbaseDB@hbaseDB.hbase_table_0
POSTHOOK: Output: hbaseDB@hbase_table_0
Found 3 items
#### A masked pattern was here ####
Modified: hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java (original)
+++ hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java Thu Aug 28 03:15:13 2014
@@ -86,7 +86,7 @@ final class CreateTableHook extends HCat
"Operation not supported. Create table as " +
"Select is not a valid operation.");
- case HiveParser.TOK_TABLEBUCKETS:
+ case HiveParser.TOK_ALTERTABLE_BUCKETS:
break;
case HiveParser.TOK_LIKETABLE:
Modified: hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java (original)
+++ hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java Thu Aug 28 03:15:13 2014
@@ -71,7 +71,7 @@ public class HCatSemanticAnalyzer extend
hook = new CreateDatabaseHook();
return hook.preAnalyze(context, ast);
- case HiveParser.TOK_ALTERTABLE_PARTITION:
+ case HiveParser.TOK_ALTERTABLE:
if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) {
return ast;
} else if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) {
@@ -163,7 +163,6 @@ public class HCatSemanticAnalyzer extend
case HiveParser.TOK_CREATETABLE:
case HiveParser.TOK_CREATEDATABASE:
- case HiveParser.TOK_ALTERTABLE_PARTITION:
// HCat will allow these operations to be performed.
// Database DDL
@@ -178,12 +177,20 @@ public class HCatSemanticAnalyzer extend
case HiveParser.TOK_CREATEINDEX:
case HiveParser.TOK_DROPINDEX:
case HiveParser.TOK_SHOWINDEXES:
+ break;
// View DDL
//case HiveParser.TOK_ALTERVIEW_ADDPARTS:
- case HiveParser.TOK_ALTERVIEW_DROPPARTS:
- case HiveParser.TOK_ALTERVIEW_PROPERTIES:
- case HiveParser.TOK_ALTERVIEW_RENAME:
+ case HiveParser.TOK_ALTERVIEW:
+ switch (ast.getChild(1).getType()) {
+ case HiveParser.TOK_ALTERVIEW_ADDPARTS:
+ case HiveParser.TOK_ALTERVIEW_DROPPARTS:
+ case HiveParser.TOK_ALTERVIEW_RENAME:
+ case HiveParser.TOK_ALTERVIEW_PROPERTIES:
+ case HiveParser.TOK_ALTERVIEW_DROPPROPERTIES:
+ }
+ break;
+
case HiveParser.TOK_CREATEVIEW:
case HiveParser.TOK_DROPVIEW:
@@ -205,20 +212,39 @@ public class HCatSemanticAnalyzer extend
case HiveParser.TOK_DESCFUNCTION:
case HiveParser.TOK_SHOWFUNCTIONS:
case HiveParser.TOK_EXPLAIN:
+ break;
// Table DDL
- case HiveParser.TOK_ALTERTABLE_ADDPARTS:
- case HiveParser.TOK_ALTERTABLE_ADDCOLS:
- case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION:
- case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES:
- case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT:
- case HiveParser.TOK_ALTERTABLE_DROPPARTS:
- case HiveParser.TOK_ALTERTABLE_PROPERTIES:
- case HiveParser.TOK_ALTERTABLE_RENAME:
- case HiveParser.TOK_ALTERTABLE_RENAMECOL:
- case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
- case HiveParser.TOK_ALTERTABLE_SERIALIZER:
- case HiveParser.TOK_ALTERTABLE_TOUCH:
+ case HiveParser.TOK_ALTERTABLE:
+ switch (ast.getChild(1).getType()) {
+ case HiveParser.TOK_ALTERTABLE_ADDPARTS:
+ case HiveParser.TOK_ALTERTABLE_ADDCOLS:
+ case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION:
+ case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES:
+ case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT:
+ case HiveParser.TOK_ALTERTABLE_DROPPARTS:
+ case HiveParser.TOK_ALTERTABLE_PROPERTIES:
+ case HiveParser.TOK_ALTERTABLE_DROPPROPERTIES:
+ case HiveParser.TOK_ALTERTABLE_RENAME:
+ case HiveParser.TOK_ALTERTABLE_RENAMECOL:
+ case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
+ case HiveParser.TOK_ALTERTABLE_SERIALIZER:
+ case HiveParser.TOK_ALTERTABLE_TOUCH:
+ case HiveParser.TOK_ALTERTABLE_ARCHIVE:
+ case HiveParser.TOK_ALTERTABLE_UNARCHIVE:
+ case HiveParser.TOK_ALTERTABLE_EXCHANGEPARTITION:
+ case HiveParser.TOK_ALTERTABLE_SKEWED:
+ case HiveParser.TOK_ALTERTABLE_FILEFORMAT:
+ case HiveParser.TOK_ALTERTABLE_PROTECTMODE:
+ case HiveParser.TOK_ALTERTABLE_LOCATION:
+ case HiveParser.TOK_ALTERTABLE_MERGEFILES:
+ case HiveParser.TOK_ALTERTABLE_RENAMEPART:
+ case HiveParser.TOK_ALTERTABLE_SKEWED_LOCATION:
+ case HiveParser.TOK_ALTERTABLE_BUCKETS:
+ case HiveParser.TOK_ALTERTABLE_COMPACT:
+ }
+ break;
+
case HiveParser.TOK_DESCTABLE:
case HiveParser.TOK_DROPTABLE:
case HiveParser.TOK_SHOW_TABLESTATUS:
Modified: hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java (original)
+++ hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java Thu Aug 28 03:15:13 2014
@@ -156,7 +156,7 @@ public class TestSemanticAnalysis extend
public void testCreateTableIfNotExists() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException {
hcatDriver.run("drop table " + TBL_NAME);
- hcatDriver.run("create table junit_sem_analysis (a int) stored as RCFILE");
+ hcatDriver.run("create table " + TBL_NAME + " (a int) stored as RCFILE");
Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
List<FieldSchema> cols = tbl.getSd().getCols();
assertEquals(1, cols.size());
Modified: hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java (original)
+++ hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java Thu Aug 28 03:15:13 2014
@@ -51,6 +51,7 @@ public class HCatPartition {
private int createTime;
private int lastAccessTime;
private StorageDescriptor sd;
+ private List<HCatFieldSchema> columns; // Cache column-list from this.sd.
private Map<String, String> parameters;
// For use from within HCatClient.getPartitions().
@@ -68,6 +69,7 @@ public class HCatPartition {
}
this.sd = partition.getSd();
+ this.columns = getColumns(this.sd);
}
// For constructing HCatPartitions afresh, as an argument to HCatClient.addPartitions().
@@ -77,6 +79,7 @@ public class HCatPartition {
this.dbName = hcatTable.getDbName();
this.sd = new StorageDescriptor(hcatTable.getSd());
this.sd.setLocation(location);
+ this.columns = getColumns(this.sd);
this.createTime = (int)(System.currentTimeMillis()/1000);
this.lastAccessTime = -1;
this.values = new ArrayList<String>(hcatTable.getPartCols().size());
@@ -98,7 +101,7 @@ public class HCatPartition {
this.dbName = rhs.dbName;
this.sd = new StorageDescriptor(rhs.sd);
this.sd.setLocation(location);
-
+ this.columns = getColumns(this.sd);
this.createTime = (int) (System.currentTimeMillis() / 1000);
this.lastAccessTime = -1;
this.values = new ArrayList<String>(hcatTable.getPartCols().size());
@@ -112,6 +115,14 @@ public class HCatPartition {
}
}
+ private static List<HCatFieldSchema> getColumns(StorageDescriptor sd) throws HCatException {
+ ArrayList<HCatFieldSchema> columns = new ArrayList<HCatFieldSchema>(sd.getColsSize());
+ for (FieldSchema fieldSchema : sd.getCols()) {
+ columns.add(HCatSchemaUtils.getHCatFieldSchema(fieldSchema));
+ }
+ return columns;
+ }
+
// For use from HCatClient.addPartitions(), to construct from user-input.
Partition toHivePartition() throws HCatException {
Partition hivePtn = new Partition();
@@ -172,11 +183,7 @@ public class HCatPartition {
*
* @return the columns
*/
- public List<HCatFieldSchema> getColumns() throws HCatException {
- ArrayList<HCatFieldSchema> columns = new ArrayList<HCatFieldSchema>(sd.getColsSize());
- for (FieldSchema fieldSchema : sd.getCols()) {
- columns.add(HCatSchemaUtils.getHCatFieldSchema(fieldSchema));
- }
+ public List<HCatFieldSchema> getColumns() {
return columns;
}
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java Thu Aug 28 03:15:13 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.securi
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
@@ -141,6 +142,47 @@ public class TestHiveAuthorizerCheckInvo
}
@Test
+ public void testCreateTableWithDb() throws HiveAuthzPluginException, HiveAccessControlException,
+ CommandNeedRetryException {
+ final String newTable = "ctTableWithDb";
+ checkCreateViewOrTableWithDb(newTable, "create table " + dbName + "." + newTable + "(i int)");
+ }
+
+ @Test
+ public void testCreateViewWithDb() throws HiveAuthzPluginException, HiveAccessControlException,
+ CommandNeedRetryException {
+ final String newTable = "ctViewWithDb";
+ checkCreateViewOrTableWithDb(newTable, "create table " + dbName + "." + newTable + "(i int)");
+ }
+
+ private void checkCreateViewOrTableWithDb(String newTable, String cmd)
+ throws HiveAuthzPluginException, HiveAccessControlException {
+ reset(mockedAuthorizer);
+ int status = driver.compile(cmd);
+ assertEquals(0, status);
+
+ List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight();
+ assertEquals("num outputs", 2, outputs.size());
+ for (HivePrivilegeObject output : outputs) {
+ switch (output.getType()) {
+ case DATABASE:
+ assertTrue("database name", output.getDbname().equalsIgnoreCase(dbName));
+ break;
+ case TABLE_OR_VIEW:
+ assertTrue("database name", output.getDbname().equalsIgnoreCase(dbName));
+ assertEqualsIgnoreCase("table name", output.getObjectName(), newTable);
+ break;
+ default:
+ fail("Unexpected type : " + output.getType());
+ }
+ }
+ }
+
+ private void assertEqualsIgnoreCase(String msg, String expected, String actual) {
+ assertEquals(msg, expected.toLowerCase(), actual.toLowerCase());
+ }
+
+ @Test
public void testInputNoColumnsUsed() throws HiveAuthzPluginException, HiveAccessControlException,
CommandNeedRetryException {
Modified: hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/src/test/resources/testconfiguration.properties?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/cbo/itests/src/test/resources/testconfiguration.properties Thu Aug 28 03:15:13 2014
@@ -87,6 +87,7 @@ minitez.query.files.shared=alter_merge_2
mapreduce2.q,\
merge1.q,\
merge2.q,\
+ metadataonly1.q,\
metadata_only_queries.q,\
optimize_nullscan.q,\
orc_analyze.q,\
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Aug 28 03:15:13 2014
@@ -250,7 +250,7 @@ public class HiveMetaStore extends Thrif
private static String currentUrl;
private Warehouse wh; // hdfs warehouse
- private final ThreadLocal<RawStore> threadLocalMS =
+ private static final ThreadLocal<RawStore> threadLocalMS =
new ThreadLocal<RawStore>() {
@Override
protected synchronized RawStore initialValue() {
@@ -265,6 +265,14 @@ public class HiveMetaStore extends Thrif
}
};
+ public static RawStore getRawStore() {
+ return threadLocalMS.get();
+ }
+
+ public static void removeRawStore() {
+ threadLocalMS.remove();
+ }
+
// Thread local configuration is needed as many threads could make changes
// to the conf using the connection hook
private final ThreadLocal<Configuration> threadLocalConf =
@@ -384,6 +392,7 @@ public class HiveMetaStore extends Thrif
}
}
+ @Override
public void init() throws MetaException {
rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL);
initListeners = MetaStoreUtils.getMetaStoreListeners(
@@ -5024,12 +5033,8 @@ public class HiveMetaStore extends Thrif
startFunction("get_aggr_stats_for: db=" + request.getDbName() + " table=" + request.getTblName());
AggrStats aggrStats = null;
try {
- //TODO: We are setting partitionCnt for which we were able to retrieve stats same as
- // incoming number from request. This is not correct, but currently no users of this api
- // rely on this. Only, current user StatsAnnotation don't care for it. StatsOptimizer
- // will care for it, so before StatsOptimizer begin using it, we need to fix this.
aggrStats = new AggrStats(getMS().get_aggr_stats_for(request.getDbName(),
- request.getTblName(), request.getPartNames(), request.getColNames()), request.getPartNames().size());
+ request.getTblName(), request.getPartNames(), request.getColNames()));
return aggrStats;
} finally {
endFunction("get_partitions_statistics_req: ", aggrStats == null, null, request.getTblName());
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Thu Aug 28 03:15:13 2014
@@ -21,19 +21,18 @@ package org.apache.hadoop.hive.metastore
import static org.apache.commons.lang.StringUtils.join;
import static org.apache.commons.lang.StringUtils.repeat;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.SQLException;
-import java.sql.Statement;
import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Date;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
@@ -43,10 +42,12 @@ import javax.jdo.datastore.JDOConnection
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -65,9 +66,8 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
-import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.datanucleus.store.schema.SchemaTool;
+import org.datanucleus.store.rdbms.query.ForwardQueryResult;
import com.google.common.collect.Lists;
@@ -97,7 +97,7 @@ class MetaStoreDirectSql {
* Whether direct SQL can be used with the current datastore backing {@link #pm}.
*/
private final boolean isCompatibleDatastore;
-
+
public MetaStoreDirectSql(PersistenceManager pm) {
this.pm = pm;
Transaction tx = pm.currentTransaction();
@@ -893,33 +893,247 @@ class MetaStoreDirectSql {
return result;
}
- public List<ColumnStatisticsObj> aggrColStatsForPartitions(String dbName, String tableName,
+ public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
List<String> partNames, List<String> colNames) throws MetaException {
- String qText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", "
- + "min(\"LONG_LOW_VALUE\"), max(\"LONG_HIGH_VALUE\"), min(\"DOUBLE_LOW_VALUE\"), max(\"DOUBLE_HIGH_VALUE\"), "
- + "min(\"BIG_DECIMAL_LOW_VALUE\"), max(\"BIG_DECIMAL_HIGH_VALUE\"), sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), "
- + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\") from \"PART_COL_STATS\""
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in ("
- + makeParams(colNames.size()) + ") AND \"PARTITION_NAME\" in ("
- + makeParams(partNames.size()) + ") group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
+ long partsFound = partsFoundForPartitions(dbName, tableName, partNames,
+ colNames);
+ List<ColumnStatisticsObj> stats = columnStatisticsObjForPartitions(dbName,
+ tableName, partNames, colNames, partsFound);
+ return new AggrStats(stats, partsFound);
+ }
+ private long partsFoundForPartitions(String dbName, String tableName,
+ List<String> partNames, List<String> colNames) throws MetaException {
+ long partsFound = 0;
boolean doTrace = LOG.isDebugEnabled();
+ String qText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " group by \"PARTITION_NAME\"";
long start = doTrace ? System.nanoTime() : 0;
Query query = pm.newQuery("javax.jdo.query.SQL", qText);
- Object qResult = query.executeWithArray(prepareParams(dbName, tableName, partNames, colNames));
- if (qResult == null) {
- query.closeAll();
- return Lists.newArrayList();
- }
- List<Object[]> list = ensureList(qResult);
- List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(list.size());
- for (Object[] row : list) {
- colStats.add(prepareCSObj(row,0));
- }
+ Object qResult = query.executeWithArray(prepareParams(dbName, tableName,
+ partNames, colNames));
long end = doTrace ? System.nanoTime() : 0;
timingTrace(doTrace, qText, start, end);
- query.closeAll();
- return colStats;
+ ForwardQueryResult fqr = (ForwardQueryResult) qResult;
+ List<Integer> colnumbers = new ArrayList<Integer>();
+ colnumbers.addAll(fqr);
+ for (Integer colnumber : colnumbers) {
+ if (colnumber == colNames.size())
+ partsFound++;
+ }
+ return partsFound;
+ }
+
+ private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(
+ String dbName, String tableName, List<String> partNames,
+ List<String> colNames, long partsFound) throws MetaException {
+ String commonPrefix = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", "
+ + "min(\"LONG_LOW_VALUE\"), max(\"LONG_HIGH_VALUE\"), min(\"DOUBLE_LOW_VALUE\"), max(\"DOUBLE_HIGH_VALUE\"), "
+ + "min(\"BIG_DECIMAL_LOW_VALUE\"), max(\"BIG_DECIMAL_HIGH_VALUE\"), sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), "
+ + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\") from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? ";
+ String qText = null;
+ long start = 0;
+ long end = 0;
+ Query query = null;
+ boolean doTrace = LOG.isDebugEnabled();
+ Object qResult = null;
+ ForwardQueryResult fqr = null;
+ // Check if the status of all the columns of all the partitions exists
+ // Extrapolation is not needed.
+ if (partsFound == partNames.size()) {
+ qText = commonPrefix
+ + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
+ start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", qText);
+ qResult = query.executeWithArray(prepareParams(dbName, tableName,
+ partNames, colNames));
+ if (qResult == null) {
+ query.closeAll();
+ return Lists.newArrayList();
+ }
+ end = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, qText, start, end);
+ List<Object[]> list = ensureList(qResult);
+ List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(
+ list.size());
+ for (Object[] row : list) {
+ colStats.add(prepareCSObj(row, 0));
+ }
+ query.closeAll();
+ return colStats;
+ } else {
+ // Extrapolation is needed for some columns.
+ // In this case, at least a column status for a partition is missing.
+ // We need to extrapolate this partition based on the other partitions
+ List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(
+ colNames.size());
+ qText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") "
+ + " from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
+ start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", qText);
+ qResult = query.executeWithArray(prepareParams(dbName, tableName,
+ partNames, colNames));
+ end = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, qText, start, end);
+ if (qResult == null) {
+ query.closeAll();
+ return Lists.newArrayList();
+ }
+ List<String> noExtraColumnNames = new ArrayList<String>();
+ Map<String, String[]> extraColumnNameTypeParts = new HashMap<String, String[]>();
+ List<Object[]> list = ensureList(qResult);
+ for (Object[] row : list) {
+ String colName = (String) row[0];
+ String colType = (String) row[1];
+ if ((Integer) row[2] == partNames.size() || (Integer) row[2] < 2) {
+ // Extrapolation is not needed for this column if
+ // count(\"PARTITION_NAME\")==partNames.size()
+ // Or, extrapolation is not possible for this column if
+ // count(\"PARTITION_NAME\")<2
+ noExtraColumnNames.add(colName);
+ } else {
+ extraColumnNameTypeParts.put(colName,
+ new String[] { colType, String.valueOf((Integer) row[2]) });
+ }
+ }
+ query.closeAll();
+ // Extrapolation is not needed for columns noExtraColumnNames
+ if (noExtraColumnNames.size() != 0) {
+ qText = commonPrefix
+ + " and \"COLUMN_NAME\" in ("+ makeParams(noExtraColumnNames.size()) + ")"
+ + " and \"PARTITION_NAME\" in ("+ makeParams(partNames.size()) +")"
+ + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
+ start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", qText);
+ qResult = query.executeWithArray(prepareParams(dbName, tableName,
+ partNames, noExtraColumnNames));
+ if (qResult == null) {
+ query.closeAll();
+ return Lists.newArrayList();
+ }
+ list = ensureList(qResult);
+ for (Object[] row : list) {
+ colStats.add(prepareCSObj(row, 0));
+ }
+ end = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, qText, start, end);
+ query.closeAll();
+ }
+ // Extrapolation is needed for extraColumnNames.
+ // give a sequence number for all the partitions
+ if (extraColumnNameTypeParts.size() != 0) {
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ // get sum for all columns to reduce the number of queries
+ Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>();
+ qText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\")"
+ + " from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " and \"COLUMN_NAME\" in (" +makeParams(extraColumnNameTypeParts.size())+ ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " group by \"COLUMN_NAME\"";
+ start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", qText);
+ List<String> extraColumnNames = new ArrayList<String>();
+ extraColumnNames.addAll(extraColumnNameTypeParts.keySet());
+ qResult = query.executeWithArray(prepareParams(dbName, tableName,
+ partNames, extraColumnNames));
+ if (qResult == null) {
+ query.closeAll();
+ return Lists.newArrayList();
+ }
+ list = ensureList(qResult);
+ // see the indexes for colstats in IExtrapolatePartStatus
+ Integer[] sumIndex = new Integer[] { 6, 10, 11 };
+ for (Object[] row : list) {
+ Map<Integer, Object> indexToObject = new HashMap<Integer, Object>();
+ for (int ind = 1; ind < row.length; ind++) {
+ indexToObject.put(sumIndex[ind - 1], row[ind]);
+ }
+ sumMap.put((String) row[0], indexToObject);
+ }
+ end = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, qText, start, end);
+ query.closeAll();
+ for (Map.Entry<String, String[]> entry : extraColumnNameTypeParts
+ .entrySet()) {
+ Object[] row = new Object[IExtrapolatePartStatus.colStatNames.length + 2];
+ String colName = entry.getKey();
+ String colType = entry.getValue()[0];
+ Long sumVal = Long.parseLong(entry.getValue()[1]);
+ // fill in colname
+ row[0] = colName;
+ // fill in coltype
+ row[1] = colType;
+ // use linear extrapolation. more complicated one can be added in the future.
+ IExtrapolatePartStatus extrapolateMethod = new LinearExtrapolatePartStatus();
+ // fill in colstatus
+ Integer[] index = IExtrapolatePartStatus.indexMaps.get(colType
+ .toLowerCase());
+ //if the colType is not the known type, long, double, etc, then get all index.
+ if (index == null) {
+ index = IExtrapolatePartStatus.indexMaps.get("default");
+ }
+ for (int colStatIndex : index) {
+ String colStatName = IExtrapolatePartStatus.colStatNames[colStatIndex];
+ // if the aggregation type is sum, we do a scale-up
+ if (IExtrapolatePartStatus.aggrTypes[colStatIndex] == IExtrapolatePartStatus.AggrType.Sum) {
+ Long val = (Long) sumMap.get(colName).get(colStatIndex);
+ if (val == null) {
+ row[2 + colStatIndex] = null;
+ } else {
+ row[2 + colStatIndex] = (Long) (val / sumVal * (partNames
+ .size()));
+ }
+ } else {
+ // if the aggregation type is min/max, we extrapolate from the
+ // left/right borders
+ qText = "select \""
+ + colStatName
+ + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+ + " and \"COLUMN_NAME\" in (" +makeParams(1)+ ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " order by \'" + colStatName + "\'";
+ start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", qText);
+ qResult = query.executeWithArray(prepareParams(dbName,
+ tableName, partNames, Arrays.asList(colName)));
+ if (qResult == null) {
+ query.closeAll();
+ return Lists.newArrayList();
+ }
+ fqr = (ForwardQueryResult) qResult;
+ Object[] min = (Object[]) (fqr.get(0));
+ Object[] max = (Object[]) (fqr.get(fqr.size() - 1));
+ end = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, qText, start, end);
+ query.closeAll();
+ if (min[0] == null || max[0] == null) {
+ row[2 + colStatIndex] = null;
+ } else {
+ row[2 + colStatIndex] = extrapolateMethod.extrapolate(min, max,
+ colStatIndex, indexMap);
+ }
+ }
+ }
+ colStats.add(prepareCSObj(row, 0));
+ }
+ }
+ return colStats;
+ }
}
private ColumnStatisticsObj prepareCSObj (Object[] row, int i) throws MetaException {
@@ -949,7 +1163,7 @@ class MetaStoreDirectSql {
return params;
}
-
+
public List<ColumnStatistics> getPartitionStats(String dbName, String tableName,
List<String> partNames, List<String> colNames) throws MetaException {
if (colNames.isEmpty() || partNames.isEmpty()) {
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Aug 28 03:15:13 2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.common.cla
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -251,6 +252,8 @@ public class ObjectStore implements RawS
expressionProxy = createExpressionProxy(hiveConf);
directSql = new MetaStoreDirectSql(pm);
}
+ LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
+ " created in the thread with id: " + Thread.currentThread().getId());
}
/**
@@ -342,6 +345,8 @@ public class ObjectStore implements RawS
@Override
public void shutdown() {
if (pm != null) {
+ LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
+ " will be shutdown");
pm.close();
}
}
@@ -5904,25 +5909,28 @@ public class ObjectStore implements RawS
@Override
- public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, String tblName,
+ public AggrStats get_aggr_stats_for(String dbName, String tblName,
final List<String> partNames, final List<String> colNames) throws MetaException, NoSuchObjectException {
-
- return new GetListHelper<ColumnStatisticsObj>(dbName, tblName, true, false) {
+ return new GetHelper<AggrStats>(dbName, tblName, true, false) {
@Override
- protected List<ColumnStatisticsObj> getSqlResult(
- GetHelper<List<ColumnStatisticsObj>> ctx) throws MetaException {
- return directSql.aggrColStatsForPartitions(dbName, tblName, partNames, colNames);
+ protected AggrStats getSqlResult(GetHelper<AggrStats> ctx)
+ throws MetaException {
+ return directSql.aggrColStatsForPartitions(dbName, tblName, partNames,
+ colNames);
}
-
@Override
- protected List<ColumnStatisticsObj> getJdoResult(
- GetHelper<List<ColumnStatisticsObj>> ctx) throws MetaException,
- NoSuchObjectException {
- // This is fast path for query optimizations, if we can find this info quickly using
+ protected AggrStats getJdoResult(GetHelper<AggrStats> ctx)
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using
// directSql, do it. No point in failing back to slow path here.
throw new MetaException("Jdo path is not implemented for stats aggr.");
}
- }.run(true);
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(true);
}
private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Thu Aug 28 03:15:13 2014
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -548,6 +549,6 @@ public interface RawStore extends Config
*/
public List<String> getFunctions(String dbName, String pattern) throws MetaException;
- public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, String tblName,
+ public AggrStats get_aggr_stats_for(String dbName, String tblName,
List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException;
}
Modified: hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Thu Aug 28 03:15:13 2014
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -712,7 +713,7 @@ public class DummyRawStoreControlledComm
}
@Override
- public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName,
+ public AggrStats get_aggr_stats_for(String dbName,
String tblName, List<String> partNames, List<String> colNames)
throws MetaException {
return null;
Modified: hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1621031&r1=1621030&r2=1621031&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Thu Aug 28 03:15:13 2014
@@ -25,6 +25,7 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -729,7 +730,7 @@ public class DummyRawStoreForJdoConnecti
}
@Override
- public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName,
+ public AggrStats get_aggr_stats_for(String dbName,
String tblName, List<String> partNames, List<String> colNames)
throws MetaException {
return null;