You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/23 22:41:01 UTC
svn commit: r1525692 [1/8] - in /hive/branches/vectorization: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/test/results/clientpositive/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/results/positive/ h...
Author: hashutosh
Date: Mon Sep 23 20:40:54 2013
New Revision: 1525692
URL: http://svn.apache.org/r1525692
Log:
Merged in with latest trunk
Added:
hive/branches/vectorization/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade.order.derby
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/derby/upgrade.order.derby
hive/branches/vectorization/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade.order.mysql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/mysql/upgrade.order.mysql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade.order.oracle
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/oracle/upgrade.order.oracle
hive/branches/vectorization/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade.order.postgres
- copied unchanged from r1525674, hive/trunk/metastore/scripts/upgrade/postgres/upgrade.order.postgres
hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
- copied unchanged from r1525674, hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
hive/branches/vectorization/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
- copied unchanged from r1525674, hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
- copied unchanged from r1525674, hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java
- copied unchanged from r1525674, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFAbs.java
- copied unchanged from r1525674, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFAbs.java
hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java
- copied unchanged from r1525674, hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java
Removed:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java
Modified:
hive/branches/vectorization/ (props changed)
hive/branches/vectorization/build-common.xml
hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/vectorization/conf/hive-default.xml.template
hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
hive/branches/vectorization/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out
hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/branches/vectorization/metastore/src/model/package.jdo
hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileRecordReader.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/PerformTestRCFileAndSeqFile.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
hive/branches/vectorization/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/binary_table_colserde.q
hive/branches/vectorization/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join0.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join15.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join18.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join27.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join30.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join31.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join_reordering_values.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/binary_output_format.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/binary_table_colserde.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucket5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/case_sensitivity.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/cast1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/cluster.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer11.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer12.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer13.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer14.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer15.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/correlationoptimizer9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ctas_colname.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ctas_hadoop20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/decimal_udf.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby11.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby1_limit.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby1_map_skew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby6_map_skew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby7_map.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby7_noskew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby8.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby8_map.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby8_noskew.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_complex_types.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_cube1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_distinct_samekey.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_position.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_rollup1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_sort_1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_sort_3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_sort_5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_sort_7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_file_format.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_multiple.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_self_join.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_auto_update.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_bitmap3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_bitmap_auto.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_compression.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_serde.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/innerjoin.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input11.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input12_hadoop20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input13.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input14_limit.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input1_limit.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input34.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input35.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input36.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input38.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input39_hadoop20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input3_limit.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input4_limit.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input8.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_dynamicserde.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_part1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_part2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_part5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_testsequencefile.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_testxpath.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input_testxpath2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/insert1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/insert_into3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/insert_into4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/insert_into5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/insert_into6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join0.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join13.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join15.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join18.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join18_multi_distinct.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join19.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join20.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join21.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join22.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join23.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join25.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join26.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join27.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join29.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join31.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join32_lessSize.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join36.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join37.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join39.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join40.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_hive_626.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_map_ppr.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_nullsafe.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_reorder.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_reorder2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_reorder3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/join_vc.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/lateral_view.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/lateral_view_cp.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/limit_pushdown.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/lineage1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/load_dyn_part1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/load_dyn_part14.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/mapjoin_distinct.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/mergejoins.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/mergejoins_mixed.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/metadataonly1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert_gby.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert_gby2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert_gby3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/multigroupby_singlemr.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/no_hooks.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/nonblock_op_deduplicate.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/nullgroup.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/nullgroup2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/nullgroup4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/orc_createas1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/parallel.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/pcr.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_clusterby.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_constant_expr.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_gby2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_gby_join.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_join2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_join_filter.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_repeated_alias.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_udf_case.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ppd_vc.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/quote1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_merge1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_merge4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/regex_col.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/sample8.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/semijoin.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoinopt13.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoinopt9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/stats0.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/stats11.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/stats4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/subq.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf_10_trims.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf_explode.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf_length.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reverse.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_explode.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union12.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union17.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union18.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union24.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union28.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union29.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union30.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union31.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union33.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/union6.q.out
hive/branches/vectorization/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input1.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input2.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input3.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input6.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input7.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input9.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/join2.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample2.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample3.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample4.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample5.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample6.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/sample7.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/subq.q.xml
hive/branches/vectorization/ql/src/test/results/compiler/plan/union.q.xml
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java
Propchange: hive/branches/vectorization/
------------------------------------------------------------------------------
Merged /hive/trunk:r1525254-1525674
Modified: hive/branches/vectorization/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/build-common.xml?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/build-common.xml (original)
+++ hive/branches/vectorization/build-common.xml Mon Sep 23 20:40:54 2013
@@ -473,6 +473,7 @@
<sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}"/>
<sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}${junit.script.extension}"/>
<sysproperty key="test.concurrency.num.threads" value="${test.concurrency.num.threads}"/>
+ <sysproperty key="hive.home" value="${hive.root}/build/dist"/>
<jvmarg line="${junit.jvm.args}"/>
<classpath refid="test.local.classpath"/>
Modified: hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Sep 23 20:40:54 2013
@@ -324,6 +324,8 @@ public class HiveConf extends Configurat
METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false),
METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms"),
METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", true),
+ METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", false),
+ METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false),
METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked"),
METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed"),
METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false),
@@ -560,7 +562,6 @@ public class HiveConf extends Configurat
HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false),
// Optimizer
- HIVEOPTCP("hive.optimize.cp", true), // column pruner
HIVEOPTINDEXFILTER("hive.optimize.index.filter", false), // automatically use indexes
HIVEINDEXAUTOUPDATE("hive.optimize.index.autoupdate", false), //automatically update stale indexes
HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown
@@ -1145,6 +1146,11 @@ public class HiveConf extends Configurat
auxJars = this.get(ConfVars.HIVEAUXJARS.varname);
}
+ if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) {
+ setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false);
+ setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true);
+ }
+
// setup list of conf vars that are not allowed to change runtime
String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString());
if (restrictListStr != null) {
Modified: hive/branches/vectorization/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/conf/hive-default.xml.template?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/conf/hive-default.xml.template (original)
+++ hive/branches/vectorization/conf/hive-default.xml.template Mon Sep 23 20:40:54 2013
@@ -461,12 +461,6 @@
</property>
<property>
- <name>hive.optimize.cp</name>
- <value>true</value>
- <description>Whether to enable column pruner</description>
-</property>
-
-<property>
<name>hive.optimize.index.filter</name>
<value>false</value>
<description>Whether to enable automatic use of indexes</description>
@@ -1994,4 +1988,16 @@
</description>
</property>
+<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ <description>
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manully migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+ </description>
+</property>
+
</configuration>
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes.q.out Mon Sep 23 20:40:54 2013
@@ -103,6 +103,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -116,6 +117,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out Mon Sep 23 20:40:54 2013
@@ -103,6 +103,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -116,6 +117,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out Mon Sep 23 20:40:54 2013
@@ -103,6 +103,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -116,6 +117,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out Mon Sep 23 20:40:54 2013
@@ -103,6 +103,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -116,6 +117,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/udf_row_sequence.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/udf_row_sequence.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/udf_row_sequence.q.out Mon Sep 23 20:40:54 2013
@@ -80,6 +80,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col1
Modified: hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Mon Sep 23 20:40:54 2013
@@ -105,14 +105,14 @@ public class HiveHBaseTableInputFormat e
throw new IOException("Cannot read more columns than the given table contains.");
}
- boolean addAll = (readColIDs.size() == 0);
+ boolean readAllColumns = ColumnProjectionUtils.isReadAllColumns(jobConf);
Scan scan = new Scan();
boolean empty = true;
// The list of families that have been added to the scan
List<String> addedFamilies = new ArrayList<String>();
- if (!addAll) {
+ if (!readAllColumns) {
for (int i : readColIDs) {
ColumnMapping colMap = columnsMapping.get(i);
if (colMap.hbaseRowKey) {
@@ -151,7 +151,7 @@ public class HiveHBaseTableInputFormat e
scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
}
- if (!addAll) {
+ if (!readAllColumns) {
break;
}
}
@@ -170,19 +170,6 @@ public class HiveHBaseTableInputFormat e
scan.setBatch(Integer.valueOf(scanBatch));
}
- // If Hive's optimizer gave us a filter to process, convert it to the
- // HBase scan form now.
- int iKey = -1;
-
- try {
- iKey = HBaseSerDe.getRowKeyColumnOffset(columnsMapping);
- } catch (SerDeException e) {
- throw new IOException(e);
- }
-
- tableSplit = convertFilter(jobConf, scan, tableSplit, iKey,
- getStorageFormatOfKey(columnsMapping.get(iKey).mappingSpec,
- jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string")));
setScan(scan);
Job job = new Job(jobConf);
TaskAttemptContext tac = ShimLoader.getHadoopShims().newTaskAttemptContext(
@@ -254,26 +241,17 @@ public class HiveHBaseTableInputFormat e
*
* @param jobConf configuration for the scan
*
- * @param scan the HBase scan object to restrict
- *
- * @param tableSplit the HBase table split to restrict, or null
- * if calculating splits
- *
* @param iKey 0-based offset of key column within Hive table
*
* @return converted table split if any
*/
- private TableSplit convertFilter(
- JobConf jobConf,
- Scan scan,
- TableSplit tableSplit,
- int iKey, boolean isKeyBinary)
- throws IOException {
+ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary)
+ throws IOException {
- String filterExprSerialized =
- jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
+ Scan scan = new Scan();
+ String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
if (filterExprSerialized == null) {
- return tableSplit;
+ return scan;
}
ExprNodeDesc filterExpr =
Utilities.deserializeExpression(filterExprSerialized, jobConf);
@@ -340,16 +318,9 @@ public class HiveHBaseTableInputFormat e
throw new IOException(comparisonOp + " is not a supported comparison operator");
}
}
- if (tableSplit != null) {
- tableSplit = new TableSplit(
- tableSplit.getTableName(),
- startRow,
- stopRow,
- tableSplit.getRegionLocation());
- }
scan.setStartRow(startRow);
scan.setStopRow(stopRow);
- return tableSplit;
+ return scan;
}
private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi,
@@ -393,7 +364,7 @@ public class HiveHBaseTableInputFormat e
private byte[] getNextBA(byte[] current){
// startRow is inclusive while stopRow is exclusive,
- //this util method returns very next bytearray which will occur after the current one
+ // this util method returns very next bytearray which will occur after the current one
// by padding current one with a trailing 0 byte.
byte[] next = new byte[current.length + 1];
System.arraycopy(current, 0, next, 0, current.length);
@@ -463,7 +434,16 @@ public class HiveHBaseTableInputFormat e
throw new IOException(e);
}
- Scan scan = new Scan();
+ // Take filter pushdown into account while calculating splits; this
+ // allows us to prune off regions immediately. Note that although
+ // the Javadoc for the superclass getSplits says that it returns one
+ // split per region, the implementation actually takes the scan
+ // definition into account and excludes regions which don't satisfy
+ // the start/stop row conditions (HBASE-1829).
+ Scan scan = createFilterScan(jobConf, iKey,
+ getStorageFormatOfKey(columnsMapping.get(iKey).mappingSpec,
+ jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string")));
+
// The list of families that have been added to the scan
List<String> addedFamilies = new ArrayList<String>();
@@ -486,18 +466,8 @@ public class HiveHBaseTableInputFormat e
}
}
}
-
- // Take filter pushdown into account while calculating splits; this
- // allows us to prune off regions immediately. Note that although
- // the Javadoc for the superclass getSplits says that it returns one
- // split per region, the implementation actually takes the scan
- // definition into account and excludes regions which don't satisfy
- // the start/stop row conditions (HBASE-1829).
- convertFilter(jobConf, scan, null, iKey,
- getStorageFormatOfKey(columnsMapping.get(iKey).mappingSpec,
- jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string")));
-
setScan(scan);
+
Job job = new Job(jobConf);
JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job);
Path [] tablePaths = FileInputFormat.getInputPaths(jobContext);
Modified: hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out (original)
+++ hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out Mon Sep 23 20:40:54 2013
@@ -188,6 +188,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -364,6 +365,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -586,6 +588,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
$INTNAME
+ TableScan
Reduce Output Operator
key expressions:
expr: UDFToDouble(_col0)
Modified: hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out (original)
+++ hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out Mon Sep 23 20:40:54 2013
@@ -113,6 +113,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -126,6 +127,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java (original)
+++ hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java Mon Sep 23 20:40:54 2013
@@ -127,11 +127,13 @@ public abstract class HCatBaseInputForma
HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>());
for (HCatFieldSchema field :
- inputJobInfo.getTableInfo().getDataColumns().getFields())
+ inputJobInfo.getTableInfo().getDataColumns().getFields()) {
allCols.append(field);
+ }
for (HCatFieldSchema field :
- inputJobInfo.getTableInfo().getPartitionColumns().getFields())
+ inputJobInfo.getTableInfo().getPartitionColumns().getFields()) {
allCols.append(field);
+ }
HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
@@ -227,6 +229,7 @@ public abstract class HCatBaseInputForma
* @see org.apache.hive.hcatalog.mapreduce.HCatBaseInputFormat#getTableSchema(org.apache.hadoop.conf.Configuration)
* @deprecated Use {@link #getTableSchema(org.apache.hadoop.conf.Configuration)}
*/
+ @Deprecated
public static HCatSchema getTableSchema(JobContext context)
throws IOException {
return getTableSchema(context.getConfiguration());
@@ -247,11 +250,13 @@ public abstract class HCatBaseInputForma
InputJobInfo inputJobInfo = getJobInfo(conf);
HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>());
for (HCatFieldSchema field :
- inputJobInfo.getTableInfo().getDataColumns().getFields())
+ inputJobInfo.getTableInfo().getDataColumns().getFields()) {
allCols.append(field);
+ }
for (HCatFieldSchema field :
- inputJobInfo.getTableInfo().getPartitionColumns().getFields())
+ inputJobInfo.getTableInfo().getPartitionColumns().getFields()) {
allCols.append(field);
+ }
return allCols;
}
Modified: hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java (original)
+++ hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java Mon Sep 23 20:40:54 2013
@@ -38,6 +38,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class TestHCatPartitioned extends HCatMapReduceTest {
@@ -106,7 +107,7 @@ public class TestHCatPartitioned extends
exc = e;
}
- assertTrue(exc != null);
+ assertNotNull(exc);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_DUPLICATE_PARTITION, ((HCatException) exc).getErrorType());
@@ -122,7 +123,7 @@ public class TestHCatPartitioned extends
exc = e;
}
- assertTrue(exc != null);
+ assertNotNull(exc);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_MISSING_PARTITION_KEY, ((HCatException) exc).getErrorType());
@@ -137,7 +138,7 @@ public class TestHCatPartitioned extends
exc = e;
}
- assertTrue(exc != null);
+ assertNotNull(exc);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES, ((HCatException) exc).getErrorType());
@@ -263,11 +264,11 @@ public class TestHCatPartitioned extends
Integer i = 0;
for (HCatRecord rec : records) {
assertEquals(5, rec.size());
- assertTrue(rec.get(0).equals(i));
- assertTrue(rec.get(1).equals("c2value" + i));
- assertTrue(rec.get(2).equals("c3value" + i));
- assertTrue(rec.get(3).equals("p1value6"));
- assertTrue(rec.get(4).equals("p0value6"));
+ assertEquals(rec.get(0), i);
+ assertEquals(rec.get(1), "c2value" + i);
+ assertEquals(rec.get(2), "c3value" + i);
+ assertEquals(rec.get(3), "p1value6");
+ assertEquals(rec.get(4), "p0value6");
i++;
}
}
Modified: hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Mon Sep 23 20:40:54 2013
@@ -207,6 +207,8 @@ public class TestHCatLoader {
while (XIter.hasNext()) {
Tuple t = XIter.next();
assertEquals(2, t.size());
+ assertNotNull(t.get(0));
+ assertNotNull(t.get(1));
assertTrue(t.get(0).getClass() == Integer.class);
assertTrue(t.get(1).getClass() == String.class);
assertEquals(t.get(0), basicInputData.get(numTuplesRead).first);
@@ -299,6 +301,9 @@ public class TestHCatLoader {
while (WIter.hasNext()) {
Tuple t = WIter.next();
assertTrue(t.size() == 3);
+ assertNotNull(t.get(0));
+ assertNotNull(t.get(1));
+ assertNotNull(t.get(2));
assertTrue(t.get(0).getClass() == Integer.class);
assertTrue(t.get(1).getClass() == String.class);
assertTrue(t.get(2).getClass() == String.class);
@@ -366,6 +371,7 @@ public class TestHCatLoader {
while (Y2Iter.hasNext()) {
Tuple t = Y2Iter.next();
assertEquals(t.size(), 1);
+ assertNotNull(t.get(0));
assertTrue(t.get(0).getClass() == Integer.class);
assertEquals(t.get(0), basicInputData.get(numTuplesRead).first);
numTuplesRead++;
@@ -375,8 +381,10 @@ public class TestHCatLoader {
while (Y3Iter.hasNext()) {
Tuple t = Y3Iter.next();
assertEquals(t.size(), 2);
+ assertNotNull(t.get(0));
assertTrue(t.get(0).getClass() == String.class);
assertEquals(t.get(0), basicInputData.get(numTuplesRead).second);
+ assertNotNull(t.get(1));
assertTrue(t.get(1).getClass() == Integer.class);
assertEquals(t.get(1), basicInputData.get(numTuplesRead).first);
numTuplesRead++;
Modified: hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql Mon Sep 23 20:40:54 2013
@@ -98,6 +98,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"("DB_N
CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
+
-- ----------------------------------------------
-- DDL Statements for indexes
-- ----------------------------------------------
@@ -284,6 +286,8 @@ ALTER TABLE "APP"."TAB_COL_STATS" ADD CO
ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
+ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID");
+
-- ----------------------------------------------
-- DDL Statements for checks
-- ----------------------------------------------
@@ -292,3 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N'));
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
Modified: hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql Mon Sep 23 20:40:54 2013
@@ -1,2 +1 @@
-- Upgrade MetaStore schema from 0.10.0 to 0.11.0
-
Modified: hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql Mon Sep 23 20:40:54 2013
@@ -1,2 +1,4 @@
-- Upgrade MetaStore schema from 0.11.0 to 0.12.0
RUN '013-HIVE-3255.derby.sql';
+RUN '014-HIVE-3764.derby.sql';
+UPDATE "APP".VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
Modified: hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql Mon Sep 23 20:40:54 2013
@@ -751,6 +751,17 @@ CREATE TABLE IF NOT EXISTS `DELEGATION_T
PRIMARY KEY (`TOKEN_IDENT`)
) ENGINE=INNODB DEFAULT CHARSET=latin1;
+--
+-- Table structure for VERSION
+--
+CREATE TABLE IF NOT EXISTS `VERSION` (
+ `VER_ID` BIGINT NOT NULL,
+ `SCHEMA_VERSION` VARCHAR(127) NOT NULL,
+ `VERSION_COMMENT` VARCHAR(255),
+ PRIMARY KEY (`VER_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
Modified: hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql Mon Sep 23 20:40:54 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
SOURCE 013-HIVE-3255.mysql.sql;
+SOURCE 014-HIVE-3764.mysql.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
Modified: hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql Mon Sep 23 20:40:54 2013
@@ -483,6 +483,13 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
+CREATE TABLE IF NOT EXISTS VERSION (
+ VER_ID NUMBER NOT NULL,
+ SCHEMA_VERSION VARCHAR(127) NOT NULL,
+ VERSION_COMMENT VARCHAR(255)
+)
+ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
+
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
@@ -707,4 +714,5 @@ CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
-- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent]
CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME);
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
Modified: hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql Mon Sep 23 20:40:54 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
@013-HIVE-3255.oracle.sql;
+@014-HIVE-3764.oracle.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
Modified: hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql Mon Sep 23 20:40:54 2013
@@ -516,6 +516,16 @@ CREATE TABLE "TAB_COL_STATS" (
);
--
+-- Table structure for VERSION
+--
+CREATE TABLE "VERSION" (
+ "VER_ID" bigint,
+ "SCHEMA_VERSION" character varying(127) NOT NULL,
+ "VERSION_COMMENT" character varying(255) NOT NULL,
+ PRIMARY KEY ("VER_ID")
+);
+
+--
-- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1379,6 +1389,9 @@ ALTER TABLE ONLY "TAB_COL_STATS" ADD CON
--
ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE;
+
+ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID");
+
--
-- Name: public; Type: ACL; Schema: -; Owner: hiveuser
--
@@ -1387,6 +1400,7 @@ REVOKE ALL ON SCHEMA public FROM PUBLIC;
GRANT ALL ON SCHEMA public TO PUBLIC;
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
--
-- PostgreSQL database dump complete
--
Modified: hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql (original)
+++ hive/branches/vectorization/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql Mon Sep 23 20:40:54 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0';
\i 013-HIVE-3255.postgres.sql;
+\i 014-HIVE-3764.postgres.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0';
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Mon Sep 23 20:40:54 2013
@@ -400,6 +400,7 @@ public class HiveMetaStore extends Thrif
RawStore ms = threadLocalMS.get();
if (ms == null) {
ms = newRawStore();
+ ms.verifySchema();
threadLocalMS.set(ms);
ms = threadLocalMS.get();
}
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Mon Sep 23 20:40:54 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.commons.lang.StringUtils.repeat;
import java.net.URI;
import java.net.URISyntaxException;
@@ -34,10 +33,11 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import javax.jdo.JDODataStoreException;
import javax.jdo.JDOHelper;
import javax.jdo.JDOObjectNotFoundException;
import javax.jdo.PersistenceManager;
@@ -119,10 +119,12 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
+import org.apache.hadoop.hive.metastore.model.MVersionTable;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
import org.apache.hadoop.util.StringUtils;
+import org.datanucleus.store.rdbms.exceptions.MissingTableException;
/**
* This class is the interface between the application logic and the database
@@ -163,6 +165,7 @@ public class ObjectStore implements RawS
int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
+ private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false);
public ObjectStore() {
}
@@ -5609,4 +5612,131 @@ public class ObjectStore implements RawS
return masterKeys;
}
+ // compare hive version and metastore version
+ @Override
+ public void verifySchema() throws MetaException {
+ // If the schema version is already checked, then go ahead and use this metastore
+ if (isSchemaVerified.get()) {
+ return;
+ }
+ checkSchema();
+ }
+
+ private synchronized void checkSchema() throws MetaException {
+ // recheck if it got verified by another thread while we were waiting
+ if (isSchemaVerified.get()) {
+ return;
+ }
+
+ boolean strictValidation =
+ HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
+ // read the schema version stored in metastore db
+ String schemaVer = getMetaStoreSchemaVersion();
+ if (schemaVer == null) {
+ // metastore has no schema version information
+ if (strictValidation) {
+ throw new MetaException("Version information not found in metastore. ");
+ } else {
+ LOG.warn("Version information not found in metastore. "
+ + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
+ " is not enabled so recording the schema version " +
+ MetaStoreSchemaInfo.getHiveSchemaVersion());
+ setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+ "Set by MetaStore");
+ }
+ } else {
+ // metastore schema version is different than Hive distribution needs
+ if (strictValidation) {
+ if (!schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) {
+ throw new MetaException("Hive Schema version "
+ + MetaStoreSchemaInfo.getHiveSchemaVersion() +
+ " does not match metastore's schema version " + schemaVer +
+ " Metastore is not upgraded or corrupt");
+ } else {
+ LOG.warn("Metastore version was " + schemaVer + " " +
+ HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
+ " is not enabled so recording the new schema version " +
+ MetaStoreSchemaInfo.getHiveSchemaVersion());
+ setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+ "Set by MetaStore");
+ }
+ }
+ }
+ isSchemaVerified.set(true);
+ return;
+ }
+
+ // load the schema version stored in metastore db
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+
+ MVersionTable mSchemaVer;
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ return null;
+ }
+ return mSchemaVer.getSchemaVersion();
+ }
+
+ @SuppressWarnings("unchecked")
+ private MVersionTable getMSchemaVersion()
+ throws NoSuchObjectException, MetaException {
+ boolean committed = false;
+ List<MVersionTable> mVerTables = new ArrayList<MVersionTable>();
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MVersionTable.class);
+
+ try {
+ mVerTables = (List<MVersionTable>)query.execute();
+ pm.retrieveAll(mVerTables);
+ } catch (JDODataStoreException e) {
+ if (e.getCause() instanceof MissingTableException) {
+ throw new MetaException("Version table not found. " +
+ "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion());
+ } else {
+ throw e;
+ }
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ if (mVerTables.isEmpty()) {
+ throw new NoSuchObjectException("No matching version found");
+ }
+ if (mVerTables.size() > 1) {
+ throw new MetaException("Metastore contains multiple versions");
+ }
+ return mVerTables.get(0);
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ MVersionTable mSchemaVer;
+ boolean commited = false;
+
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ // if the version doesn't exist, then create it
+ mSchemaVer = new MVersionTable();
+ }
+
+ mSchemaVer.setSchemaVersion(schemaVersion);
+ mSchemaVer.setVersionComment(comment);
+ try {
+ openTransaction();
+ pm.makePersistent(mSchemaVer);
+ commited = commitTransaction();
+ } finally {
+ if (!commited) {
+ rollbackTransaction();
+ }
+ }
+ }
}
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Mon Sep 23 20:40:54 2013
@@ -436,4 +436,10 @@ public interface RawStore extends Config
public abstract String[] getMasterKeys();
+ public abstract void verifySchema() throws MetaException;
+
+ public abstract String getMetaStoreSchemaVersion() throws MetaException;
+
+ public abstract void setMetaStoreSchemaVersion(String version, String comment) throws MetaException;
+
}
Modified: hive/branches/vectorization/metastore/src/model/package.jdo
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/model/package.jdo?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/model/package.jdo (original)
+++ hive/branches/vectorization/metastore/src/model/package.jdo Mon Sep 23 20:40:54 2013
@@ -899,6 +899,17 @@
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
</class>
+ <class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="VER_ID"/>
+ </datastore-identity>
+ <field name ="schemaVersion">
+ <column name="SCHEMA_VERSION" length="127" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name ="versionComment">
+ <column name="VERSION_COMMENT" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ </class>
</package>
</jdo>
Modified: hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Mon Sep 23 20:40:54 2013
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.metastore;
-import java.util.List;
import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configurable;
@@ -532,11 +532,11 @@ public class DummyRawStoreControlledComm
public boolean addToken(String tokenIdentifier, String delegationToken) {
return false;
}
-
+
public boolean removeToken(String tokenIdentifier) {
return false;
}
-
+
public String getToken(String tokenIdentifier) {
return "";
}
@@ -560,4 +560,19 @@ public class DummyRawStoreControlledComm
return new String[0];
}
+ @Override
+ public void verifySchema() throws MetaException {
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+ return objectStore.getMetaStoreSchemaVersion();
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ objectStore.setMetaStoreSchemaVersion(schemaVersion, comment);
+
+ }
+
}
Modified: hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Mon Sep 23 20:40:54 2013
@@ -593,6 +593,19 @@ public class DummyRawStoreForJdoConnecti
throws NoSuchObjectException, MetaException, InvalidObjectException {
return false;
}
+
+ @Override
+ public void verifySchema() throws MetaException {
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+ return null;
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String version, String comment) throws MetaException {
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Sep 23 20:40:54 2013
@@ -615,14 +615,8 @@ public class Driver implements CommandPr
List<Integer> neededColumnIds = tableScanOp.getNeededColumnIDs();
List<FieldSchema> columns = tbl.getCols();
List<String> cols = new ArrayList<String>();
- if (neededColumnIds != null && neededColumnIds.size() > 0) {
- for (int i = 0; i < neededColumnIds.size(); i++) {
- cols.add(columns.get(neededColumnIds.get(i)).getName());
- }
- } else {
- for (int i = 0; i < columns.size(); i++) {
- cols.add(columns.get(i).getName());
- }
+ for (int i = 0; i < neededColumnIds.size(); i++) {
+ cols.add(columns.get(neededColumnIds.get(i)).getName());
}
//map may not contain all sources, since input list may have been optimized out
//or non-existent tho such sources may still be referenced by the TableScanOperator
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Mon Sep 23 20:40:54 2013
@@ -1471,8 +1471,16 @@ public class DDLTask extends Task<DDLWor
URI originalPartitionUri = ArchiveUtils.addSlash(p.getPartitionPath().toUri());
URI test = p.getPartitionPath().toUri();
URI harPartitionDir = harHelper.getHarUri(originalPartitionUri, shim);
+ StringBuilder authority = new StringBuilder();
+ if(harPartitionDir.getUserInfo() != null) {
+ authority.append(harPartitionDir.getUserInfo()).append("@");
+ }
+ authority.append(harPartitionDir.getHost()).append(":");
+ if(harPartitionDir.getPort() != -1) {
+ authority.append(harPartitionDir.getPort());
+ }
Path harPath = new Path(harPartitionDir.getScheme(),
- harPartitionDir.getAuthority(),
+ authority.toString(),
harPartitionDir.getPath()); // make in Path to ensure no slash at the end
setArchived(p, harPath, partSpecInfo.values.size());
db.alterPartition(tblName, p);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1525692&r1=1525691&r2=1525692&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Mon Sep 23 20:40:54 2013
@@ -27,7 +27,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
@@ -70,8 +69,11 @@ public class FetchTask extends Task<Fetc
Operator<?> source = work.getSource();
if (source instanceof TableScanOperator) {
TableScanOperator ts = (TableScanOperator) source;
+ // push down projections
+ ColumnProjectionUtils.appendReadColumns(
+ job, ts.getNeededColumnIDs(), ts.getNeededColumns());
+ // push down filters
HiveInputFormat.pushFilters(job, ts);
- ColumnProjectionUtils.appendReadColumnIDs(job, ts.getNeededColumnIDs());
}
sink = work.getSink();
fetch = new FetchOperator(work, job, source, getVirtualColumns(source));