You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/06/05 20:28:15 UTC
svn commit: r1600719 [1/40] - in /hive/trunk:
contrib/src/test/results/clientpositive/
hbase-handler/src/test/results/positive/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/java/org/apache/h...
Author: hashutosh
Date: Thu Jun 5 18:28:07 2014
New Revision: 1600719
URL: http://svn.apache.org/r1600719
Log:
HIVE-4867 : Deduplicate columns appearing in both the key list and value list of ReduceSinkOperator (Navis via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java
Modified:
hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
hive/trunk/ql/src/test/queries/clientpositive/bucketmapjoin6.q
hive/trunk/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
hive/trunk/ql/src/test/results/clientpositive/ambiguous_col.q.out
hive/trunk/ql/src/test/results/clientpositive/annotate_stats_join.q.out
hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join0.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join13.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join15.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join18.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join2.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join20.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join21.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join23.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join28.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join30.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join31.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join6.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join7.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join_reordering_values.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
hive/trunk/ql/src/test/results/clientpositive/bucket5.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin6.q.out
hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
hive/trunk/ql/src/test/results/clientpositive/column_access_stats.q.out
hive/trunk/ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer1.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer10.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer11.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer12.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer13.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer14.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer15.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer2.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer4.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer5.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer9.q.out
hive/trunk/ql/src/test/results/clientpositive/count.q.out
hive/trunk/ql/src/test/results/clientpositive/create_view.q.out
hive/trunk/ql/src/test/results/clientpositive/cross_join.q.out
hive/trunk/ql/src/test/results/clientpositive/cross_product_check_1.q.out
hive/trunk/ql/src/test/results/clientpositive/ctas.q.out
hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
hive/trunk/ql/src/test/results/clientpositive/ctas_hadoop20.q.out
hive/trunk/ql/src/test/results/clientpositive/decimal_udf.q.out
hive/trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out
hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
hive/trunk/ql/src/test/results/clientpositive/escape_clusterby1.q.out
hive/trunk/ql/src/test/results/clientpositive/escape_distributeby1.q.out
hive/trunk/ql/src/test/results/clientpositive/escape_orderby1.q.out
hive/trunk/ql/src/test/results/clientpositive/escape_sortby1.q.out
hive/trunk/ql/src/test/results/clientpositive/explain_logical.q.out
hive/trunk/ql/src/test/results/clientpositive/explain_rearrange.q.out
hive/trunk/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_distinct_samekey.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_position.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_resolution.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_sort_11.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out
hive/trunk/ql/src/test/results/clientpositive/innerjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/input14.q.out
hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/input17.q.out
hive/trunk/ql/src/test/results/clientpositive/input18.q.out
hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/input20.q.out
hive/trunk/ql/src/test/results/clientpositive/input21.q.out
hive/trunk/ql/src/test/results/clientpositive/input22.q.out
hive/trunk/ql/src/test/results/clientpositive/input25.q.out
hive/trunk/ql/src/test/results/clientpositive/input26.q.out
hive/trunk/ql/src/test/results/clientpositive/input33.q.out
hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/input5.q.out
hive/trunk/ql/src/test/results/clientpositive/input_columnarserde.q.out
hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part10.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part7.q.out
hive/trunk/ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out
hive/trunk/ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into1.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into2.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into3.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into4.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into5.q.out
hive/trunk/ql/src/test/results/clientpositive/insert_into6.q.out
hive/trunk/ql/src/test/results/clientpositive/join0.q.out
hive/trunk/ql/src/test/results/clientpositive/join1.q.out
hive/trunk/ql/src/test/results/clientpositive/join10.q.out
hive/trunk/ql/src/test/results/clientpositive/join11.q.out
hive/trunk/ql/src/test/results/clientpositive/join12.q.out
hive/trunk/ql/src/test/results/clientpositive/join13.q.out
hive/trunk/ql/src/test/results/clientpositive/join14.q.out
hive/trunk/ql/src/test/results/clientpositive/join14_hadoop20.q.out
hive/trunk/ql/src/test/results/clientpositive/join15.q.out
hive/trunk/ql/src/test/results/clientpositive/join16.q.out
hive/trunk/ql/src/test/results/clientpositive/join17.q.out
hive/trunk/ql/src/test/results/clientpositive/join18.q.out
hive/trunk/ql/src/test/results/clientpositive/join18_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/join19.q.out
hive/trunk/ql/src/test/results/clientpositive/join2.q.out
hive/trunk/ql/src/test/results/clientpositive/join20.q.out
hive/trunk/ql/src/test/results/clientpositive/join21.q.out
hive/trunk/ql/src/test/results/clientpositive/join22.q.out
hive/trunk/ql/src/test/results/clientpositive/join23.q.out
hive/trunk/ql/src/test/results/clientpositive/join29.q.out
hive/trunk/ql/src/test/results/clientpositive/join3.q.out
hive/trunk/ql/src/test/results/clientpositive/join31.q.out
hive/trunk/ql/src/test/results/clientpositive/join32.q.out
hive/trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out
hive/trunk/ql/src/test/results/clientpositive/join33.q.out
hive/trunk/ql/src/test/results/clientpositive/join4.q.out
hive/trunk/ql/src/test/results/clientpositive/join40.q.out
hive/trunk/ql/src/test/results/clientpositive/join41.q.out
hive/trunk/ql/src/test/results/clientpositive/join5.q.out
hive/trunk/ql/src/test/results/clientpositive/join6.q.out
hive/trunk/ql/src/test/results/clientpositive/join7.q.out
hive/trunk/ql/src/test/results/clientpositive/join8.q.out
hive/trunk/ql/src/test/results/clientpositive/join9.q.out
hive/trunk/ql/src/test/results/clientpositive/join_alt_syntax.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
hive/trunk/ql/src/test/results/clientpositive/join_filters_overlap.q.out
hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
hive/trunk/ql/src/test/results/clientpositive/join_merging.q.out
hive/trunk/ql/src/test/results/clientpositive/join_nullsafe.q.out
hive/trunk/ql/src/test/results/clientpositive/join_rc.q.out
hive/trunk/ql/src/test/results/clientpositive/join_reorder.q.out
hive/trunk/ql/src/test/results/clientpositive/join_reorder2.q.out
hive/trunk/ql/src/test/results/clientpositive/join_reorder3.q.out
hive/trunk/ql/src/test/results/clientpositive/join_star.q.out
hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out
hive/trunk/ql/src/test/results/clientpositive/join_view.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out
hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out
hive/trunk/ql/src/test/results/clientpositive/lineage1.q.out
hive/trunk/ql/src/test/results/clientpositive/load_dyn_part14.q.out
hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/mapjoin_filter_on_outerjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/mapjoin_test_outer.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce1.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce2.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce3.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce4.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce5.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce6.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce7.q.out
hive/trunk/ql/src/test/results/clientpositive/mapreduce8.q.out
hive/trunk/ql/src/test/results/clientpositive/merge3.q.out
hive/trunk/ql/src/test/results/clientpositive/merge4.q.out
hive/trunk/ql/src/test/results/clientpositive/mergejoins.q.out
hive/trunk/ql/src/test/results/clientpositive/mergejoins_mixed.q.out
hive/trunk/ql/src/test/results/clientpositive/multiMapJoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/multiMapJoin2.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby2.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby3.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
hive/trunk/ql/src/test/results/clientpositive/multi_join_union.q.out
hive/trunk/ql/src/test/results/clientpositive/no_hooks.q.out
hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch.q.out
hive/trunk/ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out
hive/trunk/ql/src/test/results/clientpositive/optional_outer.q.out
hive/trunk/ql/src/test/results/clientpositive/orc_createas1.q.out
hive/trunk/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out
hive/trunk/ql/src/test/results/clientpositive/order.q.out
hive/trunk/ql/src/test/results/clientpositive/order2.q.out
hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/parallel_orderby.q.out
hive/trunk/ql/src/test/results/clientpositive/pcr.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join3.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join_filter.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_multi_insert.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join1.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join3.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join4.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_random.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_repeated_alias.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_vc.q.out
hive/trunk/ql/src/test/results/clientpositive/push_or.q.out
hive/trunk/ql/src/test/results/clientpositive/quotedid_skew.q.out
hive/trunk/ql/src/test/results/clientpositive/rcfile_null_value.q.out
hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out
hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out
hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out
hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
hive/trunk/ql/src/test/results/clientpositive/regexp_extract.q.out
hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/sample10.q.out
hive/trunk/ql/src/test/results/clientpositive/sample6.q.out
hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
hive/trunk/ql/src/test/results/clientpositive/script_pipe.q.out
hive/trunk/ql/src/test/results/clientpositive/select_as_omitted.q.out
hive/trunk/ql/src/test/results/clientpositive/select_transform_hint.q.out
hive/trunk/ql/src/test/results/clientpositive/semijoin.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoin.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoin_noskew.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt1.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt10.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt11.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt12.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt13.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt14.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt15.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt16.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt17.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt19.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt2.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt20.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt3.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt4.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt5.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt6.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt7.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt8.q.out
hive/trunk/ql/src/test/results/clientpositive/skewjoinopt9.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_17.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
hive/trunk/ql/src/test/results/clientpositive/sort.q.out
hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_exists.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_exists_having.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_in.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_in_having.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_notexists.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_notin.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
hive/trunk/ql/src/test/results/clientpositive/symlink_text_input_format.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/count.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/cross_join.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/ctas.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/insert_into1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/insert_into2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/join0.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/join1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapreduce1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapreduce2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mrr.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/ptf.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/script_pipe.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/subquery_exists.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/subquery_in.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_dml.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_union.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/transform_ppr1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/union3.q.out
hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
hive/trunk/ql/src/test/results/clientpositive/transform_ppr2.q.out
hive/trunk/ql/src/test/results/clientpositive/type_widening.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
hive/trunk/ql/src/test/results/clientpositive/union20.q.out
hive/trunk/ql/src/test/results/clientpositive/union22.q.out
hive/trunk/ql/src/test/results/clientpositive/union23.q.out
hive/trunk/ql/src/test/results/clientpositive/union24.q.out
hive/trunk/ql/src/test/results/clientpositive/union26.q.out
hive/trunk/ql/src/test/results/clientpositive/union3.q.out
hive/trunk/ql/src/test/results/clientpositive/union32.q.out
hive/trunk/ql/src/test/results/clientpositive/union34.q.out
hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
hive/trunk/ql/src/test/results/clientpositive/union_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/union_remove_23.q.out
hive/trunk/ql/src/test/results/clientpositive/union_remove_25.q.out
hive/trunk/ql/src/test/results/clientpositive/union_top_level.q.out
hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_non_string_partition.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorization_div0.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorization_limit.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorization_part_project.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorized_context.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
Modified: hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out Thu Jun 5 18:28:07 2014
@@ -67,9 +67,10 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
- Extract
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Modified: hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out Thu Jun 5 18:28:07 2014
@@ -53,20 +53,17 @@ STAGE PLANS:
key expressions: _col0 (type: string)
sort order: +
Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string)
Reduce Operator Tree:
- Extract
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), row_sequence() (type: bigint)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), row_sequence() (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
@@ -76,9 +73,11 @@ STAGE PLANS:
key expressions: _col1 (type: bigint)
sort order: +
Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: bigint)
+ value expressions: _col0 (type: string)
Reduce Operator Tree:
- Extract
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out Thu Jun 5 18:28:07 2014
@@ -168,9 +168,10 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
- Extract
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
@@ -273,7 +274,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: string)
+ value expressions: _col1 (type: string)
TableScan
alias: hbase_table_1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -295,7 +296,7 @@ STAGE PLANS:
Inner Join 0 to 1
condition expressions:
0
- 1 {VALUE._col0} {VALUE._col1}
+ 1 {KEY.reducesinkkey0} {VALUE._col0}
outputColumnNames: _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
@@ -317,9 +318,10 @@ STAGE PLANS:
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: string)
Reduce Operator Tree:
- Extract
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Thu Jun 5 18:28:07 2014
@@ -23,16 +23,11 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-
public abstract class AbstractMapJoinOperator <T extends MapJoinDesc> extends CommonJoinOperator<T> implements
Serializable {
@@ -46,10 +41,6 @@ public abstract class AbstractMapJoinOpe
* The ObjectInspectors for the join inputs's join keys.
*/
protected transient List<ObjectInspector>[] joinKeysObjectInspectors;
- /**
- * The standard ObjectInspectors for the join inputs's join keys.
- */
- protected transient List<ObjectInspector>[] joinKeysStandardObjectInspectors;
protected transient byte posBigTable = -1; // pos of driver alias
@@ -70,11 +61,6 @@ public abstract class AbstractMapJoinOpe
@Override
@SuppressWarnings("unchecked")
protected void initializeOp(Configuration hconf) throws HiveException {
- super.initializeOp(hconf);
-
- numMapRowsRead = 0;
- firstRow = true;
-
int tagLen = conf.getTagLength();
joinKeys = new List[tagLen];
@@ -82,8 +68,11 @@ public abstract class AbstractMapJoinOpe
JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE);
joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys,
inputObjInspectors,NOTSKIPBIGTABLE, tagLen);
- joinKeysStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
- joinKeysObjectInspectors,NOTSKIPBIGTABLE, tagLen);
+
+ super.initializeOp(hconf);
+
+ numMapRowsRead = 0;
+ firstRow = true;
// all other tables are small, and are cached in the hash table
posBigTable = (byte) conf.getPosBigTable();
@@ -96,27 +85,25 @@ public abstract class AbstractMapJoinOpe
!hasFilter(posBigTable), reporter);
storage[posBigTable] = bigPosRC;
- List<? extends StructField> structFields = ((StructObjectInspector) outputObjInspector)
- .getAllStructFieldRefs();
- if (conf.getOutputColumnNames().size() < structFields.size()) {
- List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
- for (Byte alias : order) {
- int sz = conf.getExprs().get(alias).size();
- List<Integer> retained = conf.getRetainList().get(alias);
- for (int i = 0; i < sz; i++) {
- int pos = retained.get(i);
- structFieldObjectInspectors.add(structFields.get(pos)
- .getFieldObjectInspector());
- }
- }
- outputObjInspector = ObjectInspectorFactory
- .getStandardStructObjectInspector(conf.getOutputColumnNames(),
- structFieldObjectInspectors);
- }
initializeChildren(hconf);
}
@Override
+ protected List<ObjectInspector> getValueObjectInspectors(
+ byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+ List<ObjectInspector> inspectors = aliasToObjectInspectors[alias];
+ List<Integer> retained = conf.getRetainList().get(alias);
+ if (inspectors.size() == retained.size()) {
+ return inspectors;
+ }
+ List<ObjectInspector> retainedOIs = new ArrayList<ObjectInspector>();
+ for (int index : retained) {
+ retainedOIs.add(inspectors.get(index));
+ }
+ return retainedOIs;
+ }
+
+ @Override
public OperatorType getType() {
return OperatorType.MAPJOIN;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java Thu Jun 5 18:28:07 2014
@@ -198,6 +198,11 @@ public class ColumnInfo implements Seria
}
@Override
+ public int hashCode() {
+ return internalName.hashCode() + typeName.hashCode();
+ }
+
+ @Override
public boolean equals(Object obj) {
if (!(obj instanceof ColumnInfo) || (obj == null)) {
return false;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Thu Jun 5 18:28:07 2014
@@ -160,14 +160,12 @@ public abstract class CommonJoinOperator
this.joinFilterObjectInspectors = clone.joinFilterObjectInspectors;
}
-
- protected static <T extends JoinDesc> ObjectInspector getJoinOutputObjectInspector(
- Byte[] order, List<ObjectInspector>[] aliasToObjectInspectors,
- T conf) {
+ private <T extends JoinDesc> ObjectInspector getJoinOutputObjectInspector(
+ Byte[] order, List<ObjectInspector>[] aliasToObjectInspectors, T conf) {
List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
for (Byte alias : order) {
- List<ObjectInspector> oiList = aliasToObjectInspectors[alias];
- if (oiList != null) {
+ List<ObjectInspector> oiList = getValueObjectInspectors(alias, aliasToObjectInspectors);
+ if (oiList != null && !oiList.isEmpty()) {
structFieldObjectInspectors.addAll(oiList);
}
}
@@ -178,6 +176,11 @@ public abstract class CommonJoinOperator
return joinOutputObjectInspector;
}
+ protected List<ObjectInspector> getValueObjectInspectors(
+ byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+ return aliasToObjectInspectors[alias];
+ }
+
protected Configuration hconf;
@Override
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Thu Jun 5 18:28:07 2014
@@ -34,8 +34,6 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionHandler;
import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKeyObject;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinPersistableTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinEagerRowContainer;
@@ -226,8 +224,13 @@ public class HashTableSinkOperator exten
public void processOp(Object row, int tag) throws HiveException {
byte alias = (byte)tag;
// compute keys and values as StandardObjects. Use non-optimized key (MR).
- MapJoinKey key = MapJoinKey.readFromRow(null, new MapJoinKeyObject(),
- row, joinKeys[alias], joinKeysObjectInspectors[alias], true);
+ Object[] currentKey = new Object[joinKeys[alias].size()];
+ for (int keyIndex = 0; keyIndex < joinKeys[alias].size(); ++keyIndex) {
+ currentKey[keyIndex] = joinKeys[alias].get(keyIndex).evaluate(row);
+ }
+ MapJoinKeyObject key = new MapJoinKeyObject();
+ key.readFromRow(currentKey, joinKeysObjectInspectors[alias]);
+
Object[] value = EMPTY_OBJECT_ARRAY;
if((hasFilter(alias) && filterMaps[alias].length > 0) || joinValues[alias].size() > 0) {
value = JoinUtil.computeMapJoinValues(row, joinValues[alias],
@@ -269,8 +272,10 @@ public class HashTableSinkOperator exten
flushToFile();
}
super.closeOp(abort);
+ } catch (HiveException e) {
+ throw e;
} catch (Exception e) {
- LOG.error("Error generating side-table", e);
+ throw new HiveException(e);
}
}
@@ -290,7 +295,8 @@ public class HashTableSinkOperator exten
// get the tmp URI path; it will be a hdfs path if not local mode
String dumpFilePrefix = conf.getDumpFilePrefix();
Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName);
- console.printInfo(Utilities.now() + "\tDump the side-table into file: " + path);
+ console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag +
+ " with group count: " + tableContainer.size() + " into file: " + path);
// get the hashtable file and path
FileSystem fs = path.getFileSystem(hconf);
ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(fs.create(path), 4096));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Jun 5 18:28:07 2014
@@ -19,28 +19,29 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
+import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.util.ReflectionUtils;
/**
@@ -64,6 +65,8 @@ public class MapJoinOperator extends Abs
private transient boolean hashTblInitedOnce;
private transient ReusableGetAdaptor[] hashMapRowGetters;
+ private UnwrapRowContainer[] unwrapContainer;
+
public MapJoinOperator() {
}
@@ -88,6 +91,7 @@ public class MapJoinOperator extends Abs
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
+ unwrapContainer = new UnwrapRowContainer[conf.getTagLength()];
super.initializeOp(hconf);
int tagLen = conf.getTagLength();
@@ -113,6 +117,29 @@ public class MapJoinOperator extends Abs
}
}
+ @Override
+ protected List<ObjectInspector> getValueObjectInspectors(
+ byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+ int[] valueIndex = conf.getValueIndex(alias);
+ if (valueIndex == null) {
+ return super.getValueObjectInspectors(alias, aliasToObjectInspectors);
+ }
+ unwrapContainer[alias] = new UnwrapRowContainer(valueIndex);
+
+ List<ObjectInspector> inspectors = aliasToObjectInspectors[alias];
+
+ int bigPos = conf.getPosBigTable();
+ List<ObjectInspector> valueOI = new ArrayList<ObjectInspector>();
+ for (int i = 0; i < valueIndex.length; i++) {
+ if (valueIndex[i] >= 0) {
+ valueOI.add(joinKeysObjectInspectors[bigPos].get(valueIndex[i]));
+ } else {
+ valueOI.add(inspectors.get(i));
+ }
+ }
+ return valueOI;
+ }
+
public void generateMapMetaData() throws HiveException, SerDeException {
// generate the meta data for key
// index for key is -1
@@ -237,6 +264,10 @@ public class MapJoinOperator extends Abs
adaptor.setFromOther(firstSetKey);
}
MapJoinRowContainer rowContainer = adaptor.getCurrentRows();
+ if (rowContainer != null && unwrapContainer[pos] != null) {
+ Object[] currentKey = adaptor.getCurrentKey();
+ rowContainer = unwrapContainer[pos].setInternal(rowContainer, currentKey);
+ }
// there is no join-value or join-key has all null elements
if (rowContainer == null || adaptor.hasAnyNulls(fieldCount, nullsafes)) {
if (!noOuterJoin) {
@@ -267,7 +298,7 @@ public class MapJoinOperator extends Abs
}
}
} catch (Exception e) {
- String msg = "Unxpected exception: " + e.getMessage();
+ String msg = "Unexpected exception: " + e.getMessage();
LOG.error(msg, e);
throw new HiveException(msg, e);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Thu Jun 5 18:28:07 2014
@@ -383,7 +383,7 @@ public class MapOperator extends Operato
if (!onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) {
children.add(op);
childrenOpToOpCtxMap.put(op, opCtx);
- LOG.info("dump " + op.getName() + " "
+ LOG.info("dump " + op + " "
+ opCtxMap.get(inp).rowObjectInspector.getTypeName());
}
current = opCtx; // just need for TestOperators.testMapOperator
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Thu Jun 5 18:28:07 2014
@@ -32,7 +32,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -98,6 +97,11 @@ public abstract class Operator<T extends
private boolean useBucketizedHiveInputFormat;
+ // dummy operator (for not increasing seqId)
+ private Operator(String name) {
+ id = name;
+ }
+
public Operator() {
id = String.valueOf(seqId.getAndIncrement());
childOperators = new ArrayList<Operator<? extends OperatorDesc>>();
@@ -334,7 +338,7 @@ public abstract class Operator<T extends
return;
}
- LOG.info("Initializing Self " + id + " " + getName());
+ LOG.info("Initializing Self " + this);
if (inputOIs != null) {
inputObjInspectors = inputOIs;
@@ -1288,4 +1292,14 @@ public abstract class Operator<T extends
public ObjectInspector getGroupKeyObjectInspector() {
return groupKeyOI;
}
+
+ public static Operator createDummy() {
+ return new DummyOperator();
+ }
+
+ private static class DummyOperator extends Operator {
+ public DummyOperator() { super("dummy"); }
+ public void processOp(Object row, int tag) { }
+ public OperatorType getType() { return null; }
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Thu Jun 5 18:28:07 2014
@@ -57,7 +57,7 @@ public class ReduceSinkOperator extends
implements Serializable, TopNHash.BinaryCollector {
static {
- PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases");
+ PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases", "valueIndex");
}
private static final long serialVersionUID = 1L;
@@ -96,6 +96,8 @@ public class ReduceSinkOperator extends
transient String[] inputAliases; // input aliases of this RS for join (used for PPD)
private boolean skipTag = false;
+ private transient int[] valueIndex; // index for value(+ from keys, - from values)
+
public void setInputAliases(String[] inputAliases) {
this.inputAliases = inputAliases;
}
@@ -470,4 +472,12 @@ public class ReduceSinkOperator extends
public void setSkipTag(boolean value) {
this.skipTag = value;
}
+
+ public void setValueIndex(int[] valueIndex) {
+ this.valueIndex = valueIndex;
+ }
+
+ public int[] getValueIndex() {
+ return valueIndex;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java Thu Jun 5 18:28:07 2014
@@ -27,7 +27,7 @@ import java.util.ArrayList;
public class RowSchema implements Serializable {
private static final long serialVersionUID = 1L;
- private ArrayList<ColumnInfo> signature;
+ private ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>();
public RowSchema() {
}
@@ -52,13 +52,11 @@ public class RowSchema implements Serial
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append('(');
- if (signature != null) {
- for (ColumnInfo col: signature) {
- if (sb.length() > 1) {
- sb.append(',');
- }
- sb.append(col.toString());
+ for (ColumnInfo col: signature) {
+ if (sb.length() > 1) {
+ sb.append(',');
}
+ sb.append(col.toString());
}
sb.append(')');
return sb.toString();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Jun 5 18:28:07 2014
@@ -221,6 +221,13 @@ public final class Utilities {
}
}
+ public static String removeValueTag(String column) {
+ if (column.startsWith(ReduceField.VALUE + ".")) {
+ return column.substring(6);
+ }
+ return column;
+ }
+
private Utilities() {
// prevent instantiation
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Thu Jun 5 18:28:07 2014
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.Set;
@@ -36,9 +35,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.WriteBuffers;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.BytesWritable;
@@ -143,9 +140,13 @@ public class HashMapWrapper extends Abst
}
private class GetAdaptor implements ReusableGetAdaptor {
+
+ private Object[] currentKey;
+ private List<ObjectInspector> vectorKeyOIs;
+
private MapJoinKey key;
private MapJoinRowContainer currentValue;
- private Output output;
+ private final Output output = new Output();
private boolean isFirstKey = true;
public GetAdaptor(MapJoinKey key) {
@@ -155,11 +156,17 @@ public class HashMapWrapper extends Abst
@Override
public void setFromVector(VectorHashKeyWrapper kw, VectorExpressionWriter[] keyOutputWriters,
VectorHashKeyWrapperBatch keyWrapperBatch) throws HiveException {
- if (output == null) {
- output = new Output(0);
+ if (currentKey == null) {
+ currentKey = new Object[keyOutputWriters.length];
+ vectorKeyOIs = new ArrayList<ObjectInspector>();
+ for (int i = 0; i < keyOutputWriters.length; i++) {
+ vectorKeyOIs.add(keyOutputWriters[i].getObjectInspector());
+ }
}
- key = MapJoinKey.readFromVector(
- output, key, kw, keyOutputWriters, keyWrapperBatch, !isFirstKey);
+ for (int i = 0; i < keyOutputWriters.length; i++) {
+ currentKey[i] = keyWrapperBatch.getWritableKeyValue(kw, i, keyOutputWriters[i]);
+ }
+ key = MapJoinKey.readFromVector(output, key, currentKey, vectorKeyOIs, !isFirstKey);
isFirstKey = false;
this.currentValue = mHash.get(key);
}
@@ -167,10 +174,13 @@ public class HashMapWrapper extends Abst
@Override
public void setFromRow(Object row, List<ExprNodeEvaluator> fields,
List<ObjectInspector> ois) throws HiveException {
- if (output == null) {
- output = new Output(0);
+ if (currentKey == null) {
+ currentKey = new Object[fields.size()];
+ }
+ for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
+ currentKey[keyIndex] = fields.get(keyIndex).evaluate(row);
}
- key = MapJoinKey.readFromRow(output, key, row, fields, ois, !isFirstKey);
+ key = MapJoinKey.readFromRow(output, key, currentKey, ois, !isFirstKey);
isFirstKey = false;
this.currentValue = mHash.get(key);
}
@@ -193,6 +203,11 @@ public class HashMapWrapper extends Abst
public MapJoinRowContainer getCurrentRows() {
return currentValue;
}
+
+ @Override
+ public Object[] getCurrentKey() {
+ return currentKey;
+ }
}
@Override
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java Thu Jun 5 18:28:07 2014
@@ -9,7 +9,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.debug.Utils;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
@@ -293,9 +292,18 @@ public class MapJoinBytesTableContainer
/** Implementation of ReusableGetAdaptor that has Output for key serialization; row
* container is also created once and reused for every row. */
private class GetAdaptor implements ReusableGetAdaptor {
- private ReusableRowContainer currentValue = new ReusableRowContainer();
- private Output output;
+
+ private Object[] currentKey;
private boolean[] nulls;
+ private List<ObjectInspector> vectorKeyOIs;
+
+ private final ReusableRowContainer currentValue;
+ private final Output output;
+
+ public GetAdaptor() {
+ currentValue = new ReusableRowContainer();
+ output = new Output();
+ }
@Override
public void setFromVector(VectorHashKeyWrapper kw,
@@ -303,16 +311,20 @@ public class MapJoinBytesTableContainer
VectorHashKeyWrapperBatch keyWrapperBatch) throws HiveException {
if (nulls == null) {
nulls = new boolean[keyOutputWriters.length];
+ currentKey = new Object[keyOutputWriters.length];
+ vectorKeyOIs = new ArrayList<ObjectInspector>();
+ for (int i = 0; i < keyOutputWriters.length; i++) {
+ vectorKeyOIs.add(keyOutputWriters[i].getObjectInspector());
+ }
} else {
assert nulls.length == keyOutputWriters.length;
}
- try {
- output = MapJoinKey.serializeVector(
- output, kw, keyOutputWriters, keyWrapperBatch, nulls, sortableSortOrders);
- } catch (SerDeException e) {
- throw new HiveException(e);
+ for (int i = 0; i < keyOutputWriters.length; i++) {
+ currentKey[i] = keyWrapperBatch.getWritableKeyValue(kw, i, keyOutputWriters[i]);
+ nulls[i] = currentKey[i] == null;
}
- this.currentValue.setFromOutput(output);
+ currentValue.setFromOutput(
+ MapJoinKey.serializeRow(output, currentKey, vectorKeyOIs, sortableSortOrders));
}
@Override
@@ -320,28 +332,23 @@ public class MapJoinBytesTableContainer
List<ObjectInspector> ois) throws HiveException {
if (nulls == null) {
nulls = new boolean[fields.size()];
+ currentKey = new Object[fields.size()];
}
- Object[] fieldObjs = new Object[fields.size()];
for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
- fieldObjs[keyIndex] = fields.get(keyIndex).evaluate(row);
- nulls[keyIndex] = (fieldObjs[keyIndex] == null);
+ currentKey[keyIndex] = fields.get(keyIndex).evaluate(row);
+ nulls[keyIndex] = currentKey[keyIndex] == null;
}
- try {
- output = MapJoinKey.serializeRow(output, fieldObjs, ois, sortableSortOrders);
- } catch (SerDeException e) {
- throw new HiveException(e);
- }
- this.currentValue.setFromOutput(output);
+ currentValue.setFromOutput(
+ MapJoinKey.serializeRow(output, currentKey, ois, sortableSortOrders));
}
-
@Override
public void setFromOther(ReusableGetAdaptor other) {
assert other instanceof GetAdaptor;
GetAdaptor other2 = (GetAdaptor)other;
- output = other2.output;
nulls = other2.nulls;
- this.currentValue.setFromOutput(output);
+ currentKey = other2.currentKey;
+ currentValue.setFromOutput(other2.output);
}
@Override
@@ -359,6 +366,11 @@ public class MapJoinBytesTableContainer
public MapJoinRowContainer getCurrentRows() {
return currentValue.isEmpty() ? null : currentValue;
}
+
+ @Override
+ public Object[] getCurrentKey() {
+ return currentKey;
+ }
}
/** Row container that gets and deserializes the rows on demand from bytes provided. */
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java Thu Jun 5 18:28:07 2014
@@ -27,9 +27,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.common.type.Decimal128;
-import org.apache.hadoop.hive.ql.debug.Utils;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
@@ -38,10 +35,7 @@ import org.apache.hadoop.hive.serde2.Byt
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.StringWrapper;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -66,7 +60,7 @@ public abstract class MapJoinKey {
@SuppressWarnings("deprecation")
public static MapJoinKey read(Output output, MapJoinKey key,
MapJoinObjectSerDeContext context, Writable writable, boolean mayReuseKey)
- throws SerDeException {
+ throws SerDeException, HiveException {
SerDe serde = context.getSerDe();
Object obj = serde.deserialize(writable);
boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
@@ -106,7 +100,7 @@ public abstract class MapJoinKey {
}
private static byte[] serialize(Output byteStream,
- Object obj, ObjectInspector oi, boolean checkTypes) throws SerDeException {
+ Object obj, ObjectInspector oi, boolean checkTypes) throws HiveException {
if (null == obj || !(oi instanceof StructObjectInspector)) {
return null; // not supported
}
@@ -141,29 +135,23 @@ public abstract class MapJoinKey {
return true;
}
- public static MapJoinKey readFromVector(Output output, MapJoinKey key, VectorHashKeyWrapper kw,
- VectorExpressionWriter[] keyOutputWriters, VectorHashKeyWrapperBatch keyWrapperBatch,
- boolean mayReuseKey) throws HiveException {
+ public static MapJoinKey readFromVector(Output output, MapJoinKey key, Object[] keyObject,
+ List<ObjectInspector> keyOIs, boolean mayReuseKey) throws HiveException {
boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
if (useOptimized || key == null) {
- byte[] structBytes = null;
- try {
- if (keyOutputWriters.length <= 8) {
- output = serializeVector(output, kw, keyOutputWriters, keyWrapperBatch, null, null);
- structBytes = Arrays.copyOf(output.getData(), output.getLength());
- }
- } catch (SerDeException e) {
- throw new HiveException(e);
+ if (keyObject.length <= 8) {
+ output = serializeRow(output, keyObject, keyOIs, null);
+ return MapJoinKeyBytes.fromBytes(key, mayReuseKey,
+ Arrays.copyOf(output.getData(), output.getLength()));
}
- if (structBytes != null) {
- return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
- } else if (useOptimized) {
+ if (useOptimized) {
throw new HiveException(
- "Failed to serialize " + kw + " even though optimized keys are used");
+ "Failed to serialize " + Arrays.toString(keyObject) +
+ " even though optimized keys are used");
}
}
MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
- result.readFromVector(kw, keyOutputWriters, keyWrapperBatch);
+ result.setKeyObjects(keyObject);
return result;
}
@@ -189,37 +177,28 @@ public abstract class MapJoinKey {
return serializeRow(byteStream, fieldData, fieldOis, sortableSortOrders);
}
- public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object row,
- List<ExprNodeEvaluator> fields, List<ObjectInspector> keyFieldsOI, boolean mayReuseKey)
- throws HiveException {
- Object[] fieldObjs = new Object[fields.size()];
- for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
- fieldObjs[keyIndex] = fields.get(keyIndex).evaluate(row);
- }
+ public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object[] keyObject,
+ List<ObjectInspector> keyFieldsOI, boolean mayReuseKey) throws HiveException {
boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
if (useOptimized || key == null) {
- try {
- byte[] structBytes = null;
- if (fieldObjs.length <= 8) {
- if (fieldObjs.length == 0) {
- structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
- } else {
- output = serializeRow(output, fieldObjs, keyFieldsOI, null);
- structBytes = Arrays.copyOf(output.getData(), output.getLength());
- }
- }
- if (structBytes != null) {
- return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
- } else if (useOptimized) {
- throw new HiveException(
- "Failed to serialize " + row + " even though optimized keys are used");
+ if (keyObject.length <= 8) {
+ byte[] structBytes;
+ if (keyObject.length == 0) {
+ structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
+ } else {
+ output = serializeRow(output, keyObject, keyFieldsOI, null);
+ structBytes = Arrays.copyOf(output.getData(), output.getLength());
}
- } catch (SerDeException ex) {
- throw new HiveException("Serialization error", ex);
+ return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
+ }
+ if (useOptimized) {
+ throw new HiveException(
+ "Failed to serialize " + Arrays.toString(keyObject) +
+ " even though optimized keys are used");
}
}
MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
- result.readFromRow(fieldObjs, keyFieldsOI);
+ result.readFromRow(keyObject, keyFieldsOI);
return result;
}
@@ -231,18 +210,22 @@ public abstract class MapJoinKey {
* @param byteStream Output to reuse. Can be null, in that case a new one would be created.
*/
public static Output serializeRow(Output byteStream, Object[] fieldData,
- List<ObjectInspector> fieldOis, boolean[] sortableSortOrders) throws SerDeException {
+ List<ObjectInspector> fieldOis, boolean[] sortableSortOrders) throws HiveException {
if (byteStream == null) {
byteStream = new Output();
} else {
byteStream.reset();
}
- if (fieldData.length == 0) {
- byteStream.reset();
- } else if (sortableSortOrders == null) {
- LazyBinarySerDe.serializeStruct(byteStream, fieldData, fieldOis);
- } else {
- BinarySortableSerDe.serializeStruct(byteStream, fieldData, fieldOis, sortableSortOrders);
+ try {
+ if (fieldData.length == 0) {
+ byteStream.reset();
+ } else if (sortableSortOrders == null) {
+ LazyBinarySerDe.serializeStruct(byteStream, fieldData, fieldOis);
+ } else {
+ BinarySortableSerDe.serializeStruct(byteStream, fieldData, fieldOis, sortableSortOrders);
+ }
+ } catch (SerDeException e) {
+ throw new HiveException("Serialization error", e);
}
return byteStream;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java Thu Jun 5 18:28:07 2014
@@ -46,10 +46,19 @@ public class MapJoinKeyObject extends Ma
public MapJoinKeyObject(Object[] key) {
this.key = key;
}
+
public MapJoinKeyObject() {
this(EMPTY_OBJECT_ARRAY);
}
+ public Object[] getKeyObjects() {
+ return key;
+ }
+
+ public void setKeyObjects(Object[] key) {
+ this.key = key;
+ }
+
public int getKeyLength() {
return key.length;
}
@@ -118,7 +127,7 @@ public class MapJoinKeyObject extends Ma
container.write(out);
}
- protected void readFromRow(Object[] fieldObjs, List<ObjectInspector> keyFieldsOI)
+ public void readFromRow(Object[] fieldObjs, List<ObjectInspector> keyFieldsOI)
throws HiveException {
if (key == null || key.length != fieldObjs.length) {
key = new Object[fieldObjs.length];
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java Thu Jun 5 18:28:07 2014
@@ -65,6 +65,11 @@ public interface MapJoinTableContainer {
* @return The container w/the rows corresponding to a key set via a previous set... call.
*/
MapJoinRowContainer getCurrentRows();
+
+ /**
+ * @return key
+ */
+ Object[] getCurrentKey();
}
/**
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java?rev=1600719&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java Thu Jun 5 18:28:07 2014
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.persistence;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.SerDeException;
+
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UnwrapRowContainer
+ implements MapJoinRowContainer, AbstractRowContainer.RowIterator<List<Object>> {
+
+ private final int[] index;
+ private final List<Object> unwrapped;
+
+ private transient Object[] currentKey;
+ private transient MapJoinRowContainer internal;
+
+ private transient RowIterator<List<Object>> iterator;
+
+ public UnwrapRowContainer(int[] valueIndex) {
+ index = valueIndex;
+ unwrapped = new ArrayList<Object>();
+ }
+
+ public MapJoinRowContainer setInternal(MapJoinRowContainer internal, Object[] currentKey) {
+ this.internal = internal;
+ this.currentKey = currentKey;
+ return this;
+ }
+
+ @Override
+ public List<Object> first() throws HiveException {
+ iterator = internal.rowIter();
+ return unwrap(iterator.first());
+ }
+
+ @Override
+ public List<Object> next() throws HiveException {
+ return unwrap(iterator.next());
+ }
+
+ private List<Object> unwrap(List<Object> values) {
+ if (values == null) {
+ return null;
+ }
+ unwrapped.clear();
+ for (int i = 0; i < index.length; i++) {
+ if (index[i] >= 0) {
+ unwrapped.add(currentKey[index[i]]);
+ } else {
+ int vindex = -index[i] - 1;
+ if (vindex < values.size()) {
+ unwrapped.add(values.get(vindex));
+ }
+ }
+ }
+ return unwrapped;
+ }
+
+ @Override
+ public RowIterator<List<Object>> rowIter() throws HiveException {
+ return this;
+ }
+
+ @Override
+ public void addRow(List<Object> t) throws HiveException {
+ internal.addRow(t);
+ }
+
+ @Override
+ public int rowCount() throws HiveException {
+ return internal.rowCount();
+ }
+
+ @Override
+ public void clearRows() throws HiveException {
+ internal.clearRows();
+ }
+
+ @Override
+ public byte getAliasFilter() throws HiveException {
+ return internal.getAliasFilter();
+ }
+
+ @Override
+ public MapJoinRowContainer copy() throws HiveException {
+ internal = internal.copy();
+ return this;
+ }
+
+ @Override
+ public void addRow(Object[] value) throws HiveException {
+ internal.addRow(value);
+ }
+
+ @Override
+ public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out)
+ throws IOException, SerDeException {
+ internal.write(valueContext, out);
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Thu Jun 5 18:28:07 2014
@@ -375,6 +375,7 @@ public final class ColumnPrunerProcFacto
Object... nodeOutputs) throws SemanticException {
ReduceSinkOperator op = (ReduceSinkOperator) nd;
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
+ RowResolver resolver = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
ReduceSinkDesc conf = op.getConf();
List<String> colLists = new ArrayList<String>();
@@ -395,35 +396,20 @@ public final class ColumnPrunerProcFacto
childCols = cppCtx.getPrunedColList(child);
}
+ List<ExprNodeDesc> valCols = conf.getValueCols();
+ List<String> valColNames = conf.getOutputValueColumnNames();
+
if (childCols != null) {
- /*
- * in the case of count(or sum) distinct if we are not able to map
- * a parameter column references back to the ReduceSink value columns
- * we give up and assume all columns are needed.
- */
- boolean hasUnresolvedReference = false;
- boolean[] flags = new boolean[conf.getValueCols().size()];
+ boolean[] flags = new boolean[valCols.size()];
Map<String, ExprNodeDesc> exprMap = op.getColumnExprMap();
+
for (String childCol : childCols) {
- ExprNodeDesc desc = exprMap.get(childCol);
- int index = conf.getValueCols().indexOf(desc);
+ int index = valColNames.indexOf(Utilities.removeValueTag(childCol));
if (index < 0) {
- hasUnresolvedReference = desc == null || ExprNodeDescUtils.indexOf(desc, conf.getKeyCols()) < 0;
- if ( hasUnresolvedReference ) {
- break;
- }
continue;
}
flags[index] = true;
- colLists = Utilities.mergeUniqElems(colLists, desc.getCols());
- }
-
- if ( hasUnresolvedReference ) {
- for (ExprNodeDesc val : conf.getValueCols()) {
- colLists = Utilities.mergeUniqElems(colLists, val.getCols());
- }
- cppCtx.getPrunedColLists().put(op, colLists);
- return null;
+ colLists = Utilities.mergeUniqElems(colLists, valCols.get(index).getCols());
}
Collections.sort(colLists);
@@ -434,8 +420,7 @@ public final class ColumnPrunerProcFacto
// Reduce Sink contains the columns needed - no need to aggregate from
// children
- ArrayList<ExprNodeDesc> vals = conf.getValueCols();
- for (ExprNodeDesc val : vals) {
+ for (ExprNodeDesc val : valCols) {
colLists = Utilities.mergeUniqElems(colLists, val.getCols());
}
@@ -597,10 +582,6 @@ public final class ColumnPrunerProcFacto
// do we need to prune the select operator?
List<ExprNodeDesc> originalColList = op.getConf().getColList();
- List<String> columns = new ArrayList<String>();
- for (ExprNodeDesc expr : originalColList) {
- Utilities.mergeUniqElems(columns, expr.getCols());
- }
// by now, 'prunedCols' are columns used by child operators, and 'columns'
// are columns used by this select operator.
List<String> originalOutputColumnNames = conf.getOutputColumnNames();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Thu Jun 5 18:28:07 2014
@@ -358,7 +358,7 @@ public class ConvertJoinMapJoin implemen
ParseContext parseContext = context.parseContext;
MapJoinOperator mapJoinOp = MapJoinProcessor.
convertJoinOpMapJoinOp(context.conf, parseContext.getOpParseCtx(),
- joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true);
+ joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true, true);
Operator<? extends OperatorDesc> parentBigTableOp
= mapJoinOp.getParentOperators().get(bigTablePosition);