You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/08/11 04:06:32 UTC
svn commit: r802976 [1/17] - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/lib/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/...
Author: namit
Date: Tue Aug 11 02:06:23 2009
New Revision: 802976
URL: http://svn.apache.org/viewvc?rev=802976&view=rev
Log:
HIVE-578. Refactor partition pruning
(Ashish Thusoo via namit)
Added:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprPrunerInfo.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java (contents, props changed)
- copied, changed from r802958, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/groupby_map_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/groupby_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/louter_join_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/outer_join_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/router_join_ppr.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/transform_ppr1.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/transform_ppr2.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/union_ppr.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/transform_ppr2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union_ppr.q.out
Removed:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/filterDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/tableScanDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java
hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/cast1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/create_genericudaf.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby10.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby11.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/implicit_cast1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input11.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input12.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input13.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input14.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input18.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input22.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input2_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input30.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input31.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input32.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_columnarserde.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_dynamicserde.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_part9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testsequencefile.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join0.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join10.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join11.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join12.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join13.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join14.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join15.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join16.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join17.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join21.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join22.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join25.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join26.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join27.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join28.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join29.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join30.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join31.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join32.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join33.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join34.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join35.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join36.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_rc.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/mapreduce8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/no_hooks.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/noalias_subq1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullscript.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/order.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/order2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_constant_expr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_join.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_join3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_multi_insert.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_random.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_union.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/quote1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/quote2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/regexp_extract.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/select_as_omitted.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sort.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/subq.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/type_cast_1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_10_trims.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_abs.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_ascii.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_case_thrift.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_coalesce.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_elt.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_hash.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_instr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_length.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_like.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_locate.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_lower.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_lpad_rpad.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_parse_url.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_repeat.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_reverse.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_size.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_space.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_split.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_when.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union13.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union16.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union17.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union18.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union19.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union21.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out
hadoop/hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Aug 11 02:06:23 2009
@@ -201,6 +201,9 @@
HIVE-617. Add 'jar' service to hive client shell.
(Edward Capriolo via rmurthy)
+ HIVE-578. Refactor partition pruning
+ (Ashish Thusoo via namit)
+
OPTIMIZATIONS
HIVE-279. Predicate Pushdown support (Prasad Chakka via athusoo).
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Aug 11 02:06:23 2009
@@ -153,8 +153,8 @@
// Optimizer
HIVEOPTCP("hive.optimize.cp", true), // column pruner
- HIVEOPTPPD("hive.optimize.ppd", true); // predicate pushdown
-
+ HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown
+ HIVEOPTPPR("hive.optimize.pruner", true); // partition pruner
public final String varname;
public final String defaultVal;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java Tue Aug 11 02:06:23 2009
@@ -37,25 +37,34 @@
private String internalName;
/**
- * isVirtual indicates whether the column is a virtual column or not. Virtual columns
- * are the ones that are not stored in the tables. For now these are just the partitioning
- * columns.
+ * Store the alias of the table where available.
*/
- private boolean isVirtual;
+ private String tabAlias;
+
+ /**
+ * Indicates whether the column is a partition column.
+ */
+ private boolean isPartitionCol;
transient private TypeInfo type;
public ColumnInfo() {
}
- public ColumnInfo(String internalName, TypeInfo type) {
+ public ColumnInfo(String internalName, TypeInfo type,
+ String tabAlias, boolean isPartitionCol) {
this.internalName = internalName;
this.type = type;
+ this.tabAlias = tabAlias;
+ this.isPartitionCol = isPartitionCol;
}
- public ColumnInfo(String internalName, Class type) {
+ public ColumnInfo(String internalName, Class type,
+ String tabAlias, boolean isPartitionCol) {
this.internalName = internalName;
this.type = TypeInfoFactory.getPrimitiveTypeInfoFromPrimitiveWritable(type);
+ this.tabAlias = tabAlias;
+ this.isPartitionCol = isPartitionCol;
}
public TypeInfo getType() {
@@ -74,6 +83,13 @@
this.internalName = internalName;
}
+ public String getTabAlias() {
+ return this.tabAlias;
+ }
+
+ public boolean getIsPartitionCol() {
+ return this.isPartitionCol;
+ }
/**
* Returns the string representation of the ColumnInfo.
*/
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java Tue Aug 11 02:06:23 2009
@@ -105,7 +105,8 @@
* @throws SemanticException
*/
public void walk(Node nd) throws SemanticException {
- opStack.push(nd);
+ if (opStack.empty() || nd != opStack.peek())
+ opStack.push(nd);
if((nd.getChildren() == null)
|| getDispatchedList().containsAll(nd.getChildren())) {
@@ -121,6 +122,5 @@
getToWalk().add(0, nd);
getToWalk().removeAll(nd.getChildren());
getToWalk().addAll(0, nd.getChildren());
- opStack.pop();
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Tue Aug 11 02:06:23 2009
@@ -128,7 +128,8 @@
ArrayList<exprNodeDesc> valueCols = new ArrayList<exprNodeDesc>();
for (ColumnInfo ci : fsRS.getSignature()) {
- valueCols.add(new exprNodeColumnDesc(ci.getType(), ci.getInternalName()));
+ valueCols.add(new exprNodeColumnDesc(ci.getType(), ci.getInternalName(), ci.getTabAlias(),
+ ci.getIsPartitionCol()));
}
// create a dummy tableScan operator
@@ -155,13 +156,15 @@
for(ColumnInfo colInfo: interim_rwsch.getColumnInfos()) {
String [] info = interim_rwsch.reverseLookup(colInfo.getInternalName());
out_rwsch.put(info[0], info[1],
- new ColumnInfo(pos.toString(), colInfo.getType()));
+ new ColumnInfo(pos.toString(), colInfo.getType(), info[0],
+ colInfo.getIsPartitionCol()));
pos = Integer.valueOf(pos.intValue() + 1);
}
Operator extract =
OperatorFactory.getAndMakeChild(
- new extractDesc(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, Utilities.ReduceField.VALUE.toString())),
+ new extractDesc(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
+ Utilities.ReduceField.VALUE.toString(), "", false)),
new RowSchema(out_rwsch.getColumnInfos()));
tableDesc ts = (tableDesc)fsConf.getTableInfo().clone();
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Tue Aug 11 02:06:23 2009
@@ -392,29 +392,38 @@
Path tblDir = null;
tableDesc tblDesc = null;
- // Generate the map work for this alias_id
- PartitionPruner pruner = parseCtx.getAliasToPruner().get(alias_id);
- Set<Partition> parts = null;
+ PrunedPartitionList partsList = null;
+
try {
- // pass both confirmed and unknown partitions through the map-reduce framework
- PartitionPruner.PrunedPartitionList partsList = pruner.prune();
-
- parts = partsList.getConfirmedPartns();
- parts.addAll(partsList.getUnknownPartns());
- partitionDesc aliasPartnDesc = null;
- if (parts.isEmpty()) {
- if (!partsList.getDeniedPartns().isEmpty())
- aliasPartnDesc = Utilities.getPartitionDesc(partsList.getDeniedPartns().iterator().next());
+ if (!opProcCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEOPTPPD) ||
+ !opProcCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEOPTPPR)) {
+ partsList = parseCtx.getAliasToPruner().get(alias_id).prune();
}
else {
- aliasPartnDesc = Utilities.getPartitionDesc(parts.iterator().next());
+ partsList = org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(
+ parseCtx.getTopToTable().get(topOp),
+ parseCtx.getOpToPartPruner().get(topOp));
}
- plan.getAliasToPartnInfo().put(alias_id, aliasPartnDesc);
} catch (HiveException e) {
- // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils
LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
throw new SemanticException(e.getMessage(), e);
}
+
+ // Generate the map work for this alias_id
+ Set<Partition> parts = null;
+ // pass both confirmed and unknown partitions through the map-reduce framework
+
+ parts = partsList.getConfirmedPartns();
+ parts.addAll(partsList.getUnknownPartns());
+ partitionDesc aliasPartnDesc = null;
+ if (parts.isEmpty()) {
+ if (!partsList.getDeniedPartns().isEmpty())
+ aliasPartnDesc = Utilities.getPartitionDesc(partsList.getDeniedPartns().iterator().next());
+ }
+ else {
+ aliasPartnDesc = Utilities.getPartitionDesc(parts.iterator().next());
+ }
+ plan.getAliasToPartnInfo().put(alias_id, aliasPartnDesc);
SamplePruner samplePruner = parseCtx.getAliasToSamplePruner().get(alias_id);
for (Partition part : parts) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Tue Aug 11 02:06:23 2009
@@ -178,10 +178,11 @@
String outputCol = oldValueInfo.getInternalName();
if (outputRS.get(key, field) == null) {
outputColumnNames.add(outputCol);
- exprNodeDesc colDesc = new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName());
+ exprNodeDesc colDesc = new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName(),
+ valueInfo.getTabAlias(), valueInfo.getIsPartitionCol());
values.add(colDesc);
outputRS.put(key, field, new ColumnInfo(outputCol,
- valueInfo.getType()));
+ valueInfo.getType(), valueInfo.getTabAlias(), valueInfo.getIsPartitionCol()));
colExprMap.put(outputCol, colDesc);
}
}
@@ -283,11 +284,11 @@
String[] nm = inputRR.reverseLookup(internalName);
ColumnInfo valueInfo = inputRR.get(nm[0], nm[1]);
exprNodeDesc colDesc = new exprNodeColumnDesc(valueInfo.getType(),
- valueInfo.getInternalName());
+ valueInfo.getInternalName(), nm[0], valueInfo.getIsPartitionCol());
exprs.add(colDesc);
outputs.add(internalName);
outputRS .put(nm[0], nm[1], new ColumnInfo(internalName,
- valueInfo.getType()));
+ valueInfo.getType(), nm[0], valueInfo.getIsPartitionCol()));
colExprMap.put(internalName, colDesc);
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Tue Aug 11 02:06:23 2009
@@ -25,61 +25,58 @@
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.ppd.PredicatePushDown;
+import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcessor;
/**
* Implementation of the optimizer
*/
public class Optimizer {
- private ParseContext pctx;
- private List<Transform> transformations;
-
- /**
- * empty constructor
- */
- public Optimizer() {
- }
+ private ParseContext pctx;
+ private List<Transform> transformations;
/**
* create the list of transformations
* @param hiveConf
*/
- public void initialize(HiveConf hiveConf) {
- transformations = new ArrayList<Transform>();
- if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCP)) {
- transformations.add(new ColumnPruner());
- }
+ public void initialize(HiveConf hiveConf) {
+ transformations = new ArrayList<Transform>();
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCP)) {
+ transformations.add(new ColumnPruner());
+ }
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) {
transformations.add(new PredicatePushDown());
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPR)) {
+ transformations.add(new PartitionPruner());
+ }
}
transformations.add(new UnionProcessor());
- transformations.add(new MapJoinProcessor());
- }
-
- /**
- * invoke all the transformations one-by-one, and alter the query plan
- * @return ParseContext
- * @throws SemanticException
- */
- public ParseContext optimize() throws SemanticException {
- for (Transform t : transformations)
- pctx = t.transform(pctx);
+ transformations.add(new MapJoinProcessor());
+ }
+
+ /**
+ * invoke all the transformations one-by-one, and alter the query plan
+ * @return ParseContext
+ * @throws SemanticException
+ */
+ public ParseContext optimize() throws SemanticException {
+ for (Transform t : transformations)
+ pctx = t.transform(pctx);
return pctx;
- }
-
- /**
- * @return the pctx
- */
- public ParseContext getPctx() {
- return pctx;
- }
+ }
+
+ /**
+ * @return the pctx
+ */
+ public ParseContext getPctx() {
+ return pctx;
+ }
+
+ /**
+ * @param pctx the pctx to set
+ */
+ public void setPctx(ParseContext pctx) {
+ this.pctx = pctx;
+ }
- /**
- * @param pctx the pctx to set
- */
- public void setPctx(ParseContext pctx) {
- this.pctx = pctx;
- }
-
-
}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
+/**
+ * The processor context for partition pruner. This contains the table
+ * alias that is being currently processed.
+ */
+public class ExprProcCtx implements NodeProcessorCtx {
+
+ /**
+ * The table alias that is being currently processed.
+ */
+ String tabAlias;
+
+ /**
+ * Flag to hold whether there are any non partition columns accessed in the
+ * expression.
+ */
+ boolean hasNonPartCols;
+
+ public ExprProcCtx(String tabAlias) {
+ this.tabAlias = tabAlias;
+ this.hasNonPartCols = false;
+ }
+
+ public String getTabAlias() {
+ return tabAlias;
+ }
+
+ public void setTabAlias(String tabAlias) {
+ this.tabAlias = tabAlias;
+ }
+
+ public boolean getHasNonPartCols() {
+ return this.hasNonPartCols;
+ }
+
+ public void setHasNonPartCols(boolean val) {
+ this.hasNonPartCols = val;
+ }
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
+import org.apache.hadoop.hive.ql.udf.UDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.UDFOPOr;
+import org.apache.hadoop.hive.ql.udf.UDFOPNot;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+
+/**
+ * Expression processor factory for partition pruning. Each processor tries
+ * to convert the expression subtree into a partition pruning expression.
+ * This expression is then used to figure out whether a particular partition
+ * should be scanned or not.
+ */
+public class ExprProcFactory {
+
+ /**
+ * Processor for column expressions.
+ */
+ public static class ColumnExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc newcd = null;
+ exprNodeColumnDesc cd = (exprNodeColumnDesc) nd;
+ ExprProcCtx epc = (ExprProcCtx) procCtx;
+ if (cd.getTabAlias().equalsIgnoreCase(epc.getTabAlias()) && cd.getIsParititonCol())
+ newcd = cd.clone();
+ else {
+ newcd = new exprNodeConstantDesc(cd.getTypeInfo(), null);
+ epc.setHasNonPartCols(true);
+ }
+
+ return newcd;
+ }
+
+ }
+
+ /**
+ * Process function descriptors.
+ */
+ public static class FuncExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc newfd = null;
+ exprNodeFuncDesc fd = (exprNodeFuncDesc) nd;
+
+ boolean unknown = false;
+ // Check if any of the children is unknown for non logical operators
+ if (!fd.getUDFMethod().getDeclaringClass().equals(UDFOPAnd.class)
+ && !fd.getUDFMethod().getDeclaringClass().equals(UDFOPOr.class)
+ && !fd.getUDFMethod().getDeclaringClass().equals(UDFOPNot.class))
+ for(Object child: nodeOutputs) {
+ exprNodeDesc child_nd = (exprNodeDesc)child;
+ if (child_nd instanceof exprNodeConstantDesc &&
+ ((exprNodeConstantDesc)child_nd).getValue() == null) {
+ unknown = true;
+ }
+ }
+
+ if (fd.getUDFClass().getAnnotation(UDFType.class) != null &&
+ (fd.getUDFClass().getAnnotation(UDFType.class).deterministic() == false ||
+ unknown))
+ newfd = new exprNodeConstantDesc(fd.getTypeInfo(), null);
+ else {
+ // Create the list of children
+ ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>();
+ for(Object child: nodeOutputs) {
+ children.add((exprNodeDesc) child);
+ }
+ // Create a copy of the function descriptor
+ newfd = new exprNodeFuncDesc(fd.getMethodName(),
+ fd.getTypeInfo(), fd.getUDFClass(),
+ fd.getUDFMethod(), children);
+ }
+
+ return newfd;
+ }
+
+ }
+
+ /**
+ * If all children are candidates and refer only to one table alias then this expr is a candidate
+ * else it is not a candidate but its children could be final candidates
+ */
+ public static class GenericFuncExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc newfd = null;
+ exprNodeGenericFuncDesc fd = (exprNodeGenericFuncDesc) nd;
+
+ boolean unknown = false;
+ // Check if any of the children is unknown
+ for(Object child: nodeOutputs) {
+ exprNodeDesc child_nd = (exprNodeDesc)child;
+ if (child_nd instanceof exprNodeConstantDesc &&
+ ((exprNodeConstantDesc)child_nd).getValue() == null) {
+ unknown = true;
+ }
+ }
+
+ if (unknown)
+ newfd = new exprNodeConstantDesc(fd.getTypeInfo(), null);
+ else {
+ // Create the list of children
+ ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>();
+ for(Object child: nodeOutputs) {
+ children.add((exprNodeDesc) child);
+ }
+ // Create a copy of the function descriptor
+ newfd = new exprNodeGenericFuncDesc(fd.getTypeInfo(), fd.getGenericUDFClass(), children);
+ }
+
+ return newfd;
+ }
+
+ }
+
+ public static class FieldExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeFieldDesc fnd = (exprNodeFieldDesc)nd;
+ boolean unknown = false;
+ int idx = 0;
+ exprNodeDesc left_nd = null;
+ for(Object child: nodeOutputs) {
+ exprNodeDesc child_nd = (exprNodeDesc) child;
+ if (child_nd instanceof exprNodeConstantDesc &&
+ ((exprNodeConstantDesc)child_nd).getValue() == null)
+ unknown = true;
+ left_nd = child_nd;
+ }
+
+ assert(idx == 0);
+
+ exprNodeDesc newnd = null;
+ if (unknown) {
+ newnd = new exprNodeConstantDesc(fnd.getTypeInfo(), null);
+ }
+ else {
+ newnd = new exprNodeFieldDesc(fnd.getTypeInfo(), left_nd, fnd.getFieldName(), fnd.getIsList());
+ }
+ return newnd;
+ }
+
+ }
+
+ /**
+ * Processor for constants and null expressions. For such expressions
+ * the processor simply clones the exprNodeDesc and returns it.
+ */
+ public static class DefaultExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ if (nd instanceof exprNodeConstantDesc)
+ return ((exprNodeConstantDesc)nd).clone();
+ else if (nd instanceof exprNodeNullDesc)
+ return ((exprNodeNullDesc)nd).clone();
+
+ assert(false);
+ return null;
+ }
+ }
+
+ public static NodeProcessor getDefaultExprProcessor() {
+ return new DefaultExprProcessor();
+ }
+
+ public static NodeProcessor getFuncProcessor() {
+ return new FuncExprProcessor();
+ }
+
+ public static NodeProcessor getGenericFuncProcessor() {
+ return new GenericFuncExprProcessor();
+ }
+
+ public static NodeProcessor getFieldProcessor() {
+ return new FieldExprProcessor();
+ }
+
+ public static NodeProcessor getColumnProcessor() {
+ return new ColumnExprProcessor();
+ }
+
+ /**
+ * Generates the partition pruner for the expression tree
+ * @param tabAlias The table alias of the partition table that is being considered for pruning
+ * @param pred The predicate from which the partition pruner needs to be generated
+ * @return hasNonPartCols returns true/false depending upon whether this pred has a non partition column
+ * @throws SemanticException
+ */
+ public static exprNodeDesc genPruner(String tabAlias, exprNodeDesc pred,
+ boolean hasNonPartCols) throws SemanticException {
+ // Create the walker, the rules dispatcher and the context.
+ ExprProcCtx pprCtx= new ExprProcCtx(tabAlias);
+
+ // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher
+ // generates the plan from the operator tree
+ Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
+ exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor());
+ exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor());
+ exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
+ exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
+
+ // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, pprCtx);
+ GraphWalker egw = new DefaultGraphWalker(disp);
+
+ List<Node> startNodes = new ArrayList<Node>();
+ startNodes.add(pred);
+
+ HashMap<Node, Object> outputMap = new HashMap<Node, Object>();
+ egw.startWalking(startNodes, outputMap);
+ hasNonPartCols = pprCtx.getHasNonPartCols();
+
+ // Get the exprNodeDesc corresponding to the first start node;
+ return (exprNodeDesc)outputMap.get(pred);
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprPrunerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprPrunerInfo.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprPrunerInfo.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprPrunerInfo.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
+/**
+ * The processor context for partition pruner. This contains the table
+ * alias that is being currently processed.
+ */
+public class ExprPrunerInfo implements NodeProcessorCtx {
+
+ /**
+ * The table alias that is being currently processed.
+ */
+ String tabAlias;
+
+ public String getTabAlias() {
+ return tabAlias;
+ }
+
+ public void setTabAlias(String tabAlias) {
+ this.tabAlias = tabAlias;
+ }
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.Map;
+import java.util.Stack;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+
+/**
+ * Operator factory for partition pruning processing of operator graph
+ * We find all the filter operators that appear just beneath the table scan
+ * operators. We then pass the filter to the partition pruner to construct
+ * a pruner for that table alias and store a mapping from the table scan
+ * operator to that pruner. We call that pruner later during plan generation.
+ */
+public class OpProcFactory {
+
+ /**
+ * Determines the partition pruner for the filter. This is called only when the filter
+ * follows a table scan operator.
+ */
+ public static class FilterPPR implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ OpWalkerCtx owc = (OpWalkerCtx)procCtx;
+ FilterOperator fop = (FilterOperator) nd;
+ FilterOperator fop2 = null;
+
+ // The stack contains either ... TS, Filter or
+ // ... TS, Filter, Filter with the head of the stack being the rightmost
+ // symbol. So we just pop out the two elements from the top and if the
+ // second one of them is not a table scan then the operator on the top of
+ // the stack is the Table scan operator.
+ Node tmp = stack.pop();
+ Node tmp2 = stack.pop();
+ TableScanOperator top = null;
+ if (tmp2 instanceof TableScanOperator) {
+ top = (TableScanOperator)tmp2;
+ }
+ else {
+ top = (TableScanOperator) stack.peek();
+ fop2 = (FilterOperator) tmp2;
+ }
+ stack.push(tmp2);
+ stack.push(tmp);
+
+ // If fop2 exists (i.e this is not the top level filter and fop2 is not
+ // a sampling filter then we ignore the current filter
+ if (fop2 != null && !fop2.getConf().getIsSamplingPred())
+ return null;
+
+ // ignore the predicate in case it is not a sampling predicate
+ if (fop.getConf().getIsSamplingPred()) {
+ return null;
+ }
+
+ // Otherwise this is not a sampling predicate and we need to
+ exprNodeDesc predicate = fop.getConf().getPredicate();
+ String alias = top.getConf().getAlias();
+
+ // Generate the partition pruning predicate
+ boolean hasNonPartCols = false;
+ exprNodeDesc ppr_pred = ExprProcFactory.genPruner(alias, predicate, hasNonPartCols);
+ owc.addHasNonPartCols(hasNonPartCols);
+
+ // Add the pruning predicate to the table scan operator
+ addPruningPred(owc.getOpToPartPruner(), top, ppr_pred);
+
+ return null;
+ }
+
+ private void addPruningPred(Map<TableScanOperator, exprNodeDesc> opToPPR,
+ TableScanOperator top, exprNodeDesc new_ppr_pred) {
+ exprNodeDesc old_ppr_pred = opToPPR.get(top);
+ exprNodeDesc ppr_pred = null;
+ if (old_ppr_pred != null) {
+ // or the old_ppr_pred and the new_ppr_pred
+ ppr_pred = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("OR", old_ppr_pred, new_ppr_pred);
+ }
+ else {
+ ppr_pred = new_ppr_pred;
+ }
+
+ // Put the mapping from table scan operator to ppr_pred
+ opToPPR.put(top, ppr_pred);
+
+ return;
+ }
+ }
+
+
+ /**
+ * Default processor which just merges its children
+ */
+ public static class DefaultPPR implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ // Nothing needs to be done.
+ return null;
+ }
+ }
+
+ public static NodeProcessor getFilterProc() {
+ return new FilterPPR();
+ }
+
+ public static NodeProcessor getDefaultProc() {
+ return new DefaultPPR();
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.HashMap;
+
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+
+/**
+ * Context class for operator tree walker for partition pruner.
+ */
+public class OpWalkerCtx implements NodeProcessorCtx {
+
+ private boolean hasNonPartCols;
+
+ /**
+ * Map from tablescan operator to partition pruning predicate
+ * that is initialized from the ParseContext
+ */
+ private HashMap<TableScanOperator, exprNodeDesc> opToPartPruner;
+
+ /**
+ * Constructor
+ */
+ public OpWalkerCtx(HashMap<TableScanOperator, exprNodeDesc> opToPartPruner) {
+ this.opToPartPruner = opToPartPruner;
+ this.hasNonPartCols = false;
+ }
+
+ public HashMap<TableScanOperator, exprNodeDesc> getOpToPartPruner() {
+ return this.opToPartPruner;
+ }
+
+ public void addHasNonPartCols(boolean val) {
+ hasNonPartCols = (hasNonPartCols || val);
+ }
+
+ public boolean getHasNonPartCols() {
+ return hasNonPartCols;
+ }
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=802976&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Aug 11 02:06:23 2009
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.optimizer.Transform;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * The transformation step that does partition pruning.
+ *
+ */
+public class PartitionPruner implements Transform {
+
+ // The log
+ private static final Log LOG = LogFactory.getLog("hive.ql.optimizer.ppr.PartitionPruner");
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.ParseContext)
+ */
+ @Override
+ public ParseContext transform(ParseContext pctx) throws SemanticException {
+
+ // create a the context for walking operators
+ OpWalkerCtx opWalkerCtx = new OpWalkerCtx(pctx.getOpToPartPruner());
+
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ opRules.put(new RuleRegExp("R1", "(TS%FIL%)|(TS%FIL%FIL%)"),
+ OpProcFactory.getFilterProc());
+
+ // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, opWalkerCtx);
+ GraphWalker ogw = new DefaultGraphWalker(disp);
+
+ // Create a list of topop nodes
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(pctx.getTopOps().values());
+ ogw.startWalking(topNodes, null);
+ pctx.setHasNonPartCols(opWalkerCtx.getHasNonPartCols());
+
+ return pctx;
+ }
+
+ public static PrunedPartitionList prune(Table tab, exprNodeDesc prunerExpr) throws HiveException {
+ LOG.trace("Started pruning partiton");
+ LOG.trace("tabname = " + tab.getName());
+ LOG.trace("prune Expression = " + prunerExpr);
+
+ LinkedHashSet<Partition> true_parts = new LinkedHashSet<Partition>();
+ LinkedHashSet<Partition> unkn_parts = new LinkedHashSet<Partition>();
+ LinkedHashSet<Partition> denied_parts = new LinkedHashSet<Partition>();
+
+ try {
+ StructObjectInspector rowObjectInspector = (StructObjectInspector)tab.getDeserializer().getObjectInspector();
+ Object[] rowWithPart = new Object[2];
+
+ if(tab.isPartitioned()) {
+ for(String partName: Hive.get().getPartitionNames(tab.getDbName(), tab.getName(), (short) -1)) {
+ // Set all the variables here
+ LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
+ // Create the row object
+ ArrayList<String> partNames = new ArrayList<String>();
+ ArrayList<String> partValues = new ArrayList<String>();
+ ArrayList<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
+ for(Map.Entry<String,String>entry : partSpec.entrySet()) {
+ partNames.add(entry.getKey());
+ partValues.add(entry.getValue());
+ partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+ StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
+
+ rowWithPart[1] = partValues;
+ ArrayList<StructObjectInspector> ois = new ArrayList<StructObjectInspector>(2);
+ ois.add(rowObjectInspector);
+ ois.add(partObjectInspector);
+ StructObjectInspector rowWithPartObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(ois);
+
+ // evaluate the expression tree
+ if (prunerExpr != null) {
+ ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(prunerExpr);
+ ObjectInspector evaluateResultOI = evaluator.initialize(rowWithPartObjectInspector);
+ Object evaluateResultO = evaluator.evaluate(rowWithPart);
+ Boolean r = (Boolean) ((PrimitiveObjectInspector)evaluateResultOI).getPrimitiveJavaObject(evaluateResultO);
+ LOG.trace("prune result for partition " + partSpec + ": " + r);
+ if (Boolean.FALSE.equals(r)) {
+ if (denied_parts.isEmpty()) {
+ Partition part = Hive.get().getPartition(tab, partSpec, Boolean.FALSE);
+ denied_parts.add(part);
+ }
+ LOG.trace("pruned partition: " + partSpec);
+ } else {
+ Partition part = Hive.get().getPartition(tab, partSpec, Boolean.FALSE);
+ if (Boolean.TRUE.equals(r)) {
+ LOG.debug("retained partition: " + partSpec);
+ true_parts.add(part);
+ } else {
+ LOG.debug("unknown partition: " + partSpec);
+ unkn_parts.add(part);
+ }
+ }
+ } else {
+ // is there is no parition pruning, all of them are needed
+ true_parts.add(Hive.get().getPartition(tab, partSpec, Boolean.FALSE));
+ }
+ }
+ } else {
+ true_parts.addAll(Hive.get().getPartitions(tab));
+ }
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+
+ // Now return the set of partitions
+ return new PrunedPartitionList(true_parts, unkn_parts, denied_parts);
+ }
+}
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java (from r802958, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java&r1=802958&r2=802976&rev=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java Tue Aug 11 02:06:23 2009
@@ -20,16 +20,10 @@
import java.util.*;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
@@ -56,7 +50,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-public class PartitionPruner {
+public class ASTPartitionPruner {
// The log
@SuppressWarnings("nls")
@@ -76,11 +70,11 @@
// a map-reduce job
private boolean onlyContainsPartCols;
- public PartitionPruner() {
+ public ASTPartitionPruner() {
}
/** Creates a new instance of PartitionPruner */
- public PartitionPruner(String tableAlias, QBMetaData metaData) {
+ public ASTPartitionPruner(String tableAlias, QBMetaData metaData) {
this.tableAlias = tableAlias;
this.metaData = metaData;
this.tab = metaData.getTableForAlias(tableAlias);
@@ -301,7 +295,8 @@
if (t.isPartitionKey(colName)) {
// Set value to null if it's not partition column
if (tabAlias.equalsIgnoreCase(tableAlias)) {
- desc = new ExprNodeTempDesc(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, colName));
+ desc = new ExprNodeTempDesc(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
+ colName, tabAlias, true));
} else {
desc = new ExprNodeTempDesc(new exprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, null));
}
@@ -427,147 +422,13 @@
}
}
- /**
- * list of the partitions satisfying the pruning criteria - contains both confirmed and unknown partitions
- */
- public static class PrunedPartitionList {
- // confirmed partitions - satisfy the partition criteria
- private Set<Partition> confirmedPartns;
-
- // unknown partitions - may/may not satisfy the partition criteria
- private Set<Partition> unknownPartns;
-
- // denied partitions - do not satisfy the partition criteria
- private Set<Partition> deniedPartns;
-
- /**
- * @param confirmedPartns confirmed paritions
- * @param unknownPartns unknown partitions
- */
- public PrunedPartitionList(Set<Partition> confirmedPartns, Set<Partition> unknownPartns, Set<Partition> deniedPartns) {
- this.confirmedPartns = confirmedPartns;
- this.unknownPartns = unknownPartns;
- this.deniedPartns = deniedPartns;
- }
-
- /**
- * get confirmed partitions
- * @return confirmedPartns confirmed paritions
- */
- public Set<Partition> getConfirmedPartns() {
- return confirmedPartns;
- }
-
- /**
- * get unknown partitions
- * @return unknownPartns unknown paritions
- */
- public Set<Partition> getUnknownPartns() {
- return unknownPartns;
- }
-
- /**
- * get denied partitions
- * @return deniedPartns denied paritions
- */
- public Set<Partition> getDeniedPartns() {
- return deniedPartns;
- }
-
- /**
- * set confirmed partitions
- * @param confirmedPartns confirmed paritions
- */
- public void setConfirmedPartns(Set<Partition> confirmedPartns) {
- this.confirmedPartns = confirmedPartns;
- }
-
- /**
- * set unknown partitions
- * @param unknownPartns unknown partitions
- */
- public void setUnknownPartns(Set<Partition> unknownPartns) {
- this.unknownPartns = unknownPartns;
- }
- }
-
/**
* From the table metadata prune the partitions to return the partitions.
* Evaluate the parition pruner for each partition and return confirmed and unknown partitions separately
*/
@SuppressWarnings("nls")
public PrunedPartitionList prune() throws HiveException {
- LOG.trace("Started pruning partiton");
- LOG.trace("tabname = " + this.tab.getName());
- LOG.trace("prune Expression = " + this.prunerExpr);
-
- LinkedHashSet<Partition> true_parts = new LinkedHashSet<Partition>();
- LinkedHashSet<Partition> unkn_parts = new LinkedHashSet<Partition>();
- LinkedHashSet<Partition> denied_parts = new LinkedHashSet<Partition>();
-
- try {
- StructObjectInspector rowObjectInspector = (StructObjectInspector)this.tab.getDeserializer().getObjectInspector();
- Object[] rowWithPart = new Object[2];
-
- if(tab.isPartitioned()) {
- for(String partName: Hive.get().getPartitionNames(MetaStoreUtils.DEFAULT_DATABASE_NAME, tab.getName(), (short) -1)) {
- // Set all the variables here
- LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
- LOG.debug("part name: " + partName);
- // Create the row object
- ArrayList<String> partNames = new ArrayList<String>();
- ArrayList<String> partValues = new ArrayList<String>();
- ArrayList<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
- for(Map.Entry<String,String>entry : partSpec.entrySet()) {
- partNames.add(entry.getKey());
- partValues.add(entry.getValue());
- partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- }
- StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
-
- rowWithPart[1] = partValues;
- ArrayList<StructObjectInspector> ois = new ArrayList<StructObjectInspector>(2);
- ois.add(rowObjectInspector);
- ois.add(partObjectInspector);
- StructObjectInspector rowWithPartObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(ois);
-
- // evaluate the expression tree
- if (this.prunerExpr != null) {
- ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(this.prunerExpr);
- ObjectInspector evaluateResultOI = evaluator.initialize(rowWithPartObjectInspector);
- Object evaluateResultO = evaluator.evaluate(rowWithPart);
- Boolean r = (Boolean) ((PrimitiveObjectInspector)evaluateResultOI).getPrimitiveJavaObject(evaluateResultO);
- LOG.trace("prune result for partition " + partSpec + ": " + r);
- if (Boolean.FALSE.equals(r)) {
- if (denied_parts.isEmpty()) {
- Partition part = Hive.get().getPartition(tab, partSpec, Boolean.FALSE);
- denied_parts.add(part);
- }
- LOG.trace("pruned partition: " + partSpec);
- } else {
- Partition part = Hive.get().getPartition(tab, partSpec, Boolean.FALSE);
- if (Boolean.TRUE.equals(r)) {
- LOG.debug("retained partition: " + partSpec);
- true_parts.add(part);
- } else {
- LOG.debug("unknown partition: " + partSpec);
- unkn_parts.add(part);
- }
- }
- } else {
- // is there is no parition pruning, all of them are needed
- true_parts.add(Hive.get().getPartition(tab, partSpec, Boolean.FALSE));
- }
- }
- } else {
- true_parts.addAll(Hive.get().getPartitions(tab));
- }
- } catch (Exception e) {
- throw new HiveException(e);
- }
-
- // Now return the set of partitions
- return new PrunedPartitionList(true_parts, unkn_parts, denied_parts);
+ return org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(this.tab, this.prunerExpr);
}
public Table getTable() {
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java
------------------------------------------------------------------------------
svn:mergeinfo =
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=802976&r1=802975&r2=802976&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java Tue Aug 11 02:06:23 2009
@@ -27,10 +27,13 @@
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.loadFileDesc;
import org.apache.hadoop.hive.ql.plan.loadTableDesc;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
/**
@@ -46,12 +49,14 @@
public class ParseContext {
private QB qb;
private ASTNode ast;
- private HashMap<String, PartitionPruner> aliasToPruner;
+ private HashMap<String, ASTPartitionPruner> aliasToPruner;
+ private HashMap<TableScanOperator, exprNodeDesc> opToPartPruner;
private HashMap<String, SamplePruner> aliasToSamplePruner;
private HashMap<String, Operator<? extends Serializable>> topOps;
private HashMap<String, Operator<? extends Serializable>> topSelOps;
private LinkedHashMap<Operator<? extends Serializable>, OpParseContext> opParseCtx;
private Map<JoinOperator, QBJoinTree> joinContext;
+ private HashMap<TableScanOperator, Table> topToTable;
private List<loadTableDesc> loadTableWork;
private List<loadFileDesc> loadFileWork;
private Context ctx;
@@ -60,6 +65,12 @@
private int destTableId;
private UnionProcContext uCtx;
private List<MapJoinOperator> listMapJoinOpsNoReducer; // list of map join operators with no reducer
+
+ // is set to true if the expression only contains partitioning columns and not any other column reference.
+ // This is used to optimize select * from table where ... scenario, when the where condition only references
+ // partitioning columns - the partitions are identified and streamed directly to the client without requiring
+ // a map-reduce job
+ private boolean hasNonPartCols;
public ParseContext() {
}
@@ -71,6 +82,8 @@
* current parse tree
* @param aliasToPruner
* partition pruner list
+ * @param opToPartPruner
+ * map from table scan operator to partition pruner
* @param aliasToSamplePruner
* sample pruner list
* @param loadFileWork
@@ -85,15 +98,19 @@
* @param topSelOps
* list of operators for the selects introduced for column pruning
* @param listMapJoinOpsNoReducer
- * list of map join operators with no reducer
+ * list of map join operators with no reducer
+ * @param hasNonPartCols
+ * the query has non partition columns
*/
public ParseContext(HiveConf conf, QB qb, ASTNode ast,
- HashMap<String, PartitionPruner> aliasToPruner,
+ HashMap<String, ASTPartitionPruner> aliasToPruner,
+ HashMap<TableScanOperator, exprNodeDesc> opToPartPruner,
HashMap<String, SamplePruner> aliasToSamplePruner,
HashMap<String, Operator<? extends Serializable>> topOps,
HashMap<String, Operator<? extends Serializable>> topSelOps,
LinkedHashMap<Operator<? extends Serializable>, OpParseContext> opParseCtx,
Map<JoinOperator, QBJoinTree> joinContext,
+ HashMap<TableScanOperator, Table> topToTable,
List<loadTableDesc> loadTableWork, List<loadFileDesc> loadFileWork,
Context ctx, HashMap<String, String> idToTableNameMap, int destTableId, UnionProcContext uCtx,
List<MapJoinOperator> listMapJoinOpsNoReducer) {
@@ -101,8 +118,10 @@
this.qb = qb;
this.ast = ast;
this.aliasToPruner = aliasToPruner;
+ this.opToPartPruner = opToPartPruner;
this.aliasToSamplePruner = aliasToSamplePruner;
this.joinContext = joinContext;
+ this.topToTable = topToTable;
this.loadFileWork = loadFileWork;
this.loadTableWork = loadTableWork;
this.opParseCtx = opParseCtx;
@@ -113,6 +132,7 @@
this.destTableId = destTableId;
this.uCtx = uCtx;
this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer;
+ this.hasNonPartCols = false;
}
/**
@@ -178,7 +198,7 @@
/**
* @return the aliasToPruner
*/
- public HashMap<String, PartitionPruner> getAliasToPruner() {
+ public HashMap<String, ASTPartitionPruner> getAliasToPruner() {
return aliasToPruner;
}
@@ -186,11 +206,40 @@
* @param aliasToPruner
* the aliasToPruner to set
*/
- public void setAliasToPruner(HashMap<String, PartitionPruner> aliasToPruner) {
+ public void setAliasToPruner(HashMap<String, ASTPartitionPruner> aliasToPruner) {
this.aliasToPruner = aliasToPruner;
}
/**
+ * @return the opToPartPruner
+ */
+ public HashMap<TableScanOperator, exprNodeDesc> getOpToPartPruner() {
+ return opToPartPruner;
+ }
+
+ /**
+ * @param opToPartPruner
+ * the opToPartPruner to set
+ */
+ public void setOpToPartPruner(HashMap<TableScanOperator, exprNodeDesc> opToPartPruner) {
+ this.opToPartPruner = opToPartPruner;
+ }
+
+ /**
+ * @return the topToTable
+ */
+ public HashMap<TableScanOperator, Table> getTopToTable() {
+ return topToTable;
+ }
+
+ /**
+ * @param topToTable
+ * the topToTable to set
+ */
+ public void setTopToTable(HashMap<TableScanOperator, Table> topToTable) {
+ this.topToTable = topToTable;
+ }
+ /**
* @return the aliasToSamplePruner
*/
public HashMap<String, SamplePruner> getAliasToSamplePruner() {
@@ -335,4 +384,19 @@
List<MapJoinOperator> listMapJoinOpsNoReducer) {
this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer;
}
+
+ /**
+ * Sets the hasNonPartCols flag
+ * @param val
+ */
+ public void setHasNonPartCols(boolean val) {
+ this.hasNonPartCols = val;
+ }
+
+ /**
+ * Gets the value of the hasNonPartCols flag
+ */
+ public boolean getHasNonPartCols() {
+ return this.hasNonPartCols;
+ }
}