You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2014/05/09 01:41:38 UTC
svn commit: r1593464 - in /hive/branches/branch-0.13: itests/qtest/
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez/
Author: khorgath
Date: Thu May 8 23:41:38 2014
New Revision: 1593464
URL: http://svn.apache.org/r1593464
Log:
HIVE-6826 : Hive-tez has issues when different partitions work off of different input types (Vikram Dixit K, reviewed by Gunther Hagleitner)
Added:
hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_schema_evolution.q
hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out
Modified:
hive/branches/branch-0.13/itests/qtest/pom.xml
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Modified: hive/branches/branch-0.13/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/itests/qtest/pom.xml?rev=1593464&r1=1593463&r2=1593464&view=diff
==============================================================================
--- hive/branches/branch-0.13/itests/qtest/pom.xml (original)
+++ hive/branches/branch-0.13/itests/qtest/pom.xml Thu May 8 23:41:38 2014
@@ -38,7 +38,7 @@
<execute.beeline.tests>false</execute.beeline.tests>
<minimr.query.files>stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q</minimr.query.files>
<minimr.query.negative.files>cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q</minimr.query.negative.files>
- <minitez.query.files>tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q</minitez.query.files>
+ <minitez.query.files>tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q</minitez.query.files>
<minitez.query.files.shared>cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q</minitez.query.files.shared>
<beeline.positive.exclude>add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rena
me.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_o
verwrite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q</beeline.positive.exclude>
</properties>
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java?rev=1593464&r1=1593463&r2=1593464&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java Thu May 8 23:41:38 2014
@@ -35,6 +35,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.serializer.SerializationFactory;
@@ -77,6 +78,7 @@ import com.google.common.collect.Multima
public class CustomPartitionVertex implements VertexManagerPlugin {
private static final Log LOG = LogFactory.getLog(CustomPartitionVertex.class.getName());
+ public static final String GROUP_SPLITS = "hive.enable.custom.grouped.splits";
VertexManagerPluginContext context;
@@ -152,14 +154,15 @@ public class CustomPartitionVertex imple
* TezGroupedSplits.
*/
- // This assumes that Grouping will always be used.
- // Changing the InputFormat - so that the correct one is initialized in MRInput.
- this.conf.set("mapred.input.format.class", TezGroupedSplitsInputFormat.class.getName());
- MRInputUserPayloadProto updatedPayload = MRInputUserPayloadProto
- .newBuilder(protoPayload)
- .setConfigurationBytes(MRHelpers.createByteStringFromConf(conf))
- .build();
- inputDescriptor.setUserPayload(updatedPayload.toByteArray());
+ if (conf.getBoolean(GROUP_SPLITS, true)) {
+ // Changing the InputFormat - so that the correct one is initialized in MRInput.
+ this.conf.set("mapred.input.format.class", TezGroupedSplitsInputFormat.class.getName());
+ MRInputUserPayloadProto updatedPayload = MRInputUserPayloadProto
+ .newBuilder(protoPayload)
+ .setConfigurationBytes(MRHelpers.createByteStringFromConf(conf))
+ .build();
+ inputDescriptor.setUserPayload(updatedPayload.toByteArray());
+ }
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
@@ -315,24 +318,25 @@ public class CustomPartitionVertex imple
}
private void groupSplits () throws IOException {
- estimateBucketSizes();
bucketToGroupedSplitMap =
- ArrayListMultimap.<Integer, InputSplit>create(bucketToInitialSplitMap);
-
- Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
- for (int bucketId : bucketSplitMap.keySet()) {
- Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
- TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();
-
- InputSplit[] groupedSplits = grouper.getGroupedSplits(conf,
- inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
- HiveInputFormat.class.getName());
- LOG.info("Original split size is " +
- inputSplitCollection.toArray(new InputSplit[0]).length +
- " grouped split size is " + groupedSplits.length);
- bucketToGroupedSplitMap.removeAll(bucketId);
- for (InputSplit inSplit : groupedSplits) {
- bucketToGroupedSplitMap.put(bucketId, inSplit);
+ ArrayListMultimap.<Integer, InputSplit>create(bucketToInitialSplitMap);
+ if (conf.getBoolean(GROUP_SPLITS, true)) {
+ estimateBucketSizes();
+ Map<Integer, Collection<InputSplit>> bucketSplitMap = bucketToInitialSplitMap.asMap();
+ for (int bucketId : bucketSplitMap.keySet()) {
+ Collection<InputSplit>inputSplitCollection = bucketSplitMap.get(bucketId);
+ TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper();
+
+ InputSplit[] groupedSplits = grouper.getGroupedSplits(conf,
+ inputSplitCollection.toArray(new InputSplit[0]), bucketToNumTaskMap.get(bucketId),
+ HiveInputFormat.class.getName());
+ LOG.info("Original split size is " +
+ inputSplitCollection.toArray(new InputSplit[0]).length +
+ " grouped split size is " + groupedSplits.length);
+ bucketToGroupedSplitMap.removeAll(bucketId);
+ for (InputSplit inSplit : groupedSplits) {
+ bucketToGroupedSplitMap.put(bucketId, inSplit);
+ }
}
}
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1593464&r1=1593463&r2=1593464&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Thu May 8 23:41:38 2014
@@ -23,6 +23,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -50,12 +51,14 @@ import org.apache.hadoop.hive.ql.exec.mr
import org.apache.hadoop.hive.ql.exec.mr.ExecReducer;
import org.apache.hadoop.hive.ql.exec.tez.tools.TezMergedLogicalInput;
import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.ReduceWork;
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty;
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
@@ -412,7 +415,7 @@ public class DagUtils {
Vertex map = null;
// use tez to combine splits
- boolean useTezGroupedSplits = false;
+ boolean useTezGroupedSplits = true;
int numTasks = -1;
Class amSplitGeneratorClass = null;
@@ -428,8 +431,44 @@ public class DagUtils {
}
}
}
+
+ // we cannot currently allow grouping of splits where each split is a different input format
+ // or has different deserializers similar to the checks in CombineHiveInputFormat. We do not
+ // need the check for the opList because we will not process different opLists at this time.
+ // Long term fix would be to have a custom input format
+ // logic that groups only the splits that share the same input format
+ Class<?> previousInputFormatClass = null;
+ Class<?> previousDeserializerClass = null;
+ for (String path : mapWork.getPathToPartitionInfo().keySet()) {
+ PartitionDesc pd = mapWork.getPathToPartitionInfo().get(path);
+ Class<?> currentDeserializerClass = pd.getDeserializer(conf).getClass();
+ Class<?> currentInputFormatClass = pd.getInputFileFormatClass();
+ if (previousInputFormatClass == null) {
+ previousInputFormatClass = currentInputFormatClass;
+ }
+ if (previousDeserializerClass == null) {
+ previousDeserializerClass = currentDeserializerClass;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Current input format class = "+currentInputFormatClass+", previous input format class = "
+ + previousInputFormatClass + ", verifying " + " current deserializer class = "
+ + currentDeserializerClass + " previous deserializer class = " + previousDeserializerClass);
+ }
+ if ((currentInputFormatClass != previousInputFormatClass) ||
+ (currentDeserializerClass != previousDeserializerClass)) {
+ useTezGroupedSplits = false;
+ break;
+ }
+ }
if (vertexHasCustomInput) {
- useTezGroupedSplits = false;
+ // if it is the case of different input formats for different partitions, we cannot group
+ // in the custom vertex for now. Long term, this can be improved to group the buckets that
+ // share the same input format.
+ if (useTezGroupedSplits == false) {
+ conf.setBoolean(CustomPartitionVertex.GROUP_SPLITS, false);
+ } else {
+ conf.setBoolean(CustomPartitionVertex.GROUP_SPLITS, true);
+ }
// grouping happens in execution phase. Setting the class to TezGroupedSplitsInputFormat
// here would cause pre-mature grouping which would be incorrect.
inputFormatClass = HiveInputFormat.class;
@@ -437,13 +476,17 @@ public class DagUtils {
// mapreduce.tez.input.initializer.serialize.event.payload should be set to false when using
// this plug-in to avoid getting a serialized event at run-time.
conf.setBoolean("mapreduce.tez.input.initializer.serialize.event.payload", false);
- } else {
+ } else if (useTezGroupedSplits) {
// we'll set up tez to combine spits for us iff the input format
// is HiveInputFormat
if (inputFormatClass == HiveInputFormat.class) {
- useTezGroupedSplits = true;
conf.setClass("mapred.input.format.class", TezGroupedSplitsInputFormat.class, InputFormat.class);
+ } else {
+ conf.setClass("mapred.input.format.class", CombineHiveInputFormat.class, InputFormat.class);
+ useTezGroupedSplits = false;
}
+ } else {
+ conf.setClass("mapred.input.format.class", CombineHiveInputFormat.class, InputFormat.class);
}
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_AM_SPLIT_GENERATION)) {
Added: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_schema_evolution.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_schema_evolution.q?rev=1593464&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_schema_evolution.q (added)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_schema_evolution.q Thu May 8 23:41:38 2014
@@ -0,0 +1,14 @@
+create table test (key int, value string) partitioned by (p int) stored as textfile;
+
+insert into table test partition (p=1) select * from src limit 10;
+
+alter table test set fileformat orc;
+
+insert into table test partition (p=2) select * from src limit 10;
+
+describe test;
+
+select * from test where p=1 and key > 0;
+select * from test where p=2 and key > 0;
+select * from test where key > 0;
+
Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out?rev=1593464&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out (added)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out Thu May 8 23:41:38 2014
@@ -0,0 +1,141 @@
+PREHOOK: query: create table test (key int, value string) partitioned by (p int) stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table test (key int, value string) partitioned by (p int) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: insert into table test partition (p=1) select * from src limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test@p=1
+POSTHOOK: query: insert into table test partition (p=1) select * from src limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test@p=1
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table test set fileformat orc
+PREHOOK: type: ALTERTABLE_FILEFORMAT
+PREHOOK: Input: default@test
+PREHOOK: Output: default@test
+POSTHOOK: query: alter table test set fileformat orc
+POSTHOOK: type: ALTERTABLE_FILEFORMAT
+POSTHOOK: Input: default@test
+POSTHOOK: Output: default@test
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert into table test partition (p=2) select * from src limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test@p=2
+POSTHOOK: query: insert into table test partition (p=2) select * from src limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test@p=2
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe test
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test
+POSTHOOK: query: describe test
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+key int
+value string
+p int
+
+# Partition Information
+# col_name data_type comment
+
+p int
+PREHOOK: query: select * from test where p=1 and key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Input: default@test@p=1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test where p=1 and key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Input: default@test@p=1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+484 val_484 1
+98 val_98 1
+278 val_278 1
+255 val_255 1
+409 val_409 1
+165 val_165 1
+27 val_27 1
+311 val_311 1
+86 val_86 1
+238 val_238 1
+PREHOOK: query: select * from test where p=2 and key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test where p=2 and key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+484 val_484 2
+98 val_98 2
+278 val_278 2
+255 val_255 2
+409 val_409 2
+165 val_165 2
+27 val_27 2
+311 val_311 2
+86 val_86 2
+238 val_238 2
+PREHOOK: query: select * from test where key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Input: default@test@p=1
+PREHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test where key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Input: default@test@p=1
+POSTHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+484 val_484 2
+98 val_98 2
+278 val_278 2
+255 val_255 2
+409 val_409 2
+165 val_165 2
+27 val_27 2
+311 val_311 2
+86 val_86 2
+238 val_238 2
+484 val_484 1
+98 val_98 1
+278 val_278 1
+255 val_255 1
+409 val_409 1
+165 val_165 1
+27 val_27 1
+311 val_311 1
+86 val_86 1
+238 val_238 1