You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/13 14:12:58 UTC
svn commit: r1513451 - in /hive/trunk: build-common.xml
ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
ql/src/test/queries/clientpositive/schemeAuthority2.q
ql/src/test/results/clientpositive/schemeAuthority2.q.out
Author: hashutosh
Date: Tue Aug 13 12:12:58 2013
New Revision: 1513451
URL: http://svn.apache.org/r1513451
Log:
HIVE-5023 : Hive get wrong result when partition has the same path but different schema or authority (Shuaishuai Nie via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority2.q
hive/trunk/ql/src/test/results/clientpositive/schemeAuthority2.q.out
Modified:
hive/trunk/build-common.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1513451&r1=1513450&r2=1513451&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Tue Aug 13 12:12:58 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1513451&r1=1513450&r2=1513451&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Aug 13 12:12:58 2013
@@ -480,7 +480,7 @@ public class MapOperator extends Operato
}
private Path normalizePath(String onefile) {
- return new Path(new Path(onefile).toUri().getPath());
+ return new Path(onefile);
}
public void process(Writable value) throws HiveException {
Added: hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority2.q?rev=1513451&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority2.q Tue Aug 13 12:12:58 2013
@@ -0,0 +1,12 @@
+dfs -mkdir file:///tmp/test;
+dfs -mkdir hdfs:///tmp/test;
+
+create external table dynPart (key string) partitioned by (value string, value2 string) row format delimited fields terminated by '\\t' stored as textfile;
+insert overwrite local directory "/tmp/test" select key from src where (key = 10) order by key;
+insert overwrite directory "/tmp/test" select key from src where (key = 20) order by key;
+alter table dynPart add partition (value='0', value2='clusterA') location 'file:///tmp/test';
+alter table dynPart add partition (value='0', value2='clusterB') location 'hdfs:///tmp/test';
+select value2, key from dynPart where value='0';
+
+dfs -rmr file:///tmp/test;
+dfs -rmr hdfs:///tmp/test;
Added: hive/trunk/ql/src/test/results/clientpositive/schemeAuthority2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/schemeAuthority2.q.out?rev=1513451&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/schemeAuthority2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/schemeAuthority2.q.out Tue Aug 13 12:12:58 2013
@@ -0,0 +1,46 @@
+PREHOOK: query: create external table dynPart (key string) partitioned by (value string, value2 string) row format delimited fields terminated by '\\t' stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create external table dynPart (key string) partitioned by (value string, value2 string) row format delimited fields terminated by '\\t' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dynPart
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=0/value2=clusterA
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=0/value2=clusterB
+PREHOOK: query: select value2, key from dynPart where value='0'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dynpart
+PREHOOK: Input: default@dynpart@value=0/value2=clusterA
+PREHOOK: Input: default@dynpart@value=0/value2=clusterB
+#### A masked pattern was here ####
+POSTHOOK: query: select value2, key from dynPart where value='0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Input: default@dynpart@value=0/value2=clusterA
+POSTHOOK: Input: default@dynpart@value=0/value2=clusterB
+#### A masked pattern was here ####
+clusterB 20
+clusterA 10
+#### A masked pattern was here ####