You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/24 22:38:37 UTC
svn commit: r1471661 - in /hive/branches/branch-0.11: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: hashutosh
Date: Wed Apr 24 20:38:37 2013
New Revision: 1471661
URL: http://svn.apache.org/r1471661
Log:
HIVE-4275 : Hive does not differentiate scheme and authority in file uris (Vikram Dixit via Ashutosh Chauhan)
Added:
hive/branches/branch-0.11/ql/src/test/queries/clientpositive/schemeAuthority.q
hive/branches/branch-0.11/ql/src/test/results/clientpositive/schemeAuthority.q.out
Modified:
hive/branches/branch-0.11/build-common.xml
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
Modified: hive/branches/branch-0.11/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/build-common.xml?rev=1471661&r1=1471660&r2=1471661&view=diff
==============================================================================
--- hive/branches/branch-0.11/build-common.xml (original)
+++ hive/branches/branch-0.11/build-common.xml Wed Apr 24 20:38:37 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1471661&r1=1471660&r2=1471661&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Wed Apr 24 20:38:37 2013
@@ -138,7 +138,6 @@ public class MapOperator extends Operato
public void setOp(Operator<? extends OperatorDesc> op) {
this.op = op;
}
-
}
private static class MapOpCtx {
@@ -468,8 +467,8 @@ public class MapOperator extends Operato
public void setChildren(Configuration hconf) throws HiveException {
- Path fpath = new Path((new Path(HiveConf.getVar(hconf,
- HiveConf.ConfVars.HADOOPMAPFILENAME))).toUri().getPath());
+ Path fpath = new Path(HiveConf.getVar(hconf,
+ HiveConf.ConfVars.HADOOPMAPFILENAME));
ArrayList<Operator<? extends OperatorDesc>> children =
new ArrayList<Operator<? extends OperatorDesc>>();
@@ -481,7 +480,7 @@ public class MapOperator extends Operato
try {
for (String onefile : conf.getPathToAliases().keySet()) {
MapOpCtx opCtx = initObjectInspector(conf, hconf, onefile, convertedOI);
- Path onepath = new Path(new Path(onefile).toUri().getPath());
+ Path onepath = new Path(onefile);
List<String> aliases = conf.getPathToAliases().get(onefile);
for (String onealias : aliases) {
@@ -513,7 +512,7 @@ public class MapOperator extends Operato
// didn't find match for input file path in configuration!
// serious problem ..
LOG.error("Configuration does not have any alias for path: "
- + fpath.toUri().getPath());
+ + fpath.toUri());
throw new HiveException("Configuration and input path are inconsistent");
}
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1471661&r1=1471660&r2=1471661&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Apr 24 20:38:37 2013
@@ -690,11 +690,6 @@ public final class Utilities {
return new PartitionDesc(part, tblDesc);
}
- public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?> work) {
- mr.addMapWork(tbl.getDataLocation().getPath(), alias, work, new PartitionDesc(
- getTableDesc(tbl), (LinkedHashMap<String, String>) null));
- }
-
private static String getOpTreeSkel_helper(Operator<?> op, String indent) {
if (op == null) {
return "";
Modified: hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1471661&r1=1471660&r2=1471661&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Wed Apr 24 20:38:37 2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.ScriptDesc;
@@ -132,6 +133,11 @@ public class TestExecDriver extends Test
mr = PlanUtils.getMapRedWork();
}
+ public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?> work) {
+ mr.addMapWork(tbl.getDataLocation().toString(), alias, work, new PartitionDesc(
+ Utilities.getTableDesc(tbl), null));
+ }
+
private static void fileDiff(String datafile, String testdir) throws Exception {
String testFileDir = conf.get("test.data.files");
System.out.println(testFileDir);
@@ -190,7 +196,7 @@ public class TestExecDriver extends Test
Operator<FilterDesc> op1 = OperatorFactory.get(getTestFilterDesc("key"),
op2);
- Utilities.addMapWork(mr, src, "a", op1);
+ addMapWork(mr, src, "a", op1);
}
@SuppressWarnings("unchecked")
@@ -209,7 +215,7 @@ public class TestExecDriver extends Test
Operator<FilterDesc> op1 = OperatorFactory.get(getTestFilterDesc("key"),
op2);
- Utilities.addMapWork(mr, src, "a", op1);
+ addMapWork(mr, src, "a", op1);
}
@SuppressWarnings("unchecked")
@@ -226,7 +232,7 @@ public class TestExecDriver extends Test
Utilities.makeList(getStringColumn("value")), outputColumns, true,
-1, 1, -1));
- Utilities.addMapWork(mr, src, "a", op1);
+ addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
@@ -254,7 +260,7 @@ public class TestExecDriver extends Test
.makeList(getStringColumn("key"), getStringColumn("value")),
outputColumns, false, -1, 1, -1));
- Utilities.addMapWork(mr, src, "a", op1);
+ addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
@@ -287,7 +293,7 @@ public class TestExecDriver extends Test
Utilities.makeList(getStringColumn("value")), outputColumns, true,
Byte.valueOf((byte) 0), 1, -1));
- Utilities.addMapWork(mr, src, "a", op1);
+ addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
@@ -296,7 +302,7 @@ public class TestExecDriver extends Test
Utilities.makeList(getStringColumn("key")), outputColumns, true,
Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1));
- Utilities.addMapWork(mr, src2, "b", op2);
+ addMapWork(mr, src2, "b", op2);
mr.getTagToValueDesc().add(op2.getConf().getValueSerializeInfo());
// reduce side work
@@ -338,7 +344,7 @@ public class TestExecDriver extends Test
.makeList(getStringColumn("key"), getStringColumn("value")),
outputColumns), op0);
- Utilities.addMapWork(mr, src, "a", op4);
+ addMapWork(mr, src, "a", op4);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
@@ -375,7 +381,7 @@ public class TestExecDriver extends Test
.makeList(getStringColumn("key"), getStringColumn("value")),
outputColumns), op0);
- Utilities.addMapWork(mr, src, "a", op4);
+ addMapWork(mr, src, "a", op4);
mr.setKeyDesc(op0.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op0.getConf().getValueSerializeInfo());
@@ -414,7 +420,7 @@ public class TestExecDriver extends Test
.makeList(getStringColumn("key"), getStringColumn("value")),
outputColumns), op0);
- Utilities.addMapWork(mr, src, "a", op4);
+ addMapWork(mr, src, "a", op4);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
Modified: hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java?rev=1471661&r1=1471660&r2=1471661&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (original)
+++ hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java Wed Apr 24 20:38:37 2013
@@ -320,7 +320,7 @@ public class TestOperators extends TestC
aliases.add("b");
LinkedHashMap<String, ArrayList<String>> pathToAliases =
new LinkedHashMap<String, ArrayList<String>>();
- pathToAliases.put("/testDir", aliases);
+ pathToAliases.put("hdfs:///testDir", aliases);
// initialize pathToTableInfo
// Default: treat the table as a single column "col"
@@ -328,7 +328,7 @@ public class TestOperators extends TestC
PartitionDesc pd = new PartitionDesc(td, null);
LinkedHashMap<String, org.apache.hadoop.hive.ql.plan.PartitionDesc> pathToPartitionInfo =
new LinkedHashMap<String, org.apache.hadoop.hive.ql.plan.PartitionDesc>();
- pathToPartitionInfo.put("/testDir", pd);
+ pathToPartitionInfo.put("hdfs:///testDir", pd);
// initialize aliasToWork
CollectDesc cd = new CollectDesc(Integer.valueOf(1));
Added: hive/branches/branch-0.11/ql/src/test/queries/clientpositive/schemeAuthority.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/queries/clientpositive/schemeAuthority.q?rev=1471661&view=auto
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/queries/clientpositive/schemeAuthority.q (added)
+++ hive/branches/branch-0.11/ql/src/test/queries/clientpositive/schemeAuthority.q Wed Apr 24 20:38:37 2013
@@ -0,0 +1,15 @@
+dfs -mkdir file:///tmp/test;
+dfs -mkdir hdfs:///tmp/test;
+
+create external table dynPart (key string) partitioned by (value string) row format delimited fields terminated by '\\t' stored as textfile;
+insert overwrite local directory "/tmp/test" select key from src where (key = 10) order by key;
+insert overwrite directory "/tmp/test" select key from src where (key = 20) order by key;
+alter table dynPart add partition (value='0') location 'file:///tmp/test';
+alter table dynPart add partition (value='1') location 'hdfs:///tmp/test';
+select count(*) from dynPart;
+select key from dynPart;
+select key from src where (key = 10) order by key;
+select key from src where (key = 20) order by key;
+
+dfs -rmr file:///tmp/test;
+dfs -rmr hdfs:///tmp/test;
Added: hive/branches/branch-0.11/ql/src/test/results/clientpositive/schemeAuthority.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/schemeAuthority.q.out?rev=1471661&view=auto
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/schemeAuthority.q.out (added)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/schemeAuthority.q.out Wed Apr 24 20:38:37 2013
@@ -0,0 +1,77 @@
+PREHOOK: query: create external table dynPart (key string) partitioned by (value string) row format delimited fields terminated by '\\t' stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create external table dynPart (key string) partitioned by (value string) row format delimited fields terminated by '\\t' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dynPart
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=0
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=1
+PREHOOK: query: select count(*) from dynPart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dynpart
+PREHOOK: Input: default@dynpart@value=0
+PREHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from dynPart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Input: default@dynpart@value=0
+POSTHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+2
+PREHOOK: query: select key from dynPart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dynpart
+PREHOOK: Input: default@dynpart@value=0
+PREHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+POSTHOOK: query: select key from dynPart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Input: default@dynpart@value=0
+POSTHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+20
+10
+PREHOOK: query: select key from src where (key = 10) order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where (key = 10) order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+10
+PREHOOK: query: select key from src where (key = 20) order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where (key = 20) order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+20
+#### A masked pattern was here ####