You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/06/06 03:17:19 UTC
svn commit: r1490101 - in /hive/trunk: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: hashutosh
Date: Thu Jun 6 01:17:19 2013
New Revision: 1490101
URL: http://svn.apache.org/r1490101
Log:
HIVE-4554 : Failed to create a table from existing file if file path has spaces (Xuefu Zhang via Ashutosh Chauhan)
Added:
hive/trunk/data/files/person age.txt
hive/trunk/ql/src/test/queries/clientpositive/load_file_with_space_in_the_name.q
hive/trunk/ql/src/test/queries/clientpositive/load_hdfs_file_with_space_in_the_name.q
hive/trunk/ql/src/test/results/clientpositive/load_file_with_space_in_the_name.q.out
hive/trunk/ql/src/test/results/clientpositive/load_hdfs_file_with_space_in_the_name.q.out
Modified:
hive/trunk/build-common.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1490101&r1=1490100&r2=1490101&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Thu Jun 6 01:17:19 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q,remote_script.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q,remote_script.q,load_hdfs_file_with_space_in_the_name.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Added: hive/trunk/data/files/person age.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/person%20age.txt?rev=1490101&view=auto
==============================================================================
--- hive/trunk/data/files/person age.txt (added)
+++ hive/trunk/data/files/person age.txt Thu Jun 6 01:17:19 2013
@@ -0,0 +1,5 @@
+John 23
+Tom 17
+Jim 31
+Boby 9
+Paul 51
\ No newline at end of file
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1490101&r1=1490100&r2=1490101&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Thu Jun 6 01:17:19 2013
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import org.antlr.runtime.tree.Tree;
+import org.apache.commons.httpclient.URIException;
+import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -82,7 +84,7 @@ public class LoadSemanticAnalyzer extend
// directory
if (!path.startsWith("/")) {
if (isLocal) {
- path = new Path(System.getProperty("user.dir"), path).toUri().toString();
+ path = URIUtil.decode( new Path(System.getProperty("user.dir"), path).toUri().toString() );
} else {
path = new Path(new Path("/user/" + System.getProperty("user.name")),
path).toString();
@@ -231,8 +233,13 @@ public class LoadSemanticAnalyzer extend
// that's just a test case.
String copyURIStr = ctx.getExternalTmpFileURI(toURI);
URI copyURI = URI.create(copyURIStr);
- rTask = TaskFactory.get(new CopyWork(fromURI.toString(), copyURIStr),
- conf);
+ try {
+ rTask = TaskFactory.get(new CopyWork(URIUtil.decode(fromURI.toString()), copyURIStr),
+ conf);
+ } catch (URIException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e
+ .getMessage()), e);
+ }
fromURI = copyURI;
}
@@ -261,8 +268,14 @@ public class LoadSemanticAnalyzer extend
}
- LoadTableDesc loadTableWork = new LoadTableDesc(fromURI.toString(),
- loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ LoadTableDesc loadTableWork;
+ try {
+ loadTableWork = new LoadTableDesc(URIUtil.decode(fromURI.toString()),
+ loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ } catch (URIException e1) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e1
+ .getMessage()), e1);
+ }
Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(),
getOutputs(), loadTableWork, null, true), conf);
Added: hive/trunk/ql/src/test/queries/clientpositive/load_file_with_space_in_the_name.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/load_file_with_space_in_the_name.q?rev=1490101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/load_file_with_space_in_the_name.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/load_file_with_space_in_the_name.q Thu Jun 6 01:17:19 2013
@@ -0,0 +1,5 @@
+-- test for loading into tables with the file with space in the name
+
+
+CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT);
+LOAD DATA LOCAL INPATH '../data/files/person age.txt' INTO TABLE load_file_with_space_in_the_name;
Added: hive/trunk/ql/src/test/queries/clientpositive/load_hdfs_file_with_space_in_the_name.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/load_hdfs_file_with_space_in_the_name.q?rev=1490101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/load_hdfs_file_with_space_in_the_name.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/load_hdfs_file_with_space_in_the_name.q Thu Jun 6 01:17:19 2013
@@ -0,0 +1,9 @@
+dfs -mkdir hdfs:///tmp/test/load_file_with_space_in_the_name;
+
+dfs -copyFromLocal ../data/files hdfs:///tmp/test/load_file_with_space_in_the_name;
+
+CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT);
+LOAD DATA INPATH 'hdfs:///tmp/test/load_file_with_space_in_the_name/files/person age.txt' INTO TABLE load_file_with_space_in_the_name;
+
+dfs -rmr hdfs:///tmp/test;
+
Added: hive/trunk/ql/src/test/results/clientpositive/load_file_with_space_in_the_name.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/load_file_with_space_in_the_name.q.out?rev=1490101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/load_file_with_space_in_the_name.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/load_file_with_space_in_the_name.q.out Thu Jun 6 01:17:19 2013
@@ -0,0 +1,17 @@
+PREHOOK: query: -- test for loading into tables with the file with space in the name
+
+
+CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- test for loading into tables with the file with space in the name
+
+
+CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@load_file_with_space_in_the_name
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/person age.txt' INTO TABLE load_file_with_space_in_the_name
+PREHOOK: type: LOAD
+PREHOOK: Output: default@load_file_with_space_in_the_name
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/person age.txt' INTO TABLE load_file_with_space_in_the_name
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@load_file_with_space_in_the_name
Added: hive/trunk/ql/src/test/results/clientpositive/load_hdfs_file_with_space_in_the_name.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/load_hdfs_file_with_space_in_the_name.q.out?rev=1490101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/load_hdfs_file_with_space_in_the_name.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/load_hdfs_file_with_space_in_the_name.q.out Thu Jun 6 01:17:19 2013
@@ -0,0 +1,12 @@
+PREHOOK: query: CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE load_file_with_space_in_the_name(name STRING, age INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@load_file_with_space_in_the_name
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+PREHOOK: Output: default@load_file_with_space_in_the_name
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@load_file_with_space_in_the_name
+#### A masked pattern was here ####