You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/26 00:59:23 UTC
svn commit: r1526325 - in /hive/trunk: ./ data/files/exported_table/
data/files/exported_table/data/ ql/src/java/org/apache/hadoop/hive/ql/plan/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: hashutosh
Date: Wed Sep 25 22:59:23 2013
New Revision: 1526325
URL: http://svn.apache.org/r1526325
Log:
HIVE-5318 : Import Throws Error when Importing from a table export Hive 0.9 to Hive 0.10 (Xuefu Zhang via Ashutosh Chauhan)
Added:
hive/trunk/data/files/exported_table/
hive/trunk/data/files/exported_table/_metadata
hive/trunk/data/files/exported_table/data/
hive/trunk/data/files/exported_table/data/data
hive/trunk/ql/src/test/queries/clientpositive/import_exported_table.q
hive/trunk/ql/src/test/results/clientpositive/import_exported_table.q.out
Modified:
hive/trunk/build-common.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1526325&r1=1526324&r2=1526325&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Wed Sep 25 22:59:23 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Added: hive/trunk/data/files/exported_table/_metadata
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/exported_table/_metadata?rev=1526325&view=auto
==============================================================================
--- hive/trunk/data/files/exported_table/_metadata (added)
+++ hive/trunk/data/files/exported_table/_metadata Wed Sep 25 22:59:23 2013
@@ -0,0 +1 @@
+{"partitions":[],"table":"{\"1\":{\"str\":\"j1_41\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"johndee\"},\"4\":{\"i32\":1371900915},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",2,{\"1\":{\"str\":\"a\"},\"2\":{\"str\":\"string\"}},{\"1\":{\"str\":\"b\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://hivebase01:8020/user/hive/warehouse/j1_41\"},\"3\":{\"str\":\"org.apache.hadoop.mapred.TextInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\"},\"3\":{\"map\":[\"str\",\"str\",2,{\"serialization.format\":\",\",\"field.delim\":\",\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",1,{\"transient_lastDdlTime\":\"1371900931\"}]},\"12\":{\"str\":\"MANAGED_TABLE\"}}","version":"0.1"
}
\ No newline at end of file
Added: hive/trunk/data/files/exported_table/data/data
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/exported_table/data/data?rev=1526325&view=auto
==============================================================================
--- hive/trunk/data/files/exported_table/data/data (added)
+++ hive/trunk/data/files/exported_table/data/data Wed Sep 25 22:59:23 2013
@@ -0,0 +1,2 @@
+johndee,1
+burks,2
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=1526325&r1=1526324&r2=1526325&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Wed Sep 25 22:59:23 2013
@@ -31,7 +31,6 @@ import org.apache.hadoop.hive.common.Jav
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -53,10 +52,10 @@ public class CreateTableDesc extends DDL
String databaseName;
String tableName;
boolean isExternal;
- ArrayList<FieldSchema> cols;
- ArrayList<FieldSchema> partCols;
- ArrayList<String> bucketCols;
- ArrayList<Order> sortCols;
+ List<FieldSchema> cols;
+ List<FieldSchema> partCols;
+ List<String> bucketCols;
+ List<Order> sortCols;
int numBuckets;
String fieldDelim;
String fieldEscape;
@@ -130,8 +129,12 @@ public class CreateTableDesc extends DDL
this.serdeProps = serdeProps;
this.tblProps = tblProps;
this.ifNotExists = ifNotExists;
- this.skewedColNames = new ArrayList<String>(skewedColNames);
- this.skewedColValues = new ArrayList<List<String>>(skewedColValues);
+ this.skewedColNames = copyList(skewedColNames);
+ this.skewedColValues = copyList(skewedColValues);
+ }
+
+ private static <T> List<T> copyList(List<T> copy) {
+ return copy == null ? null : new ArrayList<T>(copy);
}
@Explain(displayName = "columns")
@@ -166,7 +169,7 @@ public class CreateTableDesc extends DDL
this.tableName = tableName;
}
- public ArrayList<FieldSchema> getCols() {
+ public List<FieldSchema> getCols() {
return cols;
}
@@ -174,7 +177,7 @@ public class CreateTableDesc extends DDL
this.cols = cols;
}
- public ArrayList<FieldSchema> getPartCols() {
+ public List<FieldSchema> getPartCols() {
return partCols;
}
@@ -183,7 +186,7 @@ public class CreateTableDesc extends DDL
}
@Explain(displayName = "bucket columns")
- public ArrayList<String> getBucketCols() {
+ public List<String> getBucketCols() {
return bucketCols;
}
@@ -303,7 +306,7 @@ public class CreateTableDesc extends DDL
* @return the sortCols
*/
@Explain(displayName = "sort columns")
- public ArrayList<Order> getSortCols() {
+ public List<Order> getSortCols() {
return sortCols;
}
Added: hive/trunk/ql/src/test/queries/clientpositive/import_exported_table.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/import_exported_table.q?rev=1526325&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/import_exported_table.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/import_exported_table.q Wed Sep 25 22:59:23 2013
@@ -0,0 +1,10 @@
+dfs ${system:test.dfs.mkdir} hdfs:///tmp/test/;
+
+dfs -copyFromLocal ../data/files/exported_table hdfs:///tmp/test/;
+
+IMPORT FROM '/tmp/test/exported_table';
+DESCRIBE j1_41;
+SELECT * from j1_41;
+
+dfs -rmr hdfs:///tmp/test/exported_table;
+
Added: hive/trunk/ql/src/test/results/clientpositive/import_exported_table.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/import_exported_table.q.out?rev=1526325&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/import_exported_table.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/import_exported_table.q.out Wed Sep 25 22:59:23 2013
@@ -0,0 +1,22 @@
+#### A masked pattern was here ####
+PREHOOK: type: IMPORT
+#### A masked pattern was here ####
+POSTHOOK: type: IMPORT
+POSTHOOK: Output: default@j1_41
+PREHOOK: query: DESCRIBE j1_41
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE j1_41
+POSTHOOK: type: DESCTABLE
+a string None
+b int None
+PREHOOK: query: SELECT * from j1_41
+PREHOOK: type: QUERY
+PREHOOK: Input: default@j1_41
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from j1_41
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@j1_41
+#### A masked pattern was here ####
+johndee 1
+burks 2
+#### A masked pattern was here ####