You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2011/07/21 21:31:49 UTC
svn commit: r1149331 - in /hive/trunk: ./ data/files/ext_test/
ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/queries/clientpositive/
ql/src/test/results/clientpositive/
Author: cws
Date: Thu Jul 21 19:31:48 2011
New Revision: 1149331
URL: http://svn.apache.org/viewvc?rev=1149331&view=rev
Log:
HIVE-2086. Add test coverage for external table data loss issue (Jonathan Natkins via cws)
Added:
hive/trunk/data/files/ext_test/
hive/trunk/data/files/ext_test/test.dat
Modified:
hive/trunk/build-common.xml
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hive/trunk/ql/src/test/queries/clientpositive/create_like.q
hive/trunk/ql/src/test/results/clientpositive/create_like.q.out
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1149331&r1=1149330&r2=1149331&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Thu Jul 21 19:31:48 2011
@@ -458,6 +458,7 @@
<sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
<sysproperty key="test.silent" value="${test.silent}"/>
<sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
+ <sysproperty key="test.src.data.dir" value="${test.src.data.dir}"/>
<sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
<sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
<sysproperty key="fs.default.name" value="${fs.default.name}"/>
Added: hive/trunk/data/files/ext_test/test.dat
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/ext_test/test.dat?rev=1149331&view=auto
==============================================================================
--- hive/trunk/data/files/ext_test/test.dat (added)
+++ hive/trunk/data/files/ext_test/test.dat Thu Jul 21 19:31:48 2011
@@ -0,0 +1,6 @@
+1
+2
+3
+4
+5
+6
\ No newline at end of file
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1149331&r1=1149330&r2=1149331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Thu Jul 21 19:31:48 2011
@@ -938,6 +938,7 @@ public class QTestUtil {
"-I", "CreateTime",
"-I", "LastAccessTime",
"-I", "Location",
+ "-I", "LOCATION '",
"-I", "transient_lastDdlTime",
"-I", "last_modified_",
"-I", "java.lang.RuntimeException",
Modified: hive/trunk/ql/src/test/queries/clientpositive/create_like.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/create_like.q?rev=1149331&r1=1149330&r2=1149331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/create_like.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/create_like.q Thu Jul 21 19:31:48 2011
@@ -21,3 +21,15 @@ INSERT OVERWRITE TABLE table2 SELECT key
SELECT * FROM table1;
SELECT * FROM table2;
+CREATE EXTERNAL TABLE table4 (a INT) LOCATION '${system:test.src.data.dir}/files/ext_test';
+CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '${system:test.src.data.dir}/files/ext_test';
+
+SELECT * FROM table4;
+SELECT * FROM table5;
+
+DROP TABLE table5;
+SELECT * FROM table4;
+DROP TABLE table4;
+
+CREATE EXTERNAL TABLE table4 (a INT) LOCATION '${system:test.src.data.dir}/files/ext_test';
+SELECT * FROM table4;
Modified: hive/trunk/ql/src/test/results/clientpositive/create_like.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/create_like.q.out?rev=1149331&r1=1149330&r2=1149331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/create_like.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/create_like.q.out Thu Jul 21 19:31:48 2011
@@ -14,15 +14,15 @@ b string
# Detailed Table Information
Database: default
-Owner: charleschen
-CreateTime: Sat Jul 16 15:44:24 PDT 2011
+Owner: natty
+CreateTime: Mon Jul 18 21:36:36 PDT 2011
LastAccessTime: UNKNOWN
Protect Mode: None
Retention: 0
-Location: pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table1
+Location: pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table1
Table Type: MANAGED_TABLE
Table Parameters:
- transient_lastDdlTime 1310856264
+ transient_lastDdlTime 1311050196
# Storage Information
SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -50,15 +50,15 @@ b string
# Detailed Table Information
Database: default
-Owner: charleschen
-CreateTime: Sat Jul 16 15:44:24 PDT 2011
+Owner: natty
+CreateTime: Mon Jul 18 21:36:36 PDT 2011
LastAccessTime: UNKNOWN
Protect Mode: None
Retention: 0
-Location: pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table2
+Location: pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table2
Table Type: MANAGED_TABLE
Table Parameters:
- transient_lastDdlTime 1310856264
+ transient_lastDdlTime 1311050196
# Storage Information
SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -94,16 +94,16 @@ b string
# Detailed Table Information
Database: default
-Owner: charleschen
-CreateTime: Sat Jul 16 15:44:24 PDT 2011
+Owner: natty
+CreateTime: Mon Jul 18 21:36:37 PDT 2011
LastAccessTime: UNKNOWN
Protect Mode: None
Retention: 0
-Location: pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table3
+Location: pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table3
Table Type: EXTERNAL_TABLE
Table Parameters:
EXTERNAL TRUE
- transient_lastDdlTime 1310856264
+ transient_lastDdlTime 1311050197
# Storage Information
SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -140,11 +140,11 @@ POSTHOOK: Lineage: table2.b SIMPLE [(src
PREHOOK: query: SELECT * FROM table1
PREHOOK: type: QUERY
PREHOOK: Input: default@table1
-PREHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_490_2335283351138014900/-mr-10000
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_232_3888885319138532825/-mr-10000
POSTHOOK: query: SELECT * FROM table1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@table1
-POSTHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_490_2335283351138014900/-mr-10000
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_232_3888885319138532825/-mr-10000
POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -153,14 +153,137 @@ POSTHOOK: Lineage: table2.b SIMPLE [(src
PREHOOK: query: SELECT * FROM table2
PREHOOK: type: QUERY
PREHOOK: Input: default@table2
-PREHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_655_7605576315633944704/-mr-10000
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_724_8379773793855049431/-mr-10000
POSTHOOK: query: SELECT * FROM table2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@table2
-POSTHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_655_7605576315633944704/-mr-10000
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_724_8379773793855049431/-mr-10000
POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
100 val_100
100 val_100
+PREHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table5
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_220_5774951150754164269/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_220_5774951150754164269/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: SELECT * FROM table5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table5
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_497_977305450352332337/-mr-10000
+POSTHOOK: query: SELECT * FROM table5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table5
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_497_977305450352332337/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: DROP TABLE table5
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table5
+PREHOOK: Output: default@table5
+POSTHOOK: query: DROP TABLE table5
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table5
+POSTHOOK: Output: default@table5
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-01_834_8386395890966387972/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-01_834_8386395890966387972/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: DROP TABLE table4
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table4
+PREHOOK: Output: default@table4
+POSTHOOK: query: DROP TABLE table4
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-02_667_3267843075880450853/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-02_667_3267843075880450853/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6