You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/02/08 08:12:26 UTC
hive git commit: HIVE-17063 : insert overwrite partition onto a
external table fail when drop partition first (Wang Haihua, Deepak Jaiswal,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 637224fcc -> 1faadb074
HIVE-17063 : insert overwrite partition onto a external table fail when drop partition first (Wang Haihua, Deepak Jaiswal, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1faadb07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1faadb07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1faadb07
Branch: refs/heads/master
Commit: 1faadb0744975538a0ec4b2de63f05caf37b6f40
Parents: 637224f
Author: djaiswal <dj...@apache.org>
Authored: Thu Feb 8 00:10:57 2018 -0800
Committer: djaiswal <dj...@apache.org>
Committed: Thu Feb 8 00:10:57 2018 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 22 ++++++++++--
.../insert_after_drop_partition.q | 7 ++++
.../llap/insert_after_drop_partition.q.out | 36 ++++++++++++++++++++
4 files changed, 64 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 2a22db9..974bfac 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -555,6 +555,7 @@ minillaplocal.query.files=\
is_distinct_from.q,\
infer_bucket_sort_bucketed_table.q,\
input16_cc.q,\
+ insert_after_drop_partition.q,\
insert_dir_distcp.q,\
insert_into_with_schema.q,\
insert_values_orig_table.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 109f4c7..c8d1589 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -61,8 +61,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.common.JavaUtils;
@@ -3670,7 +3672,15 @@ private void constructOneLBLocationMap(FileStatus fSta,
"Unable to move source " + srcStatus.getPath() + " to destination " + destFile;
if (null == pool) {
- if(!destFs.rename(srcStatus.getPath(), destFile)) {
+ boolean success = false;
+ if (destFs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem)destFs).rename(srcStatus.getPath(), destFile, Options.Rename.OVERWRITE);
+ success = true;
+ } else {
+ destFs.delete(destFile, false);
+ success = destFs.rename(srcStatus.getPath(), destFile);
+ }
+ if(!success) {
throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest:"
+ destf + " returned false");
}
@@ -3681,7 +3691,15 @@ private void constructOneLBLocationMap(FileStatus fSta,
SessionState.setCurrentSessionState(parentSession);
final String group = srcStatus.getGroup();
try {
- if (!destFs.rename(srcStatus.getPath(), destFile)) {
+ boolean success = false;
+ if (destFs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem)destFs).rename(srcStatus.getPath(), destFile, Options.Rename.OVERWRITE);
+ success = true;
+ } else {
+ destFs.delete(destFile, false);
+ success = destFs.rename(srcStatus.getPath(), destFile);
+ }
+ if (!success) {
throw new IOException(
"rename for src path: " + srcStatus.getPath() + " to dest path:"
+ destFile + " returned false");
http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/test/queries/clientpositive/insert_after_drop_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/insert_after_drop_partition.q b/ql/src/test/queries/clientpositive/insert_after_drop_partition.q
new file mode 100644
index 0000000..0087560
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_after_drop_partition.q
@@ -0,0 +1,7 @@
+create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string);
+
+insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10;
+
+alter table insert_after_drop_partition drop partition (insertdate='2008-01-01');
+
+insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out b/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out
new file mode 100644
index 0000000..abe0f8d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out
@@ -0,0 +1,36 @@
+PREHOOK: query: create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_after_drop_partition
+POSTHOOK: query: create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_after_drop_partition
+PREHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+POSTHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table insert_after_drop_partition drop partition (insertdate='2008-01-01')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@insert_after_drop_partition
+PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+POSTHOOK: query: alter table insert_after_drop_partition drop partition (insertdate='2008-01-01')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@insert_after_drop_partition
+POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+PREHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+POSTHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01
+POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]