You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2017/03/11 06:15:00 UTC

hive git commit: HIVE-16156: FileSinkOperator should delete existing output target when renaming (Reviewed by Sergey)

Repository: hive
Updated Branches:
  refs/heads/master 8dda898ba -> 76b65baa7


HIVE-16156: FileSinkOperator should delete existing output target when renaming (Reviewed by Sergey)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76b65baa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76b65baa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76b65baa

Branch: refs/heads/master
Commit: 76b65baa7b5f5c0e5c1f99cf0621247f65fb0b00
Parents: 8dda898
Author: Xuefu Zhang <xu...@uber.com>
Authored: Fri Mar 10 22:14:49 2017 -0800
Committer: Xuefu Zhang <xu...@uber.com>
Committed: Fri Mar 10 22:14:49 2017 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/FileSinkOperator.java    | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/76b65baa/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 3bbe92d..a9d03d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -229,8 +229,18 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
             }
           }
           if (needToRename && outPaths[idx] != null && !fs.rename(outPaths[idx], finalPaths[idx])) {
-            throw new HiveException("Unable to rename output from: " +
+            FileStatus fileStatus = FileUtils.getFileStatusOrNull(fs, finalPaths[idx]);
+            if (fileStatus != null) {
+              LOG.warn("Target path " + finalPaths[idx] + " with a size " + fileStatus.getLen() + " exists. Trying to delete it.");
+              if (!fs.delete(finalPaths[idx], true)) {
+                throw new HiveException("Unable to delete existing target output: " + finalPaths[idx]);
+              }
+            }
+
+            if (!fs.rename(outPaths[idx], finalPaths[idx])) {
+              throw new HiveException("Unable to rename output from: " +
                 outPaths[idx] + " to: " + finalPaths[idx]);
+            }
           }
           updateProgress();
         } catch (IOException e) {