You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rb...@apache.org on 2020/04/07 05:31:43 UTC
[hive] branch master updated: HIVE-23140: Optimise file move in
CTAS (Rajesh Balamohan reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
rbalamohan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7122748 HIVE-23140: Optimise file move in CTAS (Rajesh Balamohan reviewed by Ashutosh Chauhan)
7122748 is described below
commit 71227488eedeab20a7efbfbfbe3f2480a6d1c83b
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Tue Apr 7 11:01:26 2020 +0530
HIVE-23140: Optimise file move in CTAS (Rajesh Balamohan reviewed by Ashutosh Chauhan)
---
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 900642e..e25dc54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1497,7 +1497,12 @@ public final class Utilities {
// for CTAS or Create MV statements
perfLogger.PerfLogBegin("FileSinkOperator", "moveSpecifiedFileStatus");
LOG.debug("CTAS/Create MV: Files being renamed: " + filesKept.toString());
- Utilities.moveSpecifiedFileStatus(fs, tmpPath, specPath, filesKept);
+ if (emptyBuckets.isEmpty()) {
+ fs.rename(tmpPath, specPath);
+ } else {
+ LOG.info("Duplicate files present. Moving files sequentially");
+ Utilities.moveSpecifiedFileStatus(fs, tmpPath, specPath, filesKept);
+ }
perfLogger.PerfLogEnd("FileSinkOperator", "moveSpecifiedFileStatus");
} else {
// for rest of the statement e.g. INSERT, LOAD etc