You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2021/10/27 08:06:21 UTC
[hive] branch branch-3.1 updated: HIVE-25600: Compaction job
creates redundant base/delta folder within base/delta folder (Nikhil Gupta,
reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ce5e13d HIVE-25600: Compaction job creates redundant base/delta folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan)
ce5e13d is described below
commit ce5e13da3554df8538d46e10dce87b4ef04c3119
Author: guptanikhil007 <gu...@gmail.com>
AuthorDate: Wed Oct 27 13:36:10 2021 +0530
HIVE-25600: Compaction job creates redundant base/delta folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
Closes (#2705)
---
.../org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 95870ad..474f6c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -1184,6 +1184,16 @@ public class CompactorMR {
Path tmpPath = fileStatus.getPath();
//newPath is the base/delta dir
Path newPath = new Path(finalLocation, tmpPath.getName());
+ /* rename(A, B) has "interesting" behavior if A and B are directories. If B doesn't exist,
+ * it does the expected operation and everything that was in A is now in B. If B exists,
+ * it will make A a child of B.
+ * This issue can happen if the previous MR job succeeded but HMS was unable to persist compaction result.
+ * We will delete the directory B if it exists to avoid the above issue
+ */
+ if (fs.exists(newPath)) {
+ LOG.info(String.format("Final path %s already exists. Deleting the path to avoid redundant base creation", newPath.toString()));
+ fs.delete(newPath, true);
+ }
/* Create the markers in the tmp location and rename everything in the end to prevent race condition between
* marker creation and split read. */
AcidUtils.OrcAcidVersion.writeVersionFile(tmpPath, fs);
@@ -1192,6 +1202,7 @@ public class CompactorMR {
}
fs.delete(tmpLocation, true);
}
+
private void createCompactorMarker(JobConf conf, Path finalLocation, FileSystem fs)
throws IOException {
if(conf.getBoolean(IS_MAJOR, false)) {