You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2023/01/19 19:14:08 UTC
[hive] branch master updated: HIVE-26904: QB commitCompaction should not fail when tmp table dirs are already cleaned up (Quanlong Huang, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ed3b1839b14 HIVE-26904: QB commitCompaction should not fail when tmp table dirs are already cleaned up (Quanlong Huang, reviewed by Denys Kuzmenko)
ed3b1839b14 is described below
commit ed3b1839b143ab4d590284c9439134faaa750013
Author: Quanlong Huang <hu...@gmail.com>
AuthorDate: Fri Jan 20 03:13:59 2023 +0800
HIVE-26904: QB commitCompaction should not fail when tmp table dirs are already cleaned up (Quanlong Huang, reviewed by Denys Kuzmenko)
Closes #3910
---
.../hive/ql/txn/compactor/MinorQueryCompactor.java | 4 ++--
.../ql/txn/compactor/MmMajorQueryCompactor.java | 2 +-
.../ql/txn/compactor/MmMinorQueryCompactor.java | 4 ++--
.../hive/ql/txn/compactor/QueryCompactor.java | 28 ++++++++++++++++------
4 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java
index ba618c4a1ed..23a30393a9f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java
@@ -76,8 +76,8 @@ final class MinorQueryCompactor extends QueryCompactor {
@Override
protected void commitCompaction(String dest, String tmpTableName, HiveConf conf,
ValidWriteIdList actualWriteIds, long compactorTxnId) throws IOException, HiveException {
- Util.cleanupEmptyDir(conf, AcidUtils.DELTA_PREFIX + tmpTableName + "_result");
- Util.cleanupEmptyDir(conf, AcidUtils.DELETE_DELTA_PREFIX + tmpTableName + "_result");
+ Util.cleanupEmptyTableDir(conf, AcidUtils.DELTA_PREFIX + tmpTableName + "_result");
+ Util.cleanupEmptyTableDir(conf, AcidUtils.DELETE_DELTA_PREFIX + tmpTableName + "_result");
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java
index a382918d502..421c06ba302 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java
@@ -74,7 +74,7 @@ final class MmMajorQueryCompactor extends QueryCompactor {
@Override
protected void commitCompaction(String dest, String tmpTableName, HiveConf conf,
ValidWriteIdList actualWriteIds, long compactorTxnId) throws IOException, HiveException {
- Util.cleanupEmptyDir(conf, tmpTableName);
+ Util.cleanupEmptyTableDir(conf, tmpTableName);
}
private List<String> getCreateQueries(String tmpTableName, Table table,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMinorQueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMinorQueryCompactor.java
index 4a9642f6af8..4c7ee742631 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMinorQueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMinorQueryCompactor.java
@@ -66,11 +66,11 @@ final class MmMinorQueryCompactor extends QueryCompactor {
}
/**
- * Move files from "result table" directory to table/partition to compact's directory.
+ * Clean up the empty table dir of 'tmpTableName'.
*/
@Override protected void commitCompaction(String dest, String tmpTableName, HiveConf conf,
ValidWriteIdList actualWriteIds, long compactorTxnId) throws IOException, HiveException {
- Util.cleanupEmptyDir(conf, tmpTableName);
+ Util.cleanupEmptyTableDir(conf, tmpTableName);
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
index 34161c9b04f..a1af5fc0c9b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
@@ -43,6 +43,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -162,10 +163,7 @@ abstract class QueryCompactor implements Compactor {
private void removeResultDirs(List<Path> resultDirPaths, HiveConf conf) throws IOException {
for (Path path : resultDirPaths) {
LOG.info("Compaction failed, removing directory: " + path.toString());
- FileSystem fs = path.getFileSystem(conf);
- if (!fs.listFiles(path, false).hasNext()) {
- fs.delete(path, true);
- }
+ Util.cleanupEmptyDir(conf, path);
}
}
@@ -245,16 +243,32 @@ abstract class QueryCompactor implements Compactor {
* @throws IOException the directory cannot be deleted
* @throws HiveException the table is not found
*/
- static void cleanupEmptyDir(HiveConf conf, String tmpTableName) throws IOException, HiveException {
+ static void cleanupEmptyTableDir(HiveConf conf, String tmpTableName)
+ throws IOException, HiveException {
org.apache.hadoop.hive.ql.metadata.Table tmpTable = Hive.get().getTable(tmpTableName);
if (tmpTable != null) {
- Path path = new Path(tmpTable.getSd().getLocation());
- FileSystem fs = path.getFileSystem(conf);
+ cleanupEmptyDir(conf, new Path(tmpTable.getSd().getLocation()));
+ }
+ }
+
+ /**
+ * Remove the directory if it's empty.
+ * @param conf the Hive configuration
+ * @param path path of the directory
+ * @throws IOException if any IO error occurs
+ */
+ static void cleanupEmptyDir(HiveConf conf, Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(conf);
+ try {
if (!fs.listFiles(path, false).hasNext()) {
fs.delete(path, true);
}
+ } catch (FileNotFoundException e) {
+ // Ignore the case when the dir was already removed
+ LOG.warn("Ignored exception during cleanup {}", path, e);
}
}
+
/**
* Remove the delta directories of aborted transactions.
*/