You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/09/10 09:44:40 UTC
(doris) branch master updated: [bugfix](hive)Delete the temporarily created folder (#40424)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c7f57ae1eab [bugfix](hive)Delete the temporarily created folder (#40424)
c7f57ae1eab is described below
commit c7f57ae1eabee22f192af4bba4f796b241a24660
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Tue Sep 10 17:44:31 2024 +0800
[bugfix](hive)Delete the temporarily created folder (#40424)
## Proposed changes
Delete the temporarily created folder, otherwise it will cause too many
folders on hdfs:
> The directory item limit of xxx is exceeded: limit=xxx items=xxx
---
.../doris/datasource/hive/HMSTransaction.java | 2 ++
.../doris/datasource/hive/HmsCommitTest.java | 22 ++++++++++++++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
index ac610237cdd..6183c277c1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
@@ -1484,6 +1484,8 @@ public class HMSTransaction implements Transaction {
runS3cleanWhenSuccess();
doAddPartitionsTask();
doUpdateStatisticsTasks();
+ //delete write path
+ pruneAndDeleteStagingDirectories();
doNothing();
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
index 7e99667b731..395a063fbc8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.backup.Status;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.info.SimpleTableInfo;
@@ -169,15 +170,21 @@ public class HmsCommitTest {
@Test
public void testAppendPartitionForUnPartitionedTable() throws IOException {
genQueryID();
- System.out.println(DebugUtil.printId(connectContext.queryId()));
List<THivePartitionUpdate> pus = new ArrayList<>();
pus.add(createRandomAppend(null));
pus.add(createRandomAppend(null));
pus.add(createRandomAppend(null));
+ new MockUp<HMSTransaction.HmsCommitter>(HMSTransaction.HmsCommitter.class) {
+ @Mock
+ private void doNothing() {
+ Assert.assertEquals(Status.ErrCode.NOT_FOUND, fs.exists(getWritePath()).getErrCode());
+ }
+ };
commit(dbName, tbWithoutPartition, pus);
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
assertNumRows(3, table);
+
genQueryID();
List<THivePartitionUpdate> pus2 = new ArrayList<>();
pus2.add(createRandomAppend(null));
@@ -204,6 +211,12 @@ public class HmsCommitTest {
@Test
public void testNewPartitionForPartitionedTable() throws IOException {
+ new MockUp<HMSTransaction.HmsCommitter>(HMSTransaction.HmsCommitter.class) {
+ @Mock
+ private void doNothing() {
+ Assert.assertEquals(Status.ErrCode.NOT_FOUND, fs.exists(getWritePath()).getErrCode());
+ }
+ };
genQueryID();
List<THivePartitionUpdate> pus = new ArrayList<>();
pus.add(createRandomNew("a"));
@@ -377,6 +390,11 @@ public class HmsCommitTest {
genOnePartitionUpdate("c3=" + partition, TUpdateMode.OVERWRITE);
}
+ private String getWritePath() {
+ String queryId = DebugUtil.printId(ConnectContext.get().queryId());
+ return writeLocation + queryId + "/";
+ }
+
public void commit(String dbName,
String tableName,
List<THivePartitionUpdate> hivePUs) {
@@ -385,7 +403,7 @@ public class HmsCommitTest {
HiveInsertCommandContext ctx = new HiveInsertCommandContext();
String queryId = DebugUtil.printId(ConnectContext.get().queryId());
ctx.setQueryId(queryId);
- ctx.setWritePath(writeLocation + queryId + "/");
+ ctx.setWritePath(getWritePath());
hmsTransaction.beginInsertTable(ctx);
hmsTransaction.finishInsertTable(new SimpleTableInfo(dbName, tableName));
hmsTransaction.commit();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org