You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/06/17 14:20:31 UTC

(doris) branch branch-2.1 updated: [fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402) (#36403)

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.1 by this push:
     new 4ae8607b2e6 [fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402) (#36403)
4ae8607b2e6 is described below

commit 4ae8607b2e6e165718c39e9a4db584395784796c
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Mon Jun 17 22:20:23 2024 +0800

    [fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402) (#36403)
    
    bp #36402
---
 .../doris/datasource/hive/HiveMetaStoreClientHelper.java       | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index e7dfda25c3e..aa3ff1ccee8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -66,6 +66,7 @@ import org.apache.hudi.common.table.TableSchemaResolver;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -819,6 +820,15 @@ public class HiveMetaStoreClientHelper {
     public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
         String hudiBasePath = table.getRemoteTable().getSd().getLocation();
         Configuration conf = getConfiguration(table);
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("try setting 'fs.xxx.impl.disable.cache' to true for hudi's base path: {}", hudiBasePath);
+        }
+        URI hudiBasePathUri = URI.create(hudiBasePath);
+        String scheme = hudiBasePathUri.getScheme();
+        if (!Strings.isNullOrEmpty(scheme)) {
+            // Avoid using Cache in Hadoop FileSystem, which may cause FE OOM.
+            conf.set("fs." + scheme + ".impl.disable.cache", "true");
+        }
         return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
                 () -> HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org