You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/05/28 06:28:48 UTC
(doris) branch branch-2.1 updated: [branch-2.1: [Fix](Catalog)Ensure preExecutionAuthenticator is properly initialized (#51279)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4e3700eccf8 [branch-2.1: [Fix](Catalog)Ensure preExecutionAuthenticator is properly initialized (#51279)
4e3700eccf8 is described below
commit 4e3700eccf8b524058e4ab8779c59a5325f93b80
Author: Calvin Kirs <gu...@selectdb.com>
AuthorDate: Wed May 28 14:28:41 2025 +0800
[branch-2.1: [Fix](Catalog)Ensure preExecutionAuthenticator is properly initialized (#51279)
[branch-2.1: [Fix](Catalog)Ensure preExecutionAuthenticator is properly
initialized
cherry pick https://github.com/apache/doris/pull/50839
https://github.com/apache/doris/pull/50623
---------
Co-authored-by: kang <35...@users.noreply.github.com>
Co-authored-by: lik40 <li...@chinatelecom.cn>
---
.../security/authentication/HadoopAuthenticator.java | 7 +++++++
.../authentication/PreExecutionAuthenticator.java | 15 +++++++++++++++
.../org/apache/doris/datasource/ExternalCatalog.java | 19 +++++++++++++++++++
.../doris/datasource/hive/HMSExternalCatalog.java | 14 +++++++-------
.../apache/doris/datasource/hive/HiveMetadataOps.java | 2 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 10 +++++++++-
.../iceberg/IcebergHadoopExternalCatalog.java | 13 ++++++++++---
7 files changed, 68 insertions(+), 12 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java
index c3cab5f410b..88d32a593e1 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java
@@ -20,6 +20,7 @@ package org.apache.doris.common.security.authentication;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
public interface HadoopAuthenticator {
@@ -31,6 +32,12 @@ public interface HadoopAuthenticator {
return getUGI().doAs(action);
} catch (InterruptedException e) {
throw new IOException(e);
+ } catch (UndeclaredThrowableException e) {
+ if (e.getCause() instanceof RuntimeException) {
+ throw (RuntimeException) e.getCause();
+ } else {
+ throw new RuntimeException(e.getCause());
+ }
}
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java
index 6260833b7db..0d7cf60c6f7 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java
@@ -17,6 +17,8 @@
package org.apache.doris.common.security.authentication;
+import org.apache.hadoop.conf.Configuration;
+
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.Callable;
@@ -40,6 +42,19 @@ public class PreExecutionAuthenticator {
public PreExecutionAuthenticator() {
}
+ /**
+ * Constructor to initialize the PreExecutionAuthenticator object.
+ * This constructor is responsible for initializing the Hadoop authenticator required for Kerberos authentication
+ * based on the provided configuration information.
+ *
+ * @param configuration Configuration information used to obtain Kerberos authentication settings
+ */
+ public PreExecutionAuthenticator(Configuration configuration) {
+ AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(configuration);
+ this.hadoopAuthenticator = HadoopAuthenticator.getHadoopAuthenticator(config);
+ }
+
+
/**
* Executes the specified task with necessary authentication.
* <p>If a HadoopAuthenticator is set, the task will be executed within a
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 875c8142a68..81d3e15d90f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -171,6 +171,17 @@ public abstract class ExternalCatalog
this.comment = Strings.nullToEmpty(comment);
}
+ /**
+ * Initializes the PreExecutionAuthenticator instance.
+ * This method ensures that the authenticator is created only once in a thread-safe manner.
+ * If additional authentication logic is required, it should be extended and implemented in subclasses.
+ */
+ protected synchronized void initPreExecutionAuthenticator() {
+ if (preExecutionAuthenticator == null) {
+ preExecutionAuthenticator = new PreExecutionAuthenticator();
+ }
+ }
+
public Configuration getConfiguration() {
// build configuration is costly, so we cache it.
if (cachedConf != null) {
@@ -208,6 +219,11 @@ public abstract class ExternalCatalog
}
}
+ public ExternalMetadataOps getMetadataOps() {
+ makeSureInitialized();
+ return metadataOps;
+ }
+
// Will be called when creating catalog(so when as replaying)
// to add some default properties if missing.
public void setDefaultPropsIfMissing(boolean isReplay) {
@@ -1128,6 +1144,9 @@ public abstract class ExternalCatalog
}
public PreExecutionAuthenticator getPreExecutionAuthenticator() {
+ if (null == preExecutionAuthenticator) {
+ throw new RuntimeException("PreExecutionAuthenticator is null, please confirm it is initialized.");
+ }
return preExecutionAuthenticator;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 61a7c030477..defa2a8ebe9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -25,7 +25,6 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
-import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopAuthenticator;
import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
import org.apache.doris.common.util.Util;
@@ -170,14 +169,15 @@ public class HMSExternalCatalog extends ExternalCatalog {
}
@Override
- protected void initLocalObjectsImpl() {
- this.preExecutionAuthenticator = new PreExecutionAuthenticator();
- if (this.authenticator == null) {
- AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(getConfiguration());
- this.authenticator = HadoopAuthenticator.getHadoopAuthenticator(config);
- this.preExecutionAuthenticator.setHadoopAuthenticator(authenticator);
+ protected synchronized void initPreExecutionAuthenticator() {
+ if (preExecutionAuthenticator == null) {
+ preExecutionAuthenticator = new PreExecutionAuthenticator(getConfiguration());
}
+ }
+ @Override
+ protected void initLocalObjectsImpl() {
+ initPreExecutionAuthenticator();
HiveConf hiveConf = null;
JdbcClientConfig jdbcClientConfig = null;
String hiveMetastoreType = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_TYPE, "");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index 9bde47fb199..530731db81f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -72,7 +72,7 @@ public class HiveMetadataOps implements ExternalMetadataOps {
this(catalog, createCachedClient(hiveConf,
Math.max(MIN_CLIENT_POOL_SIZE, Config.max_external_cache_loader_thread_pool_size),
jdbcClientConfig));
- hadoopAuthenticator = catalog.getAuthenticator();
+ hadoopAuthenticator = catalog.getPreExecutionAuthenticator().getHadoopAuthenticator();
client.setHadoopAuthenticator(hadoopAuthenticator);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 523c31c3f74..7282de52aa0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -40,6 +40,7 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog {
public static final String ICEBERG_HADOOP = "hadoop";
public static final String ICEBERG_GLUE = "glue";
public static final String ICEBERG_DLF = "dlf";
+ public static final String ICEBERG_S3_TABLES = "s3tables";
public static final String EXTERNAL_CATALOG_NAME = "external_catalog.name";
protected String icebergCatalogType;
protected Catalog catalog;
@@ -51,9 +52,16 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog {
// Create catalog based on catalog type
protected abstract void initCatalog();
+ @Override
+ protected synchronized void initPreExecutionAuthenticator() {
+ if (preExecutionAuthenticator == null) {
+ preExecutionAuthenticator = new PreExecutionAuthenticator(getConfiguration());
+ }
+ }
+
@Override
protected void initLocalObjectsImpl() {
- preExecutionAuthenticator = new PreExecutionAuthenticator();
+ initPreExecutionAuthenticator();
initCatalog();
IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog);
transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
index 5053e129ef6..d8c9b7e1e28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
@@ -53,15 +53,22 @@ public class IcebergHadoopExternalCatalog extends IcebergExternalCatalog {
@Override
protected void initCatalog() {
icebergCatalogType = ICEBERG_HADOOP;
- HadoopCatalog hadoopCatalog = new HadoopCatalog();
+
Configuration conf = getConfiguration();
initS3Param(conf);
// initialize hadoop catalog
Map<String, String> catalogProperties = catalogProperty.getProperties();
String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
+ HadoopCatalog hadoopCatalog = new HadoopCatalog();
hadoopCatalog.setConf(conf);
catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- hadoopCatalog.initialize(getName(), catalogProperties);
- catalog = hadoopCatalog;
+ try {
+ this.catalog = preExecutionAuthenticator.execute(() -> {
+ hadoopCatalog.initialize(getName(), catalogProperties);
+ return hadoopCatalog;
+ });
+ } catch (Exception e) {
+ throw new RuntimeException("Hadoop catalog init error!", e);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org