You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2025/08/04 08:34:53 UTC
(iceberg) branch main updated: Core: introduce shared authentication refresh executor (#12563)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 76b2f875ff Core: introduce shared authentication refresh executor (#12563)
76b2f875ff is described below
commit 76b2f875ff14f0ddeb80bfa63e819eeec3047975
Author: Alexandre Dutra <ad...@apache.org>
AuthorDate: Mon Aug 4 10:34:48 2025 +0200
Core: introduce shared authentication refresh executor (#12563)
Currently, each `AuthManager` instance manages its own refresh pool. During the lifecycle of the catalog session, a few `AuthManager` instances may be created: one for the catalog itself, and a few more in components like the S3 signer client, and credential refreshers. Each one of them is currently creating their own pool, which is a waste of resources.
This PR introduces a shared refresh pool for use by `AuthManager`s, thus allowing to minimize the amount of threads allocated for auth session refreshes. The shared pool is closed by a shutdown hook.
---
.../java/org/apache/iceberg/SystemConfigs.java | 8 +++++
.../iceberg/rest/auth/RefreshingAuthManager.java | 3 ++
.../java/org/apache/iceberg/util/ThreadPools.java | 36 ++++++++++++++++++++++
3 files changed, 47 insertions(+)
diff --git a/core/src/main/java/org/apache/iceberg/SystemConfigs.java b/core/src/main/java/org/apache/iceberg/SystemConfigs.java
index 6dd4471752..ad40c17e30 100644
--- a/core/src/main/java/org/apache/iceberg/SystemConfigs.java
+++ b/core/src/main/java/org/apache/iceberg/SystemConfigs.java
@@ -53,6 +53,14 @@ public class SystemConfigs {
Math.max(2, 4 * Runtime.getRuntime().availableProcessors()),
Integer::parseUnsignedInt);
+ /** Sets the core size of the thread pool used for refreshing authentication data. */
+ public static final ConfigEntry<Integer> AUTH_REFRESH_THREAD_POOL_SIZE =
+ new ConfigEntry<>(
+ "iceberg.rest.auth.refresh.num-threads",
+ "ICEBERG_AUTH_REFRESH_NUM_THREADS",
+ 1,
+ Integer::parseUnsignedInt);
+
/** Whether to use the shared worker pool when planning table scans. */
public static final ConfigEntry<Boolean> SCAN_THREAD_POOL_ENABLED =
new ConfigEntry<>(
diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java b/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
index 8c2d9918b8..d51a8c3b24 100644
--- a/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
+++ b/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
@@ -30,7 +30,10 @@ import org.slf4j.LoggerFactory;
/**
* An {@link AuthManager} that provides machinery for refreshing authentication data asynchronously,
* using a background thread pool.
+ *
+ * @deprecated since 1.10.0, will be removed in 1.11.0; use {@link ThreadPools#authRefreshPool()}.
*/
+@Deprecated
public abstract class RefreshingAuthManager implements AuthManager {
private static final Logger LOG = LoggerFactory.getLogger(RefreshingAuthManager.class);
diff --git a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
index 0d7d174a10..a96f020fe8 100644
--- a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
+++ b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
@@ -18,12 +18,14 @@
*/
package org.apache.iceberg.util;
+import java.time.Duration;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.iceberg.SystemConfigs;
import org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors;
import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -51,6 +53,9 @@ public class ThreadPools {
private static final ExecutorService DELETE_WORKER_POOL =
newExitingWorkerPool("iceberg-delete-worker-pool", DELETE_WORKER_THREAD_POOL_SIZE);
+ public static final int AUTH_REFRESH_THREAD_POOL_SIZE =
+ SystemConfigs.AUTH_REFRESH_THREAD_POOL_SIZE.value();
+
/**
* Return an {@link ExecutorService} that uses the "worker" thread-pool.
*
@@ -82,6 +87,20 @@ public class ThreadPools {
return DELETE_WORKER_POOL;
}
+ /**
+ * A shared {@link ScheduledExecutorService} that REST catalogs can use for refreshing their
+ * authentication data.
+ */
+ public static ScheduledExecutorService authRefreshPool() {
+ return AuthRefreshPoolHolder.INSTANCE;
+ }
+
+ private static class AuthRefreshPoolHolder {
+ private static final ScheduledExecutorService INSTANCE =
+ ThreadPools.newExitingScheduledPool(
+ "auth-session-refresh", AUTH_REFRESH_THREAD_POOL_SIZE, Duration.ZERO);
+ }
+
/**
* Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link
* MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook
@@ -152,6 +171,23 @@ public class ThreadPools {
return new ScheduledThreadPoolExecutor(poolSize, newDaemonThreadFactory(namePrefix));
}
+ /**
+ * Create a new {@link ScheduledExecutorService} with the given name and pool size.
+ *
+ * <p>Threads used by this service will be daemon threads.
+ *
+ * <p>The service registers a shutdown hook to ensure that it terminates when the JVM exits. This
+ * is suitable for long-lived thread pools that should be automatically cleaned up on JVM
+ * shutdown.
+ */
+ public static ScheduledExecutorService newExitingScheduledPool(
+ String namePrefix, int poolSize, Duration terminationTimeout) {
+ return MoreExecutors.getExitingScheduledExecutorService(
+ (ScheduledThreadPoolExecutor) newScheduledPool(namePrefix, poolSize),
+ terminationTimeout.toMillis(),
+ TimeUnit.MILLISECONDS);
+ }
+
private static ThreadFactory newDaemonThreadFactory(String namePrefix) {
return new ThreadFactoryBuilder().setDaemon(true).setNameFormat(namePrefix + "-%d").build();
}