You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2025/08/04 08:34:53 UTC

(iceberg) branch main updated: Core: introduce shared authentication refresh executor (#12563)

This is an automated email from the ASF dual-hosted git repository.

etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 76b2f875ff Core: introduce shared authentication refresh executor (#12563)
76b2f875ff is described below

commit 76b2f875ff14f0ddeb80bfa63e819eeec3047975
Author: Alexandre Dutra <ad...@apache.org>
AuthorDate: Mon Aug 4 10:34:48 2025 +0200

    Core: introduce shared authentication refresh executor (#12563)
    
    Currently, each `AuthManager` instance manages its own refresh pool. During the lifecycle of the catalog session, a few `AuthManager` instances may be created: one for the catalog itself, and a few more in components like the S3 signer client, and credential refreshers. Each one of them is currently creating their own pool, which is a waste of resources.
    
    This PR introduces a shared refresh pool for use by `AuthManager`s, thus allowing to minimize the amount of threads allocated for auth session refreshes. The shared pool is closed by a shutdown hook.
---
 .../java/org/apache/iceberg/SystemConfigs.java     |  8 +++++
 .../iceberg/rest/auth/RefreshingAuthManager.java   |  3 ++
 .../java/org/apache/iceberg/util/ThreadPools.java  | 36 ++++++++++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/core/src/main/java/org/apache/iceberg/SystemConfigs.java b/core/src/main/java/org/apache/iceberg/SystemConfigs.java
index 6dd4471752..ad40c17e30 100644
--- a/core/src/main/java/org/apache/iceberg/SystemConfigs.java
+++ b/core/src/main/java/org/apache/iceberg/SystemConfigs.java
@@ -53,6 +53,14 @@ public class SystemConfigs {
           Math.max(2, 4 * Runtime.getRuntime().availableProcessors()),
           Integer::parseUnsignedInt);
 
+  /** Sets the core size of the thread pool used for refreshing authentication data. */
+  public static final ConfigEntry<Integer> AUTH_REFRESH_THREAD_POOL_SIZE =
+      new ConfigEntry<>(
+          "iceberg.rest.auth.refresh.num-threads",
+          "ICEBERG_AUTH_REFRESH_NUM_THREADS",
+          1,
+          Integer::parseUnsignedInt);
+
   /** Whether to use the shared worker pool when planning table scans. */
   public static final ConfigEntry<Boolean> SCAN_THREAD_POOL_ENABLED =
       new ConfigEntry<>(
diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java b/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
index 8c2d9918b8..d51a8c3b24 100644
--- a/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
+++ b/core/src/main/java/org/apache/iceberg/rest/auth/RefreshingAuthManager.java
@@ -30,7 +30,10 @@ import org.slf4j.LoggerFactory;
 /**
  * An {@link AuthManager} that provides machinery for refreshing authentication data asynchronously,
  * using a background thread pool.
+ *
+ * @deprecated since 1.10.0, will be removed in 1.11.0; use {@link ThreadPools#authRefreshPool()}.
  */
+@Deprecated
 public abstract class RefreshingAuthManager implements AuthManager {
 
   private static final Logger LOG = LoggerFactory.getLogger(RefreshingAuthManager.class);
diff --git a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
index 0d7d174a10..a96f020fe8 100644
--- a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
+++ b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java
@@ -18,12 +18,14 @@
  */
 package org.apache.iceberg.util;
 
+import java.time.Duration;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import org.apache.iceberg.SystemConfigs;
 import org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors;
 import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -51,6 +53,9 @@ public class ThreadPools {
   private static final ExecutorService DELETE_WORKER_POOL =
       newExitingWorkerPool("iceberg-delete-worker-pool", DELETE_WORKER_THREAD_POOL_SIZE);
 
+  public static final int AUTH_REFRESH_THREAD_POOL_SIZE =
+      SystemConfigs.AUTH_REFRESH_THREAD_POOL_SIZE.value();
+
   /**
    * Return an {@link ExecutorService} that uses the "worker" thread-pool.
    *
@@ -82,6 +87,20 @@ public class ThreadPools {
     return DELETE_WORKER_POOL;
   }
 
+  /**
+   * A shared {@link ScheduledExecutorService} that REST catalogs can use for refreshing their
+   * authentication data.
+   */
+  public static ScheduledExecutorService authRefreshPool() {
+    return AuthRefreshPoolHolder.INSTANCE;
+  }
+
+  private static class AuthRefreshPoolHolder {
+    private static final ScheduledExecutorService INSTANCE =
+        ThreadPools.newExitingScheduledPool(
+            "auth-session-refresh", AUTH_REFRESH_THREAD_POOL_SIZE, Duration.ZERO);
+  }
+
   /**
    * Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link
    * MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook
@@ -152,6 +171,23 @@ public class ThreadPools {
     return new ScheduledThreadPoolExecutor(poolSize, newDaemonThreadFactory(namePrefix));
   }
 
+  /**
+   * Create a new {@link ScheduledExecutorService} with the given name and pool size.
+   *
+   * <p>Threads used by this service will be daemon threads.
+   *
+   * <p>The service registers a shutdown hook to ensure that it terminates when the JVM exits. This
+   * is suitable for long-lived thread pools that should be automatically cleaned up on JVM
+   * shutdown.
+   */
+  public static ScheduledExecutorService newExitingScheduledPool(
+      String namePrefix, int poolSize, Duration terminationTimeout) {
+    return MoreExecutors.getExitingScheduledExecutorService(
+        (ScheduledThreadPoolExecutor) newScheduledPool(namePrefix, poolSize),
+        terminationTimeout.toMillis(),
+        TimeUnit.MILLISECONDS);
+  }
+
   private static ThreadFactory newDaemonThreadFactory(String namePrefix) {
     return new ThreadFactoryBuilder().setDaemon(true).setNameFormat(namePrefix + "-%d").build();
   }