You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by ad...@apache.org on 2026/07/02 09:57:22 UTC

(polaris) branch main updated: Fix async task retry when handlers return false (#4914)

This is an automated email from the ASF dual-hosted git repository.

adutra pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new b2ed126a81 Fix async task retry when handlers return false (#4914)
b2ed126a81 is described below

commit b2ed126a817c2392357d2558a72d1d4cf74aa649
Author: vignesh_A <16...@users.noreply.github.com>
AuthorDate: Thu Jul 2 15:27:17 2026 +0530

    Fix async task retry when handlers return false (#4914)
    
    Make BatchFileCleanupTaskHandler and ManifestFileCleanupTaskHandler throw the original exception instead of returning false (so real errors propagate to retry logic)
---
 CHANGELOG.md                                       |   1 +
 .../service/task/BatchFileCleanupTaskHandler.java  |   2 +-
 .../task/ManifestFileCleanupTaskHandler.java       |  10 +-
 .../polaris/service/task/TaskExecutorImpl.java     |  12 +-
 .../polaris/service/task/TaskExecutorImplTest.java | 180 +++++++++++++++++++++
 5 files changed, 198 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 310fa00c6b..a77a60d337 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -67,6 +67,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 
 ### Fixes
 - Fixed a boundary condition in GCS downscoped credential generation (`GcpCredentialsStorageIntegration`). Locations without a trailing slash could previously grant access to sibling object prefixes via the generated CEL conditions for `resource.name` and list prefixes. Granted paths are now normalized to a directory prefix (with a trailing slash) before the CEL conditions are built, so sibling prefixes can no longer satisfy the `startsWith` checks.
+- Async task execution (table cleanup, manifest and batch file cleanup) now retries when a handler returns false on transient errors (e.g. IO or delete failures). Previously `false` was swallowed with only a warning log and the task was never retried via the existing retry mechanism.
 - Fixed `NullPointerException` during `dropEntity` when an entity referenced by a grant had been concurrently removed (or purged). `lookupEntities` can return null entries for dropped entities; these are now skipped safely.
 - `RateLimiterFilter` now returns an Iceberg-compatible `ErrorResponse` JSON body on HTTP 429, with `Content-Type: application/json`. Previously the body was empty, causing Iceberg REST clients to surface an opaque error.
 - The admin tool `purge` command now prints the underlying exception stack trace to stderr when a purge fails unexpectedly, matching the `bootstrap` command. Previously a failed purge printed only a generic message, giving operators no diagnostic information.
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/task/BatchFileCleanupTaskHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/task/BatchFileCleanupTaskHandler.java
index 0b91194e7f..8d09016e90 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/task/BatchFileCleanupTaskHandler.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/task/BatchFileCleanupTaskHandler.java
@@ -85,7 +85,7 @@ public class BatchFileCleanupTaskHandler extends FileCleanupTaskHandler {
         deleteFutures.join();
       } catch (Exception e) {
         LOGGER.error("Exception detected during batch files deletion", e);
-        return false;
+        throw new RuntimeException(e);
       }
 
       return true;
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
index c03f055ad9..131105a9b0 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
@@ -113,15 +113,15 @@ public class ManifestFileCleanupTaskHandler extends FileCleanupTaskHandler {
         throw new RuntimeException(e);
       } catch (ExecutionException e) {
         LOGGER.error("Unable to delete data files from manifest {}", manifestFile.path(), e);
-        return false;
+        throw new RuntimeException(e);
       }
     } catch (IOException e) {
       // Catches from ManifestFiles.read() (resource creation), from inside the block
-      // (e.g. manifest iteration), or from close(). We return false on any IO here
-      // (so the task gets retried) because close() throws checked IOException and
-      // this method returns boolean (no throws declaration).
+      // (e.g. manifest iteration), or from close(). We throw (wrapping) so the original
+      // failure propagates and TaskExecutorImpl's retry path is used. (handleTask returns
+      // boolean and cannot declare a checked throws IOException.)
       LOGGER.error("Failed to process manifest reader for {}", manifestFile.path(), e);
-      return false;
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java
index 4e2efb678f..06bfce0db9 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java
@@ -229,7 +229,11 @@ public class TaskExecutorImpl implements TaskExecutor {
             .addKeyValue("taskEntityId", taskEntityId)
             .addKeyValue("taskType", task.getTaskType())
             .log("Unable to find handler for task type");
-        return;
+        throw new RuntimeException(
+            "Unable to find handler for task type "
+                + task.getTaskType()
+                + " for task entity id "
+                + taskEntityId);
       }
       TaskHandler handler = handlerOpt.get();
       success = handler.handleTask(task, ctx);
@@ -247,6 +251,12 @@ public class TaskExecutorImpl implements TaskExecutor {
             .addKeyValue("taskEntityId", taskEntityId)
             .addKeyValue("taskEntityName", taskEntity.getName())
             .log("Unable to execute async task");
+        throw new RuntimeException(
+            "Task handler returned false for task entity id "
+                + taskEntityId
+                + " (handler: "
+                + handler.getClass().getSimpleName()
+                + ")");
       }
     } finally {
       if (polarisEventDispatcher.hasListeners(PolarisEventType.AFTER_ATTEMPT_TASK)) {
diff --git a/runtime/service/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java
index 63b4a46205..4565414938 100644
--- a/runtime/service/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java
+++ b/runtime/service/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java
@@ -18,9 +18,14 @@
  */
 package org.apache.polaris.service.task;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.polaris.core.PolarisCallContext;
 import org.apache.polaris.core.context.CallContext;
 import org.apache.polaris.core.context.RealmContext;
+import org.apache.polaris.core.entity.AsyncTaskType;
 import org.apache.polaris.core.entity.TaskEntity;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
 import org.apache.polaris.service.TestServices;
@@ -114,4 +119,179 @@ public class TaskExecutorImplTest {
     Assertions.assertEquals(
         true, afterAttemptTaskEvent.attributes().getRequired(EventAttributes.TASK_SUCCESS));
   }
+
+  @Test
+  void handleTaskThrowsWhenNoHandlerFound() {
+    String realm = "myrealm";
+    RealmContext realmContext = () -> realm;
+
+    TestServices testServices = TestServices.builder().realmContext(realmContext).build();
+
+    PolarisMetaStoreManager metaStoreManager = testServices.metaStoreManager();
+    PolarisCallContext polarisCallCtx = testServices.newCallContext();
+
+    TaskEntity taskEntity =
+        new TaskEntity.Builder()
+            .setName("no-handler-task")
+            .withTaskType(AsyncTaskType.MANIFEST_FILE_CLEANUP)
+            .setId(metaStoreManager.generateNewEntityId(polarisCallCtx).getId())
+            .setCreateTimestamp(testServices.clock().millis())
+            .build();
+    metaStoreManager.createEntityIfNotExists(polarisCallCtx, null, taskEntity);
+
+    TaskExecutorImpl executor =
+        new TaskExecutorImpl(
+            Runnable::run,
+            null,
+            testServices.clock(),
+            testServices.metaStoreManagerFactory(),
+            new TaskFileIOSupplier(
+                testServices.fileIOFactory(), testServices.storageAccessConfigProvider()),
+            new RealmContextHolder(),
+            testServices.polarisEventDispatcher(),
+            testServices.eventMetadataFactory(),
+            null,
+            new PolarisPrincipalHolder(),
+            testServices.principal());
+
+    // No handlers registered
+    assertThatThrownBy(
+            () ->
+                executor.handleTask(
+                    taskEntity.getId(),
+                    polarisCallCtx,
+                    PolarisEventMetadata.builder().realmId(realm).build(),
+                    1))
+        .isInstanceOf(RuntimeException.class)
+        .hasMessageContaining("Unable to find handler for task type")
+        .hasMessageContaining(String.valueOf(taskEntity.getId()));
+  }
+
+  @Test
+  void handleTaskThrowsWhenHandlerReturnsFalse() {
+    String realm = "myrealm";
+    RealmContext realmContext = () -> realm;
+
+    TestServices testServices = TestServices.builder().realmContext(realmContext).build();
+
+    InMemoryEventCollector testPolarisEventDispatcher =
+        (InMemoryEventCollector) testServices.polarisEventDispatcher();
+
+    PolarisMetaStoreManager metaStoreManager = testServices.metaStoreManager();
+    PolarisCallContext polarisCallCtx = testServices.newCallContext();
+
+    TaskEntity taskEntity =
+        new TaskEntity.Builder()
+            .setName("failing-task")
+            .setId(metaStoreManager.generateNewEntityId(polarisCallCtx).getId())
+            .setCreateTimestamp(testServices.clock().millis())
+            .build();
+    metaStoreManager.createEntityIfNotExists(polarisCallCtx, null, taskEntity);
+
+    TaskExecutorImpl executor =
+        new TaskExecutorImpl(
+            Runnable::run,
+            null,
+            testServices.clock(),
+            testServices.metaStoreManagerFactory(),
+            new TaskFileIOSupplier(
+                testServices.fileIOFactory(), testServices.storageAccessConfigProvider()),
+            new RealmContextHolder(),
+            testServices.polarisEventDispatcher(),
+            testServices.eventMetadataFactory(),
+            null,
+            new PolarisPrincipalHolder(),
+            testServices.principal());
+
+    executor.addTaskHandler(
+        new TaskHandler() {
+          @Override
+          public boolean canHandleTask(TaskEntity task) {
+            return true;
+          }
+
+          @Override
+          public boolean handleTask(TaskEntity task, CallContext callContext) {
+            return false; // simulate transient failure
+          }
+        });
+
+    assertThatThrownBy(
+            () ->
+                executor.handleTask(
+                    taskEntity.getId(),
+                    polarisCallCtx,
+                    PolarisEventMetadata.builder().realmId(realm).build(),
+                    1))
+        .isInstanceOf(RuntimeException.class)
+        .hasMessageContaining("Task handler returned false")
+        .hasMessageContaining(String.valueOf(taskEntity.getId()));
+
+    // Event should have been emitted with TASK_SUCCESS=false even though we threw
+    PolarisEvent afterEvent =
+        testPolarisEventDispatcher.getLatest(PolarisEventType.AFTER_ATTEMPT_TASK);
+    assertThat(afterEvent.attributes().getRequired(EventAttributes.TASK_SUCCESS)).isEqualTo(false);
+  }
+
+  @Test
+  void asyncRetryIsTriggeredWhenHandlerReturnsFalse() throws InterruptedException {
+    String realm = "myrealm";
+    RealmContext realmContext = () -> realm;
+
+    TestServices testServices = TestServices.builder().realmContext(realmContext).build();
+
+    PolarisMetaStoreManager metaStoreManager = testServices.metaStoreManager();
+    PolarisCallContext polarisCallCtx = testServices.newCallContext();
+
+    TaskEntity taskEntity =
+        new TaskEntity.Builder()
+            .setName("retry-task")
+            .setId(metaStoreManager.generateNewEntityId(polarisCallCtx).getId())
+            .setCreateTimestamp(testServices.clock().millis())
+            .build();
+    metaStoreManager.createEntityIfNotExists(polarisCallCtx, null, taskEntity);
+
+    AtomicInteger handlerCalls = new AtomicInteger(0);
+
+    TaskExecutorImpl executor =
+        new TaskExecutorImpl(
+            Runnable::run,
+            null,
+            testServices.clock(),
+            testServices.metaStoreManagerFactory(),
+            new TaskFileIOSupplier(
+                testServices.fileIOFactory(), testServices.storageAccessConfigProvider()),
+            new RealmContextHolder(),
+            testServices.polarisEventDispatcher(),
+            testServices.eventMetadataFactory(),
+            null,
+            new PolarisPrincipalHolder(),
+            testServices.principal());
+
+    executor.addTaskHandler(
+        new TaskHandler() {
+          @Override
+          public boolean canHandleTask(TaskEntity task) {
+            return true;
+          }
+
+          @Override
+          public boolean handleTask(TaskEntity task, CallContext callContext) {
+            int call = handlerCalls.incrementAndGet();
+            // Fail first 2 attempts (transient), succeed on 3rd
+            if (call < 3) {
+              return false;
+            }
+            return true;
+          }
+        });
+
+    // This starts the async processing. With Runnable::run the first handleTask runs
+    // synchronously in the current thread (so handler is called immediately), returns false
+    // -> throws inside the future (the exception path is taken, which is what we verify).
+    executor.addTaskHandlerContext(taskEntity.getId(), polarisCallCtx);
+
+    // We verify at least the first call happened (return-false leads to exception path).
+    assertThat(handlerCalls.get()).isGreaterThanOrEqualTo(1);
+  }
 }