You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ya...@apache.org on 2025/06/11 16:47:34 UTC

(pinot) branch master updated: Enable ZK-based progress tracking for SegmentRelocator rebalances (#16008)

This is an automated email from the ASF dual-hosted git repository.

yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 6639f89842 Enable ZK-based progress tracking for SegmentRelocator rebalances (#16008)
6639f89842 is described below

commit 6639f89842f082cabf7a72cf4717166b61384402
Author: Yash Mayya <ya...@gmail.com>
AuthorDate: Wed Jun 11 17:47:28 2025 +0100

    Enable ZK-based progress tracking for SegmentRelocator rebalances (#16008)
---
 .../api/resources/PinotTableRestletResource.java   |  7 ++-
 .../helix/core/rebalance/RebalanceChecker.java     | 68 ++++++++++++++++++++-
 .../core/rebalance/TableRebalanceContext.java      | 39 ++++++++++--
 .../core/rebalance/TableRebalanceManager.java      | 20 +++++--
 .../rebalance/tenant/DefaultTenantRebalancer.java  |  5 +-
 .../helix/core/relocation/SegmentRelocator.java    |  5 +-
 .../helix/core/rebalance/RebalanceCheckerTest.java | 69 ++++++++++++++++++----
 .../apache/pinot/tools/PinotTableRebalancer.java   |  2 +-
 8 files changed, 184 insertions(+), 31 deletions(-)

diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index bdb82ce766..7ca3473ffe 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -696,18 +696,19 @@ public class PinotTableRestletResource {
       if (dryRun || preChecks || downtime) {
         // For dry-run, preChecks or rebalance with downtime, it's fine to run the rebalance synchronously since it
         // should be a really short operation.
-        return _tableRebalanceManager.rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, false);
+        return _tableRebalanceManager.rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, false, false);
       } else {
         // Make a dry-run first to get the target assignment
         rebalanceConfig.setDryRun(true);
         RebalanceResult dryRunResult =
-            _tableRebalanceManager.rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, false);
+            _tableRebalanceManager.rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, false, false);
 
         if (dryRunResult.getStatus() == RebalanceResult.Status.DONE) {
           // If dry-run succeeded, run rebalance asynchronously
           rebalanceConfig.setDryRun(false);
           CompletableFuture<RebalanceResult> rebalanceResultFuture =
-              _tableRebalanceManager.rebalanceTableAsync(tableNameWithType, rebalanceConfig, rebalanceJobId, true);
+              _tableRebalanceManager.rebalanceTableAsync(tableNameWithType, rebalanceConfig, rebalanceJobId, true,
+                  true);
           rebalanceResultFuture.whenComplete((rebalanceResult, throwable) -> {
             if (throwable != null) {
               String errorMsg = String.format("Caught exception/error while rebalancing table: %s", tableNameWithType);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceChecker.java
index b16c6b8197..cac0315393 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceChecker.java
@@ -111,6 +111,11 @@ public class RebalanceChecker extends ControllerPeriodicTask<Void> {
   @VisibleForTesting
   void retryRebalanceTable(String tableNameWithType, Map<String, Map<String, String>> allJobMetadata)
       throws Exception {
+    // We first check for rebalance jobs that are stuck - i.e., those that are IN_PROGRESS but haven't updated their
+    // status in ZK within the heartbeat timeout. This could occur if a controller crashes while running a rebalance job
+    // for instance. These stuck jobs are always aborted here. They will also be retried if it's allowed as per the
+    // rebalance context.
+    //
     // Skip retry for the table if rebalance job is still running or has completed, in specific:
     // 1) Skip retry if any rebalance job is actively running. Being actively running means the job is at IN_PROGRESS
     // status, and has updated its status kept in ZK within the heartbeat timeout. It's possible that more than one
@@ -118,11 +123,23 @@ public class RebalanceChecker extends ControllerPeriodicTask<Void> {
     // 2) Skip retry if the most recently started rebalance job has completed with DONE or NO_OP. It's possible that
     // jobs started earlier may be still running, but they are ignored here.
     //
-    // Otherwise, we can get a list of failed rebalance jobs, i.e. those at FAILED status; or IN_PROGRESS status but
-    // haven't updated their status kept in ZK within the heartbeat timeout. For those candidate jobs to retry:
+    // Note that it should be very unlikely to have scenarios where there are more than one rebalance jobs running
+    // for a table at the same time, or to have a rebalance job that started earlier but completed later than the one
+    // started most recently since we try to prevent new rebalances from being triggered while a rebalance is in
+    // progress by checking ZK metadata. Such scenarios can only occur if multiple rebalance jobs are triggered at the
+    // same time and the second one is started before the first one updates its status in ZK.
+    //
+    // If we detect that a retry is required based on the above criteria, we can get a list of failed rebalance jobs,
+    // i.e. those at FAILED status; or IN_PROGRESS status but haven't updated their status kept in ZK within the
+    // heartbeat timeout. For those candidate jobs to retry:
     // 1) Firstly, group them by the original jobIds they retry for so that we can skip those exceeded maxRetry.
     // 2) For the remaining jobs, we take the one started most recently and retry it with its original configs.
     // 3) If configured, we can abort the other rebalance jobs for the table by setting their status to FAILED.
+
+    if (hasStuckInProgressJobs(tableNameWithType, allJobMetadata)) {
+      abortExistingJobs(tableNameWithType, _pinotHelixResourceManager);
+    }
+
     Map<String/*original jobId*/, Set<Pair<TableRebalanceContext/*job attempts*/, Long
         /*startTime*/>>> candidateJobs = getCandidateJobs(tableNameWithType, allJobMetadata);
     if (candidateJobs.isEmpty()) {
@@ -147,7 +164,7 @@ public class RebalanceChecker extends ControllerPeriodicTask<Void> {
           retryDelayMs);
       return;
     }
-    abortExistingJobs(tableNameWithType, _pinotHelixResourceManager);
+
     // Get tableConfig only when the table needs to retry rebalance, and get it before submitting rebalance to another
     // thread, in order to avoid unnecessary ZK reads and making too many ZK reads in a short time.
     TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType);
@@ -250,6 +267,47 @@ public class RebalanceChecker extends ControllerPeriodicTask<Void> {
     return candidateJobRun;
   }
 
+  /**
+   * Check if there are any rebalance jobs that are stuck in IN_PROGRESS status, i.e., they have not updated their
+   * status in ZK within the configured heartbeat timeout.
+   * @param tableNameWithType the table name with type
+   * @param allJobMetadata the metadata of all rebalance jobs for the table
+   * @return true if there are stuck rebalance jobs, false otherwise
+   */
+  @VisibleForTesting
+  static boolean hasStuckInProgressJobs(String tableNameWithType, Map<String, Map<String, String>> allJobMetadata)
+      throws Exception {
+    for (Map.Entry<String, Map<String, String>> entry : allJobMetadata.entrySet()) {
+      String jobId = entry.getKey();
+      Map<String, String> jobMetadata = entry.getValue();
+      long statsUpdatedAt = Long.parseLong(jobMetadata.get(CommonConstants.ControllerJob.SUBMISSION_TIME_MS));
+      String jobStatsInStr = jobMetadata.get(RebalanceJobConstants.JOB_METADATA_KEY_REBALANCE_PROGRESS_STATS);
+      if (StringUtils.isEmpty(jobStatsInStr)) {
+        // Skip rebalance job as it has no job progress stats
+        continue;
+      }
+      String jobCtxInStr = jobMetadata.get(RebalanceJobConstants.JOB_METADATA_KEY_REBALANCE_CONTEXT);
+      if (StringUtils.isEmpty(jobCtxInStr)) {
+        // Skip rebalance job: {} as it has no job context
+        continue;
+      }
+      TableRebalanceProgressStats jobStats = JsonUtils.stringToObject(jobStatsInStr, TableRebalanceProgressStats.class);
+      TableRebalanceContext jobCtx = JsonUtils.stringToObject(jobCtxInStr, TableRebalanceContext.class);
+
+      if (jobStats.getStatus() == RebalanceResult.Status.IN_PROGRESS) {
+        long heartbeatTimeoutMs = jobCtx.getConfig().getHeartbeatTimeoutInMs();
+        if (System.currentTimeMillis() - statsUpdatedAt >= heartbeatTimeoutMs) {
+          LOGGER.info("Found stuck rebalance job: {} for table: {} that has not updated its status in ZK within "
+              + "heartbeat timeout: {}", jobId, tableNameWithType, heartbeatTimeoutMs);
+          return true;
+        } else {
+          return false;
+        }
+      }
+    }
+    return false;
+  }
+
   @VisibleForTesting
   static Map<String, Set<Pair<TableRebalanceContext, Long>>> getCandidateJobs(String tableNameWithType,
       Map<String, Map<String, String>> allJobMetadata)
@@ -279,6 +337,10 @@ public class RebalanceChecker extends ControllerPeriodicTask<Void> {
       }
       TableRebalanceProgressStats jobStats = JsonUtils.stringToObject(jobStatsInStr, TableRebalanceProgressStats.class);
       TableRebalanceContext jobCtx = JsonUtils.stringToObject(jobCtxInStr, TableRebalanceContext.class);
+      if (!jobCtx.getAllowRetries()) {
+        LOGGER.info("Skip rebalance job: {} as it does not allow retries", jobId);
+        continue;
+      }
       long jobStartTimeMs = jobStats.getStartTimeMs();
       if (latestStartedJob == null || latestStartedJob.getRight() < jobStartTimeMs) {
         latestStartedJob = Pair.of(jobId, jobStartTimeMs);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceContext.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceContext.java
index 48e43558cc..aea3a2ff0d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceContext.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceContext.java
@@ -27,24 +27,45 @@ public class TableRebalanceContext {
   private String _originalJobId;
   private RebalanceConfig _config;
   private int _attemptId;
+  // Default to true for all user initiated rebalances, so that they can be retried if they fail or get stuck.
+  private boolean _allowRetries = true;
 
-  public static TableRebalanceContext forInitialAttempt(String originalJobId, RebalanceConfig config) {
-    return new TableRebalanceContext(originalJobId, config, INITIAL_ATTEMPT_ID);
+  /**
+   * Creates a new TableRebalanceContext for the initial attempt of a rebalance job.
+   *
+   * @param originalJobId The original job ID for the rebalance job.
+   * @param config The rebalance configuration.
+   * @param allowRetries Whether retries are allowed for this rebalance job. This isn't part of {@link RebalanceConfig}
+   *                     because user initiated rebalances should always support retries for failed and stuck jobs.
+   * @return A new TableRebalanceContext instance.
+   */
+  public static TableRebalanceContext forInitialAttempt(String originalJobId, RebalanceConfig config,
+      boolean allowRetries) {
+    return new TableRebalanceContext(originalJobId, config, INITIAL_ATTEMPT_ID, allowRetries);
   }
 
+  /**
+   * Creates a new TableRebalanceContext for a retry attempt of a rebalance job.
+   *
+   * @param originalJobId The original job ID for the rebalance job.
+   * @param config The rebalance configuration.
+   * @param attemptId The attempt ID for the retry.
+   * @return A new TableRebalanceContext instance.
+   */
   public static TableRebalanceContext forRetry(String originalJobId, RebalanceConfig config, int attemptId) {
-    return new TableRebalanceContext(originalJobId, config, attemptId);
+    return new TableRebalanceContext(originalJobId, config, attemptId, true);
   }
 
   public TableRebalanceContext() {
     // For JSON deserialization.
   }
 
-  private TableRebalanceContext(String originalJobId, RebalanceConfig config, int attemptId) {
+  private TableRebalanceContext(String originalJobId, RebalanceConfig config, int attemptId, boolean allowRetries) {
     _jobId = createAttemptJobId(originalJobId, attemptId);
     _originalJobId = originalJobId;
     _config = config;
     _attemptId = attemptId;
+    _allowRetries = allowRetries;
   }
 
   public int getAttemptId() {
@@ -79,10 +100,18 @@ public class TableRebalanceContext {
     _config = config;
   }
 
+  public boolean getAllowRetries() {
+    return _allowRetries;
+  }
+
+  public void setAllowRetries(boolean allowRetries) {
+    _allowRetries = allowRetries;
+  }
+
   @Override
   public String toString() {
     return "TableRebalanceContext{" + "_jobId='" + _jobId + '\'' + ", _originalJobId='" + _originalJobId + '\''
-        + ", _config=" + _config + ", _attemptId=" + _attemptId + '}';
+        + ", _config=" + _config + ", _attemptId=" + _attemptId + ", _allowRetries=" + _allowRetries + "}";
   }
 
   private static String createAttemptJobId(String originalJobId, int attemptId) {
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceManager.java
index 87de12b3d7..b700582593 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceManager.java
@@ -79,12 +79,19 @@ public class TableRebalanceManager {
    * @param rebalanceConfig configuration for the rebalance operation
    * @param rebalanceJobId ID of the rebalance job, which is used to track the progress of the rebalance operation
    * @param trackRebalanceProgress whether to track rebalance progress stats in ZK
+   * @param allowRetries whether to allow retries for failed or stuck rebalance operations (through
+   *                     {@link RebalanceChecker}). Requires {@code trackRebalanceProgress} to be true.
    * @return result of the rebalance operation
    * @throws TableNotFoundException if the table does not exist
+   * @throws RebalanceInProgressException if a rebalance job is already in progress for the table (as per ZK metadata)
    */
   public RebalanceResult rebalanceTable(String tableNameWithType, RebalanceConfig rebalanceConfig,
-      String rebalanceJobId, boolean trackRebalanceProgress)
+      String rebalanceJobId, boolean trackRebalanceProgress, boolean allowRetries)
       throws TableNotFoundException, RebalanceInProgressException {
+    if (allowRetries && !trackRebalanceProgress) {
+      throw new IllegalArgumentException(
+          "Rebalance retries are only supported when rebalance progress is tracked in ZK");
+    }
     TableConfig tableConfig = _resourceManager.getTableConfig(tableNameWithType);
     if (tableConfig == null) {
       throw new TableNotFoundException("Failed to find table config for table: " + tableNameWithType);
@@ -93,7 +100,7 @@ public class TableRebalanceManager {
     ZkBasedTableRebalanceObserver zkBasedTableRebalanceObserver = null;
     if (trackRebalanceProgress) {
       zkBasedTableRebalanceObserver = new ZkBasedTableRebalanceObserver(tableNameWithType, rebalanceJobId,
-          TableRebalanceContext.forInitialAttempt(rebalanceJobId, rebalanceConfig),
+          TableRebalanceContext.forInitialAttempt(rebalanceJobId, rebalanceConfig, allowRetries),
           _resourceManager.getPropertyStore());
     }
     return rebalanceTable(tableNameWithType, tableConfig, rebalanceJobId, rebalanceConfig,
@@ -109,11 +116,14 @@ public class TableRebalanceManager {
    * @param rebalanceConfig configuration for the rebalance operation
    * @param rebalanceJobId ID of the rebalance job, which is used to track the progress of the rebalance operation
    * @param trackRebalanceProgress whether to track rebalance progress stats in ZK
+   * @param allowRetries whether to allow retries for failed or stuck rebalance operations (through
+   *                     {@link RebalanceChecker}). Requires {@code trackRebalanceProgress} to be true.
    * @return a CompletableFuture that will complete with the result of the rebalance operation
    * @throws TableNotFoundException if the table does not exist
+   * @throws RebalanceInProgressException if a rebalance job is already in progress for the table (as per ZK metadata)
    */
   public CompletableFuture<RebalanceResult> rebalanceTableAsync(String tableNameWithType,
-      RebalanceConfig rebalanceConfig, String rebalanceJobId, boolean trackRebalanceProgress)
+      RebalanceConfig rebalanceConfig, String rebalanceJobId, boolean trackRebalanceProgress, boolean allowRetries)
       throws TableNotFoundException, RebalanceInProgressException {
     TableConfig tableConfig = _resourceManager.getTableConfig(tableNameWithType);
     if (tableConfig == null) {
@@ -125,7 +135,8 @@ public class TableRebalanceManager {
     return CompletableFuture.supplyAsync(
         () -> {
           try {
-            return rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, trackRebalanceProgress);
+            return rebalanceTable(tableNameWithType, rebalanceConfig, rebalanceJobId, trackRebalanceProgress,
+                allowRetries);
           } catch (TableNotFoundException e) {
             // Should not happen since we already checked for table existence
             throw new RuntimeException(e);
@@ -147,6 +158,7 @@ public class TableRebalanceManager {
    * @param rebalanceConfig configuration for the rebalance operation
    * @param zkBasedTableRebalanceObserver observer to track rebalance progress in ZK
    * @return a CompletableFuture that will complete with the result of the rebalance operation
+   * @throws RebalanceInProgressException if a rebalance job is already in progress for the table (as per ZK metadata)
    */
   public CompletableFuture<RebalanceResult> rebalanceTableAsync(String tableNameWithType, TableConfig tableConfig,
       String rebalanceJobId, RebalanceConfig rebalanceConfig,
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/DefaultTenantRebalancer.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/DefaultTenantRebalancer.java
index 369024faa6..3f288ef5ff 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/DefaultTenantRebalancer.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/tenant/DefaultTenantRebalancer.java
@@ -61,7 +61,8 @@ public class DefaultTenantRebalancer implements TenantRebalancer {
         RebalanceConfig rebalanceConfig = RebalanceConfig.copy(config);
         rebalanceConfig.setDryRun(true);
         rebalanceResult.put(table,
-            _tableRebalanceManager.rebalanceTable(table, rebalanceConfig, createUniqueRebalanceJobIdentifier(), false));
+            _tableRebalanceManager.rebalanceTable(table, rebalanceConfig, createUniqueRebalanceJobIdentifier(), false,
+                false));
       } catch (TableNotFoundException | RebalanceInProgressException exception) {
         rebalanceResult.put(table, new RebalanceResult(null, RebalanceResult.Status.FAILED, exception.getMessage(),
             null, null, null, null, null));
@@ -203,7 +204,7 @@ public class DefaultTenantRebalancer implements TenantRebalancer {
       TenantRebalanceObserver observer) {
     try {
       observer.onTrigger(TenantRebalanceObserver.Trigger.REBALANCE_STARTED_TRIGGER, tableName, rebalanceJobId);
-      RebalanceResult result = _tableRebalanceManager.rebalanceTable(tableName, config, rebalanceJobId, true);
+      RebalanceResult result = _tableRebalanceManager.rebalanceTable(tableName, config, rebalanceJobId, true, true);
       if (result.getStatus().equals(RebalanceResult.Status.DONE)) {
         observer.onTrigger(TenantRebalanceObserver.Trigger.REBALANCE_COMPLETED_TRIGGER, tableName, null);
       } else {
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
index b2144260c0..fd10148ce3 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
@@ -232,8 +232,11 @@ public class SegmentRelocator extends ControllerPeriodicTask<Void> {
 
       // We're not using the async rebalance API here because we want to run this on a separate thread pool from the
       // rebalance thread pool that is used for user initiated rebalances.
+
+      // Retries are disabled because SegmentRelocator itself is a periodic controller task, so we don't want the
+      // RebalanceChecker to unnecessarily retry any such failed rebalances.
       RebalanceResult rebalance = _tableRebalanceManager.rebalanceTable(tableNameWithType, rebalanceConfig,
-          TableRebalancer.createUniqueRebalanceJobIdentifier(), false);
+          TableRebalancer.createUniqueRebalanceJobIdentifier(), true, false);
       switch (rebalance.getStatus()) {
         case NO_OP:
           LOGGER.info("All segments are already relocated for table: {}", tableNameWithType);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceCheckerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceCheckerTest.java
index d88b1319c1..373815c969 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceCheckerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceCheckerTest.java
@@ -51,10 +51,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.*;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.*;
 
 
 public class RebalanceCheckerTest {
@@ -87,7 +84,7 @@ public class RebalanceCheckerTest {
     TableRebalanceProgressStats stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.FAILED);
     stats.setStartTimeMs(1000);
-    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg);
+    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg, true);
     Map<String, String> jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job1", stats, jobCtx);
     allJobMetadata.put("job1", jobMetadata);
     // 3 failed retry runs for job1
@@ -104,7 +101,7 @@ public class RebalanceCheckerTest {
     stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.FAILED);
     stats.setStartTimeMs(2000);
-    jobCtx = TableRebalanceContext.forInitialAttempt("job2", jobCfg);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job2", jobCfg, true);
     jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job2", stats, jobCtx);
     allJobMetadata.put("job2", jobMetadata);
     jobMetadata = createDummyJobMetadata(tableName, "job2", 2, 2100, RebalanceResult.Status.DONE);
@@ -116,7 +113,7 @@ public class RebalanceCheckerTest {
     stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.IN_PROGRESS);
     stats.setStartTimeMs(3000);
-    jobCtx = TableRebalanceContext.forInitialAttempt("job3", jobCfg);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job3", jobCfg, true);
     jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job3", stats, jobCtx);
     jobMetadata.put(CommonConstants.ControllerJob.SUBMISSION_TIME_MS, "3000");
     allJobMetadata.put("job3", jobMetadata);
@@ -152,11 +149,37 @@ public class RebalanceCheckerTest {
     stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.DONE);
     stats.setStartTimeMs(5000);
-    jobCtx = TableRebalanceContext.forInitialAttempt("job5", jobCfg);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job5", jobCfg, true);
     jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job5", stats, jobCtx);
     allJobMetadata.put("job5", jobMetadata);
     jobs = RebalanceChecker.getCandidateJobs(tableName, allJobMetadata);
     assertEquals(jobs.size(), 0);
+
+    // Add job6 that doesn't support retries as per its rebalance context (used by system initiated rebalances in
+    // practice).
+    jobCfg = new RebalanceConfig();
+    jobCfg.setMaxAttempts(4);
+    stats = new TableRebalanceProgressStats();
+    stats.setStatus(RebalanceResult.Status.FAILED);
+    stats.setStartTimeMs(5000);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job6", jobCfg, false);
+    jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job6", stats, jobCtx);
+    allJobMetadata.put("job6", jobMetadata);
+    jobs = RebalanceChecker.getCandidateJobs(tableName, allJobMetadata);
+    assertEquals(jobs.size(), 0);
+
+    // Ensure that a job serialized using an older version of TableRebalanceContext without the allowRetries field is
+    // retriable by default.
+    jobMetadata.put(RebalanceJobConstants.JOB_METADATA_KEY_REBALANCE_CONTEXT,
+        "{\"jobId\":\"job6\",\"attemptId\":1,\"config\":{\"maxAttempts\":4,\"dryRun\":false,\"preChecks\":false,"
+            + "\"bootstrap\":false,\"downtime\":false,\"lowDiskMode\":false,\"bestEfforts\":false,"
+            + "\"reassignInstances\":false,\"includeConsuming\":false,\"batchSizePerServer\":-1,"
+            + "\"updateTargetTier\":false,\"externalViewCheckIntervalInMs\":1000,\"minAvailableReplicas\":1,"
+            + "\"heartbeatIntervalInMs\":300000,\"heartbeatTimeoutInMs\":3600000,\"retryInitialDelayInMs\":300000,"
+            + "\"minimizeDataMovement\":\"ENABLE\",\"externalViewStabilizationTimeoutInMs\":3600000},"
+            + "\"originalJobId\":\"job6\"}, tableName=table01}");
+    jobs = RebalanceChecker.getCandidateJobs(tableName, Map.of("job6", jobMetadata));
+    assertEquals(jobs.size(), 1);
   }
 
   @Test
@@ -192,6 +215,28 @@ public class RebalanceCheckerTest {
     assertNull(jobTime);
   }
 
+  @Test
+  public void testStuckInProgressJobs()
+      throws Exception {
+    String tableName = "table01";
+    Map<String, Map<String, String>> allJobMetadata = new HashMap<>();
+
+    assertFalse(RebalanceChecker.hasStuckInProgressJobs(tableName, allJobMetadata));
+
+    RebalanceConfig jobCfg = new RebalanceConfig();
+    jobCfg.setHeartbeatTimeoutInMs(10_000);
+    TableRebalanceProgressStats stats = new TableRebalanceProgressStats();
+    stats.setStatus(RebalanceResult.Status.IN_PROGRESS);
+    // Even though allowRetries is false, we still abort stuck jobs (heartbeat timeout exceeded).
+    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg, false);
+    Map<String, String> jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job1", stats, jobCtx);
+    jobMetadata.put(CommonConstants.ControllerJob.SUBMISSION_TIME_MS,
+        String.valueOf(System.currentTimeMillis() - 20_000));
+    allJobMetadata.put("job1", jobMetadata);
+
+    assertTrue(RebalanceChecker.hasStuckInProgressJobs(tableName, allJobMetadata));
+  }
+
   @Test
   public void testRetryRebalance()
       throws Exception {
@@ -208,7 +253,7 @@ public class RebalanceCheckerTest {
     TableRebalanceProgressStats stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.FAILED);
     stats.setStartTimeMs(1000);
-    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg);
+    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg, true);
     Map<String, String> jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job1", stats, jobCtx);
     allJobMetadata.put("job1", jobMetadata);
     // 3 failed retry runs for job1
@@ -225,7 +270,7 @@ public class RebalanceCheckerTest {
     stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.FAILED);
     stats.setStartTimeMs(2000);
-    jobCtx = TableRebalanceContext.forInitialAttempt("job2", jobCfg);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job2", jobCfg, true);
     jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job2", stats, jobCtx);
     allJobMetadata.put("job2", jobMetadata);
     jobMetadata = createDummyJobMetadata(tableName, "job2", 2, 2100, RebalanceResult.Status.DONE);
@@ -237,7 +282,7 @@ public class RebalanceCheckerTest {
     stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.IN_PROGRESS);
     stats.setStartTimeMs(3000);
-    jobCtx = TableRebalanceContext.forInitialAttempt("job3", jobCfg);
+    jobCtx = TableRebalanceContext.forInitialAttempt("job3", jobCfg, true);
     jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job3", stats, jobCtx);
     jobMetadata.put(CommonConstants.ControllerJob.SUBMISSION_TIME_MS, "3000");
     allJobMetadata.put("job3", jobMetadata);
@@ -284,7 +329,7 @@ public class RebalanceCheckerTest {
     TableRebalanceProgressStats stats = new TableRebalanceProgressStats();
     stats.setStatus(RebalanceResult.Status.FAILED);
     stats.setStartTimeMs(nowMs);
-    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg);
+    TableRebalanceContext jobCtx = TableRebalanceContext.forInitialAttempt("job1", jobCfg, true);
     Map<String, String> jobMetadata = ZkBasedTableRebalanceObserver.createJobMetadata(tableName, "job1", stats, jobCtx);
     allJobMetadata.put("job1", jobMetadata);
 
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/PinotTableRebalancer.java b/pinot-tools/src/main/java/org/apache/pinot/tools/PinotTableRebalancer.java
index f935a22ba7..54778de2fe 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/PinotTableRebalancer.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/PinotTableRebalancer.java
@@ -71,7 +71,7 @@ public class PinotTableRebalancer extends PinotZKChanger {
     }
 
     ZkBasedTableRebalanceObserver rebalanceObserver = new ZkBasedTableRebalanceObserver(tableNameWithType, jobId,
-        TableRebalanceContext.forInitialAttempt(jobId, _rebalanceConfig), _propertyStore);
+        TableRebalanceContext.forInitialAttempt(jobId, _rebalanceConfig, true), _propertyStore);
 
     return new TableRebalancer(_helixManager, rebalanceObserver, null, null, null)
         .rebalance(tableConfig, _rebalanceConfig, jobId);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org