You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2025/01/29 06:33:53 UTC

(pinot) branch master updated: Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)

This is an automated email from the ASF dual-hosted git repository.

kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 3525116ac2 Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)
3525116ac2 is described below

commit 3525116ac277e39e28331d955e968a13df62f8bf
Author: 9aman <35...@users.noreply.github.com>
AuthorDate: Wed Jan 29 12:03:47 2025 +0530

    Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)
---
 .../common/metadata/segment/SegmentZKMetadata.java |  35 +++
 .../metadata/segment/SegmentZKMetadataUtils.java   | 141 +++++----
 .../resources/TableLLCSegmentUploadResponse.java   |   6 +-
 .../common/utils/FileUploadDownloadClient.java     |  25 ++
 .../common/utils/PauselessConsumptionUtils.java    |  11 +-
 .../pinot/controller/BaseControllerStarter.java    |   7 +-
 .../pinot/controller/api/upload/ZKOperator.java    |  14 +-
 .../helix/core/PinotHelixResourceManager.java      |  23 +-
 .../realtime/PinotLLCRealtimeSegmentManager.java   | 324 +++++++++++++--------
 .../helix/core/util/FailureInjectionUtils.java     |  33 +--
 .../RealtimeSegmentValidationManager.java          |   9 +-
 .../PinotLLCRealtimeSegmentManagerTest.java        | 152 +++++++++-
 .../tests/BasePauselessRealtimeIngestionTest.java  | 296 +++++++++++++++++++
 ...ltimeIngestionCommitEndMetadataFailureTest.java |  67 +++++
 ...altimeIngestionIdealStateUpdateFailureTest.java |  57 ++++
 .../PauselessRealtimeIngestionIntegrationTest.java | 159 ++--------
 ...stionNewSegmentMetadataCreationFailureTest.java |  57 ++++
 .../utils/FailureInjectingControllerStarter.java   |  31 +-
 ...ureInjectingPinotLLCRealtimeSegmentManager.java |  66 +++++
 .../realtime/utils/PauselessRealtimeTestUtils.java |  91 ++++++
 .../pinot/server/api/resources/TablesResource.java | 179 ++++++++----
 21 files changed, 1342 insertions(+), 441 deletions(-)

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
index 69cfe0a63a..9474007fc5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
@@ -18,6 +18,10 @@
  */
 package org.apache.pinot.common.metadata.segment;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
@@ -33,6 +37,8 @@ import org.slf4j.LoggerFactory;
 
 public class SegmentZKMetadata implements ZKMetadata {
   private static final Logger LOGGER = LoggerFactory.getLogger(SegmentZKMetadata.class);
+  private static final String SEGMENT_NAME_KEY = "segmentName";
+  private static final String SIMPLE_FIELDS_KEY = "simpleFields";
   private static final String NULL = "null";
 
   private final ZNRecord _znRecord;
@@ -58,6 +64,16 @@ public class SegmentZKMetadata implements ZKMetadata {
     return _znRecord.getId();
   }
 
+  public Map<String, String> getSimpleFields() {
+    return _simpleFields;
+  }
+
+  public void setSimpleFields(Map<String, String> simpleFields) {
+    _simpleFields = simpleFields;
+    _startTimeMsCached = false;
+    _endTimeMsCached = false;
+  }
+
   public long getStartTimeMs() {
     if (!_startTimeMsCached) {
       long startTimeMs = -1;
@@ -370,6 +386,25 @@ public class SegmentZKMetadata implements ZKMetadata {
     return metadataMap;
   }
 
+  public String toJsonString() {
+    ObjectNode objectNode = JsonUtils.newObjectNode();
+    objectNode.put(SEGMENT_NAME_KEY, getSegmentName());
+    objectNode.set(SIMPLE_FIELDS_KEY, JsonUtils.objectToJsonNode(_simpleFields));
+    return objectNode.toString();
+  }
+
+  public static SegmentZKMetadata fromJsonString(String jsonString)
+      throws IOException {
+    JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
+    String segmentName = jsonNode.get(SEGMENT_NAME_KEY).asText();
+    JsonNode simpleFieldsJsonNode = jsonNode.get(SIMPLE_FIELDS_KEY);
+    Map<String, String> simpleFields = JsonUtils.jsonNodeToObject(simpleFieldsJsonNode, new TypeReference<>() {
+    });
+    ZNRecord znRecord = new ZNRecord(segmentName);
+    znRecord.setSimpleFields(simpleFields);
+    return new SegmentZKMetadata(znRecord);
+  }
+
   @Override
   public ZNRecord toZNRecord() {
     // Convert to TreeMap to keep the keys sorted. The de-serialized ZNRecord has simple fields stored as LinkedHashMap.
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
similarity index 54%
rename from pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
rename to pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
index d481b669a4..58de48a843 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.controller.helix.core.util;
+package org.apache.pinot.common.metadata.segment;
 
 import com.google.common.base.Preconditions;
 import java.util.HashMap;
@@ -24,20 +24,20 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
-import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.common.utils.LLCSegmentName;
 import org.apache.pinot.segment.spi.ColumnMetadata;
 import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.creator.SegmentVersion;
 import org.apache.pinot.segment.spi.partition.PartitionFunction;
 import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
 import org.apache.pinot.spi.data.DateTimeFieldSpec;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.joda.time.Interval;
 
 
-public class ZKMetadataUtils {
-  private ZKMetadataUtils() {
+public class SegmentZKMetadataUtils {
+  private SegmentZKMetadataUtils() {
   }
 
   /**
@@ -47,7 +47,7 @@ public class ZKMetadataUtils {
       String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes) {
     SegmentZKMetadata segmentZKMetadata = new SegmentZKMetadata(segmentMetadata.getName());
     updateSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, crypterName,
-        segmentSizeInBytes, true);
+        segmentSizeInBytes, null, true);
     return segmentZKMetadata;
   }
 
@@ -57,7 +57,16 @@ public class ZKMetadataUtils {
   public static void refreshSegmentZKMetadata(String tableNameWithType, SegmentZKMetadata segmentZKMetadata,
       SegmentMetadata segmentMetadata, String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes) {
     updateSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, crypterName,
-        segmentSizeInBytes, false);
+        segmentSizeInBytes, null, false);
+  }
+
+  /**
+   * Updates the segment ZK metadata for a committing segment.
+   */
+  public static void updateCommittingSegmentZKMetadata(String realtimeTableName, SegmentZKMetadata segmentZKMetadata,
+      SegmentMetadata segmentMetadata, String downloadUrl, long segmentSizeInBytes, String endOffset) {
+    updateSegmentZKMetadata(realtimeTableName, segmentZKMetadata, segmentMetadata, downloadUrl, null,
+        segmentSizeInBytes, endOffset, false);
   }
 
   public static void updateSegmentZKTimeInterval(SegmentZKMetadata segmentZKMetadata,
@@ -77,47 +86,96 @@ public class ZKMetadataUtils {
 
   private static void updateSegmentZKMetadata(String tableNameWithType, SegmentZKMetadata segmentZKMetadata,
       SegmentMetadata segmentMetadata, String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes,
-      boolean newSegment) {
-    if (newSegment) {
-      segmentZKMetadata.setPushTime(System.currentTimeMillis());
-    } else {
-      segmentZKMetadata.setRefreshTime(System.currentTimeMillis());
-    }
+      @Nullable String endOffset, boolean newSegment) {
+    String segmentName = segmentZKMetadata.getSegmentName();
 
-    if (segmentMetadata.getTimeInterval() != null) {
-      segmentZKMetadata.setStartTime(segmentMetadata.getTimeInterval().getStartMillis());
-      segmentZKMetadata.setEndTime(segmentMetadata.getTimeInterval().getEndMillis());
-      segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
-      ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor(segmentMetadata.getTimeColumn());
-      if (isValidTimeMetadata(timeColumnMetadata)) {
-        segmentZKMetadata.setRawStartTime(timeColumnMetadata.getMinValue().toString());
-        segmentZKMetadata.setRawEndTime(timeColumnMetadata.getMaxValue().toString());
+    if (endOffset != null) {
+      // For committing segment
+
+      segmentZKMetadata.setEndOffset(endOffset);
+      segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
+
+      // For committing segment, use current time as start/end time if total docs is 0
+      if (segmentMetadata.getTotalDocs() > 0) {
+        Interval timeInterval = segmentMetadata.getTimeInterval();
+        Preconditions.checkState(timeInterval != null, "Failed to find time info for table: %s, segment: %s",
+            tableNameWithType, segmentName);
+        segmentZKMetadata.setStartTime(timeInterval.getStartMillis());
+        segmentZKMetadata.setEndTime(timeInterval.getEndMillis());
+      } else {
+        long now = System.currentTimeMillis();
+        segmentZKMetadata.setStartTime(now);
+        segmentZKMetadata.setEndTime(now);
       }
+      segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
     } else {
-      segmentZKMetadata.setStartTime(-1);
-      segmentZKMetadata.setEndTime(-1);
-      segmentZKMetadata.setTimeUnit(null);
+      // For uploaded segment
+
+      // Set segment status, start/end offset info for real-time table
+      if (TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
+        segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.UPLOADED);
+
+        // For new segment, start/end offset must exist if the segment name follows LLC segment name convention
+        if (newSegment && LLCSegmentName.isLLCSegment(segmentMetadata.getName())) {
+          Preconditions.checkArgument(
+              segmentMetadata.getStartOffset() != null && segmentMetadata.getEndOffset() != null,
+              "New uploaded LLC segment must have start/end offset in the segment metadata");
+        }
+
+        // NOTE:
+        // - If start/end offset is available in the uploaded segment, update them in the segment ZK metadata
+        // - If not, keep the existing start/end offset in the segment ZK metadata unchanged
+        if (segmentMetadata.getStartOffset() != null) {
+          segmentZKMetadata.setStartOffset(segmentMetadata.getStartOffset());
+        }
+        if (segmentMetadata.getEndOffset() != null) {
+          segmentZKMetadata.setEndOffset(segmentMetadata.getEndOffset());
+        }
+      }
+
+      segmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
+      if (newSegment) {
+        segmentZKMetadata.setPushTime(System.currentTimeMillis());
+      } else {
+        segmentZKMetadata.setRefreshTime(System.currentTimeMillis());
+      }
+      Interval timeInterval = segmentMetadata.getTimeInterval();
+      if (timeInterval != null) {
+        segmentZKMetadata.setStartTime(timeInterval.getStartMillis());
+        segmentZKMetadata.setEndTime(timeInterval.getEndMillis());
+        segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
+        ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor(segmentMetadata.getTimeColumn());
+        if (isValidTimeMetadata(timeColumnMetadata)) {
+          segmentZKMetadata.setRawStartTime(timeColumnMetadata.getMinValue().toString());
+          segmentZKMetadata.setRawEndTime(timeColumnMetadata.getMaxValue().toString());
+        }
+      } else {
+        segmentZKMetadata.setStartTime(-1);
+        segmentZKMetadata.setEndTime(-1);
+        segmentZKMetadata.setTimeUnit(null);
+      }
     }
-    segmentZKMetadata.setIndexVersion(
-        segmentMetadata.getVersion() != null ? segmentMetadata.getVersion().name() : null);
+
+    SegmentVersion segmentVersion = segmentMetadata.getVersion();
+    segmentZKMetadata.setIndexVersion(segmentVersion != null ? segmentVersion.toString() : null);
     segmentZKMetadata.setTotalDocs(segmentMetadata.getTotalDocs());
-    segmentZKMetadata.setSizeInBytes(segmentSizeInBytes);
     segmentZKMetadata.setCrc(Long.parseLong(segmentMetadata.getCrc()));
-    segmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
     segmentZKMetadata.setDownloadUrl(downloadUrl);
     segmentZKMetadata.setCrypterName(crypterName);
+    segmentZKMetadata.setSizeInBytes(segmentSizeInBytes);
 
     // Set partition metadata
     Map<String, ColumnPartitionMetadata> columnPartitionMap = new HashMap<>();
-    segmentMetadata.getColumnMetadataMap().forEach((column, columnMetadata) -> {
+    for (Map.Entry<String, ColumnMetadata> entry : segmentMetadata.getColumnMetadataMap().entrySet()) {
+      ColumnMetadata columnMetadata = entry.getValue();
       PartitionFunction partitionFunction = columnMetadata.getPartitionFunction();
       if (partitionFunction != null) {
         ColumnPartitionMetadata columnPartitionMetadata =
             new ColumnPartitionMetadata(partitionFunction.getName(), partitionFunction.getNumPartitions(),
                 columnMetadata.getPartitions(), partitionFunction.getFunctionConfig());
-        columnPartitionMap.put(column, columnPartitionMetadata);
+        columnPartitionMap.put(entry.getKey(), columnPartitionMetadata);
       }
-    });
+    }
     segmentZKMetadata.setPartitionMetadata(
         !columnPartitionMap.isEmpty() ? new SegmentPartitionMetadata(columnPartitionMap) : null);
 
@@ -130,27 +188,6 @@ public class ZKMetadataUtils {
       customMap.putAll(segmentMetadata.getCustomMap());
     }
     segmentZKMetadata.setCustomMap(customMap);
-
-    // Set fields specific to realtime table
-    if (TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
-      segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.UPLOADED);
-
-      // For new segment, start/end offset must exist if the segment name follows LLC segment name convention
-      if (newSegment && LLCSegmentName.isLLCSegment(segmentMetadata.getName())) {
-        Preconditions.checkArgument(segmentMetadata.getStartOffset() != null && segmentMetadata.getEndOffset() != null,
-            "New uploaded LLC segment must have start/end offset in the segment metadata");
-      }
-
-      // NOTE:
-      // - If start/end offset is available in the uploaded segment, update them in the segment ZK metadata
-      // - If not, keep the existing start/end offset in the segment ZK metadata unchanged
-      if (segmentMetadata.getStartOffset() != null) {
-        segmentZKMetadata.setStartOffset(segmentMetadata.getStartOffset());
-      }
-      if (segmentMetadata.getEndOffset() != null) {
-        segmentZKMetadata.setEndOffset(segmentMetadata.getEndOffset());
-      }
-    }
   }
 
   private static boolean isValidTimeMetadata(ColumnMetadata timeColumnMetadata) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
index aa3dbe5c0b..2bbbb52f46 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
@@ -26,8 +26,8 @@ public class TableLLCSegmentUploadResponse {
   private final long _crc;
   private final String _downloadUrl;
 
-  public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
-      @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
+  public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName, @JsonProperty("crc") long crc,
+      @JsonProperty("downloadUrl") String downloadUrl) {
     _segmentName = segmentName;
     _crc = crc;
     _downloadUrl = downloadUrl;
@@ -37,7 +37,7 @@ public class TableLLCSegmentUploadResponse {
     return _segmentName;
   }
 
-  public Long getCrc() {
+  public long getCrc() {
     return _crc;
   }
 
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
index 24e6c701d5..6936e6dcb6 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
@@ -55,6 +55,7 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
 import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.apache.pinot.common.auth.AuthProviderUtils;
 import org.apache.pinot.common.exception.HttpErrorStatusException;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.common.restlet.resources.EndReplaceSegmentsRequest;
 import org.apache.pinot.common.restlet.resources.StartReplaceSegmentsRequest;
 import org.apache.pinot.common.restlet.resources.TableLLCSegmentUploadResponse;
@@ -991,6 +992,30 @@ public class FileUploadDownloadClient implements AutoCloseable {
     return tableLLCSegmentUploadResponse;
   }
 
+  /**
+   * Used by controllers to send requests to servers: Controller periodic task uses this endpoint to ask servers
+   * to upload committed llc segment to segment store if missing.
+   * @param uri The uri to ask servers to upload segment to segment store
+   * @return {@link SegmentZKMetadata} - segment download url, crc, other metadata
+   * @throws URISyntaxException
+   * @throws IOException
+   * @throws HttpErrorStatusException
+   */
+  public SegmentZKMetadata uploadLLCToSegmentStoreWithZKMetadata(String uri)
+      throws URISyntaxException, IOException, HttpErrorStatusException {
+    ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.post(new URI(uri)).setVersion(HttpVersion.HTTP_1_1);
+    // sendRequest checks the response status code
+    SimpleHttpResponse response = HttpClient.wrapAndThrowHttpException(
+        _httpClient.sendRequest(requestBuilder.build(), HttpClient.DEFAULT_SOCKET_TIMEOUT_MS));
+    SegmentZKMetadata segmentZKMetadata = SegmentZKMetadata.fromJsonString(response.getResponse());
+    if (StringUtils.isEmpty(segmentZKMetadata.getDownloadUrl())) {
+      throw new HttpErrorStatusException(
+          String.format("Returned segment download url is empty after requesting servers to upload by the path: %s",
+              uri), response.getStatusCode());
+    }
+    return segmentZKMetadata;
+  }
+
   /**
    * Send segment uri.
    *
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
index 36449a5422..89b3a2cd2b 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
@@ -19,7 +19,7 @@
 package org.apache.pinot.common.utils;
 
 import java.util.Optional;
-import javax.validation.constraints.NotNull;
+import javax.annotation.Nullable;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
 import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
@@ -39,8 +39,11 @@ public class PauselessConsumptionUtils {
    * @return true if pauseless consumption is explicitly enabled, false otherwise
    * @throws NullPointerException if tableConfig is null
    */
-  public static boolean isPauselessEnabled(@NotNull TableConfig tableConfig) {
-    return Optional.ofNullable(tableConfig.getIngestionConfig()).map(IngestionConfig::getStreamIngestionConfig)
-        .map(StreamIngestionConfig::isPauselessConsumptionEnabled).orElse(false);
+  public static boolean isPauselessEnabled(@Nullable TableConfig tableConfig) {
+    return Optional.ofNullable(tableConfig)
+        .map(TableConfig::getIngestionConfig)
+        .map(IngestionConfig::getStreamIngestionConfig)
+        .map(StreamIngestionConfig::isPauselessConsumptionEnabled)
+        .orElse(false);
   }
 }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index 171e850638..a0ce503d41 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -487,8 +487,7 @@ public abstract class BaseControllerStarter implements ServiceStartable {
 
     // Helix resource manager must be started in order to create PinotLLCRealtimeSegmentManager
     LOGGER.info("Starting realtime segment manager");
-    _pinotLLCRealtimeSegmentManager =
-        new PinotLLCRealtimeSegmentManager(_helixResourceManager, _config, _controllerMetrics);
+    _pinotLLCRealtimeSegmentManager = createPinotLLCRealtimeSegmentManager();
     // TODO: Need to put this inside HelixResourceManager when HelixControllerLeadershipManager is removed.
     _helixResourceManager.registerPinotLLCRealtimeSegmentManager(_pinotLLCRealtimeSegmentManager);
 
@@ -623,6 +622,10 @@ public abstract class BaseControllerStarter implements ServiceStartable {
     _serviceStatusCallbackList.add(generateServiceStatusCallback(_helixParticipantManager));
   }
 
+  protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+    return new PinotLLCRealtimeSegmentManager(_helixResourceManager, _config, _controllerMetrics);
+  }
+
   /**
    * This method is used to fix table/schema names.
    * TODO: in the next release, maybe 2.0.0, we can remove this method. Meanwhile we can delete the orphan schemas
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
index eeee40b20d..4da9c72a56 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
@@ -33,6 +33,7 @@ import org.apache.helix.model.IdealState;
 import org.apache.helix.zookeeper.datamodel.ZNRecord;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifier;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
 import org.apache.pinot.common.metrics.ControllerMeter;
 import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.common.utils.FileUploadDownloadClient;
@@ -40,7 +41,6 @@ import org.apache.pinot.common.utils.FileUploadDownloadClient.FileUploadType;
 import org.apache.pinot.controller.ControllerConf;
 import org.apache.pinot.controller.api.exception.ControllerApplicationException;
 import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
-import org.apache.pinot.controller.helix.core.util.ZKMetadataUtils;
 import org.apache.pinot.segment.spi.SegmentMetadata;
 import org.apache.pinot.spi.filesystem.PinotFSFactory;
 import org.slf4j.Logger;
@@ -309,11 +309,11 @@ public class ZKOperator {
         if (customMapModifier == null) {
           // If no modifier is provided, use the custom map from the segment metadata
           segmentZKMetadata.setCustomMap(null);
-          ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+          SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
               segmentDownloadURIStr, crypterName, segmentSizeInBytes);
         } else {
           // If modifier is provided, first set the custom map from the segment metadata, then apply the modifier
-          ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+          SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
               segmentDownloadURIStr, crypterName, segmentSizeInBytes);
           segmentZKMetadata.setCustomMap(customMapModifier.modifyMap(segmentZKMetadata.getCustomMap()));
         }
@@ -446,11 +446,11 @@ public class ZKOperator {
           if (customMapModifier == null) {
             // If no modifier is provided, use the custom map from the segment metadata
             segmentZKMetadata.setCustomMap(null);
-            ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+            SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
                 segmentDownloadURIStr, crypterName, segmentSizeInBytes);
           } else {
             // If modifier is provided, first set the custom map from the segment metadata, then apply the modifier
-            ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+            SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
                 segmentDownloadURIStr, crypterName, segmentSizeInBytes);
             segmentZKMetadata.setCustomMap(customMapModifier.modifyMap(segmentZKMetadata.getCustomMap()));
           }
@@ -502,7 +502,7 @@ public class ZKOperator {
     SegmentZKMetadata newSegmentZKMetadata;
     try {
       newSegmentZKMetadata =
-          ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, segmentDownloadURIStr,
+          SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, segmentDownloadURIStr,
               crypterName, segmentSizeInBytes);
     } catch (IllegalArgumentException e) {
       throw new ControllerApplicationException(LOGGER,
@@ -590,7 +590,7 @@ public class ZKOperator {
       long segmentSizeInBytes = segmentUploadMetadata.getSegmentSizeInBytes();
       File segmentFile = segmentUploadMetadata.getEncryptionInfo().getRight();
       try {
-        newSegmentZKMetadata = ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata,
+        newSegmentZKMetadata = SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata,
             segmentDownloadURIStr, crypterName, segmentSizeInBytes);
         segmentZKMetadataMap.put(segmentName, newSegmentZKMetadata);
         segmentNames.add(segmentName);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 77762c2ee8..1f983ce671 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -114,6 +114,7 @@ import org.apache.pinot.common.metadata.ZKMetadataProvider;
 import org.apache.pinot.common.metadata.controllerjob.ControllerJobType;
 import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
 import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.common.minion.MinionTaskMetadataUtils;
 import org.apache.pinot.common.restlet.resources.EndReplaceSegmentsRequest;
@@ -154,7 +155,6 @@ import org.apache.pinot.controller.helix.core.rebalance.RebalanceResult;
 import org.apache.pinot.controller.helix.core.rebalance.TableRebalanceContext;
 import org.apache.pinot.controller.helix.core.rebalance.TableRebalancer;
 import org.apache.pinot.controller.helix.core.rebalance.ZkBasedTableRebalanceObserver;
-import org.apache.pinot.controller.helix.core.util.ZKMetadataUtils;
 import org.apache.pinot.controller.helix.starter.HelixConfig;
 import org.apache.pinot.segment.spi.SegmentMetadata;
 import org.apache.pinot.spi.config.DatabaseConfig;
@@ -405,6 +405,15 @@ public class PinotHelixResourceManager {
     return _propertyStore;
   }
 
+  /**
+   * Get the realtime segment manager
+   *
+   * @return Realtime segment manager
+   */
+  public PinotLLCRealtimeSegmentManager getRealtimeSegmentManager() {
+    return _pinotLLCRealtimeSegmentManager;
+  }
+
   /**
    * Get the linage manager.
    *
@@ -2329,9 +2338,9 @@ public class PinotHelixResourceManager {
   public void addNewSegment(String tableNameWithType, SegmentMetadata segmentMetadata, String downloadUrl) {
     // NOTE: must first set the segment ZK metadata before assigning segment to instances because segment assignment
     // might need them to determine the partition of the segment, and server will need them to download the segment
-    SegmentZKMetadata segmentZkmetadata =
-        ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, downloadUrl, null, -1);
-    ZNRecord znRecord = segmentZkmetadata.toZNRecord();
+    SegmentZKMetadata segmentZKMetadata =
+        SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, downloadUrl, null, -1);
+    ZNRecord znRecord = segmentZKMetadata.toZNRecord();
 
     String segmentName = segmentMetadata.getName();
     String segmentZKMetadataPath =
@@ -2589,7 +2598,7 @@ public class PinotHelixResourceManager {
   }
 
   public void updateZkTimeInterval(SegmentZKMetadata segmentZKMetadata, DateTimeFieldSpec timeColumnFieldSpec) {
-    ZKMetadataUtils.updateSegmentZKTimeInterval(segmentZKMetadata, timeColumnFieldSpec);
+    SegmentZKMetadataUtils.updateSegmentZKTimeInterval(segmentZKMetadata, timeColumnFieldSpec);
   }
 
   @VisibleForTesting
@@ -2601,8 +2610,8 @@ public class PinotHelixResourceManager {
     // ZK metadata to refresh the segment (server will compare the segment ZK metadata with the local metadata to decide
     // whether to download the new segment; broker will update the segment partition info & time boundary based on the
     // segment ZK metadata)
-    ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, null,
-        -1);
+    SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl,
+        null, -1);
     if (!ZKMetadataProvider.setSegmentZKMetadata(_propertyStore, tableNameWithType, segmentZKMetadata,
         expectedVersion)) {
       throw new RuntimeException(
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index 2b9cf8f954..8f5382ae5e 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -60,6 +60,7 @@ import org.apache.pinot.common.messages.IngestionMetricsRemoveMessage;
 import org.apache.pinot.common.metadata.ZKMetadataProvider;
 import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
 import org.apache.pinot.common.metrics.ControllerGauge;
 import org.apache.pinot.common.metrics.ControllerMeter;
 import org.apache.pinot.common.metrics.ControllerMetrics;
@@ -67,6 +68,7 @@ import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
 import org.apache.pinot.common.restlet.resources.TableLLCSegmentUploadResponse;
 import org.apache.pinot.common.utils.FileUploadDownloadClient;
 import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.common.utils.PauselessConsumptionUtils;
 import org.apache.pinot.common.utils.URIUtils;
 import org.apache.pinot.common.utils.helix.HelixHelper;
 import org.apache.pinot.controller.ControllerConf;
@@ -86,10 +88,7 @@ import org.apache.pinot.controller.helix.core.retention.strategy.TimeRetentionSt
 import org.apache.pinot.controller.validation.RealtimeSegmentValidationManager;
 import org.apache.pinot.core.data.manager.realtime.SegmentCompletionUtils;
 import org.apache.pinot.core.util.PeerServerSegmentFinder;
-import org.apache.pinot.segment.spi.ColumnMetadata;
-import org.apache.pinot.segment.spi.creator.SegmentVersion;
 import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
 import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
 import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
 import org.apache.pinot.spi.config.table.PauseState;
@@ -114,7 +113,6 @@ import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateM
 import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
 import org.apache.pinot.spi.utils.IngestionConfigUtils;
 import org.apache.pinot.spi.utils.JsonUtils;
-import org.apache.pinot.spi.utils.StringUtil;
 import org.apache.pinot.spi.utils.TimeUtils;
 import org.apache.pinot.spi.utils.builder.TableNameBuilder;
 import org.apache.pinot.spi.utils.retry.RetryPolicies;
@@ -404,7 +402,7 @@ public class PinotLLCRealtimeSegmentManager {
   }
 
   @VisibleForTesting
-  SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+  protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
     try {
       ZNRecord znRecord =
           _propertyStore.get(ZKMetadataProvider.constructPropertyStorePathForSegment(realtimeTableName, segmentName),
@@ -532,7 +530,7 @@ public class PinotLLCRealtimeSegmentManager {
     /*
      * Update zookeeper in 3 steps.
      *
-     * Step 1: Update PROPERTYSTORE to change the old segment metadata status to COMMITTING/ DONE
+     * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/COMMITTING
      * Step 2: Update PROPERTYSTORE to create the new segment metadata with status IN_PROGRESS
      * Step 3: Update IDEALSTATES to include new segment in CONSUMING state, and change old segment to ONLINE state.
      */
@@ -543,6 +541,8 @@ public class PinotLLCRealtimeSegmentManager {
     SegmentZKMetadata committingSegmentZKMetadata =
         updateCommittingSegmentMetadata(realtimeTableName, committingSegmentDescriptor, isStartMetadata);
 
+    preProcessNewSegmentZKMetadata();
+
     // Step-2: Create new segment metadata if needed
     LOGGER.info("Creating new segment metadata with status IN_PROGRESS: {}", committingSegmentName);
     long startTimeNs2 = System.nanoTime();
@@ -550,6 +550,8 @@ public class PinotLLCRealtimeSegmentManager {
         createNewSegmentMetadata(tableConfig, idealState, committingSegmentDescriptor, committingSegmentZKMetadata,
             instancePartitions);
 
+    preProcessCommitIdealStateUpdate();
+
     // Step-3: Update IdealState
     LOGGER.info("Updating Idealstate for previous: {} and new segment: {}", committingSegmentName,
         newConsumingSegmentName);
@@ -588,6 +590,21 @@ public class PinotLLCRealtimeSegmentManager {
     }
   }
 
+  @VisibleForTesting
+  protected void preProcessNewSegmentZKMetadata() {
+    // No-op
+  }
+
+  @VisibleForTesting
+  protected void preProcessCommitIdealStateUpdate() {
+    // No-op
+  }
+
+  @VisibleForTesting
+  protected void preProcessCommitSegmentEndMetadata() {
+    // No-op
+  }
+
   // Step 1: Update committing segment metadata
   private SegmentZKMetadata updateCommittingSegmentMetadata(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor, boolean isStartMetadata) {
@@ -603,8 +620,8 @@ public class PinotLLCRealtimeSegmentManager {
 
   // Step 2: Create new segment metadata
   private String createNewSegmentMetadata(TableConfig tableConfig, IdealState idealState,
-      CommittingSegmentDescriptor committingSegmentDescriptor,
-      SegmentZKMetadata committingSegmentZKMetadata, InstancePartitions instancePartitions) {
+      CommittingSegmentDescriptor committingSegmentDescriptor, SegmentZKMetadata committingSegmentZKMetadata,
+      InstancePartitions instancePartitions) {
     String committingSegmentName = committingSegmentDescriptor.getSegmentName();
 
     String realtimeTableName = tableConfig.getTableName();
@@ -615,9 +632,10 @@ public class PinotLLCRealtimeSegmentManager {
       LLCSegmentName committingLLCSegment = new LLCSegmentName(committingSegmentName);
       int committingSegmentPartitionGroupId = committingLLCSegment.getPartitionGroupId();
 
-      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
-          streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
-      ).collect(Collectors.toList());
+      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig)
+          .stream()
+          .map(streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig))
+          .collect(Collectors.toList());
       Set<Integer> partitionIds = getPartitionIds(streamConfigs, idealState);
 
       if (partitionIds.contains(committingSegmentPartitionGroupId)) {
@@ -676,14 +694,17 @@ public class PinotLLCRealtimeSegmentManager {
   }
 
   /**
-   * Invoked after the realtime segment has been built and uploaded.
+   * Invoked after the realtime segment has been built and uploaded during pauseless ingestion.
    * Updates the metadata like CRC, download URL, etc. in the Zookeeper metadata for the committing segment.
    */
   public void commitSegmentEndMetadata(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     Preconditions.checkState(!_isStopping, "Segment manager is stopping");
+
+    preProcessCommitSegmentEndMetadata();
+
+    _numCompletingSegments.addAndGet(1);
     try {
-      _numCompletingSegments.addAndGet(1);
       // Validate segment location only for metadata commit
       if (StringUtils.isBlank(committingSegmentDescriptor.getSegmentLocation())) {
         LOGGER.warn("Committing segment: {} was not uploaded to deep store",
@@ -691,6 +712,18 @@ public class PinotLLCRealtimeSegmentManager {
         _controllerMetrics.addMeteredTableValue(realtimeTableName, ControllerMeter.SEGMENT_MISSING_DEEP_STORE_LINK, 1);
       }
       String committingSegmentName = committingSegmentDescriptor.getSegmentName();
+      // When segment completion begins, the zk metadata and ideal state are updated.
+      // This is followed by updating zk metadata for the committing segment with crc, size, download url etc.
+      // during the commit end metadata call.
+      // We allow only {@link PinotLLCRealtimeSegmentManager::MAX_SEGMENT_COMPLETION_TIME_MILLIS} ms for a segment to
+      // complete, after which the segment is eligible for repairs by the
+      // {@link org.apache.pinot.controller.validation.RealtimeSegmentValidationManager}
+      if (isExceededMaxSegmentCompletionTime(realtimeTableName, committingSegmentName, getCurrentTimeMs())) {
+        LOGGER.error("Exceeded max segment completion time. Skipping ZK Metadata update for segment: {}",
+            committingSegmentName);
+        throw new HelixHelper.PermanentUpdaterException(
+            "Exceeded max segment completion time for segment " + committingSegmentName);
+      }
       Stat stat = new Stat();
       SegmentZKMetadata committingSegmentZKMetadata =
           getSegmentZKMetadata(realtimeTableName, committingSegmentName, stat);
@@ -718,7 +751,6 @@ public class PinotLLCRealtimeSegmentManager {
         "Segment status for segment: %s should be IN_PROGRESS, found: %s", segmentName,
         committingSegmentZKMetadata.getStatus());
 
-    // TODO Issue 5953 remove the long parsing once metadata is set correctly.
     committingSegmentZKMetadata.setEndOffset(committingSegmentDescriptor.getNextOffset());
     committingSegmentZKMetadata.setStatus(Status.COMMITTING);
 
@@ -726,7 +758,6 @@ public class PinotLLCRealtimeSegmentManager {
     return committingSegmentZKMetadata;
   }
 
-
   /**
    * Updates segment ZK metadata for the committing segment.
    */
@@ -744,38 +775,12 @@ public class PinotLLCRealtimeSegmentManager {
     Preconditions.checkState(segmentMetadata != null, "Failed to find segment metadata from descriptor for segment: %s",
         segmentName);
 
-    // TODO Issue 5953 remove the long parsing once metadata is set correctly.
-    committingSegmentZKMetadata.setEndOffset(committingSegmentDescriptor.getNextOffset());
-    committingSegmentZKMetadata.setStatus(Status.DONE);
-    // If the download url set by the server is a peer download url format with peer scheme, put
-    // METADATA_URI_FOR_PEER_DOWNLOAD in zk; otherwise just use the location in the descriptor.
-    committingSegmentZKMetadata.setDownloadUrl(isPeerURL(committingSegmentDescriptor.getSegmentLocation())
-        ? CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD : committingSegmentDescriptor.getSegmentLocation());
-    committingSegmentZKMetadata.setCrc(Long.valueOf(segmentMetadata.getCrc()));
-    if (segmentMetadata.getTotalDocs() > 0) {
-      Preconditions.checkNotNull(segmentMetadata.getTimeInterval(),
-          "start/end time information is not correctly written to the segment for table: " + realtimeTableName);
-      committingSegmentZKMetadata.setStartTime(segmentMetadata.getTimeInterval().getStartMillis());
-      committingSegmentZKMetadata.setEndTime(segmentMetadata.getTimeInterval().getEndMillis());
-    } else {
-      // Set current time as start/end time if total docs is 0
-      long now = System.currentTimeMillis();
-      committingSegmentZKMetadata.setStartTime(now);
-      committingSegmentZKMetadata.setEndTime(now);
-    }
-    committingSegmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
-    SegmentVersion segmentVersion = segmentMetadata.getVersion();
-    if (segmentVersion != null) {
-      committingSegmentZKMetadata.setIndexVersion(segmentVersion.name());
-    }
-    committingSegmentZKMetadata.setTotalDocs(segmentMetadata.getTotalDocs());
-    committingSegmentZKMetadata.setSizeInBytes(committingSegmentDescriptor.getSegmentSizeBytes());
-
-    // Update the partition group metadata based on the segment metadata
-    // NOTE: When the stream partition changes, or the records are not properly partitioned from the stream, the
-    //       partition of the segment (based on the actual consumed records) can be different from the stream partition.
-    committingSegmentZKMetadata.setPartitionMetadata(getPartitionMetadataFromSegmentMetadata(segmentMetadata));
-
+    String segmentLocation = committingSegmentDescriptor.getSegmentLocation();
+    String downloadUrl =
+        isPeerURL(segmentLocation) ? CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD : segmentLocation;
+    SegmentZKMetadataUtils.updateCommittingSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata,
+        segmentMetadata, downloadUrl, committingSegmentDescriptor.getSegmentSizeBytes(),
+        committingSegmentDescriptor.getNextOffset());
     persistSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata, stat.getVersion());
     return committingSegmentZKMetadata;
   }
@@ -851,22 +856,6 @@ public class PinotLLCRealtimeSegmentManager {
     }
   }
 
-  @Nullable
-  private SegmentPartitionMetadata getPartitionMetadataFromSegmentMetadata(SegmentMetadataImpl segmentMetadata) {
-    for (Map.Entry<String, ColumnMetadata> entry : segmentMetadata.getColumnMetadataMap().entrySet()) {
-      // NOTE: There is at most one partition column.
-      ColumnMetadata columnMetadata = entry.getValue();
-      PartitionFunction partitionFunction = columnMetadata.getPartitionFunction();
-      if (partitionFunction != null) {
-        ColumnPartitionMetadata columnPartitionMetadata =
-            new ColumnPartitionMetadata(partitionFunction.getName(), partitionFunction.getNumPartitions(),
-                columnMetadata.getPartitions(), columnMetadata.getPartitionFunction().getFunctionConfig());
-        return new SegmentPartitionMetadata(Collections.singletonMap(entry.getKey(), columnPartitionMetadata));
-      }
-    }
-    return null;
-  }
-
   public long getCommitTimeoutMS(String realtimeTableName) {
     long commitTimeoutMS = SegmentCompletionProtocol.getMaxSegmentCommitTimeMs();
     if (_propertyStore == null) {
@@ -1036,7 +1025,7 @@ public class PinotLLCRealtimeSegmentManager {
    * leader of the table.
    *
    * During segment commit, we update zookeeper in 3 steps
-   * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/ COMMITTING
+   * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/COMMITTING
    * Step 2: Update PROPERTYSTORE to create the new segment metadata with status IN_PROGRESS
    * Step 3: Update IDEALSTATES to include new segment in CONSUMING state, and change old segment to ONLINE state.
    *
@@ -1260,7 +1249,8 @@ public class PinotLLCRealtimeSegmentManager {
    * elapsed since segment metadata update
    */
   @VisibleForTesting
-  boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName, long currentTimeMs) {
+  protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+      long currentTimeMs) {
     Stat stat = new Stat();
     getSegmentZKMetadata(realtimeTableName, segmentName, stat);
     if (currentTimeMs > stat.getMtime() + MAX_SEGMENT_COMPLETION_TIME_MILLIS) {
@@ -1339,7 +1329,7 @@ public class PinotLLCRealtimeSegmentManager {
 
     // Walk over all partitions that we have metadata for, and repair any partitions necessary.
     // Possible things to repair:
-    // 1. The latest metadata is in DONE state, but the idealstate says segment is CONSUMING:
+    // 1. The latest metadata is in DONE/COMMITTING state, but the idealstate says segment is CONSUMING:
     //    a. Create metadata for next segment and find hosts to assign it to.
     //    b. update current segment in idealstate to ONLINE (only if partition is present in newPartitionGroupMetadata)
     //    c. add new segment in idealstate to CONSUMING on the hosts (only if partition is present in
@@ -1356,6 +1346,14 @@ public class PinotLLCRealtimeSegmentManager {
     //       not have the same offset).
     //       In latter case, report data loss.
     long currentTimeMs = getCurrentTimeMs();
+
+    // This is the expected segment status after completion of first of the 3 steps of the segment commit protocol
+    // The status in step one is updated to
+    // 1. DONE for normal consumption
+    // 2. COMMITTING for pauseless consumption
+    Status statusPostSegmentMetadataUpdate =
+        PauselessConsumptionUtils.isPauselessEnabled(tableConfig) ? Status.COMMITTING : Status.DONE;
+
     for (Map.Entry<Integer, SegmentZKMetadata> entry : latestSegmentZKMetadataMap.entrySet()) {
       int partitionId = entry.getKey();
       SegmentZKMetadata latestSegmentZKMetadata = entry.getValue();
@@ -1366,9 +1364,9 @@ public class PinotLLCRealtimeSegmentManager {
       if (instanceStateMap != null) {
         // Latest segment of metadata is in idealstate.
         if (instanceStateMap.containsValue(SegmentStateModel.CONSUMING)) {
-          if (latestSegmentZKMetadata.getStatus() == Status.DONE) {
+          if (latestSegmentZKMetadata.getStatus() == statusPostSegmentMetadataUpdate) {
 
-            // step-1 of commmitSegmentMetadata is done (i.e. marking old segment as DONE)
+            // step-1 of commmitSegmentMetadata is done (i.e. marking old segment as DONE/COMMITTING)
             // but step-2 is not done (i.e. adding new metadata for the next segment)
             // and ideal state update (i.e. marking old segment as ONLINE and new segment as CONSUMING) is not done
             // either.
@@ -1376,8 +1374,8 @@ public class PinotLLCRealtimeSegmentManager {
               continue;
             }
             if (partitionIdToStartOffset.containsKey(partitionId)) {
-              LOGGER.info("Repairing segment: {} which is DONE in segment ZK metadata, but is CONSUMING in IdealState",
-                  latestSegmentName);
+              LOGGER.info("Repairing segment: {} which is {} in segment ZK metadata, but is CONSUMING in IdealState",
+                  latestSegmentName, statusPostSegmentMetadataUpdate);
 
               LLCSegmentName newLLCSegmentName = getNextLLCSegmentName(latestLLCSegmentName, currentTimeMs);
               String newSegmentName = newLLCSegmentName.getSegmentName();
@@ -1449,7 +1447,7 @@ public class PinotLLCRealtimeSegmentManager {
       } else {
         // idealstate does not have an entry for the segment (but metadata is present)
         // controller has failed between step-2 and step-3 of commitSegmentMetadata.
-        // i.e. after updating old segment metadata (old segment metadata state = DONE)
+        // i.e. after updating old segment metadata (old segment metadata state = DONE/COMMITTING)
         // and creating new segment metadata (new segment metadata state = IN_PROGRESS),
         // but before updating ideal state (new segment ideal missing from ideal state)
         if (!isExceededMaxSegmentCompletionTime(realtimeTableName, latestSegmentName, currentTimeMs)) {
@@ -1661,22 +1659,9 @@ public class PinotLLCRealtimeSegmentManager {
     //  2. Update the LLC segment ZK metadata by adding deep store download url.
     for (SegmentZKMetadata segmentZKMetadata : segmentsZKMetadata) {
       String segmentName = segmentZKMetadata.getSegmentName();
-      try {
-        // Only fix the committed (DONE) LLC segment without deep store copy (empty download URL)
-        if (segmentZKMetadata.getStatus() != Status.DONE
-            || !CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD.equals(segmentZKMetadata.getDownloadUrl())) {
-          continue;
-        }
-        // Skip the fix for the segment if it is already out of retention.
-        if (retentionStrategy.isPurgeable(realtimeTableName, segmentZKMetadata)) {
-          LOGGER.info("Skipped deep store uploading of LLC segment {} which is already out of retention",
-              segmentName);
-          continue;
-        }
-      } catch (Exception e) {
-        LOGGER.warn("Failed checking segment deep store URL for segment {}", segmentName);
+      if (shouldSkipSegmentForDeepStoreUpload(tableConfig, segmentZKMetadata, retentionStrategy)) {
+        continue;
       }
-
       // Skip the fix if an upload is already queued for this segment
       if (!_deepStoreUploadExecutorPendingSegments.add(segmentName)) {
         int queueSize = _deepStoreUploadExecutorPendingSegments.size();
@@ -1707,39 +1692,9 @@ public class PinotLLCRealtimeSegmentManager {
 
           // Randomly ask one server to upload
           URI uri = peerSegmentURIs.get(RANDOM.nextInt(peerSegmentURIs.size()));
-          try {
-            String serverUploadRequestUrl = StringUtil.join("/", uri.toString(), "uploadLLCSegment");
-            serverUploadRequestUrl =
-                String.format("%s?uploadTimeoutMs=%d", serverUploadRequestUrl, _deepstoreUploadRetryTimeoutMs);
-            LOGGER.info("Ask server to upload LLC segment {} to deep store by this path: {}", segmentName,
-                serverUploadRequestUrl);
-            TableLLCSegmentUploadResponse tableLLCSegmentUploadResponse
-                = _fileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl);
-            String segmentDownloadUrl =
-                moveSegmentFile(rawTableName, segmentName, tableLLCSegmentUploadResponse.getDownloadUrl(), pinotFS);
-            LOGGER.info("Updating segment {} download url in ZK to be {}", segmentName, segmentDownloadUrl);
-            // Update segment ZK metadata by adding the download URL
-            segmentZKMetadata.setDownloadUrl(segmentDownloadUrl);
-            // Update ZK crc to that of the server segment crc if unmatched
-            if (tableLLCSegmentUploadResponse.getCrc() != segmentZKMetadata.getCrc()) {
-              LOGGER.info("Updating segment {} crc in ZK to be {} from previous {}", segmentName,
-                  tableLLCSegmentUploadResponse.getCrc(), segmentZKMetadata.getCrc());
-              segmentZKMetadata.setCrc(tableLLCSegmentUploadResponse.getCrc());
-            }
-          } catch (Exception e) {
-            // this is a fallback call for backward compatibility to the original API /upload in pinot-server
-            // should be deprecated in the long run
-            String serverUploadRequestUrl = StringUtil.join("/", uri.toString(), "upload");
-            serverUploadRequestUrl =
-                String.format("%s?uploadTimeoutMs=%d", serverUploadRequestUrl, _deepstoreUploadRetryTimeoutMs);
-            LOGGER.info("Ask server to upload LLC segment {} to deep store by this path: {}", segmentName,
-                serverUploadRequestUrl);
-            String tempSegmentDownloadUrl = _fileUploadDownloadClient.uploadToSegmentStore(serverUploadRequestUrl);
-            String segmentDownloadUrl = moveSegmentFile(rawTableName, segmentName, tempSegmentDownloadUrl, pinotFS);
-            LOGGER.info("Updating segment {} download url in ZK to be {}", segmentName, segmentDownloadUrl);
-            // Update segment ZK metadata by adding the download URL
-            segmentZKMetadata.setDownloadUrl(segmentDownloadUrl);
-          }
+          // upload the segment to deep store and update the segmentZKMetadata
+          // the updated metadata is then persisted to ZK.
+          uploadToDeepStoreWithFallback(uri, segmentName, rawTableName, segmentZKMetadata, pinotFS);
           // TODO: add version check when persist segment ZK metadata
           persistSegmentZKMetadata(realtimeTableName, segmentZKMetadata, -1);
           LOGGER.info("Successfully uploaded LLC segment {} to deep store with download url: {}", segmentName,
@@ -1764,6 +1719,137 @@ public class PinotLLCRealtimeSegmentManager {
     }
   }
 
+  private void uploadToDeepStoreWithFallback(URI uri, String segmentName, String rawTableName,
+      SegmentZKMetadata segmentZKMetadata, PinotFS pinotFS) {
+    String serverUploadRequestUrl = getUploadUrl(uri, "uploadCommittedSegment");
+    LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+    try {
+      SegmentZKMetadata uploadedMetadata =
+          _fileUploadDownloadClient.uploadLLCToSegmentStoreWithZKMetadata(serverUploadRequestUrl);
+      handleMetadataUpload(rawTableName, segmentName, segmentZKMetadata, uploadedMetadata, pinotFS);
+      return;
+    } catch (Exception e) {
+      if (segmentZKMetadata.getStatus() == Status.COMMITTING) {
+        throw new RuntimeException("Failed to ask server to upload COMMITTING segment: " + segmentName, e);
+      }
+      LOGGER.warn("Failed to ask server to upload segment: {} by path: {}, falling back to other upload methods",
+          segmentName, serverUploadRequestUrl, e);
+    }
+
+    serverUploadRequestUrl = getUploadUrl(uri, "uploadLLCSegment");
+    LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+    try {
+      TableLLCSegmentUploadResponse response =
+          _fileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl);
+      handleLLCUpload(segmentName, rawTableName, segmentZKMetadata, response, pinotFS);
+      return;
+    } catch (Exception e) {
+      LOGGER.warn("Failed to ask server to upload segment: {} by path: {}, falling back to other upload methods",
+          segmentName, serverUploadRequestUrl, e);
+    }
+
+    serverUploadRequestUrl = getUploadUrl(uri, "upload");
+    LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+    try {
+      String segmentLocation = _fileUploadDownloadClient.uploadToSegmentStore(serverUploadRequestUrl);
+      handleBasicUpload(rawTableName, segmentName, segmentZKMetadata, segmentLocation, pinotFS);
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to ask server to upload segment: " + segmentName, e);
+    }
+  }
+
+  private String getUploadUrl(URI uri, String endpoint) {
+    return String.format("%s/%s?uploadTimeoutMs=%d", uri.toString(), endpoint, _deepstoreUploadRetryTimeoutMs);
+  }
+
+  private void handleMetadataUpload(String rawTableName, String segmentName, SegmentZKMetadata currentMetadata,
+      SegmentZKMetadata uploadedMetadata, PinotFS pinotFS)
+      throws Exception {
+    if (currentMetadata.getStatus() == Status.COMMITTING) {
+      LOGGER.info("Updating ZK metadata for committing segment: {}", segmentName);
+      currentMetadata.setSimpleFields(uploadedMetadata.getSimpleFields());
+    } else if (currentMetadata.getCrc() != uploadedMetadata.getCrc()) {
+      LOGGER.info("Updating CRC in ZK metadata for segment: {} from: {} to: {}", segmentName, currentMetadata.getCrc(),
+          uploadedMetadata.getCrc());
+      currentMetadata.setCrc(uploadedMetadata.getCrc());
+    }
+    moveSegmentAndSetDownloadUrl(rawTableName, segmentName, uploadedMetadata.getDownloadUrl(), pinotFS,
+        currentMetadata);
+  }
+
+  private void moveSegmentAndSetDownloadUrl(String rawTableName, String segmentName, String segmentLocation,
+      PinotFS pinotFS, SegmentZKMetadata segmentZKMetadata)
+      throws Exception {
+    String newDownloadUrl = moveSegmentFile(rawTableName, segmentName, segmentLocation, pinotFS);
+    LOGGER.info("Updating download url in ZK metadata for segment: {} to: {}", segmentName, newDownloadUrl);
+    segmentZKMetadata.setDownloadUrl(newDownloadUrl);
+  }
+
+  private void handleLLCUpload(String segmentName, String rawTableName, SegmentZKMetadata currentMetadata,
+      TableLLCSegmentUploadResponse response, PinotFS pinotFS)
+      throws Exception {
+    long currentCrc = currentMetadata.getCrc();
+    long newCrc = response.getCrc();
+    if (currentCrc != newCrc) {
+      LOGGER.info("Updating CRC in ZK metadata for segment: {} from: {} to: {}", segmentName, currentCrc, newCrc);
+      currentMetadata.setCrc(newCrc);
+    }
+    moveSegmentAndSetDownloadUrl(rawTableName, segmentName, response.getDownloadUrl(), pinotFS, currentMetadata);
+  }
+
+  private void handleBasicUpload(String rawTableName, String segmentName, SegmentZKMetadata metadata,
+      String segmentLocation, PinotFS pinotFS)
+      throws Exception {
+    moveSegmentAndSetDownloadUrl(rawTableName, segmentName, segmentLocation, pinotFS, metadata);
+  }
+
+  private boolean shouldSkipSegmentForDeepStoreUpload(TableConfig tableConfig, SegmentZKMetadata segmentZKMetadata,
+      RetentionStrategy retentionStrategy) {
+    String realtimeTableName = tableConfig.getTableName();
+    String segmentName = segmentZKMetadata.getSegmentName();
+
+    try {
+      // Skip the fix for the segment if it is already out of retention.
+      if (retentionStrategy.isPurgeable(realtimeTableName, segmentZKMetadata)) {
+        LOGGER.info("Skipped deep store uploading of LLC segment {} which is already out of retention", segmentName);
+        return true;
+      }
+
+      return PauselessConsumptionUtils.isPauselessEnabled(tableConfig) ? shouldSkipForPauselessMode(realtimeTableName,
+          segmentZKMetadata) : shouldSkipForNonPauselessMode(segmentZKMetadata);
+    } catch (Exception e) {
+      LOGGER.warn("Failed checking segment deep store URL for segment {}", segmentName, e);
+      return false;
+    }
+  }
+
+  private boolean shouldSkipForPauselessMode(String realtimeTableName, SegmentZKMetadata segmentZKMetadata) {
+    String segmentName = segmentZKMetadata.getSegmentName();
+
+    // Skip if max completion time not exceeded
+    if (!isExceededMaxSegmentCompletionTime(realtimeTableName, segmentName, getCurrentTimeMs())) {
+      LOGGER.info("Segment: {} has not exceeded max completion time: {}. Not fixing upload failures", segmentName,
+          MAX_SEGMENT_COMPLETION_TIME_MILLIS);
+      return true;
+    }
+
+    // Fix the upload URL in case the controller could not finish commit end metadata step of the segment commit
+    // protocol within the required completion time
+    if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.COMMITTING) {
+      LOGGER.info("Fixing upload URL for segment: {} which is in COMMITTING state", segmentName);
+      return false;
+    }
+
+    // handle upload failure cases in which the segment is marked DONE but the upload url is missing.
+    return shouldSkipForNonPauselessMode(segmentZKMetadata);
+  }
+
+  private boolean shouldSkipForNonPauselessMode(SegmentZKMetadata segmentZKMetadata) {
+    // Only fix the committed (DONE) LLC segment without deep store copy (empty download URL)
+    return segmentZKMetadata.getStatus() != Status.DONE
+        || !CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD.equals(segmentZKMetadata.getDownloadUrl());
+  }
+
   @VisibleForTesting
   boolean deepStoreUploadExecutorPendingSegmentsIsEmpty() {
     return _deepStoreUploadExecutorPendingSegments.isEmpty();
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
similarity index 52%
copy from pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
copy to pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
index aa3dbe5c0b..2f19a2b9d9 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
@@ -16,32 +16,23 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.common.restlet.resources;
+package org.apache.pinot.controller.helix.core.util;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
 
 
-public class TableLLCSegmentUploadResponse {
-  private final String _segmentName;
-  private final long _crc;
-  private final String _downloadUrl;
+public class FailureInjectionUtils {
+  public static final String FAULT_BEFORE_COMMIT_END_METADATA = "FaultBeforeCommitEndMetadata";
+  public static final String FAULT_BEFORE_IDEAL_STATE_UPDATE = "FaultBeforeIdealStateUpdate";
+  public static final String FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION = "FaultBeforeNewSegmentCreation";
 
-  public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
-      @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
-    _segmentName = segmentName;
-    _crc = crc;
-    _downloadUrl = downloadUrl;
+  private FailureInjectionUtils() {
   }
 
-  public String getSegmentName() {
-    return _segmentName;
-  }
-
-  public Long getCrc() {
-    return _crc;
-  }
-
-  public String getDownloadUrl() {
-    return _downloadUrl;
+  public static void injectFailure(String faultTypeKey, Map<String, String> managerConfigs) {
+    String faultTypeConfig = managerConfigs.getOrDefault(faultTypeKey, "false");
+    if (Boolean.parseBoolean(faultTypeConfig)) {
+      throw new RuntimeException("Injecting failure: " + faultTypeKey);
+    }
   }
 }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
index abc94b8aa8..5ea86bd9fb 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
@@ -110,13 +110,16 @@ public class RealtimeSegmentValidationManager extends ControllerPeriodicTask<Rea
     List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
         streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
     ).collect(Collectors.toList());
-    if (context._runSegmentLevelValidation) {
-      runSegmentLevelValidation(tableConfig);
-    }
 
     if (shouldEnsureConsuming(tableNameWithType)) {
       _llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig, streamConfigs, context._offsetCriteria);
     }
+
+    if (context._runSegmentLevelValidation) {
+      runSegmentLevelValidation(tableConfig);
+    } else {
+      LOGGER.info("Skipping segment-level validation for table: {}", tableConfig.getTableName());
+    }
   }
 
   /**
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index d5969e611f..ba1d9345da 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -210,8 +210,9 @@ public class PinotLLCRealtimeSegmentManagerTest {
 
     // Commit a segment for partition group 0
     String committingSegment = new LLCSegmentName(RAW_TABLE_NAME, 0, 0, CURRENT_TIME_MS).getSegmentName();
-    CommittingSegmentDescriptor committingSegmentDescriptor = new CommittingSegmentDescriptor(committingSegment,
-        new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString(), SEGMENT_SIZE_IN_BYTES);
+    String nextOffset = new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString();
+    CommittingSegmentDescriptor committingSegmentDescriptor =
+        new CommittingSegmentDescriptor(committingSegment, nextOffset, SEGMENT_SIZE_IN_BYTES);
     committingSegmentDescriptor.setSegmentMetadata(mockSegmentMetadata());
     segmentManager.commitSegmentMetadata(REALTIME_TABLE_NAME, committingSegmentDescriptor);
 
@@ -231,8 +232,7 @@ public class PinotLLCRealtimeSegmentManagerTest {
     SegmentZKMetadata committedSegmentZKMetadata = segmentManager._segmentZKMetadataMap.get(committingSegment);
     assertEquals(committedSegmentZKMetadata.getStatus(), Status.DONE);
     assertEquals(committedSegmentZKMetadata.getStartOffset(), PARTITION_OFFSET.toString());
-    assertEquals(committedSegmentZKMetadata.getEndOffset(),
-        new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString());
+    assertEquals(committedSegmentZKMetadata.getEndOffset(), nextOffset);
     assertEquals(committedSegmentZKMetadata.getCreationTime(), CURRENT_TIME_MS);
     assertEquals(committedSegmentZKMetadata.getCrc(), Long.parseLong(CRC));
     assertEquals(committedSegmentZKMetadata.getIndexVersion(), SEGMENT_VERSION.name());
@@ -241,8 +241,7 @@ public class PinotLLCRealtimeSegmentManagerTest {
 
     SegmentZKMetadata consumingSegmentZKMetadata = segmentManager._segmentZKMetadataMap.get(consumingSegment);
     assertEquals(consumingSegmentZKMetadata.getStatus(), Status.IN_PROGRESS);
-    assertEquals(consumingSegmentZKMetadata.getStartOffset(),
-        new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString());
+    assertEquals(consumingSegmentZKMetadata.getStartOffset(), nextOffset);
     assertEquals(committedSegmentZKMetadata.getCreationTime(), CURRENT_TIME_MS);
 
     // Turn one instance of the consuming segment OFFLINE and commit the segment
@@ -1114,6 +1113,9 @@ public class PinotLLCRealtimeSegmentManagerTest {
     // its final location. This is the expected segment location.
     String expectedSegmentLocation =
         segmentManager.createSegmentPath(RAW_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName()).toString();
+    SegmentZKMetadata segmentZKMetadataCopy =
+        new SegmentZKMetadata(new ZNRecord(segmentsZKMetadata.get(0).toZNRecord()));
+
     when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl0)).thenReturn(
         new TableLLCSegmentUploadResponse(segmentsZKMetadata.get(0).getSegmentName(), 12345678L,
             tempSegmentFileLocation.getPath()));
@@ -1178,6 +1180,135 @@ public class PinotLLCRealtimeSegmentManagerTest {
     assertNull(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(4), null).getDownloadUrl());
   }
 
+  @Test
+  public void testUploadCommittedSegment()
+      throws HttpErrorStatusException, IOException, URISyntaxException {
+    // mock the behavior for PinotHelixResourceManager
+    PinotHelixResourceManager pinotHelixResourceManager = mock(PinotHelixResourceManager.class);
+    HelixManager helixManager = mock(HelixManager.class);
+    HelixAdmin helixAdmin = mock(HelixAdmin.class);
+    ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore =
+        (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
+    when(pinotHelixResourceManager.getHelixZkManager()).thenReturn(helixManager);
+    when(helixManager.getClusterManagmentTool()).thenReturn(helixAdmin);
+    when(helixManager.getClusterName()).thenReturn(CLUSTER_NAME);
+    when(pinotHelixResourceManager.getPropertyStore()).thenReturn(zkHelixPropertyStore);
+
+    // init fake PinotLLCRealtimeSegmentManager
+    ControllerConf controllerConfig = new ControllerConf();
+    controllerConfig.setProperty(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT,
+        true);
+    controllerConfig.setDataDir(TEMP_DIR.toString());
+    FakePinotLLCRealtimeSegmentManager segmentManager =
+        new FakePinotLLCRealtimeSegmentManager(pinotHelixResourceManager, controllerConfig);
+    Assert.assertTrue(segmentManager.isDeepStoreLLCSegmentUploadRetryEnabled());
+
+    // Set up a new table with 2 replicas, 5 instances, 5 partition.
+    setUpNewTable(segmentManager, 2, 5, 5);
+    SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
+        new SegmentsValidationAndRetentionConfig();
+    segmentsValidationAndRetentionConfig.setRetentionTimeUnit(TimeUnit.DAYS.toString());
+    segmentsValidationAndRetentionConfig.setRetentionTimeValue("3");
+    segmentManager._tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
+    List<SegmentZKMetadata> segmentsZKMetadata = new ArrayList<>(segmentManager._segmentZKMetadataMap.values());
+    Assert.assertEquals(segmentsZKMetadata.size(), 5);
+
+    // Set up external view for this table
+    ExternalView externalView = new ExternalView(REALTIME_TABLE_NAME);
+    when(helixAdmin.getResourceExternalView(CLUSTER_NAME, REALTIME_TABLE_NAME)).thenReturn(externalView);
+
+    // Change 1st segment status to be DONE, but with default peer download url.
+    // Verify later the download url is fixed after upload success.
+    segmentsZKMetadata.get(0).setStatus(Status.DONE);
+    segmentsZKMetadata.get(0).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+    // set up the external view for 1st segment
+    String instance0 = "instance0";
+    int adminPort = 2077;
+    externalView.setState(segmentsZKMetadata.get(0).getSegmentName(), instance0, "ONLINE");
+    InstanceConfig instanceConfig0 = new InstanceConfig(instance0);
+    instanceConfig0.setHostName(instance0);
+    instanceConfig0.getRecord().setIntField(Instance.ADMIN_PORT_KEY, adminPort);
+    when(helixAdmin.getInstanceConfig(CLUSTER_NAME, instance0)).thenReturn(instanceConfig0);
+    // mock the request/response for 1st segment upload
+    String serverUploadRequestUrl0 =
+        String.format("http://%s:%d/segments/%s/%s/uploadCommittedSegment?uploadTimeoutMs=-1", instance0, adminPort,
+            REALTIME_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName());
+    // tempSegmentFileLocation is the location where the segment uploader will upload the segment. This usually ends
+    // with a random UUID
+    File tempSegmentFileLocation = new File(TEMP_DIR, segmentsZKMetadata.get(0).getSegmentName() + UUID.randomUUID());
+    FileUtils.write(tempSegmentFileLocation, "test");
+    // After the deep-store retry task gets the segment location returned by Pinot server, it will move the segment to
+    // its final location. This is the expected segment location.
+    String expectedSegmentLocation =
+        segmentManager.createSegmentPath(RAW_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName()).toString();
+    SegmentZKMetadata segmentZKMetadataCopy =
+        new SegmentZKMetadata(new ZNRecord(segmentsZKMetadata.get(0).toZNRecord()));
+
+    segmentZKMetadataCopy.setDownloadUrl(tempSegmentFileLocation.getPath());
+    when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStoreWithZKMetadata(
+        serverUploadRequestUrl0)).thenReturn(segmentZKMetadataCopy);
+
+    // Change 2nd segment status to be DONE, but with default peer download url.
+    // Verify later the download url isn't fixed after upload failure.
+    segmentsZKMetadata.get(1).setStatus(Status.DONE);
+    segmentsZKMetadata.get(1).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+    // set up the external view for 2nd segment
+    String instance1 = "instance1";
+    externalView.setState(segmentsZKMetadata.get(1).getSegmentName(), instance1, "ONLINE");
+    InstanceConfig instanceConfig1 = new InstanceConfig(instance1);
+    instanceConfig1.setHostName(instance1);
+    instanceConfig1.getRecord().setIntField(Instance.ADMIN_PORT_KEY, adminPort);
+    when(helixAdmin.getInstanceConfig(CLUSTER_NAME, instance1)).thenReturn(instanceConfig1);
+    // mock the request/response for 2nd segment upload
+    String serverUploadRequestUrl1 =
+        String.format("http://%s:%d/segments/%s/%s/uploadCommittedSegment?uploadTimeoutMs=-1", instance1, adminPort,
+            REALTIME_TABLE_NAME, segmentsZKMetadata.get(1).getSegmentName());
+    when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl1)).thenThrow(
+        new HttpErrorStatusException("failed to upload segment",
+            Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()));
+
+    // Change 3rd segment status to be DONE, but with default peer download url.
+    // Verify later the download url isn't fixed because no ONLINE replica found in any server.
+    segmentsZKMetadata.get(2).setStatus(Status.DONE);
+    segmentsZKMetadata.get(2).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+    // set up the external view for 3rd segment
+    String instance2 = "instance2";
+    externalView.setState(segmentsZKMetadata.get(2).getSegmentName(), instance2, "OFFLINE");
+
+    // Change 4th segment status to be DONE and with segment download url.
+    // Verify later the download url is still the same.
+    String defaultDownloadUrl = "canItBeDownloaded";
+    segmentsZKMetadata.get(3).setStatus(Status.DONE);
+    segmentsZKMetadata.get(3).setDownloadUrl(defaultDownloadUrl);
+
+    // Keep 5th segment status as IN_PROGRESS.
+
+    List<String> segmentNames =
+        segmentsZKMetadata.stream().map(SegmentZKMetadata::getSegmentName).collect(Collectors.toList());
+    when(pinotHelixResourceManager.getTableConfig(REALTIME_TABLE_NAME)).thenReturn(segmentManager._tableConfig);
+
+    // Verify the result
+    segmentManager.uploadToDeepStoreIfMissing(segmentManager._tableConfig, segmentsZKMetadata);
+
+    // Block until all tasks have been able to complete
+    TestUtils.waitForCondition(aVoid -> segmentManager.deepStoreUploadExecutorPendingSegmentsIsEmpty(), 30_000L,
+        "Timed out waiting for upload retry tasks to finish");
+
+    assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(0), null).getDownloadUrl(),
+        expectedSegmentLocation);
+    assertFalse(tempSegmentFileLocation.exists(),
+        "Deep-store retry task should move the file from temp location to permanent location");
+
+    assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(1), null).getDownloadUrl(),
+        METADATA_URI_FOR_PEER_DOWNLOAD);
+    assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(2), null).getDownloadUrl(),
+        METADATA_URI_FOR_PEER_DOWNLOAD);
+    assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(3), null).getDownloadUrl(),
+        defaultDownloadUrl);
+    assertNull(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(4), null).getDownloadUrl());
+  }
+
+
   @Test
   public void testDeleteTmpSegmentFiles()
       throws Exception {
@@ -1370,7 +1501,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
     }
 
     @Override
-    SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+    protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName,
+        @Nullable Stat stat) {
       Preconditions.checkState(_segmentZKMetadataMap.containsKey(segmentName));
       if (stat != null) {
         stat.setVersion(_segmentZKMetadataVersionMap.get(segmentName));
@@ -1430,7 +1562,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
     }
 
     @Override
-    boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName, long currentTimeMs) {
+    protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+        long currentTimeMs) {
       return _exceededMaxSegmentCompletionTime;
     }
 
@@ -1453,7 +1586,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
     }
 
     @Override
-    SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+    protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName,
+        @Nullable Stat stat) {
       SegmentZKMetadata segmentZKMetadata = super.getSegmentZKMetadata(realtimeTableName, segmentName, stat);
       switch (_scenario) {
         case ZK_VERSION_CHANGED:
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java
new file mode 100644
index 0000000000..5d7f7caa4a
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.common.utils.PauselessConsumptionUtils;
+import org.apache.pinot.controller.BaseControllerStarter;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
+import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionConfig;
+import org.apache.pinot.integration.tests.realtime.utils.FailureInjectingControllerStarter;
+import org.apache.pinot.integration.tests.realtime.utils.FailureInjectingPinotLLCRealtimeSegmentManager;
+import org.apache.pinot.integration.tests.realtime.utils.PauselessRealtimeTestUtils;
+import org.apache.pinot.server.starter.helix.HelixInstanceDataManagerConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
+import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+
+import static org.apache.pinot.spi.stream.StreamConfigProperties.SEGMENT_COMPLETION_FSM_SCHEME;
+import static org.testng.Assert.assertTrue;
+
+
+public abstract class BasePauselessRealtimeIngestionTest extends BaseClusterIntegrationTest {
+  protected static final int NUM_REALTIME_SEGMENTS = 48;
+  protected static final long DEFAULT_COUNT_STAR_RESULT = 115545L;
+  protected static final String DEFAULT_TABLE_NAME_2 = DEFAULT_TABLE_NAME + "_2";
+  private static final long MAX_SEGMENT_COMPLETION_TIME_MILLIS = 10_000;
+
+  protected List<File> _avroFiles;
+  protected boolean _failureEnabled = false;
+  private static final Logger LOGGER = LoggerFactory.getLogger(BasePauselessRealtimeIngestionTest.class);
+
+  protected abstract String getFailurePoint();
+
+  protected abstract int getExpectedSegmentsWithFailure();
+
+  protected abstract int getExpectedZKMetadataWithFailure();
+
+  protected abstract long getCountStarResultWithFailure();
+
+  @Override
+  public BaseControllerStarter createControllerStarter() {
+    return new FakePauselessControllerStarter();
+  }
+
+  @Override
+  protected void overrideControllerConf(Map<String, Object> properties) {
+    properties.put(ControllerConf.ControllerPeriodicTasksConf.PINOT_TASK_MANAGER_SCHEDULER_ENABLED, true);
+    properties.put(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT, true);
+    properties.put(SegmentCompletionConfig.FSM_SCHEME + "pauseless",
+        "org.apache.pinot.controller.helix.core.realtime.PauselessSegmentCompletionFSM");
+    properties.put(ControllerConf.ControllerPeriodicTasksConf.REALTIME_SEGMENT_VALIDATION_INITIAL_DELAY_IN_SECONDS,
+        500);
+  }
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    try {
+      LOGGER.info("Set segment.store.uri: {} for server with scheme: {}", _controllerConfig.getDataDir(),
+          new URI(_controllerConfig.getDataDir()).getScheme());
+      serverConf.setProperty("pinot.server.instance.segment.store.uri",
+          "file:" + _controllerConfig.getDataDir());
+      serverConf.setProperty("pinot.server.instance." + HelixInstanceDataManagerConfig.UPLOAD_SEGMENT_TO_DEEP_STORE,
+          "true");
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @BeforeClass
+  public void setUp()
+      throws Exception {
+    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
+    startZk();
+    startController();
+    startBroker();
+    startServer();
+
+    setupNonPauselessTable();
+    injectFailure();
+    setupPauselessTable();
+    waitForAllDocsLoaded(600_000L);
+  }
+
+  private void setupNonPauselessTable()
+      throws Exception {
+    _avroFiles = unpackAvroData(_tempDir);
+    startKafka();
+    pushAvroIntoKafka(_avroFiles);
+
+    Schema schema = createSchema();
+    schema.setSchemaName(DEFAULT_TABLE_NAME_2);
+    addSchema(schema);
+
+    TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
+    tableConfig.setTableName(DEFAULT_TABLE_NAME_2);
+    tableConfig.getValidationConfig().setRetentionTimeUnit("DAYS");
+    tableConfig.getValidationConfig().setRetentionTimeValue("100000");
+    addTableConfig(tableConfig);
+
+    waitForDocsLoaded(600_000L, true, tableConfig.getTableName());
+    TestUtils.waitForCondition((aVoid) -> {
+      List<SegmentZKMetadata> segmentZKMetadataList =
+          _helixResourceManager.getSegmentsZKMetadata(tableConfig.getTableName());
+      return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+    }, 1000, 100000, "Some segments still have missing url");
+  }
+
+  private void setupPauselessTable()
+      throws Exception {
+    Schema schema = createSchema();
+    schema.setSchemaName(DEFAULT_TABLE_NAME);
+    addSchema(schema);
+
+    TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
+    tableConfig.getValidationConfig().setRetentionTimeUnit("DAYS");
+    tableConfig.getValidationConfig().setRetentionTimeValue("100000");
+
+    IngestionConfig ingestionConfig = new IngestionConfig();
+    ingestionConfig.setStreamIngestionConfig(
+        new StreamIngestionConfig(List.of(tableConfig.getIndexingConfig().getStreamConfigs())));
+    ingestionConfig.getStreamIngestionConfig().setPauselessConsumptionEnabled(true);
+    ingestionConfig.getStreamIngestionConfig()
+        .getStreamConfigMaps()
+        .get(0)
+        .put(SEGMENT_COMPLETION_FSM_SCHEME, "pauseless");
+    tableConfig.getIndexingConfig().setStreamConfigs(null);
+    tableConfig.setIngestionConfig(ingestionConfig);
+
+    addTableConfig(tableConfig);
+  }
+
+  protected void injectFailure() {
+    PinotLLCRealtimeSegmentManager realtimeSegmentManager = _helixResourceManager.getRealtimeSegmentManager();
+    if (realtimeSegmentManager instanceof FailureInjectingPinotLLCRealtimeSegmentManager) {
+      ((FailureInjectingPinotLLCRealtimeSegmentManager) realtimeSegmentManager).enableTestFault(getFailurePoint());
+    }
+    _failureEnabled = true;
+  }
+
+  protected void disableFailure() {
+    _failureEnabled = false;
+    PinotLLCRealtimeSegmentManager realtimeSegmentManager = _helixResourceManager.getRealtimeSegmentManager();
+    if (realtimeSegmentManager instanceof FailureInjectingPinotLLCRealtimeSegmentManager) {
+      ((FailureInjectingPinotLLCRealtimeSegmentManager) realtimeSegmentManager).disableTestFault(getFailurePoint());
+    }
+  }
+
+  @AfterClass
+  public void tearDown()
+      throws IOException {
+    LOGGER.info("Tearing down...");
+    dropRealtimeTable(getTableName());
+    stopServer();
+    stopBroker();
+    stopController();
+    stopKafka();
+    stopZk();
+    FileUtils.deleteDirectory(_tempDir);
+  }
+
+  protected long getCountStarResult() {
+    return _failureEnabled ? getCountStarResultWithFailure() : DEFAULT_COUNT_STAR_RESULT;
+  }
+
+  protected void runValidationAndVerify()
+      throws Exception {
+    String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+    String tableNameWithType2 = TableNameBuilder.REALTIME.tableNameWithType(DEFAULT_TABLE_NAME_2);
+
+    PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, getExpectedSegmentsWithFailure(), _helixManager);
+
+    TestUtils.waitForCondition((aVoid) -> {
+      List<SegmentZKMetadata> segmentZKMetadataList =
+          _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+      return segmentZKMetadataList.size() == getExpectedZKMetadataWithFailure();
+    }, 1000, 100000, "New Segment ZK Metadata not created");
+
+    Thread.sleep(MAX_SEGMENT_COMPLETION_TIME_MILLIS);
+    disableFailure();
+
+    _controllerStarter.getRealtimeSegmentValidationManager().run();
+
+    waitForAllDocsLoaded(600_000L);
+    waitForDocsLoaded(600_000L, true, tableNameWithType2);
+
+    PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+    PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType2, NUM_REALTIME_SEGMENTS, _helixManager);
+
+    TestUtils.waitForCondition((aVoid) -> {
+      List<SegmentZKMetadata> segmentZKMetadataList =
+          _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+      return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+    }, 1000, 100000, "Some segments still have missing url");
+
+    PauselessRealtimeTestUtils.compareZKMetadataForSegments(
+        _helixResourceManager.getSegmentsZKMetadata(tableNameWithType),
+        _helixResourceManager.getSegmentsZKMetadata(tableNameWithType2));
+  }
+
+  /**
+   * Basic test to verify segment assignment and metadata without any failures
+   */
+  protected void testBasicSegmentAssignment() {
+    String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+
+    PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+    assertTrue(PauselessConsumptionUtils.isPauselessEnabled(getRealtimeTableConfig()));
+
+    TestUtils.waitForCondition((aVoid) -> {
+      List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+      return !hasSegmentsInStatus(segmentZKMetadataList, CommonConstants.Segment.Realtime.Status.COMMITTING);
+    }, 1000, 100000, "Some segments have status COMMITTING");
+
+    TestUtils.waitForCondition((aVoid) -> {
+      List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+      return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+    }, 1000, 100000, "Some segments still have missing url");
+  }
+
+  private boolean hasSegmentsInStatus(List<SegmentZKMetadata> segmentZKMetadataList,
+      CommonConstants.Segment.Realtime.Status prohibitedStatus) {
+    for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
+      if (segmentZKMetadata.getStatus() == prohibitedStatus) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static class FakePauselessLLCRealtimeSegmentManager extends FailureInjectingPinotLLCRealtimeSegmentManager {
+
+    public FakePauselessLLCRealtimeSegmentManager(
+        PinotHelixResourceManager helixResourceManager,
+        ControllerConf controllerConf, ControllerMetrics controllerMetrics) {
+      super(helixResourceManager, controllerConf, controllerMetrics);
+    }
+
+    @Override
+    protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+        long currentTimeMs) {
+      Stat stat = new Stat();
+      getSegmentZKMetadata(realtimeTableName, segmentName, stat);
+      if (currentTimeMs > stat.getMtime() + MAX_SEGMENT_COMPLETION_TIME_MILLIS) {
+        LOGGER.info("Segment: {} exceeds the max completion time: {}ms, metadata update time: {}, current time: {}",
+            segmentName, MAX_SEGMENT_COMPLETION_TIME_MILLIS, stat.getMtime(), currentTimeMs);
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  private static class FakePauselessControllerStarter extends FailureInjectingControllerStarter {
+
+    @Override
+    protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+      return new FakePauselessLLCRealtimeSegmentManager(_helixResourceManager, getConfig(), _controllerMetrics);
+    }
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java
new file mode 100644
index 0000000000..0c816b6563
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.util.List;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.apache.pinot.integration.tests.realtime.utils.PauselessRealtimeTestUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertNull;
+
+
+public class PauselessRealtimeIngestionCommitEndMetadataFailureTest
+    extends BasePauselessRealtimeIngestionTest {
+
+  @Override
+  protected String getFailurePoint() {
+    return FailureInjectionUtils.FAULT_BEFORE_COMMIT_END_METADATA;
+  }
+
+  @Override
+  protected int getExpectedSegmentsWithFailure() {
+    return NUM_REALTIME_SEGMENTS;  // All segments still appear in ideal state
+  }
+
+  @Override
+  protected int getExpectedZKMetadataWithFailure() {
+    return NUM_REALTIME_SEGMENTS;
+  }
+
+  @Override
+  protected long getCountStarResultWithFailure() {
+    return DEFAULT_COUNT_STAR_RESULT;
+  }
+
+  @Test
+  public void testSegmentAssignment()
+      throws Exception {
+    String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+    PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+
+    List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+    for (SegmentZKMetadata metadata : segmentZKMetadataList) {
+      assertNull(metadata.getDownloadUrl());
+    }
+
+    runValidationAndVerify();
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java
new file mode 100644
index 0000000000..a0ca117cf7
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.testng.annotations.Test;
+
+
+public class PauselessRealtimeIngestionIdealStateUpdateFailureTest
+    extends BasePauselessRealtimeIngestionTest {
+
+  private static final int NUM_REALTIME_SEGMENTS_WITH_FAILURE = 2;
+  private static final int NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE = 4;
+  private static final long DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE = 5000;
+
+  @Override
+  protected String getFailurePoint() {
+    return FailureInjectionUtils.FAULT_BEFORE_IDEAL_STATE_UPDATE;
+  }
+
+  @Override
+  protected int getExpectedSegmentsWithFailure() {
+    return NUM_REALTIME_SEGMENTS_WITH_FAILURE;
+  }
+
+  @Override
+  protected int getExpectedZKMetadataWithFailure() {
+    return NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE;
+  }
+
+  @Override
+  protected long getCountStarResultWithFailure() {
+    return DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE;
+  }
+
+  @Test
+  public void testSegmentAssignment()
+      throws Exception {
+    runValidationAndVerify();
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
index 4e9fcac0ab..bce6e67e35 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
@@ -18,159 +18,44 @@
  */
 package org.apache.pinot.integration.tests;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.io.FileUtils;
-import org.apache.helix.model.IdealState;
-import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
-import org.apache.pinot.common.utils.PauselessConsumptionUtils;
-import org.apache.pinot.common.utils.helix.HelixHelper;
-import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionConfig;
-import org.apache.pinot.server.starter.helix.HelixInstanceDataManagerConfig;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
-import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.builder.TableNameBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import static org.apache.pinot.spi.stream.StreamConfigProperties.SEGMENT_COMPLETION_FSM_SCHEME;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
 
-
-public class PauselessRealtimeIngestionIntegrationTest extends BaseClusterIntegrationTest {
-
-  private static final int NUM_REALTIME_SEGMENTS = 48;
-  private static final Logger LOGGER = LoggerFactory.getLogger(PauselessRealtimeIngestionIntegrationTest.class);
-  private List<File> _avroFiles;
-
-  protected void overrideControllerConf(Map<String, Object> properties) {
-    properties.put(ControllerConf.ControllerPeriodicTasksConf.PINOT_TASK_MANAGER_SCHEDULER_ENABLED, true);
-    properties.put(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT, true);
-    properties.put(SegmentCompletionConfig.FSM_SCHEME + "pauseless",
-        "org.apache.pinot.controller.helix.core.realtime.PauselessSegmentCompletionFSM");
-  }
+public class PauselessRealtimeIngestionIntegrationTest extends BasePauselessRealtimeIngestionTest {
 
   @Override
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    // Set segment store uri to the one used by controller as data dir (i.e. deep store)
-    try {
-      LOGGER.info("Set segment.store.uri: {} for server with scheme: {}", _controllerConfig.getDataDir(),
-          new URI(_controllerConfig.getDataDir()).getScheme());
-    } catch (URISyntaxException e) {
-      throw new RuntimeException(e);
-    }
-    serverConf.setProperty("pinot.server.instance.segment.store.uri", "file:" + _controllerConfig.getDataDir());
-    serverConf.setProperty("pinot.server.instance." + HelixInstanceDataManagerConfig.UPLOAD_SEGMENT_TO_DEEP_STORE,
-        "true");
-  }
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    // Start a customized controller with more frequent realtime segment validation
-    startController();
-    startBroker();
-    startServer();
-
-    _avroFiles = unpackAvroData(_tempDir);
-    startKafka();
-    pushAvroIntoKafka(_avroFiles);
-
-    Schema schema = createSchema();
-    addSchema(schema);
-    TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
-    // Replace stream config from indexing config to ingestion config
-    IngestionConfig ingestionConfig = new IngestionConfig();
-    ingestionConfig.setStreamIngestionConfig(
-        new StreamIngestionConfig(List.of(tableConfig.getIndexingConfig().getStreamConfigs())));
-    ingestionConfig.getStreamIngestionConfig().setPauselessConsumptionEnabled(true);
-    tableConfig.getIndexingConfig().setStreamConfigs(null);
-    tableConfig.setIngestionConfig(ingestionConfig);
-    addTableConfig(tableConfig);
-
-    waitForAllDocsLoaded(600_000L);
+  protected String getFailurePoint() {
+    return null;  // No failure point for basic test
   }
 
-  @Test(description = "Ensure that all the segments are ingested, built and uploaded when pauseless consumption is "
-      + "enabled")
-  public void testSegmentAssignment()
-      throws Exception {
-    String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
-    verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS);
-    assertTrue(PauselessConsumptionUtils.isPauselessEnabled(getRealtimeTableConfig()));
-    TestUtils.waitForCondition((aVoid) -> {
-      List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
-      return assertNoSegmentInProhibitedStatus(segmentZKMetadataList,
-          CommonConstants.Segment.Realtime.Status.COMMITTING);
-    }, 1000, 100000, "Some segments have status COMMITTING");
-    TestUtils.waitForCondition((aVoid) -> {
-      List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
-      return assertUrlPresent(segmentZKMetadataList);
-    }, 1000, 100000, "Some segments still have missing url");
+  @Override
+  protected int getExpectedSegmentsWithFailure() {
+    return NUM_REALTIME_SEGMENTS;  // Always expect full segments
   }
 
-  @AfterClass
-  public void tearDown()
-      throws IOException {
-    LOGGER.info("Tearing down...");
-    dropRealtimeTable(getTableName());
-    stopServer();
-    stopBroker();
-    stopController();
-    stopKafka();
-    stopZk();
-    FileUtils.deleteDirectory(_tempDir);
+  @Override
+  protected int getExpectedZKMetadataWithFailure() {
+    return NUM_REALTIME_SEGMENTS;  // Always expect full metadata
   }
 
-  private void verifyIdealState(String tableName, int numSegmentsExpected) {
-    IdealState idealState = HelixHelper.getTableIdealState(_helixManager, tableName);
-    Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields();
-    assertEquals(segmentAssignment.size(), numSegmentsExpected);
+  @Override
+  protected long getCountStarResultWithFailure() {
+    return DEFAULT_COUNT_STAR_RESULT;  // Always expect full count
   }
 
-  private boolean assertUrlPresent(List<SegmentZKMetadata> segmentZKMetadataList) {
-    for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
-      if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.DONE
-          && segmentZKMetadata.getDownloadUrl() == null) {
-        System.out.println("URl not found for segment: " + segmentZKMetadata.getSegmentName());
-        return false;
-      }
-    }
-    return true;
+  @Override
+  protected void injectFailure() {
+    // Do nothing - no failure to inject
   }
 
-  private boolean assertNoSegmentInProhibitedStatus(List<SegmentZKMetadata> segmentZKMetadataList,
-      CommonConstants.Segment.Realtime.Status prohibitedStatus) {
-    for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
-      if (segmentZKMetadata.getStatus() == prohibitedStatus) {
-        return false;
-      }
-    }
-    return true;
+  @Override
+  protected void disableFailure() {
+    // Do nothing - no failure to disable
   }
 
-  @Override
-  protected Map<String, String> getStreamConfigs() {
-    Map<String, String> streamConfigMap = getStreamConfigMap();
-    streamConfigMap.put(SEGMENT_COMPLETION_FSM_SCHEME, "pauseless");
-    return streamConfigMap;
+  @Test(description = "Ensure that all the segments are ingested, built and uploaded when pauseless consumption is "
+      + "enabled")
+  public void testSegmentAssignment() {
+    testBasicSegmentAssignment();
   }
 }
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java
new file mode 100644
index 0000000000..7af627ad70
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.testng.annotations.Test;
+
+
+public class PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest
+    extends BasePauselessRealtimeIngestionTest {
+
+  private static final int NUM_REALTIME_SEGMENTS_WITH_FAILURE = 2;
+  private static final int NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE = 2;
+  private static final long DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE = 5000;
+
+  @Override
+  protected String getFailurePoint() {
+    return FailureInjectionUtils.FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION;
+  }
+
+  @Override
+  protected int getExpectedSegmentsWithFailure() {
+    return NUM_REALTIME_SEGMENTS_WITH_FAILURE;
+  }
+
+  @Override
+  protected int getExpectedZKMetadataWithFailure() {
+    return NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE;
+  }
+
+  @Override
+  protected long getCountStarResultWithFailure() {
+    return DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE;
+  }
+
+  @Test
+  public void testSegmentAssignment()
+      throws Exception {
+    runValidationAndVerify();
+  }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
similarity index 54%
copy from pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
copy to pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
index aa3dbe5c0b..468b956cb4 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
@@ -16,32 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.common.restlet.resources;
+package org.apache.pinot.integration.tests.realtime.utils;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.pinot.controller.ControllerStarter;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
 
 
-public class TableLLCSegmentUploadResponse {
-  private final String _segmentName;
-  private final long _crc;
-  private final String _downloadUrl;
-
-  public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
-      @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
-    _segmentName = segmentName;
-    _crc = crc;
-    _downloadUrl = downloadUrl;
-  }
-
-  public String getSegmentName() {
-    return _segmentName;
-  }
-
-  public Long getCrc() {
-    return _crc;
-  }
-
-  public String getDownloadUrl() {
-    return _downloadUrl;
+public class FailureInjectingControllerStarter extends ControllerStarter {
+  @Override
+  protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+    return new FailureInjectingPinotLLCRealtimeSegmentManager(_helixResourceManager, getConfig(), _controllerMetrics);
   }
 }
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java
new file mode 100644
index 0000000000..66573e34b5
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.realtime.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+
+
+public class FailureInjectingPinotLLCRealtimeSegmentManager extends PinotLLCRealtimeSegmentManager {
+  @VisibleForTesting
+  private final Map<String, String> _failureConfig;
+
+  public FailureInjectingPinotLLCRealtimeSegmentManager(PinotHelixResourceManager helixResourceManager,
+      ControllerConf controllerConf, ControllerMetrics controllerMetrics) {
+    super(helixResourceManager, controllerConf, controllerMetrics);
+    _failureConfig = new HashMap<>();
+  }
+
+  @VisibleForTesting
+  public void enableTestFault(String faultType) {
+    _failureConfig.put(faultType, "true");
+  }
+
+  @VisibleForTesting
+  public void disableTestFault(String faultType) {
+    _failureConfig.remove(faultType);
+  }
+
+  @Override
+  protected void preProcessNewSegmentZKMetadata() {
+    FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION,
+        _failureConfig);
+  }
+
+  @Override
+  protected void preProcessCommitIdealStateUpdate() {
+    FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_IDEAL_STATE_UPDATE, _failureConfig);
+  }
+
+  @Override
+  protected void preProcessCommitSegmentEndMetadata() {
+    FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_COMMIT_END_METADATA, _failureConfig);
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java
new file mode 100644
index 0000000000..fc7f5264c0
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.realtime.utils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.helix.HelixManager;
+import org.apache.helix.model.IdealState;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.common.utils.helix.HelixHelper;
+import org.apache.pinot.spi.utils.CommonConstants;
+
+import static org.testng.Assert.assertEquals;
+
+
+public class PauselessRealtimeTestUtils {
+
+  private PauselessRealtimeTestUtils() {
+  }
+
+  public static void verifyIdealState(String tableName, int numSegmentsExpected, HelixManager helixManager) {
+    IdealState idealState = HelixHelper.getTableIdealState(helixManager, tableName);
+    Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields();
+    assertEquals(segmentAssignment.size(), numSegmentsExpected);
+  }
+
+  public static boolean assertUrlPresent(List<SegmentZKMetadata> segmentZKMetadataList) {
+    for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
+      if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.COMMITTING
+          && segmentZKMetadata.getDownloadUrl() == null) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  public static void compareZKMetadataForSegments(List<SegmentZKMetadata> segmentsZKMetadata,
+      List<SegmentZKMetadata> segmentsZKMetadata1) {
+    Map<String, SegmentZKMetadata> segmentZKMetadataMap =
+        getPartitionSegmentNumberToMetadataMap(segmentsZKMetadata);
+    Map<String, SegmentZKMetadata> segmentZKMetadataMap1 =
+        getPartitionSegmentNumberToMetadataMap(segmentsZKMetadata1);
+
+    segmentZKMetadataMap.forEach((segmentKey, segmentZKMetadata) -> {
+      SegmentZKMetadata segmentZKMetadata1 = segmentZKMetadataMap1.get(segmentKey);
+      compareSegmentZkMetadata(segmentZKMetadata, segmentZKMetadata1);
+    });
+  }
+
+  private static Map<String, SegmentZKMetadata> getPartitionSegmentNumberToMetadataMap(
+      List<SegmentZKMetadata> segmentsZKMetadata) {
+    Map<String, SegmentZKMetadata> segmentZKMetadataMap = new HashMap<>();
+    for (SegmentZKMetadata segmentZKMetadata : segmentsZKMetadata) {
+      LLCSegmentName llcSegmentName = new LLCSegmentName(segmentZKMetadata.getSegmentName());
+      String segmentKey = llcSegmentName.getPartitionGroupId() + "_" + llcSegmentName.getSequenceNumber();
+      segmentZKMetadataMap.put(segmentKey, segmentZKMetadata);
+    }
+    return segmentZKMetadataMap;
+  }
+
+  private static void compareSegmentZkMetadata(SegmentZKMetadata segmentZKMetadata,
+      SegmentZKMetadata segmentZKMetadata1) {
+    if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.IN_PROGRESS) {
+      return;
+    }
+    assertEquals(segmentZKMetadata.getStatus(), segmentZKMetadata1.getStatus());
+    assertEquals(segmentZKMetadata.getStartOffset(), segmentZKMetadata1.getStartOffset());
+    assertEquals(segmentZKMetadata.getEndOffset(), segmentZKMetadata1.getEndOffset());
+    assertEquals(segmentZKMetadata.getTotalDocs(), segmentZKMetadata1.getTotalDocs());
+    assertEquals(segmentZKMetadata.getStartTimeMs(), segmentZKMetadata1.getStartTimeMs());
+    assertEquals(segmentZKMetadata.getEndTimeMs(), segmentZKMetadata1.getEndTimeMs());
+  }
+}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 8568a5178c..f7038ed11f 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -29,6 +29,7 @@ import io.swagger.annotations.Authorization;
 import io.swagger.annotations.SecurityDefinition;
 import io.swagger.annotations.SwaggerDefinition;
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -61,6 +62,7 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.helix.model.IdealState;
 import org.apache.pinot.common.metadata.ZKMetadataProvider;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
 import org.apache.pinot.common.response.server.TableIndexMetadataResponse;
 import org.apache.pinot.common.restlet.resources.ResourceUtils;
 import org.apache.pinot.common.restlet.resources.SegmentConsumerInfo;
@@ -823,33 +825,8 @@ public class TablesResource {
 
     File segmentTarFile = null;
     try {
-      // Create the tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
-      File segmentTarUploadDir =
-          new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
-      segmentTarUploadDir.mkdir();
-
-      segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
-          tableNameWithType + "_" + segmentName + "_" + UUID.randomUUID() + TarCompressionUtils.TAR_GZ_FILE_EXTENSION,
-          "Invalid table / segment name: %s, %s", tableNameWithType, segmentName);
-
-      TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
-          segmentTarFile);
-
-      // Use segment uploader to upload the segment tar file to segment store and return the segment download url.
-      SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
-      URI segmentDownloadUrl;
-      if (timeoutMs <= 0) {
-        // Use default timeout if passed timeout is not positive
-        segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
-      } else {
-        segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
-      }
-      if (segmentDownloadUrl == null) {
-        throw new WebApplicationException(
-            String.format("Failed to upload table %s segment %s to segment store", realtimeTableName, segmentName),
-            Response.Status.INTERNAL_SERVER_ERROR);
-      }
-      return segmentDownloadUrl.toString();
+      segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+      return uploadSegment(segmentTarFile, tableNameWithType, segmentName, timeoutMs);
     } finally {
       FileUtils.deleteQuietly(segmentTarFile);
       tableDataManager.releaseSegment(segmentDataManager);
@@ -871,6 +848,7 @@ public class TablesResource {
    * @return full url where the segment is uploaded, crc, segmentName. Can add more segment metadata in the future.
    * @throws Exception if an error occurred during the segment upload.
    */
+  @Deprecated
   @POST
   @Path("/segments/{realtimeTableNameWithType}/{segmentName}/uploadLLCSegment")
   @Produces(MediaType.APPLICATION_JSON)
@@ -917,44 +895,139 @@ public class TablesResource {
           String.format("Table %s segment %s does not exist", realtimeTableNameWithType, segmentName),
           Response.Status.NOT_FOUND);
     }
-    String crc = segmentDataManager.getSegment().getSegmentMetadata().getCrc();
 
     File segmentTarFile = null;
     try {
-      // Create the tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
-      File segmentTarUploadDir =
-          new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
-      segmentTarUploadDir.mkdir();
+      segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+      String downloadUrl = uploadSegment(segmentTarFile, realtimeTableNameWithType, segmentName, timeoutMs);
+      return new TableLLCSegmentUploadResponse(segmentName,
+          Long.parseLong(segmentDataManager.getSegment().getSegmentMetadata().getCrc()), downloadUrl);
+    } finally {
+      FileUtils.deleteQuietly(segmentTarFile);
+      tableDataManager.releaseSegment(segmentDataManager);
+    }
+  }
 
-      segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
-          realtimeTableNameWithType + "_" + segmentName + "_" + UUID.randomUUID()
-              + TarCompressionUtils.TAR_GZ_FILE_EXTENSION, "Invalid table / segment name: %s, %s",
-          realtimeTableNameWithType, segmentName);
+  /**
+   * Upload a real-time committed segment to segment store and return the segment ZK metadata in json format.
+   * This endpoint is used when segment store copy is unavailable for real-time committed segments.
+   * Please note that invocation of this endpoint may cause query performance to suffer, since we tar up the segment to
+   * upload it.
+   *
+   * @see <a href="https://tinyurl.com/f63ru4sb></a>
+   * @param realtimeTableName table name with type.
+   * @param segmentName name of the segment to be uploaded
+   * @param timeoutMs timeout for the segment upload to the deep-store. If this is negative, the default timeout
+   *                  would be used.
+   * @return segment ZK metadata in json format.
+   * @throws Exception if an error occurred during the segment upload.
+   */
+  @POST
+  @Path("/segments/{realtimeTableName}/{segmentName}/uploadCommittedSegment")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ApiOperation(value = "Upload a real-time committed segment to segment store and return the segment ZK metadata",
+      notes = "Upload a real-time committed segment to segment store and return the segment ZK metadata")
+  @ApiResponses(value = {
+      @ApiResponse(code = 200, message = "Success"),
+      @ApiResponse(code = 400, message = "Bad request", response = ErrorInfo.class),
+      @ApiResponse(code = 404, message = "Table or segment not found", response = ErrorInfo.class),
+      @ApiResponse(code = 500, message = "Internal server error", response = ErrorInfo.class)
+  })
+  public String uploadCommittedSegment(
+      @ApiParam(value = "Name of the real-time table", required = true) @PathParam("realtimeTableName")
+      String realtimeTableName,
+      @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") String segmentName,
+      @QueryParam("uploadTimeoutMs") @DefaultValue("-1") int timeoutMs, @Context HttpHeaders headers)
+      throws Exception {
+    realtimeTableName = DatabaseUtils.translateTableName(realtimeTableName, headers);
+    LOGGER.info("Received a request to upload committed segment: {} for table: {}", segmentName, realtimeTableName);
 
-      TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
-          segmentTarFile);
+    // Check it's real-time table
+    if (!TableNameBuilder.isRealtimeTableResource(realtimeTableName)) {
+      throw new WebApplicationException(
+          "Cannot upload committed segment for a non-realtime table: " + realtimeTableName,
+          Response.Status.BAD_REQUEST);
+    }
 
-      // Use segment uploader to upload the segment tar file to segment store and return the segment download url.
-      SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
-      URI segmentDownloadUrl;
-      if (timeoutMs <= 0) {
-        // Use default timeout if passed timeout is not positive
-        segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
-      } else {
-        segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
-      }
-      if (segmentDownloadUrl == null) {
-        throw new WebApplicationException(
-            String.format("Failed to upload table %s segment %s to segment store", realtimeTableNameWithType,
-                segmentName), Response.Status.INTERNAL_SERVER_ERROR);
-      }
-      return new TableLLCSegmentUploadResponse(segmentName, Long.parseLong(crc), segmentDownloadUrl.toString());
+    // Check the segment is low level consumer segment
+    if (!LLCSegmentName.isLLCSegment(segmentName)) {
+      throw new WebApplicationException(String.format("Segment: %s is not a low level consumer segment", segmentName),
+          Response.Status.BAD_REQUEST);
+    }
+
+    TableDataManager tableDataManager =
+        ServerResourceUtils.checkGetTableDataManager(_serverInstance, realtimeTableName);
+    SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
+    if (segmentDataManager == null) {
+      throw new WebApplicationException(
+          String.format("Failed to find table: %s, segment: %s", realtimeTableName, segmentName),
+          Response.Status.NOT_FOUND);
+    }
+    if (!(segmentDataManager instanceof ImmutableSegmentDataManager)) {
+      throw new WebApplicationException(
+          String.format("Table: %s, segment: %s hasn't been sealed", realtimeTableName, segmentName),
+          Response.Status.NOT_FOUND);
+    }
+
+    File segmentTarFile = null;
+    try {
+      segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+      String downloadUrl = uploadSegment(segmentTarFile, realtimeTableName, segmentName, timeoutMs);
+
+      // Fetch existing segment ZK Metadata
+      SegmentZKMetadata segmentZKMetadata =
+          ZKMetadataProvider.getSegmentZKMetadata(_serverInstance.getHelixManager().getHelixPropertyStore(),
+              realtimeTableName, segmentName);
+      Preconditions.checkState(segmentZKMetadata != null,
+          "Failed to find segment ZK metadata for table: %s, segment: %s", realtimeTableName, segmentName);
+
+      // Update the Segment ZK Metadata with the segment metadata present on the server
+      ImmutableSegment segment = ((ImmutableSegmentDataManager) segmentDataManager).getSegment();
+      SegmentZKMetadataUtils.updateCommittingSegmentZKMetadata(realtimeTableName, segmentZKMetadata,
+          segment.getSegmentMetadata(), downloadUrl, segment.getSegmentSizeBytes(), segmentZKMetadata.getEndOffset());
+      return segmentZKMetadata.toJsonString();
     } finally {
       FileUtils.deleteQuietly(segmentTarFile);
       tableDataManager.releaseSegment(segmentDataManager);
     }
   }
 
+  /**
+   * Creates a tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
+   */
+  private File createSegmentTarFile(TableDataManager tableDataManager, String segmentName)
+      throws IOException {
+    File segmentTarUploadDir =
+        new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
+    segmentTarUploadDir.mkdir();
+    String tableNameWithType = tableDataManager.getTableName();
+    File segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
+        tableNameWithType + "_" + segmentName + "_" + UUID.randomUUID() + TarCompressionUtils.TAR_GZ_FILE_EXTENSION,
+        "Invalid table / segment name: %s, %s", tableNameWithType, segmentName);
+    TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
+        segmentTarFile);
+    return segmentTarFile;
+  }
+
+  /**
+   * Uploads a segment tar file to the segment store and returns the segment download url.
+   */
+  private String uploadSegment(File segmentTarFile, String tableNameWithType, String segmentName, int timeoutMs) {
+    SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
+    URI segmentDownloadUrl;
+    if (timeoutMs <= 0) {
+      // Use default timeout if passed timeout is not positive
+      segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
+    } else {
+      segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
+    }
+    if (segmentDownloadUrl == null) {
+      throw new WebApplicationException(
+          String.format("Failed to upload table: %s, segment: %s to segment store", tableNameWithType, segmentName),
+          Response.Status.INTERNAL_SERVER_ERROR);
+    }
+    return segmentDownloadUrl.toString();
+  }
 
   @GET
   @Path("tables/{realtimeTableName}/consumingSegmentsInfo")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org