You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2025/01/29 06:33:53 UTC
(pinot) branch master updated: Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3525116ac2 Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)
3525116ac2 is described below
commit 3525116ac277e39e28331d955e968a13df62f8bf
Author: 9aman <35...@users.noreply.github.com>
AuthorDate: Wed Jan 29 12:03:47 2025 +0530
Pauseless Ingestion #2: Handle Failure scenarios without DR (#14798)
---
.../common/metadata/segment/SegmentZKMetadata.java | 35 +++
.../metadata/segment/SegmentZKMetadataUtils.java | 141 +++++----
.../resources/TableLLCSegmentUploadResponse.java | 6 +-
.../common/utils/FileUploadDownloadClient.java | 25 ++
.../common/utils/PauselessConsumptionUtils.java | 11 +-
.../pinot/controller/BaseControllerStarter.java | 7 +-
.../pinot/controller/api/upload/ZKOperator.java | 14 +-
.../helix/core/PinotHelixResourceManager.java | 23 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 324 +++++++++++++--------
.../helix/core/util/FailureInjectionUtils.java | 33 +--
.../RealtimeSegmentValidationManager.java | 9 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 152 +++++++++-
.../tests/BasePauselessRealtimeIngestionTest.java | 296 +++++++++++++++++++
...ltimeIngestionCommitEndMetadataFailureTest.java | 67 +++++
...altimeIngestionIdealStateUpdateFailureTest.java | 57 ++++
.../PauselessRealtimeIngestionIntegrationTest.java | 159 ++--------
...stionNewSegmentMetadataCreationFailureTest.java | 57 ++++
.../utils/FailureInjectingControllerStarter.java | 31 +-
...ureInjectingPinotLLCRealtimeSegmentManager.java | 66 +++++
.../realtime/utils/PauselessRealtimeTestUtils.java | 91 ++++++
.../pinot/server/api/resources/TablesResource.java | 179 ++++++++----
21 files changed, 1342 insertions(+), 441 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
index 69cfe0a63a..9474007fc5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
@@ -18,6 +18,10 @@
*/
package org.apache.pinot.common.metadata.segment;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
@@ -33,6 +37,8 @@ import org.slf4j.LoggerFactory;
public class SegmentZKMetadata implements ZKMetadata {
private static final Logger LOGGER = LoggerFactory.getLogger(SegmentZKMetadata.class);
+ private static final String SEGMENT_NAME_KEY = "segmentName";
+ private static final String SIMPLE_FIELDS_KEY = "simpleFields";
private static final String NULL = "null";
private final ZNRecord _znRecord;
@@ -58,6 +64,16 @@ public class SegmentZKMetadata implements ZKMetadata {
return _znRecord.getId();
}
+ public Map<String, String> getSimpleFields() {
+ return _simpleFields;
+ }
+
+ public void setSimpleFields(Map<String, String> simpleFields) {
+ _simpleFields = simpleFields;
+ _startTimeMsCached = false;
+ _endTimeMsCached = false;
+ }
+
public long getStartTimeMs() {
if (!_startTimeMsCached) {
long startTimeMs = -1;
@@ -370,6 +386,25 @@ public class SegmentZKMetadata implements ZKMetadata {
return metadataMap;
}
+ public String toJsonString() {
+ ObjectNode objectNode = JsonUtils.newObjectNode();
+ objectNode.put(SEGMENT_NAME_KEY, getSegmentName());
+ objectNode.set(SIMPLE_FIELDS_KEY, JsonUtils.objectToJsonNode(_simpleFields));
+ return objectNode.toString();
+ }
+
+ public static SegmentZKMetadata fromJsonString(String jsonString)
+ throws IOException {
+ JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
+ String segmentName = jsonNode.get(SEGMENT_NAME_KEY).asText();
+ JsonNode simpleFieldsJsonNode = jsonNode.get(SIMPLE_FIELDS_KEY);
+ Map<String, String> simpleFields = JsonUtils.jsonNodeToObject(simpleFieldsJsonNode, new TypeReference<>() {
+ });
+ ZNRecord znRecord = new ZNRecord(segmentName);
+ znRecord.setSimpleFields(simpleFields);
+ return new SegmentZKMetadata(znRecord);
+ }
+
@Override
public ZNRecord toZNRecord() {
// Convert to TreeMap to keep the keys sorted. The de-serialized ZNRecord has simple fields stored as LinkedHashMap.
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
similarity index 54%
rename from pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
rename to pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
index d481b669a4..58de48a843 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadataUtils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.controller.helix.core.util;
+package org.apache.pinot.common.metadata.segment;
import com.google.common.base.Preconditions;
import java.util.HashMap;
@@ -24,20 +24,20 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
-import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.joda.time.Interval;
-public class ZKMetadataUtils {
- private ZKMetadataUtils() {
+public class SegmentZKMetadataUtils {
+ private SegmentZKMetadataUtils() {
}
/**
@@ -47,7 +47,7 @@ public class ZKMetadataUtils {
String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes) {
SegmentZKMetadata segmentZKMetadata = new SegmentZKMetadata(segmentMetadata.getName());
updateSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, crypterName,
- segmentSizeInBytes, true);
+ segmentSizeInBytes, null, true);
return segmentZKMetadata;
}
@@ -57,7 +57,16 @@ public class ZKMetadataUtils {
public static void refreshSegmentZKMetadata(String tableNameWithType, SegmentZKMetadata segmentZKMetadata,
SegmentMetadata segmentMetadata, String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes) {
updateSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, crypterName,
- segmentSizeInBytes, false);
+ segmentSizeInBytes, null, false);
+ }
+
+ /**
+ * Updates the segment ZK metadata for a committing segment.
+ */
+ public static void updateCommittingSegmentZKMetadata(String realtimeTableName, SegmentZKMetadata segmentZKMetadata,
+ SegmentMetadata segmentMetadata, String downloadUrl, long segmentSizeInBytes, String endOffset) {
+ updateSegmentZKMetadata(realtimeTableName, segmentZKMetadata, segmentMetadata, downloadUrl, null,
+ segmentSizeInBytes, endOffset, false);
}
public static void updateSegmentZKTimeInterval(SegmentZKMetadata segmentZKMetadata,
@@ -77,47 +86,96 @@ public class ZKMetadataUtils {
private static void updateSegmentZKMetadata(String tableNameWithType, SegmentZKMetadata segmentZKMetadata,
SegmentMetadata segmentMetadata, String downloadUrl, @Nullable String crypterName, long segmentSizeInBytes,
- boolean newSegment) {
- if (newSegment) {
- segmentZKMetadata.setPushTime(System.currentTimeMillis());
- } else {
- segmentZKMetadata.setRefreshTime(System.currentTimeMillis());
- }
+ @Nullable String endOffset, boolean newSegment) {
+ String segmentName = segmentZKMetadata.getSegmentName();
- if (segmentMetadata.getTimeInterval() != null) {
- segmentZKMetadata.setStartTime(segmentMetadata.getTimeInterval().getStartMillis());
- segmentZKMetadata.setEndTime(segmentMetadata.getTimeInterval().getEndMillis());
- segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
- ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor(segmentMetadata.getTimeColumn());
- if (isValidTimeMetadata(timeColumnMetadata)) {
- segmentZKMetadata.setRawStartTime(timeColumnMetadata.getMinValue().toString());
- segmentZKMetadata.setRawEndTime(timeColumnMetadata.getMaxValue().toString());
+ if (endOffset != null) {
+ // For committing segment
+
+ segmentZKMetadata.setEndOffset(endOffset);
+ segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
+
+ // For committing segment, use current time as start/end time if total docs is 0
+ if (segmentMetadata.getTotalDocs() > 0) {
+ Interval timeInterval = segmentMetadata.getTimeInterval();
+ Preconditions.checkState(timeInterval != null, "Failed to find time info for table: %s, segment: %s",
+ tableNameWithType, segmentName);
+ segmentZKMetadata.setStartTime(timeInterval.getStartMillis());
+ segmentZKMetadata.setEndTime(timeInterval.getEndMillis());
+ } else {
+ long now = System.currentTimeMillis();
+ segmentZKMetadata.setStartTime(now);
+ segmentZKMetadata.setEndTime(now);
}
+ segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
} else {
- segmentZKMetadata.setStartTime(-1);
- segmentZKMetadata.setEndTime(-1);
- segmentZKMetadata.setTimeUnit(null);
+ // For uploaded segment
+
+ // Set segment status, start/end offset info for real-time table
+ if (TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
+ segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.UPLOADED);
+
+ // For new segment, start/end offset must exist if the segment name follows LLC segment name convention
+ if (newSegment && LLCSegmentName.isLLCSegment(segmentMetadata.getName())) {
+ Preconditions.checkArgument(
+ segmentMetadata.getStartOffset() != null && segmentMetadata.getEndOffset() != null,
+ "New uploaded LLC segment must have start/end offset in the segment metadata");
+ }
+
+ // NOTE:
+ // - If start/end offset is available in the uploaded segment, update them in the segment ZK metadata
+ // - If not, keep the existing start/end offset in the segment ZK metadata unchanged
+ if (segmentMetadata.getStartOffset() != null) {
+ segmentZKMetadata.setStartOffset(segmentMetadata.getStartOffset());
+ }
+ if (segmentMetadata.getEndOffset() != null) {
+ segmentZKMetadata.setEndOffset(segmentMetadata.getEndOffset());
+ }
+ }
+
+ segmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
+ if (newSegment) {
+ segmentZKMetadata.setPushTime(System.currentTimeMillis());
+ } else {
+ segmentZKMetadata.setRefreshTime(System.currentTimeMillis());
+ }
+ Interval timeInterval = segmentMetadata.getTimeInterval();
+ if (timeInterval != null) {
+ segmentZKMetadata.setStartTime(timeInterval.getStartMillis());
+ segmentZKMetadata.setEndTime(timeInterval.getEndMillis());
+ segmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
+ ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor(segmentMetadata.getTimeColumn());
+ if (isValidTimeMetadata(timeColumnMetadata)) {
+ segmentZKMetadata.setRawStartTime(timeColumnMetadata.getMinValue().toString());
+ segmentZKMetadata.setRawEndTime(timeColumnMetadata.getMaxValue().toString());
+ }
+ } else {
+ segmentZKMetadata.setStartTime(-1);
+ segmentZKMetadata.setEndTime(-1);
+ segmentZKMetadata.setTimeUnit(null);
+ }
}
- segmentZKMetadata.setIndexVersion(
- segmentMetadata.getVersion() != null ? segmentMetadata.getVersion().name() : null);
+
+ SegmentVersion segmentVersion = segmentMetadata.getVersion();
+ segmentZKMetadata.setIndexVersion(segmentVersion != null ? segmentVersion.toString() : null);
segmentZKMetadata.setTotalDocs(segmentMetadata.getTotalDocs());
- segmentZKMetadata.setSizeInBytes(segmentSizeInBytes);
segmentZKMetadata.setCrc(Long.parseLong(segmentMetadata.getCrc()));
- segmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
segmentZKMetadata.setDownloadUrl(downloadUrl);
segmentZKMetadata.setCrypterName(crypterName);
+ segmentZKMetadata.setSizeInBytes(segmentSizeInBytes);
// Set partition metadata
Map<String, ColumnPartitionMetadata> columnPartitionMap = new HashMap<>();
- segmentMetadata.getColumnMetadataMap().forEach((column, columnMetadata) -> {
+ for (Map.Entry<String, ColumnMetadata> entry : segmentMetadata.getColumnMetadataMap().entrySet()) {
+ ColumnMetadata columnMetadata = entry.getValue();
PartitionFunction partitionFunction = columnMetadata.getPartitionFunction();
if (partitionFunction != null) {
ColumnPartitionMetadata columnPartitionMetadata =
new ColumnPartitionMetadata(partitionFunction.getName(), partitionFunction.getNumPartitions(),
columnMetadata.getPartitions(), partitionFunction.getFunctionConfig());
- columnPartitionMap.put(column, columnPartitionMetadata);
+ columnPartitionMap.put(entry.getKey(), columnPartitionMetadata);
}
- });
+ }
segmentZKMetadata.setPartitionMetadata(
!columnPartitionMap.isEmpty() ? new SegmentPartitionMetadata(columnPartitionMap) : null);
@@ -130,27 +188,6 @@ public class ZKMetadataUtils {
customMap.putAll(segmentMetadata.getCustomMap());
}
segmentZKMetadata.setCustomMap(customMap);
-
- // Set fields specific to realtime table
- if (TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
- segmentZKMetadata.setStatus(CommonConstants.Segment.Realtime.Status.UPLOADED);
-
- // For new segment, start/end offset must exist if the segment name follows LLC segment name convention
- if (newSegment && LLCSegmentName.isLLCSegment(segmentMetadata.getName())) {
- Preconditions.checkArgument(segmentMetadata.getStartOffset() != null && segmentMetadata.getEndOffset() != null,
- "New uploaded LLC segment must have start/end offset in the segment metadata");
- }
-
- // NOTE:
- // - If start/end offset is available in the uploaded segment, update them in the segment ZK metadata
- // - If not, keep the existing start/end offset in the segment ZK metadata unchanged
- if (segmentMetadata.getStartOffset() != null) {
- segmentZKMetadata.setStartOffset(segmentMetadata.getStartOffset());
- }
- if (segmentMetadata.getEndOffset() != null) {
- segmentZKMetadata.setEndOffset(segmentMetadata.getEndOffset());
- }
- }
}
private static boolean isValidTimeMetadata(ColumnMetadata timeColumnMetadata) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
index aa3dbe5c0b..2bbbb52f46 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
@@ -26,8 +26,8 @@ public class TableLLCSegmentUploadResponse {
private final long _crc;
private final String _downloadUrl;
- public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
- @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
+ public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName, @JsonProperty("crc") long crc,
+ @JsonProperty("downloadUrl") String downloadUrl) {
_segmentName = segmentName;
_crc = crc;
_downloadUrl = downloadUrl;
@@ -37,7 +37,7 @@ public class TableLLCSegmentUploadResponse {
return _segmentName;
}
- public Long getCrc() {
+ public long getCrc() {
return _crc;
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
index 24e6c701d5..6936e6dcb6 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
@@ -55,6 +55,7 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.pinot.common.auth.AuthProviderUtils;
import org.apache.pinot.common.exception.HttpErrorStatusException;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.restlet.resources.EndReplaceSegmentsRequest;
import org.apache.pinot.common.restlet.resources.StartReplaceSegmentsRequest;
import org.apache.pinot.common.restlet.resources.TableLLCSegmentUploadResponse;
@@ -991,6 +992,30 @@ public class FileUploadDownloadClient implements AutoCloseable {
return tableLLCSegmentUploadResponse;
}
+ /**
+ * Used by controllers to send requests to servers: Controller periodic task uses this endpoint to ask servers
+ * to upload committed llc segment to segment store if missing.
+ * @param uri The uri to ask servers to upload segment to segment store
+ * @return {@link SegmentZKMetadata} - segment download url, crc, other metadata
+ * @throws URISyntaxException
+ * @throws IOException
+ * @throws HttpErrorStatusException
+ */
+ public SegmentZKMetadata uploadLLCToSegmentStoreWithZKMetadata(String uri)
+ throws URISyntaxException, IOException, HttpErrorStatusException {
+ ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.post(new URI(uri)).setVersion(HttpVersion.HTTP_1_1);
+ // sendRequest checks the response status code
+ SimpleHttpResponse response = HttpClient.wrapAndThrowHttpException(
+ _httpClient.sendRequest(requestBuilder.build(), HttpClient.DEFAULT_SOCKET_TIMEOUT_MS));
+ SegmentZKMetadata segmentZKMetadata = SegmentZKMetadata.fromJsonString(response.getResponse());
+ if (StringUtils.isEmpty(segmentZKMetadata.getDownloadUrl())) {
+ throw new HttpErrorStatusException(
+ String.format("Returned segment download url is empty after requesting servers to upload by the path: %s",
+ uri), response.getStatusCode());
+ }
+ return segmentZKMetadata;
+ }
+
/**
* Send segment uri.
*
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
index 36449a5422..89b3a2cd2b 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/PauselessConsumptionUtils.java
@@ -19,7 +19,7 @@
package org.apache.pinot.common.utils;
import java.util.Optional;
-import javax.validation.constraints.NotNull;
+import javax.annotation.Nullable;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
@@ -39,8 +39,11 @@ public class PauselessConsumptionUtils {
* @return true if pauseless consumption is explicitly enabled, false otherwise
* @throws NullPointerException if tableConfig is null
*/
- public static boolean isPauselessEnabled(@NotNull TableConfig tableConfig) {
- return Optional.ofNullable(tableConfig.getIngestionConfig()).map(IngestionConfig::getStreamIngestionConfig)
- .map(StreamIngestionConfig::isPauselessConsumptionEnabled).orElse(false);
+ public static boolean isPauselessEnabled(@Nullable TableConfig tableConfig) {
+ return Optional.ofNullable(tableConfig)
+ .map(TableConfig::getIngestionConfig)
+ .map(IngestionConfig::getStreamIngestionConfig)
+ .map(StreamIngestionConfig::isPauselessConsumptionEnabled)
+ .orElse(false);
}
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index 171e850638..a0ce503d41 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -487,8 +487,7 @@ public abstract class BaseControllerStarter implements ServiceStartable {
// Helix resource manager must be started in order to create PinotLLCRealtimeSegmentManager
LOGGER.info("Starting realtime segment manager");
- _pinotLLCRealtimeSegmentManager =
- new PinotLLCRealtimeSegmentManager(_helixResourceManager, _config, _controllerMetrics);
+ _pinotLLCRealtimeSegmentManager = createPinotLLCRealtimeSegmentManager();
// TODO: Need to put this inside HelixResourceManager when HelixControllerLeadershipManager is removed.
_helixResourceManager.registerPinotLLCRealtimeSegmentManager(_pinotLLCRealtimeSegmentManager);
@@ -623,6 +622,10 @@ public abstract class BaseControllerStarter implements ServiceStartable {
_serviceStatusCallbackList.add(generateServiceStatusCallback(_helixParticipantManager));
}
+ protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+ return new PinotLLCRealtimeSegmentManager(_helixResourceManager, _config, _controllerMetrics);
+ }
+
/**
* This method is used to fix table/schema names.
* TODO: in the next release, maybe 2.0.0, we can remove this method. Meanwhile we can delete the orphan schemas
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
index eeee40b20d..4da9c72a56 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
@@ -33,6 +33,7 @@ import org.apache.helix.model.IdealState;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifier;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
@@ -40,7 +41,6 @@ import org.apache.pinot.common.utils.FileUploadDownloadClient.FileUploadType;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
-import org.apache.pinot.controller.helix.core.util.ZKMetadataUtils;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.slf4j.Logger;
@@ -309,11 +309,11 @@ public class ZKOperator {
if (customMapModifier == null) {
// If no modifier is provided, use the custom map from the segment metadata
segmentZKMetadata.setCustomMap(null);
- ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+ SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
segmentDownloadURIStr, crypterName, segmentSizeInBytes);
} else {
// If modifier is provided, first set the custom map from the segment metadata, then apply the modifier
- ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+ SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
segmentDownloadURIStr, crypterName, segmentSizeInBytes);
segmentZKMetadata.setCustomMap(customMapModifier.modifyMap(segmentZKMetadata.getCustomMap()));
}
@@ -446,11 +446,11 @@ public class ZKOperator {
if (customMapModifier == null) {
// If no modifier is provided, use the custom map from the segment metadata
segmentZKMetadata.setCustomMap(null);
- ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+ SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
segmentDownloadURIStr, crypterName, segmentSizeInBytes);
} else {
// If modifier is provided, first set the custom map from the segment metadata, then apply the modifier
- ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
+ SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata,
segmentDownloadURIStr, crypterName, segmentSizeInBytes);
segmentZKMetadata.setCustomMap(customMapModifier.modifyMap(segmentZKMetadata.getCustomMap()));
}
@@ -502,7 +502,7 @@ public class ZKOperator {
SegmentZKMetadata newSegmentZKMetadata;
try {
newSegmentZKMetadata =
- ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, segmentDownloadURIStr,
+ SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, segmentDownloadURIStr,
crypterName, segmentSizeInBytes);
} catch (IllegalArgumentException e) {
throw new ControllerApplicationException(LOGGER,
@@ -590,7 +590,7 @@ public class ZKOperator {
long segmentSizeInBytes = segmentUploadMetadata.getSegmentSizeInBytes();
File segmentFile = segmentUploadMetadata.getEncryptionInfo().getRight();
try {
- newSegmentZKMetadata = ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata,
+ newSegmentZKMetadata = SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata,
segmentDownloadURIStr, crypterName, segmentSizeInBytes);
segmentZKMetadataMap.put(segmentName, newSegmentZKMetadata);
segmentNames.add(segmentName);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 77762c2ee8..1f983ce671 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -114,6 +114,7 @@ import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.controllerjob.ControllerJobType;
import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.minion.MinionTaskMetadataUtils;
import org.apache.pinot.common.restlet.resources.EndReplaceSegmentsRequest;
@@ -154,7 +155,6 @@ import org.apache.pinot.controller.helix.core.rebalance.RebalanceResult;
import org.apache.pinot.controller.helix.core.rebalance.TableRebalanceContext;
import org.apache.pinot.controller.helix.core.rebalance.TableRebalancer;
import org.apache.pinot.controller.helix.core.rebalance.ZkBasedTableRebalanceObserver;
-import org.apache.pinot.controller.helix.core.util.ZKMetadataUtils;
import org.apache.pinot.controller.helix.starter.HelixConfig;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.config.DatabaseConfig;
@@ -405,6 +405,15 @@ public class PinotHelixResourceManager {
return _propertyStore;
}
+ /**
+ * Get the realtime segment manager
+ *
+ * @return Realtime segment manager
+ */
+ public PinotLLCRealtimeSegmentManager getRealtimeSegmentManager() {
+ return _pinotLLCRealtimeSegmentManager;
+ }
+
/**
* Get the linage manager.
*
@@ -2329,9 +2338,9 @@ public class PinotHelixResourceManager {
public void addNewSegment(String tableNameWithType, SegmentMetadata segmentMetadata, String downloadUrl) {
// NOTE: must first set the segment ZK metadata before assigning segment to instances because segment assignment
// might need them to determine the partition of the segment, and server will need them to download the segment
- SegmentZKMetadata segmentZkmetadata =
- ZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, downloadUrl, null, -1);
- ZNRecord znRecord = segmentZkmetadata.toZNRecord();
+ SegmentZKMetadata segmentZKMetadata =
+ SegmentZKMetadataUtils.createSegmentZKMetadata(tableNameWithType, segmentMetadata, downloadUrl, null, -1);
+ ZNRecord znRecord = segmentZKMetadata.toZNRecord();
String segmentName = segmentMetadata.getName();
String segmentZKMetadataPath =
@@ -2589,7 +2598,7 @@ public class PinotHelixResourceManager {
}
public void updateZkTimeInterval(SegmentZKMetadata segmentZKMetadata, DateTimeFieldSpec timeColumnFieldSpec) {
- ZKMetadataUtils.updateSegmentZKTimeInterval(segmentZKMetadata, timeColumnFieldSpec);
+ SegmentZKMetadataUtils.updateSegmentZKTimeInterval(segmentZKMetadata, timeColumnFieldSpec);
}
@VisibleForTesting
@@ -2601,8 +2610,8 @@ public class PinotHelixResourceManager {
// ZK metadata to refresh the segment (server will compare the segment ZK metadata with the local metadata to decide
// whether to download the new segment; broker will update the segment partition info & time boundary based on the
// segment ZK metadata)
- ZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl, null,
- -1);
+ SegmentZKMetadataUtils.refreshSegmentZKMetadata(tableNameWithType, segmentZKMetadata, segmentMetadata, downloadUrl,
+ null, -1);
if (!ZKMetadataProvider.setSegmentZKMetadata(_propertyStore, tableNameWithType, segmentZKMetadata,
expectedVersion)) {
throw new RuntimeException(
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index 2b9cf8f954..8f5382ae5e 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -60,6 +60,7 @@ import org.apache.pinot.common.messages.IngestionMetricsRemoveMessage;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
@@ -67,6 +68,7 @@ import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
import org.apache.pinot.common.restlet.resources.TableLLCSegmentUploadResponse;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.common.utils.PauselessConsumptionUtils;
import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.common.utils.helix.HelixHelper;
import org.apache.pinot.controller.ControllerConf;
@@ -86,10 +88,7 @@ import org.apache.pinot.controller.helix.core.retention.strategy.TimeRetentionSt
import org.apache.pinot.controller.validation.RealtimeSegmentValidationManager;
import org.apache.pinot.core.data.manager.realtime.SegmentCompletionUtils;
import org.apache.pinot.core.util.PeerServerSegmentFinder;
-import org.apache.pinot.segment.spi.ColumnMetadata;
-import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.PauseState;
@@ -114,7 +113,6 @@ import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateM
import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
import org.apache.pinot.spi.utils.IngestionConfigUtils;
import org.apache.pinot.spi.utils.JsonUtils;
-import org.apache.pinot.spi.utils.StringUtil;
import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.spi.utils.retry.RetryPolicies;
@@ -404,7 +402,7 @@ public class PinotLLCRealtimeSegmentManager {
}
@VisibleForTesting
- SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+ protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
try {
ZNRecord znRecord =
_propertyStore.get(ZKMetadataProvider.constructPropertyStorePathForSegment(realtimeTableName, segmentName),
@@ -532,7 +530,7 @@ public class PinotLLCRealtimeSegmentManager {
/*
* Update zookeeper in 3 steps.
*
- * Step 1: Update PROPERTYSTORE to change the old segment metadata status to COMMITTING/ DONE
+ * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/COMMITTING
* Step 2: Update PROPERTYSTORE to create the new segment metadata with status IN_PROGRESS
* Step 3: Update IDEALSTATES to include new segment in CONSUMING state, and change old segment to ONLINE state.
*/
@@ -543,6 +541,8 @@ public class PinotLLCRealtimeSegmentManager {
SegmentZKMetadata committingSegmentZKMetadata =
updateCommittingSegmentMetadata(realtimeTableName, committingSegmentDescriptor, isStartMetadata);
+ preProcessNewSegmentZKMetadata();
+
// Step-2: Create new segment metadata if needed
LOGGER.info("Creating new segment metadata with status IN_PROGRESS: {}", committingSegmentName);
long startTimeNs2 = System.nanoTime();
@@ -550,6 +550,8 @@ public class PinotLLCRealtimeSegmentManager {
createNewSegmentMetadata(tableConfig, idealState, committingSegmentDescriptor, committingSegmentZKMetadata,
instancePartitions);
+ preProcessCommitIdealStateUpdate();
+
// Step-3: Update IdealState
LOGGER.info("Updating Idealstate for previous: {} and new segment: {}", committingSegmentName,
newConsumingSegmentName);
@@ -588,6 +590,21 @@ public class PinotLLCRealtimeSegmentManager {
}
}
+ @VisibleForTesting
+ protected void preProcessNewSegmentZKMetadata() {
+ // No-op
+ }
+
+ @VisibleForTesting
+ protected void preProcessCommitIdealStateUpdate() {
+ // No-op
+ }
+
+ @VisibleForTesting
+ protected void preProcessCommitSegmentEndMetadata() {
+ // No-op
+ }
+
// Step 1: Update committing segment metadata
private SegmentZKMetadata updateCommittingSegmentMetadata(String realtimeTableName,
CommittingSegmentDescriptor committingSegmentDescriptor, boolean isStartMetadata) {
@@ -603,8 +620,8 @@ public class PinotLLCRealtimeSegmentManager {
// Step 2: Create new segment metadata
private String createNewSegmentMetadata(TableConfig tableConfig, IdealState idealState,
- CommittingSegmentDescriptor committingSegmentDescriptor,
- SegmentZKMetadata committingSegmentZKMetadata, InstancePartitions instancePartitions) {
+ CommittingSegmentDescriptor committingSegmentDescriptor, SegmentZKMetadata committingSegmentZKMetadata,
+ InstancePartitions instancePartitions) {
String committingSegmentName = committingSegmentDescriptor.getSegmentName();
String realtimeTableName = tableConfig.getTableName();
@@ -615,9 +632,10 @@ public class PinotLLCRealtimeSegmentManager {
LLCSegmentName committingLLCSegment = new LLCSegmentName(committingSegmentName);
int committingSegmentPartitionGroupId = committingLLCSegment.getPartitionGroupId();
- List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
- streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
- ).collect(Collectors.toList());
+ List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig)
+ .stream()
+ .map(streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig))
+ .collect(Collectors.toList());
Set<Integer> partitionIds = getPartitionIds(streamConfigs, idealState);
if (partitionIds.contains(committingSegmentPartitionGroupId)) {
@@ -676,14 +694,17 @@ public class PinotLLCRealtimeSegmentManager {
}
/**
- * Invoked after the realtime segment has been built and uploaded.
+ * Invoked after the realtime segment has been built and uploaded during pauseless ingestion.
* Updates the metadata like CRC, download URL, etc. in the Zookeeper metadata for the committing segment.
*/
public void commitSegmentEndMetadata(String realtimeTableName,
CommittingSegmentDescriptor committingSegmentDescriptor) {
Preconditions.checkState(!_isStopping, "Segment manager is stopping");
+
+ preProcessCommitSegmentEndMetadata();
+
+ _numCompletingSegments.addAndGet(1);
try {
- _numCompletingSegments.addAndGet(1);
// Validate segment location only for metadata commit
if (StringUtils.isBlank(committingSegmentDescriptor.getSegmentLocation())) {
LOGGER.warn("Committing segment: {} was not uploaded to deep store",
@@ -691,6 +712,18 @@ public class PinotLLCRealtimeSegmentManager {
_controllerMetrics.addMeteredTableValue(realtimeTableName, ControllerMeter.SEGMENT_MISSING_DEEP_STORE_LINK, 1);
}
String committingSegmentName = committingSegmentDescriptor.getSegmentName();
+ // When segment completion begins, the zk metadata and ideal state are updated.
+ // This is followed by updating zk metadata for the committing segment with crc, size, download url etc.
+ // during the commit end metadata call.
+ // We allow only {@link PinotLLCRealtimeSegmentManager::MAX_SEGMENT_COMPLETION_TIME_MILLIS} ms for a segment to
+ // complete, after which the segment is eligible for repairs by the
+ // {@link org.apache.pinot.controller.validation.RealtimeSegmentValidationManager}
+ if (isExceededMaxSegmentCompletionTime(realtimeTableName, committingSegmentName, getCurrentTimeMs())) {
+ LOGGER.error("Exceeded max segment completion time. Skipping ZK Metadata update for segment: {}",
+ committingSegmentName);
+ throw new HelixHelper.PermanentUpdaterException(
+ "Exceeded max segment completion time for segment " + committingSegmentName);
+ }
Stat stat = new Stat();
SegmentZKMetadata committingSegmentZKMetadata =
getSegmentZKMetadata(realtimeTableName, committingSegmentName, stat);
@@ -718,7 +751,6 @@ public class PinotLLCRealtimeSegmentManager {
"Segment status for segment: %s should be IN_PROGRESS, found: %s", segmentName,
committingSegmentZKMetadata.getStatus());
- // TODO Issue 5953 remove the long parsing once metadata is set correctly.
committingSegmentZKMetadata.setEndOffset(committingSegmentDescriptor.getNextOffset());
committingSegmentZKMetadata.setStatus(Status.COMMITTING);
@@ -726,7 +758,6 @@ public class PinotLLCRealtimeSegmentManager {
return committingSegmentZKMetadata;
}
-
/**
* Updates segment ZK metadata for the committing segment.
*/
@@ -744,38 +775,12 @@ public class PinotLLCRealtimeSegmentManager {
Preconditions.checkState(segmentMetadata != null, "Failed to find segment metadata from descriptor for segment: %s",
segmentName);
- // TODO Issue 5953 remove the long parsing once metadata is set correctly.
- committingSegmentZKMetadata.setEndOffset(committingSegmentDescriptor.getNextOffset());
- committingSegmentZKMetadata.setStatus(Status.DONE);
- // If the download url set by the server is a peer download url format with peer scheme, put
- // METADATA_URI_FOR_PEER_DOWNLOAD in zk; otherwise just use the location in the descriptor.
- committingSegmentZKMetadata.setDownloadUrl(isPeerURL(committingSegmentDescriptor.getSegmentLocation())
- ? CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD : committingSegmentDescriptor.getSegmentLocation());
- committingSegmentZKMetadata.setCrc(Long.valueOf(segmentMetadata.getCrc()));
- if (segmentMetadata.getTotalDocs() > 0) {
- Preconditions.checkNotNull(segmentMetadata.getTimeInterval(),
- "start/end time information is not correctly written to the segment for table: " + realtimeTableName);
- committingSegmentZKMetadata.setStartTime(segmentMetadata.getTimeInterval().getStartMillis());
- committingSegmentZKMetadata.setEndTime(segmentMetadata.getTimeInterval().getEndMillis());
- } else {
- // Set current time as start/end time if total docs is 0
- long now = System.currentTimeMillis();
- committingSegmentZKMetadata.setStartTime(now);
- committingSegmentZKMetadata.setEndTime(now);
- }
- committingSegmentZKMetadata.setTimeUnit(TimeUnit.MILLISECONDS);
- SegmentVersion segmentVersion = segmentMetadata.getVersion();
- if (segmentVersion != null) {
- committingSegmentZKMetadata.setIndexVersion(segmentVersion.name());
- }
- committingSegmentZKMetadata.setTotalDocs(segmentMetadata.getTotalDocs());
- committingSegmentZKMetadata.setSizeInBytes(committingSegmentDescriptor.getSegmentSizeBytes());
-
- // Update the partition group metadata based on the segment metadata
- // NOTE: When the stream partition changes, or the records are not properly partitioned from the stream, the
- // partition of the segment (based on the actual consumed records) can be different from the stream partition.
- committingSegmentZKMetadata.setPartitionMetadata(getPartitionMetadataFromSegmentMetadata(segmentMetadata));
-
+ String segmentLocation = committingSegmentDescriptor.getSegmentLocation();
+ String downloadUrl =
+ isPeerURL(segmentLocation) ? CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD : segmentLocation;
+ SegmentZKMetadataUtils.updateCommittingSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata,
+ segmentMetadata, downloadUrl, committingSegmentDescriptor.getSegmentSizeBytes(),
+ committingSegmentDescriptor.getNextOffset());
persistSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata, stat.getVersion());
return committingSegmentZKMetadata;
}
@@ -851,22 +856,6 @@ public class PinotLLCRealtimeSegmentManager {
}
}
- @Nullable
- private SegmentPartitionMetadata getPartitionMetadataFromSegmentMetadata(SegmentMetadataImpl segmentMetadata) {
- for (Map.Entry<String, ColumnMetadata> entry : segmentMetadata.getColumnMetadataMap().entrySet()) {
- // NOTE: There is at most one partition column.
- ColumnMetadata columnMetadata = entry.getValue();
- PartitionFunction partitionFunction = columnMetadata.getPartitionFunction();
- if (partitionFunction != null) {
- ColumnPartitionMetadata columnPartitionMetadata =
- new ColumnPartitionMetadata(partitionFunction.getName(), partitionFunction.getNumPartitions(),
- columnMetadata.getPartitions(), columnMetadata.getPartitionFunction().getFunctionConfig());
- return new SegmentPartitionMetadata(Collections.singletonMap(entry.getKey(), columnPartitionMetadata));
- }
- }
- return null;
- }
-
public long getCommitTimeoutMS(String realtimeTableName) {
long commitTimeoutMS = SegmentCompletionProtocol.getMaxSegmentCommitTimeMs();
if (_propertyStore == null) {
@@ -1036,7 +1025,7 @@ public class PinotLLCRealtimeSegmentManager {
* leader of the table.
*
* During segment commit, we update zookeeper in 3 steps
- * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/ COMMITTING
+ * Step 1: Update PROPERTYSTORE to change the old segment metadata status to DONE/COMMITTING
* Step 2: Update PROPERTYSTORE to create the new segment metadata with status IN_PROGRESS
* Step 3: Update IDEALSTATES to include new segment in CONSUMING state, and change old segment to ONLINE state.
*
@@ -1260,7 +1249,8 @@ public class PinotLLCRealtimeSegmentManager {
* elapsed since segment metadata update
*/
@VisibleForTesting
- boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName, long currentTimeMs) {
+ protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+ long currentTimeMs) {
Stat stat = new Stat();
getSegmentZKMetadata(realtimeTableName, segmentName, stat);
if (currentTimeMs > stat.getMtime() + MAX_SEGMENT_COMPLETION_TIME_MILLIS) {
@@ -1339,7 +1329,7 @@ public class PinotLLCRealtimeSegmentManager {
// Walk over all partitions that we have metadata for, and repair any partitions necessary.
// Possible things to repair:
- // 1. The latest metadata is in DONE state, but the idealstate says segment is CONSUMING:
+ // 1. The latest metadata is in DONE/COMMITTING state, but the idealstate says segment is CONSUMING:
// a. Create metadata for next segment and find hosts to assign it to.
// b. update current segment in idealstate to ONLINE (only if partition is present in newPartitionGroupMetadata)
// c. add new segment in idealstate to CONSUMING on the hosts (only if partition is present in
@@ -1356,6 +1346,14 @@ public class PinotLLCRealtimeSegmentManager {
// not have the same offset).
// In latter case, report data loss.
long currentTimeMs = getCurrentTimeMs();
+
+ // This is the expected segment status after completion of first of the 3 steps of the segment commit protocol
+ // The status in step one is updated to
+ // 1. DONE for normal consumption
+ // 2. COMMITTING for pauseless consumption
+ Status statusPostSegmentMetadataUpdate =
+ PauselessConsumptionUtils.isPauselessEnabled(tableConfig) ? Status.COMMITTING : Status.DONE;
+
for (Map.Entry<Integer, SegmentZKMetadata> entry : latestSegmentZKMetadataMap.entrySet()) {
int partitionId = entry.getKey();
SegmentZKMetadata latestSegmentZKMetadata = entry.getValue();
@@ -1366,9 +1364,9 @@ public class PinotLLCRealtimeSegmentManager {
if (instanceStateMap != null) {
// Latest segment of metadata is in idealstate.
if (instanceStateMap.containsValue(SegmentStateModel.CONSUMING)) {
- if (latestSegmentZKMetadata.getStatus() == Status.DONE) {
+ if (latestSegmentZKMetadata.getStatus() == statusPostSegmentMetadataUpdate) {
- // step-1 of commmitSegmentMetadata is done (i.e. marking old segment as DONE)
+ // step-1 of commmitSegmentMetadata is done (i.e. marking old segment as DONE/COMMITTING)
// but step-2 is not done (i.e. adding new metadata for the next segment)
// and ideal state update (i.e. marking old segment as ONLINE and new segment as CONSUMING) is not done
// either.
@@ -1376,8 +1374,8 @@ public class PinotLLCRealtimeSegmentManager {
continue;
}
if (partitionIdToStartOffset.containsKey(partitionId)) {
- LOGGER.info("Repairing segment: {} which is DONE in segment ZK metadata, but is CONSUMING in IdealState",
- latestSegmentName);
+ LOGGER.info("Repairing segment: {} which is {} in segment ZK metadata, but is CONSUMING in IdealState",
+ latestSegmentName, statusPostSegmentMetadataUpdate);
LLCSegmentName newLLCSegmentName = getNextLLCSegmentName(latestLLCSegmentName, currentTimeMs);
String newSegmentName = newLLCSegmentName.getSegmentName();
@@ -1449,7 +1447,7 @@ public class PinotLLCRealtimeSegmentManager {
} else {
// idealstate does not have an entry for the segment (but metadata is present)
// controller has failed between step-2 and step-3 of commitSegmentMetadata.
- // i.e. after updating old segment metadata (old segment metadata state = DONE)
+ // i.e. after updating old segment metadata (old segment metadata state = DONE/COMMITTING)
// and creating new segment metadata (new segment metadata state = IN_PROGRESS),
// but before updating ideal state (new segment ideal missing from ideal state)
if (!isExceededMaxSegmentCompletionTime(realtimeTableName, latestSegmentName, currentTimeMs)) {
@@ -1661,22 +1659,9 @@ public class PinotLLCRealtimeSegmentManager {
// 2. Update the LLC segment ZK metadata by adding deep store download url.
for (SegmentZKMetadata segmentZKMetadata : segmentsZKMetadata) {
String segmentName = segmentZKMetadata.getSegmentName();
- try {
- // Only fix the committed (DONE) LLC segment without deep store copy (empty download URL)
- if (segmentZKMetadata.getStatus() != Status.DONE
- || !CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD.equals(segmentZKMetadata.getDownloadUrl())) {
- continue;
- }
- // Skip the fix for the segment if it is already out of retention.
- if (retentionStrategy.isPurgeable(realtimeTableName, segmentZKMetadata)) {
- LOGGER.info("Skipped deep store uploading of LLC segment {} which is already out of retention",
- segmentName);
- continue;
- }
- } catch (Exception e) {
- LOGGER.warn("Failed checking segment deep store URL for segment {}", segmentName);
+ if (shouldSkipSegmentForDeepStoreUpload(tableConfig, segmentZKMetadata, retentionStrategy)) {
+ continue;
}
-
// Skip the fix if an upload is already queued for this segment
if (!_deepStoreUploadExecutorPendingSegments.add(segmentName)) {
int queueSize = _deepStoreUploadExecutorPendingSegments.size();
@@ -1707,39 +1692,9 @@ public class PinotLLCRealtimeSegmentManager {
// Randomly ask one server to upload
URI uri = peerSegmentURIs.get(RANDOM.nextInt(peerSegmentURIs.size()));
- try {
- String serverUploadRequestUrl = StringUtil.join("/", uri.toString(), "uploadLLCSegment");
- serverUploadRequestUrl =
- String.format("%s?uploadTimeoutMs=%d", serverUploadRequestUrl, _deepstoreUploadRetryTimeoutMs);
- LOGGER.info("Ask server to upload LLC segment {} to deep store by this path: {}", segmentName,
- serverUploadRequestUrl);
- TableLLCSegmentUploadResponse tableLLCSegmentUploadResponse
- = _fileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl);
- String segmentDownloadUrl =
- moveSegmentFile(rawTableName, segmentName, tableLLCSegmentUploadResponse.getDownloadUrl(), pinotFS);
- LOGGER.info("Updating segment {} download url in ZK to be {}", segmentName, segmentDownloadUrl);
- // Update segment ZK metadata by adding the download URL
- segmentZKMetadata.setDownloadUrl(segmentDownloadUrl);
- // Update ZK crc to that of the server segment crc if unmatched
- if (tableLLCSegmentUploadResponse.getCrc() != segmentZKMetadata.getCrc()) {
- LOGGER.info("Updating segment {} crc in ZK to be {} from previous {}", segmentName,
- tableLLCSegmentUploadResponse.getCrc(), segmentZKMetadata.getCrc());
- segmentZKMetadata.setCrc(tableLLCSegmentUploadResponse.getCrc());
- }
- } catch (Exception e) {
- // this is a fallback call for backward compatibility to the original API /upload in pinot-server
- // should be deprecated in the long run
- String serverUploadRequestUrl = StringUtil.join("/", uri.toString(), "upload");
- serverUploadRequestUrl =
- String.format("%s?uploadTimeoutMs=%d", serverUploadRequestUrl, _deepstoreUploadRetryTimeoutMs);
- LOGGER.info("Ask server to upload LLC segment {} to deep store by this path: {}", segmentName,
- serverUploadRequestUrl);
- String tempSegmentDownloadUrl = _fileUploadDownloadClient.uploadToSegmentStore(serverUploadRequestUrl);
- String segmentDownloadUrl = moveSegmentFile(rawTableName, segmentName, tempSegmentDownloadUrl, pinotFS);
- LOGGER.info("Updating segment {} download url in ZK to be {}", segmentName, segmentDownloadUrl);
- // Update segment ZK metadata by adding the download URL
- segmentZKMetadata.setDownloadUrl(segmentDownloadUrl);
- }
+ // upload the segment to deep store and update the segmentZKMetadata
+ // the updated metadata is then persisted to ZK.
+ uploadToDeepStoreWithFallback(uri, segmentName, rawTableName, segmentZKMetadata, pinotFS);
// TODO: add version check when persist segment ZK metadata
persistSegmentZKMetadata(realtimeTableName, segmentZKMetadata, -1);
LOGGER.info("Successfully uploaded LLC segment {} to deep store with download url: {}", segmentName,
@@ -1764,6 +1719,137 @@ public class PinotLLCRealtimeSegmentManager {
}
}
+ private void uploadToDeepStoreWithFallback(URI uri, String segmentName, String rawTableName,
+ SegmentZKMetadata segmentZKMetadata, PinotFS pinotFS) {
+ String serverUploadRequestUrl = getUploadUrl(uri, "uploadCommittedSegment");
+ LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+ try {
+ SegmentZKMetadata uploadedMetadata =
+ _fileUploadDownloadClient.uploadLLCToSegmentStoreWithZKMetadata(serverUploadRequestUrl);
+ handleMetadataUpload(rawTableName, segmentName, segmentZKMetadata, uploadedMetadata, pinotFS);
+ return;
+ } catch (Exception e) {
+ if (segmentZKMetadata.getStatus() == Status.COMMITTING) {
+ throw new RuntimeException("Failed to ask server to upload COMMITTING segment: " + segmentName, e);
+ }
+ LOGGER.warn("Failed to ask server to upload segment: {} by path: {}, falling back to other upload methods",
+ segmentName, serverUploadRequestUrl, e);
+ }
+
+ serverUploadRequestUrl = getUploadUrl(uri, "uploadLLCSegment");
+ LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+ try {
+ TableLLCSegmentUploadResponse response =
+ _fileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl);
+ handleLLCUpload(segmentName, rawTableName, segmentZKMetadata, response, pinotFS);
+ return;
+ } catch (Exception e) {
+ LOGGER.warn("Failed to ask server to upload segment: {} by path: {}, falling back to other upload methods",
+ segmentName, serverUploadRequestUrl, e);
+ }
+
+ serverUploadRequestUrl = getUploadUrl(uri, "upload");
+ LOGGER.info("Asking server to upload segment: {} by path: {}", segmentName, serverUploadRequestUrl);
+ try {
+ String segmentLocation = _fileUploadDownloadClient.uploadToSegmentStore(serverUploadRequestUrl);
+ handleBasicUpload(rawTableName, segmentName, segmentZKMetadata, segmentLocation, pinotFS);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to ask server to upload segment: " + segmentName, e);
+ }
+ }
+
+ private String getUploadUrl(URI uri, String endpoint) {
+ return String.format("%s/%s?uploadTimeoutMs=%d", uri.toString(), endpoint, _deepstoreUploadRetryTimeoutMs);
+ }
+
+ private void handleMetadataUpload(String rawTableName, String segmentName, SegmentZKMetadata currentMetadata,
+ SegmentZKMetadata uploadedMetadata, PinotFS pinotFS)
+ throws Exception {
+ if (currentMetadata.getStatus() == Status.COMMITTING) {
+ LOGGER.info("Updating ZK metadata for committing segment: {}", segmentName);
+ currentMetadata.setSimpleFields(uploadedMetadata.getSimpleFields());
+ } else if (currentMetadata.getCrc() != uploadedMetadata.getCrc()) {
+ LOGGER.info("Updating CRC in ZK metadata for segment: {} from: {} to: {}", segmentName, currentMetadata.getCrc(),
+ uploadedMetadata.getCrc());
+ currentMetadata.setCrc(uploadedMetadata.getCrc());
+ }
+ moveSegmentAndSetDownloadUrl(rawTableName, segmentName, uploadedMetadata.getDownloadUrl(), pinotFS,
+ currentMetadata);
+ }
+
+ private void moveSegmentAndSetDownloadUrl(String rawTableName, String segmentName, String segmentLocation,
+ PinotFS pinotFS, SegmentZKMetadata segmentZKMetadata)
+ throws Exception {
+ String newDownloadUrl = moveSegmentFile(rawTableName, segmentName, segmentLocation, pinotFS);
+ LOGGER.info("Updating download url in ZK metadata for segment: {} to: {}", segmentName, newDownloadUrl);
+ segmentZKMetadata.setDownloadUrl(newDownloadUrl);
+ }
+
+ private void handleLLCUpload(String segmentName, String rawTableName, SegmentZKMetadata currentMetadata,
+ TableLLCSegmentUploadResponse response, PinotFS pinotFS)
+ throws Exception {
+ long currentCrc = currentMetadata.getCrc();
+ long newCrc = response.getCrc();
+ if (currentCrc != newCrc) {
+ LOGGER.info("Updating CRC in ZK metadata for segment: {} from: {} to: {}", segmentName, currentCrc, newCrc);
+ currentMetadata.setCrc(newCrc);
+ }
+ moveSegmentAndSetDownloadUrl(rawTableName, segmentName, response.getDownloadUrl(), pinotFS, currentMetadata);
+ }
+
+ private void handleBasicUpload(String rawTableName, String segmentName, SegmentZKMetadata metadata,
+ String segmentLocation, PinotFS pinotFS)
+ throws Exception {
+ moveSegmentAndSetDownloadUrl(rawTableName, segmentName, segmentLocation, pinotFS, metadata);
+ }
+
+ private boolean shouldSkipSegmentForDeepStoreUpload(TableConfig tableConfig, SegmentZKMetadata segmentZKMetadata,
+ RetentionStrategy retentionStrategy) {
+ String realtimeTableName = tableConfig.getTableName();
+ String segmentName = segmentZKMetadata.getSegmentName();
+
+ try {
+ // Skip the fix for the segment if it is already out of retention.
+ if (retentionStrategy.isPurgeable(realtimeTableName, segmentZKMetadata)) {
+ LOGGER.info("Skipped deep store uploading of LLC segment {} which is already out of retention", segmentName);
+ return true;
+ }
+
+ return PauselessConsumptionUtils.isPauselessEnabled(tableConfig) ? shouldSkipForPauselessMode(realtimeTableName,
+ segmentZKMetadata) : shouldSkipForNonPauselessMode(segmentZKMetadata);
+ } catch (Exception e) {
+ LOGGER.warn("Failed checking segment deep store URL for segment {}", segmentName, e);
+ return false;
+ }
+ }
+
+ private boolean shouldSkipForPauselessMode(String realtimeTableName, SegmentZKMetadata segmentZKMetadata) {
+ String segmentName = segmentZKMetadata.getSegmentName();
+
+ // Skip if max completion time not exceeded
+ if (!isExceededMaxSegmentCompletionTime(realtimeTableName, segmentName, getCurrentTimeMs())) {
+ LOGGER.info("Segment: {} has not exceeded max completion time: {}. Not fixing upload failures", segmentName,
+ MAX_SEGMENT_COMPLETION_TIME_MILLIS);
+ return true;
+ }
+
+ // Fix the upload URL in case the controller could not finish commit end metadata step of the segment commit
+ // protocol within the required completion time
+ if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.COMMITTING) {
+ LOGGER.info("Fixing upload URL for segment: {} which is in COMMITTING state", segmentName);
+ return false;
+ }
+
+ // handle upload failure cases in which the segment is marked DONE but the upload url is missing.
+ return shouldSkipForNonPauselessMode(segmentZKMetadata);
+ }
+
+ private boolean shouldSkipForNonPauselessMode(SegmentZKMetadata segmentZKMetadata) {
+ // Only fix the committed (DONE) LLC segment without deep store copy (empty download URL)
+ return segmentZKMetadata.getStatus() != Status.DONE
+ || !CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD.equals(segmentZKMetadata.getDownloadUrl());
+ }
+
@VisibleForTesting
boolean deepStoreUploadExecutorPendingSegmentsIsEmpty() {
return _deepStoreUploadExecutorPendingSegments.isEmpty();
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
similarity index 52%
copy from pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
copy to pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
index aa3dbe5c0b..2f19a2b9d9 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/FailureInjectionUtils.java
@@ -16,32 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.restlet.resources;
+package org.apache.pinot.controller.helix.core.util;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
-public class TableLLCSegmentUploadResponse {
- private final String _segmentName;
- private final long _crc;
- private final String _downloadUrl;
+public class FailureInjectionUtils {
+ public static final String FAULT_BEFORE_COMMIT_END_METADATA = "FaultBeforeCommitEndMetadata";
+ public static final String FAULT_BEFORE_IDEAL_STATE_UPDATE = "FaultBeforeIdealStateUpdate";
+ public static final String FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION = "FaultBeforeNewSegmentCreation";
- public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
- @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
- _segmentName = segmentName;
- _crc = crc;
- _downloadUrl = downloadUrl;
+ private FailureInjectionUtils() {
}
- public String getSegmentName() {
- return _segmentName;
- }
-
- public Long getCrc() {
- return _crc;
- }
-
- public String getDownloadUrl() {
- return _downloadUrl;
+ public static void injectFailure(String faultTypeKey, Map<String, String> managerConfigs) {
+ String faultTypeConfig = managerConfigs.getOrDefault(faultTypeKey, "false");
+ if (Boolean.parseBoolean(faultTypeConfig)) {
+ throw new RuntimeException("Injecting failure: " + faultTypeKey);
+ }
}
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
index abc94b8aa8..5ea86bd9fb 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
@@ -110,13 +110,16 @@ public class RealtimeSegmentValidationManager extends ControllerPeriodicTask<Rea
List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
).collect(Collectors.toList());
- if (context._runSegmentLevelValidation) {
- runSegmentLevelValidation(tableConfig);
- }
if (shouldEnsureConsuming(tableNameWithType)) {
_llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig, streamConfigs, context._offsetCriteria);
}
+
+ if (context._runSegmentLevelValidation) {
+ runSegmentLevelValidation(tableConfig);
+ } else {
+ LOGGER.info("Skipping segment-level validation for table: {}", tableConfig.getTableName());
+ }
}
/**
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index d5969e611f..ba1d9345da 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -210,8 +210,9 @@ public class PinotLLCRealtimeSegmentManagerTest {
// Commit a segment for partition group 0
String committingSegment = new LLCSegmentName(RAW_TABLE_NAME, 0, 0, CURRENT_TIME_MS).getSegmentName();
- CommittingSegmentDescriptor committingSegmentDescriptor = new CommittingSegmentDescriptor(committingSegment,
- new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString(), SEGMENT_SIZE_IN_BYTES);
+ String nextOffset = new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString();
+ CommittingSegmentDescriptor committingSegmentDescriptor =
+ new CommittingSegmentDescriptor(committingSegment, nextOffset, SEGMENT_SIZE_IN_BYTES);
committingSegmentDescriptor.setSegmentMetadata(mockSegmentMetadata());
segmentManager.commitSegmentMetadata(REALTIME_TABLE_NAME, committingSegmentDescriptor);
@@ -231,8 +232,7 @@ public class PinotLLCRealtimeSegmentManagerTest {
SegmentZKMetadata committedSegmentZKMetadata = segmentManager._segmentZKMetadataMap.get(committingSegment);
assertEquals(committedSegmentZKMetadata.getStatus(), Status.DONE);
assertEquals(committedSegmentZKMetadata.getStartOffset(), PARTITION_OFFSET.toString());
- assertEquals(committedSegmentZKMetadata.getEndOffset(),
- new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString());
+ assertEquals(committedSegmentZKMetadata.getEndOffset(), nextOffset);
assertEquals(committedSegmentZKMetadata.getCreationTime(), CURRENT_TIME_MS);
assertEquals(committedSegmentZKMetadata.getCrc(), Long.parseLong(CRC));
assertEquals(committedSegmentZKMetadata.getIndexVersion(), SEGMENT_VERSION.name());
@@ -241,8 +241,7 @@ public class PinotLLCRealtimeSegmentManagerTest {
SegmentZKMetadata consumingSegmentZKMetadata = segmentManager._segmentZKMetadataMap.get(consumingSegment);
assertEquals(consumingSegmentZKMetadata.getStatus(), Status.IN_PROGRESS);
- assertEquals(consumingSegmentZKMetadata.getStartOffset(),
- new LongMsgOffset(PARTITION_OFFSET.getOffset() + NUM_DOCS).toString());
+ assertEquals(consumingSegmentZKMetadata.getStartOffset(), nextOffset);
assertEquals(committedSegmentZKMetadata.getCreationTime(), CURRENT_TIME_MS);
// Turn one instance of the consuming segment OFFLINE and commit the segment
@@ -1114,6 +1113,9 @@ public class PinotLLCRealtimeSegmentManagerTest {
// its final location. This is the expected segment location.
String expectedSegmentLocation =
segmentManager.createSegmentPath(RAW_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName()).toString();
+ SegmentZKMetadata segmentZKMetadataCopy =
+ new SegmentZKMetadata(new ZNRecord(segmentsZKMetadata.get(0).toZNRecord()));
+
when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl0)).thenReturn(
new TableLLCSegmentUploadResponse(segmentsZKMetadata.get(0).getSegmentName(), 12345678L,
tempSegmentFileLocation.getPath()));
@@ -1178,6 +1180,135 @@ public class PinotLLCRealtimeSegmentManagerTest {
assertNull(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(4), null).getDownloadUrl());
}
+ @Test
+ public void testUploadCommittedSegment()
+ throws HttpErrorStatusException, IOException, URISyntaxException {
+ // mock the behavior for PinotHelixResourceManager
+ PinotHelixResourceManager pinotHelixResourceManager = mock(PinotHelixResourceManager.class);
+ HelixManager helixManager = mock(HelixManager.class);
+ HelixAdmin helixAdmin = mock(HelixAdmin.class);
+ ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore =
+ (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
+ when(pinotHelixResourceManager.getHelixZkManager()).thenReturn(helixManager);
+ when(helixManager.getClusterManagmentTool()).thenReturn(helixAdmin);
+ when(helixManager.getClusterName()).thenReturn(CLUSTER_NAME);
+ when(pinotHelixResourceManager.getPropertyStore()).thenReturn(zkHelixPropertyStore);
+
+ // init fake PinotLLCRealtimeSegmentManager
+ ControllerConf controllerConfig = new ControllerConf();
+ controllerConfig.setProperty(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT,
+ true);
+ controllerConfig.setDataDir(TEMP_DIR.toString());
+ FakePinotLLCRealtimeSegmentManager segmentManager =
+ new FakePinotLLCRealtimeSegmentManager(pinotHelixResourceManager, controllerConfig);
+ Assert.assertTrue(segmentManager.isDeepStoreLLCSegmentUploadRetryEnabled());
+
+ // Set up a new table with 2 replicas, 5 instances, 5 partition.
+ setUpNewTable(segmentManager, 2, 5, 5);
+ SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
+ new SegmentsValidationAndRetentionConfig();
+ segmentsValidationAndRetentionConfig.setRetentionTimeUnit(TimeUnit.DAYS.toString());
+ segmentsValidationAndRetentionConfig.setRetentionTimeValue("3");
+ segmentManager._tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
+ List<SegmentZKMetadata> segmentsZKMetadata = new ArrayList<>(segmentManager._segmentZKMetadataMap.values());
+ Assert.assertEquals(segmentsZKMetadata.size(), 5);
+
+ // Set up external view for this table
+ ExternalView externalView = new ExternalView(REALTIME_TABLE_NAME);
+ when(helixAdmin.getResourceExternalView(CLUSTER_NAME, REALTIME_TABLE_NAME)).thenReturn(externalView);
+
+ // Change 1st segment status to be DONE, but with default peer download url.
+ // Verify later the download url is fixed after upload success.
+ segmentsZKMetadata.get(0).setStatus(Status.DONE);
+ segmentsZKMetadata.get(0).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+ // set up the external view for 1st segment
+ String instance0 = "instance0";
+ int adminPort = 2077;
+ externalView.setState(segmentsZKMetadata.get(0).getSegmentName(), instance0, "ONLINE");
+ InstanceConfig instanceConfig0 = new InstanceConfig(instance0);
+ instanceConfig0.setHostName(instance0);
+ instanceConfig0.getRecord().setIntField(Instance.ADMIN_PORT_KEY, adminPort);
+ when(helixAdmin.getInstanceConfig(CLUSTER_NAME, instance0)).thenReturn(instanceConfig0);
+ // mock the request/response for 1st segment upload
+ String serverUploadRequestUrl0 =
+ String.format("http://%s:%d/segments/%s/%s/uploadCommittedSegment?uploadTimeoutMs=-1", instance0, adminPort,
+ REALTIME_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName());
+ // tempSegmentFileLocation is the location where the segment uploader will upload the segment. This usually ends
+ // with a random UUID
+ File tempSegmentFileLocation = new File(TEMP_DIR, segmentsZKMetadata.get(0).getSegmentName() + UUID.randomUUID());
+ FileUtils.write(tempSegmentFileLocation, "test");
+ // After the deep-store retry task gets the segment location returned by Pinot server, it will move the segment to
+ // its final location. This is the expected segment location.
+ String expectedSegmentLocation =
+ segmentManager.createSegmentPath(RAW_TABLE_NAME, segmentsZKMetadata.get(0).getSegmentName()).toString();
+ SegmentZKMetadata segmentZKMetadataCopy =
+ new SegmentZKMetadata(new ZNRecord(segmentsZKMetadata.get(0).toZNRecord()));
+
+ segmentZKMetadataCopy.setDownloadUrl(tempSegmentFileLocation.getPath());
+ when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStoreWithZKMetadata(
+ serverUploadRequestUrl0)).thenReturn(segmentZKMetadataCopy);
+
+ // Change 2nd segment status to be DONE, but with default peer download url.
+ // Verify later the download url isn't fixed after upload failure.
+ segmentsZKMetadata.get(1).setStatus(Status.DONE);
+ segmentsZKMetadata.get(1).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+ // set up the external view for 2nd segment
+ String instance1 = "instance1";
+ externalView.setState(segmentsZKMetadata.get(1).getSegmentName(), instance1, "ONLINE");
+ InstanceConfig instanceConfig1 = new InstanceConfig(instance1);
+ instanceConfig1.setHostName(instance1);
+ instanceConfig1.getRecord().setIntField(Instance.ADMIN_PORT_KEY, adminPort);
+ when(helixAdmin.getInstanceConfig(CLUSTER_NAME, instance1)).thenReturn(instanceConfig1);
+ // mock the request/response for 2nd segment upload
+ String serverUploadRequestUrl1 =
+ String.format("http://%s:%d/segments/%s/%s/uploadCommittedSegment?uploadTimeoutMs=-1", instance1, adminPort,
+ REALTIME_TABLE_NAME, segmentsZKMetadata.get(1).getSegmentName());
+ when(segmentManager._mockedFileUploadDownloadClient.uploadLLCToSegmentStore(serverUploadRequestUrl1)).thenThrow(
+ new HttpErrorStatusException("failed to upload segment",
+ Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()));
+
+ // Change 3rd segment status to be DONE, but with default peer download url.
+ // Verify later the download url isn't fixed because no ONLINE replica found in any server.
+ segmentsZKMetadata.get(2).setStatus(Status.DONE);
+ segmentsZKMetadata.get(2).setDownloadUrl(METADATA_URI_FOR_PEER_DOWNLOAD);
+ // set up the external view for 3rd segment
+ String instance2 = "instance2";
+ externalView.setState(segmentsZKMetadata.get(2).getSegmentName(), instance2, "OFFLINE");
+
+ // Change 4th segment status to be DONE and with segment download url.
+ // Verify later the download url is still the same.
+ String defaultDownloadUrl = "canItBeDownloaded";
+ segmentsZKMetadata.get(3).setStatus(Status.DONE);
+ segmentsZKMetadata.get(3).setDownloadUrl(defaultDownloadUrl);
+
+ // Keep 5th segment status as IN_PROGRESS.
+
+ List<String> segmentNames =
+ segmentsZKMetadata.stream().map(SegmentZKMetadata::getSegmentName).collect(Collectors.toList());
+ when(pinotHelixResourceManager.getTableConfig(REALTIME_TABLE_NAME)).thenReturn(segmentManager._tableConfig);
+
+ // Verify the result
+ segmentManager.uploadToDeepStoreIfMissing(segmentManager._tableConfig, segmentsZKMetadata);
+
+ // Block until all tasks have been able to complete
+ TestUtils.waitForCondition(aVoid -> segmentManager.deepStoreUploadExecutorPendingSegmentsIsEmpty(), 30_000L,
+ "Timed out waiting for upload retry tasks to finish");
+
+ assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(0), null).getDownloadUrl(),
+ expectedSegmentLocation);
+ assertFalse(tempSegmentFileLocation.exists(),
+ "Deep-store retry task should move the file from temp location to permanent location");
+
+ assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(1), null).getDownloadUrl(),
+ METADATA_URI_FOR_PEER_DOWNLOAD);
+ assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(2), null).getDownloadUrl(),
+ METADATA_URI_FOR_PEER_DOWNLOAD);
+ assertEquals(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(3), null).getDownloadUrl(),
+ defaultDownloadUrl);
+ assertNull(segmentManager.getSegmentZKMetadata(REALTIME_TABLE_NAME, segmentNames.get(4), null).getDownloadUrl());
+ }
+
+
@Test
public void testDeleteTmpSegmentFiles()
throws Exception {
@@ -1370,7 +1501,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
}
@Override
- SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+ protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName,
+ @Nullable Stat stat) {
Preconditions.checkState(_segmentZKMetadataMap.containsKey(segmentName));
if (stat != null) {
stat.setVersion(_segmentZKMetadataVersionMap.get(segmentName));
@@ -1430,7 +1562,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
}
@Override
- boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName, long currentTimeMs) {
+ protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+ long currentTimeMs) {
return _exceededMaxSegmentCompletionTime;
}
@@ -1453,7 +1586,8 @@ public class PinotLLCRealtimeSegmentManagerTest {
}
@Override
- SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName, @Nullable Stat stat) {
+ protected SegmentZKMetadata getSegmentZKMetadata(String realtimeTableName, String segmentName,
+ @Nullable Stat stat) {
SegmentZKMetadata segmentZKMetadata = super.getSegmentZKMetadata(realtimeTableName, segmentName, stat);
switch (_scenario) {
case ZK_VERSION_CHANGED:
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java
new file mode 100644
index 0000000000..5d7f7caa4a
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasePauselessRealtimeIngestionTest.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.common.utils.PauselessConsumptionUtils;
+import org.apache.pinot.controller.BaseControllerStarter;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
+import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionConfig;
+import org.apache.pinot.integration.tests.realtime.utils.FailureInjectingControllerStarter;
+import org.apache.pinot.integration.tests.realtime.utils.FailureInjectingPinotLLCRealtimeSegmentManager;
+import org.apache.pinot.integration.tests.realtime.utils.PauselessRealtimeTestUtils;
+import org.apache.pinot.server.starter.helix.HelixInstanceDataManagerConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
+import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+
+import static org.apache.pinot.spi.stream.StreamConfigProperties.SEGMENT_COMPLETION_FSM_SCHEME;
+import static org.testng.Assert.assertTrue;
+
+
+public abstract class BasePauselessRealtimeIngestionTest extends BaseClusterIntegrationTest {
+ protected static final int NUM_REALTIME_SEGMENTS = 48;
+ protected static final long DEFAULT_COUNT_STAR_RESULT = 115545L;
+ protected static final String DEFAULT_TABLE_NAME_2 = DEFAULT_TABLE_NAME + "_2";
+ private static final long MAX_SEGMENT_COMPLETION_TIME_MILLIS = 10_000;
+
+ protected List<File> _avroFiles;
+ protected boolean _failureEnabled = false;
+ private static final Logger LOGGER = LoggerFactory.getLogger(BasePauselessRealtimeIngestionTest.class);
+
+ protected abstract String getFailurePoint();
+
+ protected abstract int getExpectedSegmentsWithFailure();
+
+ protected abstract int getExpectedZKMetadataWithFailure();
+
+ protected abstract long getCountStarResultWithFailure();
+
+ @Override
+ public BaseControllerStarter createControllerStarter() {
+ return new FakePauselessControllerStarter();
+ }
+
+ @Override
+ protected void overrideControllerConf(Map<String, Object> properties) {
+ properties.put(ControllerConf.ControllerPeriodicTasksConf.PINOT_TASK_MANAGER_SCHEDULER_ENABLED, true);
+ properties.put(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT, true);
+ properties.put(SegmentCompletionConfig.FSM_SCHEME + "pauseless",
+ "org.apache.pinot.controller.helix.core.realtime.PauselessSegmentCompletionFSM");
+ properties.put(ControllerConf.ControllerPeriodicTasksConf.REALTIME_SEGMENT_VALIDATION_INITIAL_DELAY_IN_SECONDS,
+ 500);
+ }
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ try {
+ LOGGER.info("Set segment.store.uri: {} for server with scheme: {}", _controllerConfig.getDataDir(),
+ new URI(_controllerConfig.getDataDir()).getScheme());
+ serverConf.setProperty("pinot.server.instance.segment.store.uri",
+ "file:" + _controllerConfig.getDataDir());
+ serverConf.setProperty("pinot.server.instance." + HelixInstanceDataManagerConfig.UPLOAD_SEGMENT_TO_DEEP_STORE,
+ "true");
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
+ startZk();
+ startController();
+ startBroker();
+ startServer();
+
+ setupNonPauselessTable();
+ injectFailure();
+ setupPauselessTable();
+ waitForAllDocsLoaded(600_000L);
+ }
+
+ private void setupNonPauselessTable()
+ throws Exception {
+ _avroFiles = unpackAvroData(_tempDir);
+ startKafka();
+ pushAvroIntoKafka(_avroFiles);
+
+ Schema schema = createSchema();
+ schema.setSchemaName(DEFAULT_TABLE_NAME_2);
+ addSchema(schema);
+
+ TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
+ tableConfig.setTableName(DEFAULT_TABLE_NAME_2);
+ tableConfig.getValidationConfig().setRetentionTimeUnit("DAYS");
+ tableConfig.getValidationConfig().setRetentionTimeValue("100000");
+ addTableConfig(tableConfig);
+
+ waitForDocsLoaded(600_000L, true, tableConfig.getTableName());
+ TestUtils.waitForCondition((aVoid) -> {
+ List<SegmentZKMetadata> segmentZKMetadataList =
+ _helixResourceManager.getSegmentsZKMetadata(tableConfig.getTableName());
+ return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+ }, 1000, 100000, "Some segments still have missing url");
+ }
+
+ private void setupPauselessTable()
+ throws Exception {
+ Schema schema = createSchema();
+ schema.setSchemaName(DEFAULT_TABLE_NAME);
+ addSchema(schema);
+
+ TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
+ tableConfig.getValidationConfig().setRetentionTimeUnit("DAYS");
+ tableConfig.getValidationConfig().setRetentionTimeValue("100000");
+
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ingestionConfig.setStreamIngestionConfig(
+ new StreamIngestionConfig(List.of(tableConfig.getIndexingConfig().getStreamConfigs())));
+ ingestionConfig.getStreamIngestionConfig().setPauselessConsumptionEnabled(true);
+ ingestionConfig.getStreamIngestionConfig()
+ .getStreamConfigMaps()
+ .get(0)
+ .put(SEGMENT_COMPLETION_FSM_SCHEME, "pauseless");
+ tableConfig.getIndexingConfig().setStreamConfigs(null);
+ tableConfig.setIngestionConfig(ingestionConfig);
+
+ addTableConfig(tableConfig);
+ }
+
+ protected void injectFailure() {
+ PinotLLCRealtimeSegmentManager realtimeSegmentManager = _helixResourceManager.getRealtimeSegmentManager();
+ if (realtimeSegmentManager instanceof FailureInjectingPinotLLCRealtimeSegmentManager) {
+ ((FailureInjectingPinotLLCRealtimeSegmentManager) realtimeSegmentManager).enableTestFault(getFailurePoint());
+ }
+ _failureEnabled = true;
+ }
+
+ protected void disableFailure() {
+ _failureEnabled = false;
+ PinotLLCRealtimeSegmentManager realtimeSegmentManager = _helixResourceManager.getRealtimeSegmentManager();
+ if (realtimeSegmentManager instanceof FailureInjectingPinotLLCRealtimeSegmentManager) {
+ ((FailureInjectingPinotLLCRealtimeSegmentManager) realtimeSegmentManager).disableTestFault(getFailurePoint());
+ }
+ }
+
+ @AfterClass
+ public void tearDown()
+ throws IOException {
+ LOGGER.info("Tearing down...");
+ dropRealtimeTable(getTableName());
+ stopServer();
+ stopBroker();
+ stopController();
+ stopKafka();
+ stopZk();
+ FileUtils.deleteDirectory(_tempDir);
+ }
+
+ protected long getCountStarResult() {
+ return _failureEnabled ? getCountStarResultWithFailure() : DEFAULT_COUNT_STAR_RESULT;
+ }
+
+ protected void runValidationAndVerify()
+ throws Exception {
+ String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+ String tableNameWithType2 = TableNameBuilder.REALTIME.tableNameWithType(DEFAULT_TABLE_NAME_2);
+
+ PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, getExpectedSegmentsWithFailure(), _helixManager);
+
+ TestUtils.waitForCondition((aVoid) -> {
+ List<SegmentZKMetadata> segmentZKMetadataList =
+ _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+ return segmentZKMetadataList.size() == getExpectedZKMetadataWithFailure();
+ }, 1000, 100000, "New Segment ZK Metadata not created");
+
+ Thread.sleep(MAX_SEGMENT_COMPLETION_TIME_MILLIS);
+ disableFailure();
+
+ _controllerStarter.getRealtimeSegmentValidationManager().run();
+
+ waitForAllDocsLoaded(600_000L);
+ waitForDocsLoaded(600_000L, true, tableNameWithType2);
+
+ PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+ PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType2, NUM_REALTIME_SEGMENTS, _helixManager);
+
+ TestUtils.waitForCondition((aVoid) -> {
+ List<SegmentZKMetadata> segmentZKMetadataList =
+ _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+ return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+ }, 1000, 100000, "Some segments still have missing url");
+
+ PauselessRealtimeTestUtils.compareZKMetadataForSegments(
+ _helixResourceManager.getSegmentsZKMetadata(tableNameWithType),
+ _helixResourceManager.getSegmentsZKMetadata(tableNameWithType2));
+ }
+
+ /**
+ * Basic test to verify segment assignment and metadata without any failures
+ */
+ protected void testBasicSegmentAssignment() {
+ String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+
+ PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+ assertTrue(PauselessConsumptionUtils.isPauselessEnabled(getRealtimeTableConfig()));
+
+ TestUtils.waitForCondition((aVoid) -> {
+ List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+ return !hasSegmentsInStatus(segmentZKMetadataList, CommonConstants.Segment.Realtime.Status.COMMITTING);
+ }, 1000, 100000, "Some segments have status COMMITTING");
+
+ TestUtils.waitForCondition((aVoid) -> {
+ List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+ return PauselessRealtimeTestUtils.assertUrlPresent(segmentZKMetadataList);
+ }, 1000, 100000, "Some segments still have missing url");
+ }
+
+ private boolean hasSegmentsInStatus(List<SegmentZKMetadata> segmentZKMetadataList,
+ CommonConstants.Segment.Realtime.Status prohibitedStatus) {
+ for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
+ if (segmentZKMetadata.getStatus() == prohibitedStatus) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static class FakePauselessLLCRealtimeSegmentManager extends FailureInjectingPinotLLCRealtimeSegmentManager {
+
+ public FakePauselessLLCRealtimeSegmentManager(
+ PinotHelixResourceManager helixResourceManager,
+ ControllerConf controllerConf, ControllerMetrics controllerMetrics) {
+ super(helixResourceManager, controllerConf, controllerMetrics);
+ }
+
+ @Override
+ protected boolean isExceededMaxSegmentCompletionTime(String realtimeTableName, String segmentName,
+ long currentTimeMs) {
+ Stat stat = new Stat();
+ getSegmentZKMetadata(realtimeTableName, segmentName, stat);
+ if (currentTimeMs > stat.getMtime() + MAX_SEGMENT_COMPLETION_TIME_MILLIS) {
+ LOGGER.info("Segment: {} exceeds the max completion time: {}ms, metadata update time: {}, current time: {}",
+ segmentName, MAX_SEGMENT_COMPLETION_TIME_MILLIS, stat.getMtime(), currentTimeMs);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ private static class FakePauselessControllerStarter extends FailureInjectingControllerStarter {
+
+ @Override
+ protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+ return new FakePauselessLLCRealtimeSegmentManager(_helixResourceManager, getConfig(), _controllerMetrics);
+ }
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java
new file mode 100644
index 0000000000..0c816b6563
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionCommitEndMetadataFailureTest.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.util.List;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.apache.pinot.integration.tests.realtime.utils.PauselessRealtimeTestUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertNull;
+
+
+public class PauselessRealtimeIngestionCommitEndMetadataFailureTest
+ extends BasePauselessRealtimeIngestionTest {
+
+ @Override
+ protected String getFailurePoint() {
+ return FailureInjectionUtils.FAULT_BEFORE_COMMIT_END_METADATA;
+ }
+
+ @Override
+ protected int getExpectedSegmentsWithFailure() {
+ return NUM_REALTIME_SEGMENTS; // All segments still appear in ideal state
+ }
+
+ @Override
+ protected int getExpectedZKMetadataWithFailure() {
+ return NUM_REALTIME_SEGMENTS;
+ }
+
+ @Override
+ protected long getCountStarResultWithFailure() {
+ return DEFAULT_COUNT_STAR_RESULT;
+ }
+
+ @Test
+ public void testSegmentAssignment()
+ throws Exception {
+ String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
+ PauselessRealtimeTestUtils.verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS, _helixManager);
+
+ List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
+ for (SegmentZKMetadata metadata : segmentZKMetadataList) {
+ assertNull(metadata.getDownloadUrl());
+ }
+
+ runValidationAndVerify();
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java
new file mode 100644
index 0000000000..a0ca117cf7
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIdealStateUpdateFailureTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.testng.annotations.Test;
+
+
+public class PauselessRealtimeIngestionIdealStateUpdateFailureTest
+ extends BasePauselessRealtimeIngestionTest {
+
+ private static final int NUM_REALTIME_SEGMENTS_WITH_FAILURE = 2;
+ private static final int NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE = 4;
+ private static final long DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE = 5000;
+
+ @Override
+ protected String getFailurePoint() {
+ return FailureInjectionUtils.FAULT_BEFORE_IDEAL_STATE_UPDATE;
+ }
+
+ @Override
+ protected int getExpectedSegmentsWithFailure() {
+ return NUM_REALTIME_SEGMENTS_WITH_FAILURE;
+ }
+
+ @Override
+ protected int getExpectedZKMetadataWithFailure() {
+ return NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE;
+ }
+
+ @Override
+ protected long getCountStarResultWithFailure() {
+ return DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE;
+ }
+
+ @Test
+ public void testSegmentAssignment()
+ throws Exception {
+ runValidationAndVerify();
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
index 4e9fcac0ab..bce6e67e35 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionIntegrationTest.java
@@ -18,159 +18,44 @@
*/
package org.apache.pinot.integration.tests;
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.io.FileUtils;
-import org.apache.helix.model.IdealState;
-import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
-import org.apache.pinot.common.utils.PauselessConsumptionUtils;
-import org.apache.pinot.common.utils.helix.HelixHelper;
-import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionConfig;
-import org.apache.pinot.server.starter.helix.HelixInstanceDataManagerConfig;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
-import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.builder.TableNameBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static org.apache.pinot.spi.stream.StreamConfigProperties.SEGMENT_COMPLETION_FSM_SCHEME;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-public class PauselessRealtimeIngestionIntegrationTest extends BaseClusterIntegrationTest {
-
- private static final int NUM_REALTIME_SEGMENTS = 48;
- private static final Logger LOGGER = LoggerFactory.getLogger(PauselessRealtimeIngestionIntegrationTest.class);
- private List<File> _avroFiles;
-
- protected void overrideControllerConf(Map<String, Object> properties) {
- properties.put(ControllerConf.ControllerPeriodicTasksConf.PINOT_TASK_MANAGER_SCHEDULER_ENABLED, true);
- properties.put(ControllerConf.ControllerPeriodicTasksConf.ENABLE_DEEP_STORE_RETRY_UPLOAD_LLC_SEGMENT, true);
- properties.put(SegmentCompletionConfig.FSM_SCHEME + "pauseless",
- "org.apache.pinot.controller.helix.core.realtime.PauselessSegmentCompletionFSM");
- }
+public class PauselessRealtimeIngestionIntegrationTest extends BasePauselessRealtimeIngestionTest {
@Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- // Set segment store uri to the one used by controller as data dir (i.e. deep store)
- try {
- LOGGER.info("Set segment.store.uri: {} for server with scheme: {}", _controllerConfig.getDataDir(),
- new URI(_controllerConfig.getDataDir()).getScheme());
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
- serverConf.setProperty("pinot.server.instance.segment.store.uri", "file:" + _controllerConfig.getDataDir());
- serverConf.setProperty("pinot.server.instance." + HelixInstanceDataManagerConfig.UPLOAD_SEGMENT_TO_DEEP_STORE,
- "true");
- }
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- // Start a customized controller with more frequent realtime segment validation
- startController();
- startBroker();
- startServer();
-
- _avroFiles = unpackAvroData(_tempDir);
- startKafka();
- pushAvroIntoKafka(_avroFiles);
-
- Schema schema = createSchema();
- addSchema(schema);
- TableConfig tableConfig = createRealtimeTableConfig(_avroFiles.get(0));
- // Replace stream config from indexing config to ingestion config
- IngestionConfig ingestionConfig = new IngestionConfig();
- ingestionConfig.setStreamIngestionConfig(
- new StreamIngestionConfig(List.of(tableConfig.getIndexingConfig().getStreamConfigs())));
- ingestionConfig.getStreamIngestionConfig().setPauselessConsumptionEnabled(true);
- tableConfig.getIndexingConfig().setStreamConfigs(null);
- tableConfig.setIngestionConfig(ingestionConfig);
- addTableConfig(tableConfig);
-
- waitForAllDocsLoaded(600_000L);
+ protected String getFailurePoint() {
+ return null; // No failure point for basic test
}
- @Test(description = "Ensure that all the segments are ingested, built and uploaded when pauseless consumption is "
- + "enabled")
- public void testSegmentAssignment()
- throws Exception {
- String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(getTableName());
- verifyIdealState(tableNameWithType, NUM_REALTIME_SEGMENTS);
- assertTrue(PauselessConsumptionUtils.isPauselessEnabled(getRealtimeTableConfig()));
- TestUtils.waitForCondition((aVoid) -> {
- List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
- return assertNoSegmentInProhibitedStatus(segmentZKMetadataList,
- CommonConstants.Segment.Realtime.Status.COMMITTING);
- }, 1000, 100000, "Some segments have status COMMITTING");
- TestUtils.waitForCondition((aVoid) -> {
- List<SegmentZKMetadata> segmentZKMetadataList = _helixResourceManager.getSegmentsZKMetadata(tableNameWithType);
- return assertUrlPresent(segmentZKMetadataList);
- }, 1000, 100000, "Some segments still have missing url");
+ @Override
+ protected int getExpectedSegmentsWithFailure() {
+ return NUM_REALTIME_SEGMENTS; // Always expect full segments
}
- @AfterClass
- public void tearDown()
- throws IOException {
- LOGGER.info("Tearing down...");
- dropRealtimeTable(getTableName());
- stopServer();
- stopBroker();
- stopController();
- stopKafka();
- stopZk();
- FileUtils.deleteDirectory(_tempDir);
+ @Override
+ protected int getExpectedZKMetadataWithFailure() {
+ return NUM_REALTIME_SEGMENTS; // Always expect full metadata
}
- private void verifyIdealState(String tableName, int numSegmentsExpected) {
- IdealState idealState = HelixHelper.getTableIdealState(_helixManager, tableName);
- Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields();
- assertEquals(segmentAssignment.size(), numSegmentsExpected);
+ @Override
+ protected long getCountStarResultWithFailure() {
+ return DEFAULT_COUNT_STAR_RESULT; // Always expect full count
}
- private boolean assertUrlPresent(List<SegmentZKMetadata> segmentZKMetadataList) {
- for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
- if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.DONE
- && segmentZKMetadata.getDownloadUrl() == null) {
- System.out.println("URl not found for segment: " + segmentZKMetadata.getSegmentName());
- return false;
- }
- }
- return true;
+ @Override
+ protected void injectFailure() {
+ // Do nothing - no failure to inject
}
- private boolean assertNoSegmentInProhibitedStatus(List<SegmentZKMetadata> segmentZKMetadataList,
- CommonConstants.Segment.Realtime.Status prohibitedStatus) {
- for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
- if (segmentZKMetadata.getStatus() == prohibitedStatus) {
- return false;
- }
- }
- return true;
+ @Override
+ protected void disableFailure() {
+ // Do nothing - no failure to disable
}
- @Override
- protected Map<String, String> getStreamConfigs() {
- Map<String, String> streamConfigMap = getStreamConfigMap();
- streamConfigMap.put(SEGMENT_COMPLETION_FSM_SCHEME, "pauseless");
- return streamConfigMap;
+ @Test(description = "Ensure that all the segments are ingested, built and uploaded when pauseless consumption is "
+ + "enabled")
+ public void testSegmentAssignment() {
+ testBasicSegmentAssignment();
}
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java
new file mode 100644
index 0000000000..7af627ad70
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+import org.testng.annotations.Test;
+
+
+public class PauselessRealtimeIngestionNewSegmentMetadataCreationFailureTest
+ extends BasePauselessRealtimeIngestionTest {
+
+ private static final int NUM_REALTIME_SEGMENTS_WITH_FAILURE = 2;
+ private static final int NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE = 2;
+ private static final long DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE = 5000;
+
+ @Override
+ protected String getFailurePoint() {
+ return FailureInjectionUtils.FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION;
+ }
+
+ @Override
+ protected int getExpectedSegmentsWithFailure() {
+ return NUM_REALTIME_SEGMENTS_WITH_FAILURE;
+ }
+
+ @Override
+ protected int getExpectedZKMetadataWithFailure() {
+ return NUM_REALTIME_SEGMENTS_ZK_METADATA_WITH_FAILURE;
+ }
+
+ @Override
+ protected long getCountStarResultWithFailure() {
+ return DEFAULT_COUNT_STAR_RESULT_WITH_FAILURE;
+ }
+
+ @Test
+ public void testSegmentAssignment()
+ throws Exception {
+ runValidationAndVerify();
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
similarity index 54%
copy from pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
copy to pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
index aa3dbe5c0b..468b956cb4 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableLLCSegmentUploadResponse.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingControllerStarter.java
@@ -16,32 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.restlet.resources;
+package org.apache.pinot.integration.tests.realtime.utils;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.pinot.controller.ControllerStarter;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
-public class TableLLCSegmentUploadResponse {
- private final String _segmentName;
- private final long _crc;
- private final String _downloadUrl;
-
- public TableLLCSegmentUploadResponse(@JsonProperty("segmentName") String segmentName,
- @JsonProperty("crc") Long crc, @JsonProperty("downloadUrl") String downloadUrl) {
- _segmentName = segmentName;
- _crc = crc;
- _downloadUrl = downloadUrl;
- }
-
- public String getSegmentName() {
- return _segmentName;
- }
-
- public Long getCrc() {
- return _crc;
- }
-
- public String getDownloadUrl() {
- return _downloadUrl;
+public class FailureInjectingControllerStarter extends ControllerStarter {
+ @Override
+ protected PinotLLCRealtimeSegmentManager createPinotLLCRealtimeSegmentManager() {
+ return new FailureInjectingPinotLLCRealtimeSegmentManager(_helixResourceManager, getConfig(), _controllerMetrics);
}
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java
new file mode 100644
index 0000000000..66573e34b5
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingPinotLLCRealtimeSegmentManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.realtime.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager;
+import org.apache.pinot.controller.helix.core.util.FailureInjectionUtils;
+
+
+public class FailureInjectingPinotLLCRealtimeSegmentManager extends PinotLLCRealtimeSegmentManager {
+ @VisibleForTesting
+ private final Map<String, String> _failureConfig;
+
+ public FailureInjectingPinotLLCRealtimeSegmentManager(PinotHelixResourceManager helixResourceManager,
+ ControllerConf controllerConf, ControllerMetrics controllerMetrics) {
+ super(helixResourceManager, controllerConf, controllerMetrics);
+ _failureConfig = new HashMap<>();
+ }
+
+ @VisibleForTesting
+ public void enableTestFault(String faultType) {
+ _failureConfig.put(faultType, "true");
+ }
+
+ @VisibleForTesting
+ public void disableTestFault(String faultType) {
+ _failureConfig.remove(faultType);
+ }
+
+ @Override
+ protected void preProcessNewSegmentZKMetadata() {
+ FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_NEW_SEGMENT_METADATA_CREATION,
+ _failureConfig);
+ }
+
+ @Override
+ protected void preProcessCommitIdealStateUpdate() {
+ FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_IDEAL_STATE_UPDATE, _failureConfig);
+ }
+
+ @Override
+ protected void preProcessCommitSegmentEndMetadata() {
+ FailureInjectionUtils.injectFailure(FailureInjectionUtils.FAULT_BEFORE_COMMIT_END_METADATA, _failureConfig);
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java
new file mode 100644
index 0000000000..fc7f5264c0
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/PauselessRealtimeTestUtils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.realtime.utils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.helix.HelixManager;
+import org.apache.helix.model.IdealState;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.common.utils.helix.HelixHelper;
+import org.apache.pinot.spi.utils.CommonConstants;
+
+import static org.testng.Assert.assertEquals;
+
+
+public class PauselessRealtimeTestUtils {
+
+ private PauselessRealtimeTestUtils() {
+ }
+
+ public static void verifyIdealState(String tableName, int numSegmentsExpected, HelixManager helixManager) {
+ IdealState idealState = HelixHelper.getTableIdealState(helixManager, tableName);
+ Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields();
+ assertEquals(segmentAssignment.size(), numSegmentsExpected);
+ }
+
+ public static boolean assertUrlPresent(List<SegmentZKMetadata> segmentZKMetadataList) {
+ for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
+ if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.COMMITTING
+ && segmentZKMetadata.getDownloadUrl() == null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static void compareZKMetadataForSegments(List<SegmentZKMetadata> segmentsZKMetadata,
+ List<SegmentZKMetadata> segmentsZKMetadata1) {
+ Map<String, SegmentZKMetadata> segmentZKMetadataMap =
+ getPartitionSegmentNumberToMetadataMap(segmentsZKMetadata);
+ Map<String, SegmentZKMetadata> segmentZKMetadataMap1 =
+ getPartitionSegmentNumberToMetadataMap(segmentsZKMetadata1);
+
+ segmentZKMetadataMap.forEach((segmentKey, segmentZKMetadata) -> {
+ SegmentZKMetadata segmentZKMetadata1 = segmentZKMetadataMap1.get(segmentKey);
+ compareSegmentZkMetadata(segmentZKMetadata, segmentZKMetadata1);
+ });
+ }
+
+ private static Map<String, SegmentZKMetadata> getPartitionSegmentNumberToMetadataMap(
+ List<SegmentZKMetadata> segmentsZKMetadata) {
+ Map<String, SegmentZKMetadata> segmentZKMetadataMap = new HashMap<>();
+ for (SegmentZKMetadata segmentZKMetadata : segmentsZKMetadata) {
+ LLCSegmentName llcSegmentName = new LLCSegmentName(segmentZKMetadata.getSegmentName());
+ String segmentKey = llcSegmentName.getPartitionGroupId() + "_" + llcSegmentName.getSequenceNumber();
+ segmentZKMetadataMap.put(segmentKey, segmentZKMetadata);
+ }
+ return segmentZKMetadataMap;
+ }
+
+ private static void compareSegmentZkMetadata(SegmentZKMetadata segmentZKMetadata,
+ SegmentZKMetadata segmentZKMetadata1) {
+ if (segmentZKMetadata.getStatus() == CommonConstants.Segment.Realtime.Status.IN_PROGRESS) {
+ return;
+ }
+ assertEquals(segmentZKMetadata.getStatus(), segmentZKMetadata1.getStatus());
+ assertEquals(segmentZKMetadata.getStartOffset(), segmentZKMetadata1.getStartOffset());
+ assertEquals(segmentZKMetadata.getEndOffset(), segmentZKMetadata1.getEndOffset());
+ assertEquals(segmentZKMetadata.getTotalDocs(), segmentZKMetadata1.getTotalDocs());
+ assertEquals(segmentZKMetadata.getStartTimeMs(), segmentZKMetadata1.getStartTimeMs());
+ assertEquals(segmentZKMetadata.getEndTimeMs(), segmentZKMetadata1.getEndTimeMs());
+ }
+}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 8568a5178c..f7038ed11f 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -29,6 +29,7 @@ import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -61,6 +62,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.helix.model.IdealState;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadataUtils;
import org.apache.pinot.common.response.server.TableIndexMetadataResponse;
import org.apache.pinot.common.restlet.resources.ResourceUtils;
import org.apache.pinot.common.restlet.resources.SegmentConsumerInfo;
@@ -823,33 +825,8 @@ public class TablesResource {
File segmentTarFile = null;
try {
- // Create the tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
- File segmentTarUploadDir =
- new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
- segmentTarUploadDir.mkdir();
-
- segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
- tableNameWithType + "_" + segmentName + "_" + UUID.randomUUID() + TarCompressionUtils.TAR_GZ_FILE_EXTENSION,
- "Invalid table / segment name: %s, %s", tableNameWithType, segmentName);
-
- TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
- segmentTarFile);
-
- // Use segment uploader to upload the segment tar file to segment store and return the segment download url.
- SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
- URI segmentDownloadUrl;
- if (timeoutMs <= 0) {
- // Use default timeout if passed timeout is not positive
- segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
- } else {
- segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
- }
- if (segmentDownloadUrl == null) {
- throw new WebApplicationException(
- String.format("Failed to upload table %s segment %s to segment store", realtimeTableName, segmentName),
- Response.Status.INTERNAL_SERVER_ERROR);
- }
- return segmentDownloadUrl.toString();
+ segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+ return uploadSegment(segmentTarFile, tableNameWithType, segmentName, timeoutMs);
} finally {
FileUtils.deleteQuietly(segmentTarFile);
tableDataManager.releaseSegment(segmentDataManager);
@@ -871,6 +848,7 @@ public class TablesResource {
* @return full url where the segment is uploaded, crc, segmentName. Can add more segment metadata in the future.
* @throws Exception if an error occurred during the segment upload.
*/
+ @Deprecated
@POST
@Path("/segments/{realtimeTableNameWithType}/{segmentName}/uploadLLCSegment")
@Produces(MediaType.APPLICATION_JSON)
@@ -917,44 +895,139 @@ public class TablesResource {
String.format("Table %s segment %s does not exist", realtimeTableNameWithType, segmentName),
Response.Status.NOT_FOUND);
}
- String crc = segmentDataManager.getSegment().getSegmentMetadata().getCrc();
File segmentTarFile = null;
try {
- // Create the tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
- File segmentTarUploadDir =
- new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
- segmentTarUploadDir.mkdir();
+ segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+ String downloadUrl = uploadSegment(segmentTarFile, realtimeTableNameWithType, segmentName, timeoutMs);
+ return new TableLLCSegmentUploadResponse(segmentName,
+ Long.parseLong(segmentDataManager.getSegment().getSegmentMetadata().getCrc()), downloadUrl);
+ } finally {
+ FileUtils.deleteQuietly(segmentTarFile);
+ tableDataManager.releaseSegment(segmentDataManager);
+ }
+ }
- segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
- realtimeTableNameWithType + "_" + segmentName + "_" + UUID.randomUUID()
- + TarCompressionUtils.TAR_GZ_FILE_EXTENSION, "Invalid table / segment name: %s, %s",
- realtimeTableNameWithType, segmentName);
+ /**
+ * Upload a real-time committed segment to segment store and return the segment ZK metadata in json format.
+ * This endpoint is used when segment store copy is unavailable for real-time committed segments.
+ * Please note that invocation of this endpoint may cause query performance to suffer, since we tar up the segment to
+ * upload it.
+ *
+ * @see <a href="https://tinyurl.com/f63ru4sb></a>
+ * @param realtimeTableName table name with type.
+ * @param segmentName name of the segment to be uploaded
+ * @param timeoutMs timeout for the segment upload to the deep-store. If this is negative, the default timeout
+ * would be used.
+ * @return segment ZK metadata in json format.
+ * @throws Exception if an error occurred during the segment upload.
+ */
+ @POST
+ @Path("/segments/{realtimeTableName}/{segmentName}/uploadCommittedSegment")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Upload a real-time committed segment to segment store and return the segment ZK metadata",
+ notes = "Upload a real-time committed segment to segment store and return the segment ZK metadata")
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Success"),
+ @ApiResponse(code = 400, message = "Bad request", response = ErrorInfo.class),
+ @ApiResponse(code = 404, message = "Table or segment not found", response = ErrorInfo.class),
+ @ApiResponse(code = 500, message = "Internal server error", response = ErrorInfo.class)
+ })
+ public String uploadCommittedSegment(
+ @ApiParam(value = "Name of the real-time table", required = true) @PathParam("realtimeTableName")
+ String realtimeTableName,
+ @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") String segmentName,
+ @QueryParam("uploadTimeoutMs") @DefaultValue("-1") int timeoutMs, @Context HttpHeaders headers)
+ throws Exception {
+ realtimeTableName = DatabaseUtils.translateTableName(realtimeTableName, headers);
+ LOGGER.info("Received a request to upload committed segment: {} for table: {}", segmentName, realtimeTableName);
- TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
- segmentTarFile);
+ // Check it's real-time table
+ if (!TableNameBuilder.isRealtimeTableResource(realtimeTableName)) {
+ throw new WebApplicationException(
+ "Cannot upload committed segment for a non-realtime table: " + realtimeTableName,
+ Response.Status.BAD_REQUEST);
+ }
- // Use segment uploader to upload the segment tar file to segment store and return the segment download url.
- SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
- URI segmentDownloadUrl;
- if (timeoutMs <= 0) {
- // Use default timeout if passed timeout is not positive
- segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
- } else {
- segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
- }
- if (segmentDownloadUrl == null) {
- throw new WebApplicationException(
- String.format("Failed to upload table %s segment %s to segment store", realtimeTableNameWithType,
- segmentName), Response.Status.INTERNAL_SERVER_ERROR);
- }
- return new TableLLCSegmentUploadResponse(segmentName, Long.parseLong(crc), segmentDownloadUrl.toString());
+ // Check the segment is low level consumer segment
+ if (!LLCSegmentName.isLLCSegment(segmentName)) {
+ throw new WebApplicationException(String.format("Segment: %s is not a low level consumer segment", segmentName),
+ Response.Status.BAD_REQUEST);
+ }
+
+ TableDataManager tableDataManager =
+ ServerResourceUtils.checkGetTableDataManager(_serverInstance, realtimeTableName);
+ SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
+ if (segmentDataManager == null) {
+ throw new WebApplicationException(
+ String.format("Failed to find table: %s, segment: %s", realtimeTableName, segmentName),
+ Response.Status.NOT_FOUND);
+ }
+ if (!(segmentDataManager instanceof ImmutableSegmentDataManager)) {
+ throw new WebApplicationException(
+ String.format("Table: %s, segment: %s hasn't been sealed", realtimeTableName, segmentName),
+ Response.Status.NOT_FOUND);
+ }
+
+ File segmentTarFile = null;
+ try {
+ segmentTarFile = createSegmentTarFile(tableDataManager, segmentName);
+ String downloadUrl = uploadSegment(segmentTarFile, realtimeTableName, segmentName, timeoutMs);
+
+ // Fetch existing segment ZK Metadata
+ SegmentZKMetadata segmentZKMetadata =
+ ZKMetadataProvider.getSegmentZKMetadata(_serverInstance.getHelixManager().getHelixPropertyStore(),
+ realtimeTableName, segmentName);
+ Preconditions.checkState(segmentZKMetadata != null,
+ "Failed to find segment ZK metadata for table: %s, segment: %s", realtimeTableName, segmentName);
+
+ // Update the Segment ZK Metadata with the segment metadata present on the server
+ ImmutableSegment segment = ((ImmutableSegmentDataManager) segmentDataManager).getSegment();
+ SegmentZKMetadataUtils.updateCommittingSegmentZKMetadata(realtimeTableName, segmentZKMetadata,
+ segment.getSegmentMetadata(), downloadUrl, segment.getSegmentSizeBytes(), segmentZKMetadata.getEndOffset());
+ return segmentZKMetadata.toJsonString();
} finally {
FileUtils.deleteQuietly(segmentTarFile);
tableDataManager.releaseSegment(segmentDataManager);
}
}
+ /**
+ * Creates a tar.gz segment file in the server's segmentTarUploadDir folder with a unique file name.
+ */
+ private File createSegmentTarFile(TableDataManager tableDataManager, String segmentName)
+ throws IOException {
+ File segmentTarUploadDir =
+ new File(_serverInstance.getInstanceDataManager().getSegmentFileDirectory(), SEGMENT_UPLOAD_DIR);
+ segmentTarUploadDir.mkdir();
+ String tableNameWithType = tableDataManager.getTableName();
+ File segmentTarFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(segmentTarUploadDir,
+ tableNameWithType + "_" + segmentName + "_" + UUID.randomUUID() + TarCompressionUtils.TAR_GZ_FILE_EXTENSION,
+ "Invalid table / segment name: %s, %s", tableNameWithType, segmentName);
+ TarCompressionUtils.createCompressedTarFile(new File(tableDataManager.getTableDataDir(), segmentName),
+ segmentTarFile);
+ return segmentTarFile;
+ }
+
+ /**
+ * Uploads a segment tar file to the segment store and returns the segment download url.
+ */
+ private String uploadSegment(File segmentTarFile, String tableNameWithType, String segmentName, int timeoutMs) {
+ SegmentUploader segmentUploader = _serverInstance.getInstanceDataManager().getSegmentUploader();
+ URI segmentDownloadUrl;
+ if (timeoutMs <= 0) {
+ // Use default timeout if passed timeout is not positive
+ segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName));
+ } else {
+ segmentDownloadUrl = segmentUploader.uploadSegment(segmentTarFile, new LLCSegmentName(segmentName), timeoutMs);
+ }
+ if (segmentDownloadUrl == null) {
+ throw new WebApplicationException(
+ String.format("Failed to upload table: %s, segment: %s to segment store", tableNameWithType, segmentName),
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return segmentDownloadUrl.toString();
+ }
@GET
@Path("tables/{realtimeTableName}/consumingSegmentsInfo")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org