You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/09/25 00:58:44 UTC
[pinot] branch master updated: Remove the old deprecated commit end without metadata (#11662)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 16ab9c1945 Remove the old deprecated commit end without metadata (#11662)
16ab9c1945 is described below
commit 16ab9c1945f7ac868f88c97070c4fb4b909f6ba3
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Sun Sep 24 17:58:39 2023 -0700
Remove the old deprecated commit end without metadata (#11662)
---
.../resources/LLCSegmentCompletionHandlers.java | 71 ----------------------
.../ServerSegmentCompletionProtocolHandler.java | 12 ----
2 files changed, 83 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
index 93fd7b1896..e774f91436 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
@@ -42,7 +42,6 @@ import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.common.utils.URIUtils;
-import org.apache.pinot.common.utils.fetcher.SegmentFetcherFactory;
import org.apache.pinot.controller.api.access.AccessType;
import org.apache.pinot.controller.api.access.Authenticate;
import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager;
@@ -214,57 +213,6 @@ public class LLCSegmentCompletionHandlers {
return responseStr;
}
- // TODO: remove this API. Should not allow committing without metadata
- @GET
- @Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT_END)
- @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_ADMIN_INFO)
- @Produces(MediaType.APPLICATION_JSON)
- public String segmentCommitEnd(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_LOCATION) String segmentLocation,
- @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset,
- @QueryParam(SegmentCompletionProtocol.PARAM_STREAM_PARTITION_MSG_OFFSET) String streamPartitionMsgOffset,
- @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes,
- @QueryParam(SegmentCompletionProtocol.PARAM_BUILD_TIME_MILLIS) long buildTimeMillis,
- @QueryParam(SegmentCompletionProtocol.PARAM_WAIT_TIME_MILLIS) long waitTimeMillis,
- @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_SIZE_BYTES) long segmentSizeBytes) {
- if (instanceId == null || segmentName == null || segmentLocation == null || (offset == -1
- && streamPartitionMsgOffset == null)) {
- LOGGER.error(
- "Invalid call: offset={}, segmentName={}, instanceId={}, segmentLocation={}, streamPartitionMsgOffset={}",
- offset, segmentName, instanceId, segmentLocation, streamPartitionMsgOffset);
- // TODO: memoryUsedInBytes = 0 if not present in params. Add validation when we start using it
- return SegmentCompletionProtocol.RESP_FAILED.toJsonString();
- }
-
- SegmentMetadataImpl segmentMetadata;
- try {
- segmentMetadata = extractMetadataFromSegmentFileURI(new URI(segmentLocation), segmentName);
- } catch (Exception e) {
- LOGGER.error("Caught exception while extracting metadata from segment: {} at location: {} from instance: {}",
- segmentName, segmentLocation, instanceId, e);
- return SegmentCompletionProtocol.RESP_FAILED.toJsonString();
- }
- SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params();
- requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withSegmentLocation(segmentLocation)
- .withSegmentSizeBytes(segmentSizeBytes).withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis)
- .withNumRows(numRows).withMemoryUsedBytes(memoryUsedBytes);
- extractOffsetFromParams(requestParams, streamPartitionMsgOffset, offset);
- LOGGER.info("Processing segmentCommitEnd:{}", requestParams.toString());
-
- final boolean isSuccess = true;
- final boolean isSplitCommit = true;
-
- CommittingSegmentDescriptor committingSegmentDescriptor =
- CommittingSegmentDescriptor.fromSegmentCompletionReqParamsAndMetadata(requestParams, segmentMetadata);
- SegmentCompletionProtocol.Response response = _segmentCompletionManager
- .segmentCommitEnd(requestParams, isSuccess, isSplitCommit, committingSegmentDescriptor);
- final String responseStr = response.toJsonString();
- LOGGER.info("Response to segmentCommitEnd for segment:{} is:{}", segmentName, responseStr);
- return responseStr;
- }
-
@POST
@Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT)
@Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.COMMIT_SEGMENT)
@@ -542,25 +490,6 @@ public class LLCSegmentCompletionHandlers {
}
}
- /**
- * Extracts the segment metadata from the segment file URI (could be remote). Copy the segment file to local under
- * file upload temporary directory first, then use the untarred file temporary directory to store the metadata files
- * temporarily.
- */
- private static SegmentMetadataImpl extractMetadataFromSegmentFileURI(URI segmentFileURI, String segmentName)
- throws Exception {
- File localTempFile = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(
- ControllerFilePathProvider.getInstance().getFileUploadTempDir(), getTempSegmentFileName(segmentName),
- "Invalid segment name: %s", segmentName);
-
- try {
- SegmentFetcherFactory.fetchSegmentToLocal(segmentFileURI, localTempFile);
- return extractMetadataFromLocalSegmentFile(localTempFile);
- } finally {
- FileUtils.deleteQuietly(localTempFile);
- }
- }
-
private static String getTempSegmentFileName(String segmentName) {
return segmentName + "." + UUID.randomUUID();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java b/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
index 03227f2920..6e1e24a688 100644
--- a/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
@@ -128,18 +128,6 @@ public class ServerSegmentCompletionProtocolHandler {
return request.getUrl(hostPort, protocol);
}
- // Replaced by segmentCommitEndWithMetadata().
- @Deprecated
- public SegmentCompletionProtocol.Response segmentCommitEnd(SegmentCompletionProtocol.Request.Params params) {
- SegmentCompletionProtocol.SegmentCommitEndRequest request =
- new SegmentCompletionProtocol.SegmentCommitEndRequest(params);
- String url = createSegmentCompletionUrl(request);
- if (url == null) {
- return SegmentCompletionProtocol.RESP_NOT_SENT;
- }
- return sendRequest(url);
- }
-
public SegmentCompletionProtocol.Response segmentCommitEndWithMetadata(
SegmentCompletionProtocol.Request.Params params, final Map<String, File> metadataFiles) {
SegmentCompletionProtocol.SegmentCommitEndWithMetadataRequest request =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org