You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2023/03/31 17:16:23 UTC
[pinot] branch master updated: Allow empty segmentsTo for segment replacement protocol (#10511)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6e9bfb18b3 Allow empty segmentsTo for segment replacement protocol (#10511)
6e9bfb18b3 is described below
commit 6e9bfb18b37f280ed02683c32c6560b16534ea89
Author: Seunghyun Lee <se...@startree.ai>
AuthorDate: Fri Mar 31 10:16:16 2023 -0700
Allow empty segmentsTo for segment replacement protocol (#10511)
* Allow empty segmentsTo for segment replacement protocol
This PR attempts to support the empty segmentsTo
- (s1, s2, s3) -> () is a valid replacement where we want to
delete (s1, s2, s3) at the same time.
- We already support () -> (s1, s2, s3)
* Update pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StartReplaceSegmentsRequest.java
Co-authored-by: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
---------
Co-authored-by: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
---
.../restlet/resources/StartReplaceSegmentsRequest.java | 8 ++++----
.../helix/core/PinotHelixResourceManagerStatelessTest.java | 14 ++++++++++++++
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StartReplaceSegmentsRequest.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StartReplaceSegmentsRequest.java
index 05479526bb..933eff4478 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StartReplaceSegmentsRequest.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StartReplaceSegmentsRequest.java
@@ -37,11 +37,11 @@ public class StartReplaceSegmentsRequest {
private final List<String> _segmentsTo;
public StartReplaceSegmentsRequest(@JsonProperty("segmentsFrom") @Nullable List<String> segmentsFrom,
- @JsonProperty("segmentsTo") List<String> segmentsTo) {
+ @JsonProperty("segmentsTo") @Nullable List<String> segmentsTo) {
_segmentsFrom = (segmentsFrom == null) ? Collections.emptyList() : segmentsFrom;
- _segmentsTo = segmentsTo;
- Preconditions
- .checkArgument(segmentsTo != null && !segmentsTo.isEmpty(), "'segmentsTo' should not be null or empty");
+ _segmentsTo = (segmentsTo == null) ? Collections.emptyList() : segmentsTo;
+ Preconditions.checkArgument(!_segmentsFrom.isEmpty() || !_segmentsTo.isEmpty(),
+ "'segmentsFrom' and 'segmentsTo' cannot both be empty");
}
public List<String> getSegmentsFrom() {
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
index 07c8aa8906..a611a2f549 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
@@ -970,6 +970,20 @@ public class PinotHelixResourceManagerStatelessTest extends ControllerTest {
_helixResourceManager.endReplaceSegments(OFFLINE_TABLE_NAME, lineageEntryId9);
assertEquals(segmentLineage.getLineageEntry(lineageEntryId9).getState(), LineageEntryState.COMPLETED);
+ // Test empty segmentsTo. This is a special case where we are atomically removing segments.
+ List<String> segmentsFrom10 = Arrays.asList("s13", "s14");
+ List<String> segmentsTo10 = Collections.emptyList();
+ String lineageEntryId10 =
+ _helixResourceManager.startReplaceSegments(OFFLINE_TABLE_NAME, segmentsFrom10, segmentsTo10, true);
+ segmentLineage = SegmentLineageAccessHelper.getSegmentLineage(_propertyStore, OFFLINE_TABLE_NAME);
+ assertEquals(segmentLineage.getLineageEntryIds().size(), 9);
+ assertEquals(segmentLineage.getLineageEntry(lineageEntryId10).getState(), LineageEntryState.IN_PROGRESS);
+ _helixResourceManager.endReplaceSegments(OFFLINE_TABLE_NAME, lineageEntryId10);
+ segmentLineage = SegmentLineageAccessHelper.getSegmentLineage(_propertyStore, OFFLINE_TABLE_NAME);
+ assertEquals(segmentLineage.getLineageEntry(lineageEntryId10).getSegmentsFrom(), segmentsFrom10);
+ assertEquals(segmentLineage.getLineageEntry(lineageEntryId10).getSegmentsTo(), segmentsTo10);
+ assertEquals(segmentLineage.getLineageEntry(lineageEntryId10).getState(), LineageEntryState.COMPLETED);
+
// Delete the table
_helixResourceManager.deleteOfflineTable(RAW_TABLE_NAME);
segmentLineage = SegmentLineageAccessHelper.getSegmentLineage(_propertyStore, OFFLINE_TABLE_NAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org