You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2024/02/04 00:48:04 UTC
(pinot) branch master updated: Skip instead of throwing error on 'getValidDocIdMetadata' (#12360)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0a03c542ec Skip instead of throwing error on 'getValidDocIdMetadata' (#12360)
0a03c542ec is described below
commit 0a03c542ec38fa53faa646cd8c782c8e02a7bfec
Author: Seunghyun Lee <se...@startree.ai>
AuthorDate: Sat Feb 3 16:47:59 2024 -0800
Skip instead of throwing error on 'getValidDocIdMetadata' (#12360)
* Skip instead of throwing error on 'getValidDocIdMetadata'
- Current server side API throws the error if the snapshot
file is not availble. In this case, we should skip
instead of throwing the error.
- Improve the API doc for validDocIds related APIs
* Change the API 'validDocIdMetadata -> validDocIdsMetadata'
---
...adataInfo.java => ValidDocIdsMetadataInfo.java} | 4 +-
.../api/resources/PinotTableRestletResource.java | 19 +++---
.../util/ServerSegmentMetadataReader.java | 24 ++++----
.../pinot/controller/util/TableMetadataReader.java | 10 ++--
.../apache/pinot/core/common/MinionConstants.java | 2 +-
.../UpsertCompactionTaskGenerator.java | 26 ++++----
.../UpsertCompactionTaskGeneratorTest.java | 30 +++++-----
.../pinot/server/api/resources/TablesResource.java | 69 ++++++++++++----------
.../pinot/server/api/TablesResourceTest.java | 21 +++----
9 files changed, 107 insertions(+), 98 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdMetadataInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
similarity index 94%
rename from pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdMetadataInfo.java
rename to pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
index ddec1df4db..ce54424d16 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdMetadataInfo.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
-public class ValidDocIdMetadataInfo {
+public class ValidDocIdsMetadataInfo {
private final String _segmentName;
private final long _totalValidDocs;
private final long _totalInvalidDocs;
@@ -31,7 +31,7 @@ public class ValidDocIdMetadataInfo {
private final String _segmentCrc;
private final ValidDocIdsType _validDocIdsType;
- public ValidDocIdMetadataInfo(@JsonProperty("segmentName") String segmentName,
+ public ValidDocIdsMetadataInfo(@JsonProperty("segmentName") String segmentName,
@JsonProperty("totalValidDocs") long totalValidDocs, @JsonProperty("totalInvalidDocs") long totalInvalidDocs,
@JsonProperty("totalDocs") long totalDocs, @JsonProperty("segmentCrc") String segmentCrc,
@JsonProperty("validDocIdsType") ValidDocIdsType validDocIdsType) {
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index 4505ff0bc4..aedc6a7ef8 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -80,6 +80,7 @@ import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.response.server.TableIndexMetadataResponse;
import org.apache.pinot.common.restlet.resources.TableSegmentValidationInfo;
+import org.apache.pinot.common.restlet.resources.ValidDocIdsType;
import org.apache.pinot.common.utils.helix.HelixHelper;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.access.AccessControlFactory;
@@ -951,18 +952,18 @@ public class PinotTableRestletResource {
}
@GET
- @Path("tables/{tableName}/validDocIdMetadata")
+ @Path("tables/{tableName}/validDocIdsMetadata")
@Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get the aggregate valid doc id metadata of all segments for a table", notes = "Get the "
+ "aggregate valid doc id metadata of all segments for a table")
- public String getTableAggregateValidDocIdMetadata(
+ public String getTableAggregateValidDocIdsMetadata(
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
@ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr,
@ApiParam(value = "A list of segments", allowMultiple = true) @QueryParam("segmentNames")
List<String> segmentNames,
- @ApiParam(value = "Valid doc id type", example = "SNAPSHOT|IN_MEMORY|IN_MEMORY_WITH_DELETE")
- @QueryParam("validDocIdsType") String validDocIdsType) {
+ @ApiParam(value = "Valid doc ids type")
+ @QueryParam("validDocIdsType") ValidDocIdsType validDocIdsType) {
LOGGER.info("Received a request to fetch aggregate valid doc id metadata for a table {}", tableName);
TableType tableType = Constants.validateTableType(tableTypeStr);
if (tableType == TableType.OFFLINE) {
@@ -972,21 +973,21 @@ public class PinotTableRestletResource {
String tableNameWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
- String validDocIdMetadata;
+ String validDocIdsMetadata;
try {
TableMetadataReader tableMetadataReader =
new TableMetadataReader(_executor, _connectionManager, _pinotHelixResourceManager);
JsonNode segmentsMetadataJson =
- tableMetadataReader.getAggregateValidDocIdMetadata(tableNameWithType, segmentNames, validDocIdsType,
- _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
- validDocIdMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson);
+ tableMetadataReader.getAggregateValidDocIdsMetadata(tableNameWithType, segmentNames,
+ validDocIdsType.toString(), _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
+ validDocIdsMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson);
} catch (InvalidConfigException e) {
throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.BAD_REQUEST);
} catch (IOException ioe) {
throw new ControllerApplicationException(LOGGER, "Error parsing Pinot server response: " + ioe.getMessage(),
Response.Status.INTERNAL_SERVER_ERROR, ioe);
}
- return validDocIdMetadata;
+ return validDocIdsMetadata;
}
@GET
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
index fad0559b76..f728d51635 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -42,8 +42,8 @@ import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.pinot.common.restlet.resources.TableMetadataInfo;
import org.apache.pinot.common.restlet.resources.TableSegments;
-import org.apache.pinot.common.restlet.resources.ValidDocIdMetadataInfo;
import org.apache.pinot.common.restlet.resources.ValidDocIdsBitmapResponse;
+import org.apache.pinot.common.restlet.resources.ValidDocIdsMetadataInfo;
import org.apache.pinot.common.utils.RoaringBitmapUtils;
import org.apache.pinot.spi.utils.JsonUtils;
import org.glassfish.jersey.client.ClientConfig;
@@ -209,7 +209,7 @@ public class ServerSegmentMetadataReader {
*
* @return segment metadata as a JSON string
*/
- public List<ValidDocIdMetadataInfo> getValidDocIdMetadataFromServer(String tableNameWithType,
+ public List<ValidDocIdsMetadataInfo> getValidDocIdsMetadataFromServer(String tableNameWithType,
Map<String, List<String>> serverToSegmentsMap, BiMap<String, String> serverToEndpoints,
@Nullable List<String> segmentNames, int timeoutMs, String validDocIdsType) {
List<Pair<String, String>> serverURLsAndBodies = new ArrayList<>();
@@ -226,7 +226,7 @@ public class ServerSegmentMetadataReader {
}
}
}
- serverURLsAndBodies.add(generateValidDocIdMetadataURL(tableNameWithType, segmentsToQuery, validDocIdsType,
+ serverURLsAndBodies.add(generateValidDocIdsMetadataURL(tableNameWithType, segmentsToQuery, validDocIdsType,
serverToEndpoints.get(serverToSegments.getKey())));
}
@@ -239,16 +239,16 @@ public class ServerSegmentMetadataReader {
completionServiceHelper.doMultiPostRequest(serverURLsAndBodies, tableNameWithType, false, requestHeaders,
timeoutMs, null);
- List<ValidDocIdMetadataInfo> validDocIdMetadataInfos = new ArrayList<>();
+ List<ValidDocIdsMetadataInfo> validDocIdsMetadataInfos = new ArrayList<>();
int failedParses = 0;
int returnedSegmentsCount = 0;
for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
try {
- String validDocIdMetadataList = streamResponse.getValue();
- List<ValidDocIdMetadataInfo> validDocIdMetadataInfo =
- JsonUtils.stringToObject(validDocIdMetadataList, new TypeReference<ArrayList<ValidDocIdMetadataInfo>>() {
+ String validDocIdsMetadataList = streamResponse.getValue();
+ List<ValidDocIdsMetadataInfo> validDocIdsMetadataInfo =
+ JsonUtils.stringToObject(validDocIdsMetadataList, new TypeReference<ArrayList<ValidDocIdsMetadataInfo>>() {
});
- validDocIdMetadataInfos.addAll(validDocIdMetadataInfo);
+ validDocIdsMetadataInfos.addAll(validDocIdsMetadataInfo);
returnedSegmentsCount++;
} catch (Exception e) {
failedParses++;
@@ -261,12 +261,12 @@ public class ServerSegmentMetadataReader {
}
if (segmentNames != null && returnedSegmentsCount != segmentNames.size()) {
- LOGGER.error("Unable to get validDocIdMetadata from all servers. Expected: {}, Actual: {}", segmentNames.size(),
+ LOGGER.error("Unable to get validDocIdsMetadata from all servers. Expected: {}, Actual: {}", segmentNames.size(),
returnedSegmentsCount);
}
LOGGER.info("Retrieved valid doc id metadata for {} segments from {} servers.", returnedSegmentsCount,
serverURLsAndBodies.size());
- return validDocIdMetadataInfos;
+ return validDocIdsMetadataInfos;
}
/**
@@ -354,7 +354,7 @@ public class ServerSegmentMetadataReader {
return url;
}
- private Pair<String, String> generateValidDocIdMetadataURL(String tableNameWithType, List<String> segmentNames,
+ private Pair<String, String> generateValidDocIdsMetadataURL(String tableNameWithType, List<String> segmentNames,
String validDocIdsType, String endpoint) {
tableNameWithType = URLEncoder.encode(tableNameWithType, StandardCharsets.UTF_8);
TableSegments tableSegments = new TableSegments(segmentNames);
@@ -365,7 +365,7 @@ public class ServerSegmentMetadataReader {
LOGGER.error("Failed to convert segment names to json request body: segmentNames={}", segmentNames);
throw new RuntimeException(e);
}
- String url = String.format("%s/tables/%s/validDocIdMetadata", endpoint, tableNameWithType);
+ String url = String.format("%s/tables/%s/validDocIdsMetadata", endpoint, tableNameWithType);
if (validDocIdsType != null) {
url = url + "?validDocIdsType=" + validDocIdsType;
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
index 25ca6fc7f9..389c8d2e94 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
@@ -31,7 +31,7 @@ import java.util.stream.Collectors;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.restlet.resources.TableMetadataInfo;
-import org.apache.pinot.common.restlet.resources.ValidDocIdMetadataInfo;
+import org.apache.pinot.common.restlet.resources.ValidDocIdsMetadataInfo;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -156,9 +156,9 @@ public class TableMetadataReader {
/**
* This method retrieves the aggregated valid doc id metadata for a given table.
- * @return a list of ValidDocIdMetadataInfo
+ * @return a list of ValidDocIdsMetadataInfo
*/
- public JsonNode getAggregateValidDocIdMetadata(String tableNameWithType, List<String> segmentNames,
+ public JsonNode getAggregateValidDocIdsMetadata(String tableNameWithType, List<String> segmentNames,
String validDocIdsType, int timeoutMs)
throws InvalidConfigException {
final Map<String, List<String>> serverToSegments =
@@ -168,8 +168,8 @@ public class TableMetadataReader {
ServerSegmentMetadataReader serverSegmentMetadataReader =
new ServerSegmentMetadataReader(_executor, _connectionManager);
- List<ValidDocIdMetadataInfo> aggregateTableMetadataInfo =
- serverSegmentMetadataReader.getValidDocIdMetadataFromServer(tableNameWithType, serverToSegments, endpoints,
+ List<ValidDocIdsMetadataInfo> aggregateTableMetadataInfo =
+ serverSegmentMetadataReader.getValidDocIdsMetadataFromServer(tableNameWithType, serverToSegments, endpoints,
segmentNames, timeoutMs, validDocIdsType);
return JsonUtils.objectToJsonNode(aggregateTableMetadataInfo);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
index a02b2876e3..02f00046b9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
@@ -166,7 +166,7 @@ public class MinionConstants {
public static final String INVALID_RECORDS_THRESHOLD_COUNT = "invalidRecordsThresholdCount";
/**
- * Valid doc id type
+ * Valid doc ids type
*/
public static final String VALID_DOC_IDS_TYPE = "validDocIdsType";
}
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
index ad005a1ca2..6d21d8417f 100644
--- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
+++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
@@ -31,7 +31,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.helix.task.TaskState;
import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
-import org.apache.pinot.common.restlet.resources.ValidDocIdMetadataInfo;
+import org.apache.pinot.common.restlet.resources.ValidDocIdsMetadataInfo;
import org.apache.pinot.common.restlet.resources.ValidDocIdsType;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.helix.core.minion.generator.BaseTaskGenerator;
@@ -151,15 +151,15 @@ public class UpsertCompactionTaskGenerator extends BaseTaskGenerator {
validDocIdsType));
}
- List<ValidDocIdMetadataInfo> validDocIdMetadataList =
- serverSegmentMetadataReader.getValidDocIdMetadataFromServer(tableNameWithType, serverToSegments,
+ List<ValidDocIdsMetadataInfo> validDocIdsMetadataList =
+ serverSegmentMetadataReader.getValidDocIdsMetadataFromServer(tableNameWithType, serverToSegments,
serverToEndpoints, null, 60_000, validDocIdsType.toString());
Map<String, SegmentZKMetadata> completedSegmentsMap =
completedSegments.stream().collect(Collectors.toMap(SegmentZKMetadata::getSegmentName, Function.identity()));
SegmentSelectionResult segmentSelectionResult =
- processValidDocIdMetadata(taskConfigs, completedSegmentsMap, validDocIdMetadataList);
+ processValidDocIdsMetadata(taskConfigs, completedSegmentsMap, validDocIdsMetadataList);
if (!segmentSelectionResult.getSegmentsForDeletion().isEmpty()) {
pinotHelixResourceManager.deleteSegments(tableNameWithType, segmentSelectionResult.getSegmentsForDeletion(),
@@ -195,8 +195,8 @@ public class UpsertCompactionTaskGenerator extends BaseTaskGenerator {
}
@VisibleForTesting
- public static SegmentSelectionResult processValidDocIdMetadata(Map<String, String> taskConfigs,
- Map<String, SegmentZKMetadata> completedSegmentsMap, List<ValidDocIdMetadataInfo> validDocIdMetadataInfoList) {
+ public static SegmentSelectionResult processValidDocIdsMetadata(Map<String, String> taskConfigs,
+ Map<String, SegmentZKMetadata> completedSegmentsMap, List<ValidDocIdsMetadataInfo> validDocIdsMetadataInfoList) {
double invalidRecordsThresholdPercent = Double.parseDouble(
taskConfigs.getOrDefault(UpsertCompactionTask.INVALID_RECORDS_THRESHOLD_PERCENT,
String.valueOf(DEFAULT_INVALID_RECORDS_THRESHOLD_PERCENT)));
@@ -205,19 +205,19 @@ public class UpsertCompactionTaskGenerator extends BaseTaskGenerator {
String.valueOf(DEFAULT_INVALID_RECORDS_THRESHOLD_COUNT)));
List<SegmentZKMetadata> segmentsForCompaction = new ArrayList<>();
List<String> segmentsForDeletion = new ArrayList<>();
- for (ValidDocIdMetadataInfo validDocIdMetadata : validDocIdMetadataInfoList) {
- long totalInvalidDocs = validDocIdMetadata.getTotalInvalidDocs();
- String segmentName = validDocIdMetadata.getSegmentName();
+ for (ValidDocIdsMetadataInfo validDocIdsMetadata : validDocIdsMetadataInfoList) {
+ long totalInvalidDocs = validDocIdsMetadata.getTotalInvalidDocs();
+ String segmentName = validDocIdsMetadata.getSegmentName();
// Skip segments if the crc from zk metadata and server does not match. They may be being reloaded.
SegmentZKMetadata segment = completedSegmentsMap.get(segmentName);
- if (segment.getCrc() != Long.parseLong(validDocIdMetadata.getSegmentCrc())) {
+ if (segment.getCrc() != Long.parseLong(validDocIdsMetadata.getSegmentCrc())) {
LOGGER.warn(
- "CRC mismatch for segment: {}, skipping it for compaction (segmentZKMetadata={}, validDocIdMetadata={})",
- segmentName, segment.getCrc(), validDocIdMetadata.getSegmentCrc());
+ "CRC mismatch for segment: {}, skipping it for compaction (segmentZKMetadata={}, validDocIdsMetadata={})",
+ segmentName, segment.getCrc(), validDocIdsMetadata.getSegmentCrc());
continue;
}
- long totalDocs = validDocIdMetadata.getTotalDocs();
+ long totalDocs = validDocIdsMetadata.getTotalDocs();
double invalidRecordPercent = ((double) totalInvalidDocs / totalDocs) * 100;
if (totalInvalidDocs == totalDocs) {
segmentsForDeletion.add(segment.getSegmentName());
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
index d30107056b..d74b03e815 100644
--- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
+++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
@@ -28,7 +28,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.helix.model.IdealState;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
-import org.apache.pinot.common.restlet.resources.ValidDocIdMetadataInfo;
+import org.apache.pinot.common.restlet.resources.ValidDocIdsMetadataInfo;
import org.apache.pinot.controller.helix.core.minion.ClusterInfoAccessor;
import org.apache.pinot.core.common.MinionConstants;
import org.apache.pinot.core.common.MinionConstants.UpsertCompactionTask;
@@ -182,7 +182,7 @@ public class UpsertCompactionTaskGeneratorTest {
}
@Test
- public void testProcessValidDocIdMetadata()
+ public void testProcessValidDocIdsMetadata()
throws IOException {
Map<String, String> compactionConfigs = getCompactionConfigs("1", "10");
String json = "[{" + "\"totalValidDocs\" : 50," + "\"totalInvalidDocs\" : 50," + "\"segmentName\" : \""
@@ -190,12 +190,12 @@ public class UpsertCompactionTaskGeneratorTest {
+ _completedSegment.getCrc() + "\"}," + "{" + "\"totalValidDocs\" : 0," + "\"totalInvalidDocs\" : 10,"
+ "\"segmentName\" : \"" + _completedSegment2.getSegmentName() + "\", " + "\"segmentCrc\" : \""
+ _completedSegment2.getCrc() + "\"," + "\"totalDocs\" : 10" + "}]";
- List<ValidDocIdMetadataInfo> validDocIdMetadataInfo =
- JsonUtils.stringToObject(json, new TypeReference<ArrayList<ValidDocIdMetadataInfo>>() {
+ List<ValidDocIdsMetadataInfo> validDocIdsMetadataInfo =
+ JsonUtils.stringToObject(json, new TypeReference<ArrayList<ValidDocIdsMetadataInfo>>() {
});
UpsertCompactionTaskGenerator.SegmentSelectionResult segmentSelectionResult =
- UpsertCompactionTaskGenerator.processValidDocIdMetadata(compactionConfigs, _completedSegmentsMap,
- validDocIdMetadataInfo);
+ UpsertCompactionTaskGenerator.processValidDocIdsMetadata(compactionConfigs, _completedSegmentsMap,
+ validDocIdsMetadataInfo);
assertEquals(segmentSelectionResult.getSegmentsForCompaction().get(0).getSegmentName(),
_completedSegment.getSegmentName());
assertEquals(segmentSelectionResult.getSegmentsForDeletion().get(0), _completedSegment2.getSegmentName());
@@ -203,23 +203,23 @@ public class UpsertCompactionTaskGeneratorTest {
// test with a higher invalidRecordsThresholdPercent
compactionConfigs = getCompactionConfigs("60", "10");
segmentSelectionResult =
- UpsertCompactionTaskGenerator.processValidDocIdMetadata(compactionConfigs, _completedSegmentsMap,
- validDocIdMetadataInfo);
+ UpsertCompactionTaskGenerator.processValidDocIdsMetadata(compactionConfigs, _completedSegmentsMap,
+ validDocIdsMetadataInfo);
assertTrue(segmentSelectionResult.getSegmentsForCompaction().isEmpty());
// test without an invalidRecordsThresholdPercent
compactionConfigs = getCompactionConfigs("0", "10");
segmentSelectionResult =
- UpsertCompactionTaskGenerator.processValidDocIdMetadata(compactionConfigs, _completedSegmentsMap,
- validDocIdMetadataInfo);
+ UpsertCompactionTaskGenerator.processValidDocIdsMetadata(compactionConfigs, _completedSegmentsMap,
+ validDocIdsMetadataInfo);
assertEquals(segmentSelectionResult.getSegmentsForCompaction().get(0).getSegmentName(),
_completedSegment.getSegmentName());
// test without a invalidRecordsThresholdCount
compactionConfigs = getCompactionConfigs("30", "0");
segmentSelectionResult =
- UpsertCompactionTaskGenerator.processValidDocIdMetadata(compactionConfigs, _completedSegmentsMap,
- validDocIdMetadataInfo);
+ UpsertCompactionTaskGenerator.processValidDocIdsMetadata(compactionConfigs, _completedSegmentsMap,
+ validDocIdsMetadataInfo);
assertEquals(segmentSelectionResult.getSegmentsForCompaction().get(0).getSegmentName(),
_completedSegment.getSegmentName());
@@ -230,11 +230,11 @@ public class UpsertCompactionTaskGeneratorTest {
+ "\"segmentName\" : \"" + _completedSegment2.getSegmentName() + "\", " + "\"segmentCrc\" : \""
+ _completedSegment2.getCrc() + "\","
+ "\"totalDocs\" : 10" + "}]";
- validDocIdMetadataInfo = JsonUtils.stringToObject(json, new TypeReference<ArrayList<ValidDocIdMetadataInfo>>() {
+ validDocIdsMetadataInfo = JsonUtils.stringToObject(json, new TypeReference<ArrayList<ValidDocIdsMetadataInfo>>() {
});
segmentSelectionResult =
- UpsertCompactionTaskGenerator.processValidDocIdMetadata(compactionConfigs, _completedSegmentsMap,
- validDocIdMetadataInfo);
+ UpsertCompactionTaskGenerator.processValidDocIdsMetadata(compactionConfigs, _completedSegmentsMap,
+ validDocIdsMetadataInfo);
// completedSegment is supposed to be filtered out
Assert.assertEquals(segmentSelectionResult.getSegmentsForCompaction().size(), 0);
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 52702c831f..5e458bbb28 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -476,7 +476,7 @@ public class TablesResource {
public ValidDocIdsBitmapResponse downloadValidDocIdsBitmap(
@ApiParam(value = "Name of the table with type REALTIME", required = true, example = "myTable_REALTIME")
@PathParam("tableNameWithType") String tableNameWithType,
- @ApiParam(value = "Valid doc id type", example = "SNAPSHOT|IN_MEMORY|IN_MEMORY_WITH_DELETE")
+ @ApiParam(value = "Valid doc ids type")
@QueryParam("validDocIdsType") String validDocIdsType,
@ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
@Context HttpHeaders httpHeaders) {
@@ -508,8 +508,10 @@ public class TablesResource {
MutableRoaringBitmap validDocIdSnapshot = validDocIdsSnapshotPair.getRight();
if (validDocIdSnapshot == null) {
- String msg = String.format("Missing validDocIds for table %s segment %s does not exist", tableNameWithType,
- segmentDataManager.getSegmentName());
+ String msg = String.format(
+ "Found that validDocIds is missing while fetching validDocIds for table %s segment %s while "
+ + "reading the validDocIds with validDocIdType %s",
+ tableNameWithType, segmentDataManager.getSegmentName(), validDocIdsType);
LOGGER.warn(msg);
throw new WebApplicationException(msg, Response.Status.NOT_FOUND);
}
@@ -536,7 +538,7 @@ public class TablesResource {
@ApiParam(value = "Name of the table with type REALTIME", required = true, example = "myTable_REALTIME")
@PathParam("tableNameWithType") String tableNameWithType,
@ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
- @ApiParam(value = "Valid doc id type", example = "SNAPSHOT|IN_MEMORY|IN_MEMORY_WITH_DELETE")
+ @ApiParam(value = "Valid doc ids type")
@QueryParam("validDocIdsType") String validDocIdsType, @Context HttpHeaders httpHeaders) {
segmentName = URIUtils.decode(segmentName);
LOGGER.info("Received a request to download validDocIds for segment {} table {}", segmentName, tableNameWithType);
@@ -564,8 +566,10 @@ public class TablesResource {
getValidDocIds(indexSegment, validDocIdsType);
MutableRoaringBitmap validDocIdSnapshot = validDocIdSnapshotPair.getRight();
if (validDocIdSnapshot == null) {
- String msg = String.format("Missing validDocIds for table %s segment %s does not exist", tableNameWithType,
- segmentDataManager.getSegmentName());
+ String msg = String.format(
+ "Found that validDocIds is missing while fetching validDocIds for table %s segment %s while "
+ + "reading the validDocIds with validDocIdType %s",
+ tableNameWithType, segmentDataManager.getSegmentName(), validDocIdsType);
LOGGER.warn(msg);
throw new WebApplicationException(msg, Response.Status.NOT_FOUND);
}
@@ -579,6 +583,7 @@ public class TablesResource {
}
}
+ @Deprecated
@GET
@Path("/tables/{tableNameWithType}/validDocIdMetadata")
@Produces(MediaType.APPLICATION_JSON)
@@ -588,36 +593,36 @@ public class TablesResource {
response = ErrorInfo.class), @ApiResponse(code = 404, message = "Table or segment not found", response =
ErrorInfo.class)
})
- public String getValidDocIdMetadata(
+ public String getValidDocIdsMetadata(
@ApiParam(value = "Table name including type", required = true, example = "myTable_REALTIME")
@PathParam("tableNameWithType") String tableNameWithType,
- @ApiParam(value = "Valid doc id type", example = "SNAPSHOT|IN_MEMORY|IN_MEMORY_WITH_DELETE")
+ @ApiParam(value = "Valid doc ids type")
@QueryParam("validDocIdsType") String validDocIdsType,
@ApiParam(value = "Segment name", allowMultiple = true) @QueryParam("segmentNames") List<String> segmentNames) {
return ResourceUtils.convertToJsonString(
- processValidDocIdMetadata(tableNameWithType, segmentNames, validDocIdsType));
+ processValidDocIdsMetadata(tableNameWithType, segmentNames, validDocIdsType));
}
@POST
- @Path("/tables/{tableNameWithType}/validDocIdMetadata")
+ @Path("/tables/{tableNameWithType}/validDocIdsMetadata")
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Provides segment validDocId metadata", notes = "Provides segment validDocId metadata")
+ @ApiOperation(value = "Provides segment valid doc ids metadata", notes = "Provides segment valid doc ids metadata")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error",
response = ErrorInfo.class), @ApiResponse(code = 404, message = "Table or segment not found", response =
ErrorInfo.class)
})
- public String getValidDocIdMetadata(
+ public String getValidDocIdsMetadata(
@ApiParam(value = "Table name including type", required = true, example = "myTable_REALTIME")
@PathParam("tableNameWithType") String tableNameWithType,
- @ApiParam(value = "Valid doc id type", example = "SNAPSHOT|IN_MEMORY|IN_MEMORY_WITH_DELETE")
+ @ApiParam(value = "Valid doc ids type")
@QueryParam("validDocIdsType") String validDocIdsType, TableSegments tableSegments) {
List<String> segmentNames = tableSegments.getSegments();
return ResourceUtils.convertToJsonString(
- processValidDocIdMetadata(tableNameWithType, segmentNames, validDocIdsType));
+ processValidDocIdsMetadata(tableNameWithType, segmentNames, validDocIdsType));
}
- private List<Map<String, Object>> processValidDocIdMetadata(String tableNameWithType, List<String> segments,
+ private List<Map<String, Object>> processValidDocIdsMetadata(String tableNameWithType, List<String> segments,
String validDocIdsType) {
TableDataManager tableDataManager =
ServerResourceUtils.checkGetTableDataManager(_serverInstance, tableNameWithType);
@@ -633,7 +638,7 @@ public class TablesResource {
Response.Status.NOT_FOUND);
}
}
- List<Map<String, Object>> allValidDocIdMetadata = new ArrayList<>();
+ List<Map<String, Object>> allValidDocIdsMetadata = new ArrayList<>();
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
try {
IndexSegment indexSegment = segmentDataManager.getSegment();
@@ -652,30 +657,32 @@ public class TablesResource {
final Pair<ValidDocIdsType, MutableRoaringBitmap> validDocIdSnapshotPair =
getValidDocIds(indexSegment, validDocIdsType);
String finalValidDocIdsType = validDocIdSnapshotPair.getLeft().toString();
- MutableRoaringBitmap validDocIdSnapshot = validDocIdSnapshotPair.getRight();
- if (validDocIdSnapshot == null) {
- String msg = String.format("Missing validDocIds for table %s segment %s does not exist", tableNameWithType,
- segmentDataManager.getSegmentName());
+ MutableRoaringBitmap validDocIdsSnapshot = validDocIdSnapshotPair.getRight();
+ if (validDocIdsSnapshot == null) {
+ String msg = String.format(
+ "Found that validDocIds is missing while processing validDocIdsMetadata for table %s segment %s while "
+ + "reading the validDocIds with validDocIdType %s",
+ tableNameWithType, segmentDataManager.getSegmentName(), validDocIdsType);
LOGGER.warn(msg);
- throw new WebApplicationException(msg, Response.Status.NOT_FOUND);
+ continue;
}
- Map<String, Object> validDocIdMetadata = new HashMap<>();
+ Map<String, Object> validDocIdsMetadata = new HashMap<>();
int totalDocs = indexSegment.getSegmentMetadata().getTotalDocs();
- int totalValidDocs = validDocIdSnapshot.getCardinality();
+ int totalValidDocs = validDocIdsSnapshot.getCardinality();
int totalInvalidDocs = totalDocs - totalValidDocs;
- validDocIdMetadata.put("segmentName", segmentDataManager.getSegmentName());
- validDocIdMetadata.put("totalDocs", totalDocs);
- validDocIdMetadata.put("totalValidDocs", totalValidDocs);
- validDocIdMetadata.put("totalInvalidDocs", totalInvalidDocs);
- validDocIdMetadata.put("segmentCrc", indexSegment.getSegmentMetadata().getCrc());
- validDocIdMetadata.put("validDocIdsType", finalValidDocIdsType);
- allValidDocIdMetadata.add(validDocIdMetadata);
+ validDocIdsMetadata.put("segmentName", segmentDataManager.getSegmentName());
+ validDocIdsMetadata.put("totalDocs", totalDocs);
+ validDocIdsMetadata.put("totalValidDocs", totalValidDocs);
+ validDocIdsMetadata.put("totalInvalidDocs", totalInvalidDocs);
+ validDocIdsMetadata.put("segmentCrc", indexSegment.getSegmentMetadata().getCrc());
+ validDocIdsMetadata.put("validDocIdsType", finalValidDocIdsType);
+ allValidDocIdsMetadata.add(validDocIdsMetadata);
} finally {
tableDataManager.releaseSegment(segmentDataManager);
}
}
- return allValidDocIdMetadata;
+ return allValidDocIdsMetadata;
}
private Pair<ValidDocIdsType, MutableRoaringBitmap> getValidDocIds(IndexSegment indexSegment,
diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index bca644fb16..e4db9cd419 100644
--- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -298,6 +298,7 @@ public class TablesResourceTest extends BaseResourceTest {
Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
}
+ @Deprecated
@Test
public void testValidDocIdMetadata()
throws IOException {
@@ -323,7 +324,7 @@ public class TablesResourceTest extends BaseResourceTest {
}
@Test
- public void testValidDocIdMetadataPost()
+ public void testValidDocIdsMetadataPost()
throws IOException {
IndexSegment segment = _realtimeIndexSegments.get(0);
// Verify the content of the downloaded snapshot from a realtime table.
@@ -334,18 +335,18 @@ public class TablesResourceTest extends BaseResourceTest {
List<String> segments = List.of(segment.getSegmentName());
TableSegments tableSegments = new TableSegments(segments);
- String validDocIdMetadataPath =
- "/tables/" + TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME) + "/validDocIdMetadata";
+ String validDocIdsMetadataPath =
+ "/tables/" + TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME) + "/validDocIdsMetadata";
String response =
- _webTarget.path(validDocIdMetadataPath).queryParam("segmentNames", segment.getSegmentName()).request()
+ _webTarget.path(validDocIdsMetadataPath).queryParam("segmentNames", segment.getSegmentName()).request()
.post(Entity.json(tableSegments), String.class);
- JsonNode validDocIdMetadata = JsonUtils.stringToJsonNode(response).get(0);
+ JsonNode validDocIdsMetadata = JsonUtils.stringToJsonNode(response).get(0);
- Assert.assertEquals(validDocIdMetadata.get("totalDocs").asInt(), 100000);
- Assert.assertEquals(validDocIdMetadata.get("totalValidDocs").asInt(), 8);
- Assert.assertEquals(validDocIdMetadata.get("totalInvalidDocs").asInt(), 99992);
- Assert.assertEquals(validDocIdMetadata.get("segmentCrc").asText(), "1265679343");
- Assert.assertEquals(validDocIdMetadata.get("validDocIdsType").asText(), "SNAPSHOT");
+ Assert.assertEquals(validDocIdsMetadata.get("totalDocs").asInt(), 100000);
+ Assert.assertEquals(validDocIdsMetadata.get("totalValidDocs").asInt(), 8);
+ Assert.assertEquals(validDocIdsMetadata.get("totalInvalidDocs").asInt(), 99992);
+ Assert.assertEquals(validDocIdsMetadata.get("segmentCrc").asText(), "1265679343");
+ Assert.assertEquals(validDocIdsMetadata.get("validDocIdsType").asText(), "SNAPSHOT");
}
// Verify metadata file from segments.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org