You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by an...@apache.org on 2024/02/17 06:31:32 UTC
(pinot) branch master updated: Fix bug in logging in UpsertCompaction task (#12419)
This is an automated email from the ASF dual-hosted git repository.
ankitsultana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 20f3fc6790 Fix bug in logging in UpsertCompaction task (#12419)
20f3fc6790 is described below
commit 20f3fc679013509bf24d32f9a0fbcfb441b6bbee
Author: Pratik Tibrewal <ti...@uber.com>
AuthorDate: Sat Feb 17 12:01:26 2024 +0530
Fix bug in logging in UpsertCompaction task (#12419)
---
.../api/resources/PinotTableRestletResource.java | 6 +++---
.../controller/util/CompletionServiceHelper.java | 8 ++++----
.../controller/util/ServerSegmentMetadataReader.java | 20 +++++++++++++-------
.../indexsegment/immutable/ImmutableSegmentImpl.java | 14 +++++++-------
.../pinot/server/api/resources/TablesResource.java | 2 +-
5 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index 26fb98ad50..23a405514b 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -950,8 +950,8 @@ public class PinotTableRestletResource {
@Path("tables/{tableName}/validDocIdsMetadata")
@Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA)
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Get the aggregate valid doc id metadata of all segments for a table", notes = "Get the "
- + "aggregate valid doc id metadata of all segments for a table")
+ @ApiOperation(value = "Get the aggregate validDocIds metadata of all segments for a table", notes = "Get the "
+ + "aggregate validDocIds metadata of all segments for a table")
public String getTableAggregateValidDocIdsMetadata(
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
@ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr,
@@ -959,7 +959,7 @@ public class PinotTableRestletResource {
List<String> segmentNames,
@ApiParam(value = "Valid doc ids type")
@QueryParam("validDocIdsType") @DefaultValue("SNAPSHOT") ValidDocIdsType validDocIdsType) {
- LOGGER.info("Received a request to fetch aggregate valid doc id metadata for a table {}", tableName);
+ LOGGER.info("Received a request to fetch aggregate validDocIds metadata for a table {}", tableName);
TableType tableType = Constants.validateTableType(tableTypeStr);
if (tableType == TableType.OFFLINE) {
throw new ControllerApplicationException(LOGGER, "Table type : " + tableTypeStr + " not yet supported.",
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
index f0b36b2cfa..236242660a 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
@@ -89,15 +89,15 @@ public class CompletionServiceHelper {
/**
* This method makes a MultiPost call to all given URLs and its corresponding bodies.
- * @param serverURLsAndRequestBodies server urls to send GET request.
+ * @param serverURLsAndRequestBodies server urls to send POST request.
* @param tableNameWithType table name with type suffix
* @param multiRequestPerServer it's possible that need to send multiple requests to a same server.
- * If multiRequestPerServer is set as false, return as long as one of the requests get
+ * If multiRequestPerServer is set as false, return as long as one of the requests return
* response; If multiRequestPerServer is set as true, wait until all requests
- * get response.
+ * return response.
* @param requestHeaders Headers to be set when making the http calls.
* @param timeoutMs timeout in milliseconds to wait per request.
- * @param useCase the use case initiating the multi-get request. If not null and an exception is thrown, only the
+ * @param useCase the use case initiating the multi-post request. If not null and an exception is thrown, only the
* error message and the use case are logged instead of the full stack trace.
* @return CompletionServiceResponse Map of the endpoint(server instance, or full request path if
* multiRequestPerServer is true) to the response from that endpoint.
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
index cdcc7dc78c..cc8ea07153 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -250,7 +250,7 @@ public class ServerSegmentMetadataReader {
List<ValidDocIdsMetadataInfo> validDocIdsMetadataInfos = new ArrayList<>();
int failedParses = 0;
- int returnedSegmentsCount = 0;
+ int returnedServersCount = 0;
for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
try {
String validDocIdsMetadataList = streamResponse.getValue();
@@ -258,7 +258,7 @@ public class ServerSegmentMetadataReader {
JsonUtils.stringToObject(validDocIdsMetadataList, new TypeReference<ArrayList<ValidDocIdsMetadataInfo>>() {
});
validDocIdsMetadataInfos.addAll(validDocIdsMetadataInfo);
- returnedSegmentsCount++;
+ returnedServersCount++;
} catch (Exception e) {
failedParses++;
LOGGER.error("Unable to parse server {} response due to an error: ", streamResponse.getKey(), e);
@@ -269,12 +269,18 @@ public class ServerSegmentMetadataReader {
serverURLsAndBodies.size());
}
- if (segmentNames != null && returnedSegmentsCount != segmentNames.size()) {
- LOGGER.error("Unable to get validDocIdsMetadata from all servers. Expected: {}, Actual: {}", segmentNames.size(),
- returnedSegmentsCount);
+ if (returnedServersCount != serverURLsAndBodies.size()) {
+ LOGGER.error("Unable to get validDocIdsMetadata from all servers. Expected: {}, Actual: {}",
+ serverURLsAndBodies.size(), returnedServersCount);
}
- LOGGER.info("Retrieved valid doc id metadata for {} segments from {} servers.", returnedSegmentsCount,
- serverURLsAndBodies.size());
+
+ if (segmentNames != null && !segmentNames.isEmpty() && segmentNames.size() != validDocIdsMetadataInfos.size()) {
+ LOGGER.error("Unable to get validDocIdsMetadata for all segments. Expected: {}, Actual: {}",
+ segmentNames.size(), validDocIdsMetadataInfos.size());
+ }
+
+ LOGGER.info("Retrieved validDocIds metadata for {} segments from {} servers.", validDocIdsMetadataInfos.size(),
+ returnedServersCount);
return validDocIdsMetadataInfos;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
index a214044d99..88707df9bc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -114,11 +114,11 @@ public class ImmutableSegmentImpl implements ImmutableSegment {
try {
byte[] bytes = FileUtils.readFileToByteArray(validDocIdsSnapshotFile);
MutableRoaringBitmap validDocIds = new ImmutableRoaringBitmap(ByteBuffer.wrap(bytes)).toMutableRoaringBitmap();
- LOGGER.info("Loaded valid doc ids for segment: {} with: {} valid docs", getSegmentName(),
+ LOGGER.info("Loaded validDocIds for segment: {} with: {} valid docs", getSegmentName(),
validDocIds.getCardinality());
return validDocIds;
} catch (Exception e) {
- LOGGER.warn("Caught exception while loading valid doc ids from snapshot file: {}, ignoring the snapshot",
+ LOGGER.warn("Caught exception while loading validDocIds from snapshot file: {}, ignoring the snapshot",
validDocIdsSnapshotFile);
}
}
@@ -140,10 +140,10 @@ public class ImmutableSegmentImpl implements ImmutableSegment {
}
Preconditions.checkState(tmpFile.renameTo(validDocIdsSnapshotFile),
"Failed to rename tmp snapshot file: %s to snapshot file: %s", tmpFile, validDocIdsSnapshotFile);
- LOGGER.info("Persisted valid doc ids for segment: {} with: {} valid docs", getSegmentName(),
+ LOGGER.info("Persisted validDocIds for segment: {} with: {} valid docs", getSegmentName(),
validDocIdsSnapshot.getCardinality());
} catch (Exception e) {
- LOGGER.warn("Caught exception while persisting valid doc ids to snapshot file: {}, skipping",
+ LOGGER.warn("Caught exception while persisting validDocIds to snapshot file: {}, skipping",
validDocIdsSnapshotFile, e);
}
}
@@ -157,12 +157,12 @@ public class ImmutableSegmentImpl implements ImmutableSegment {
if (validDocIdsSnapshotFile.exists()) {
try {
if (!FileUtils.deleteQuietly(validDocIdsSnapshotFile)) {
- LOGGER.warn("Cannot delete old valid doc ids snapshot file: {}, skipping", validDocIdsSnapshotFile);
+ LOGGER.warn("Cannot delete old validDocIds snapshot file: {}, skipping", validDocIdsSnapshotFile);
return;
}
- LOGGER.info("Deleted valid doc ids snapshot for segment: {}", getSegmentName());
+ LOGGER.info("Deleted validDocIds snapshot for segment: {}", getSegmentName());
} catch (Exception e) {
- LOGGER.warn("Caught exception while deleting valid doc ids snapshot file: {}, skipping",
+ LOGGER.warn("Caught exception while deleting validDocIds snapshot file: {}, skipping",
validDocIdsSnapshotFile);
}
}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index aad233bc77..0c298f8021 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -621,7 +621,7 @@ public class TablesResource {
@POST
@Path("/tables/{tableNameWithType}/validDocIdsMetadata")
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Provides segment valid doc ids metadata", notes = "Provides segment valid doc ids metadata")
+ @ApiOperation(value = "Provides segment validDocIds metadata", notes = "Provides segment validDocIds metadata")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error",
response = ErrorInfo.class), @ApiResponse(code = 404, message = "Table or segment not found", response =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org