You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "ashishkumar50 (via GitHub)" <gi...@apache.org> on 2023/06/19 10:32:31 UTC
[GitHub] [ozone] ashishkumar50 commented on a diff in pull request #4876: HDDS-8701. Recon - Improve Mismatched container info API (containers/v1/mismatch).
ashishkumar50 commented on code in PR #4876:
URL: https://github.com/apache/ozone/pull/4876#discussion_r1233865002
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java:
##########
@@ -498,72 +535,138 @@ private List<ContainerBlockMetadata> getBlocks(
return blockIds;
}
+ /**
+ * Retrieves the container mismatch insights.
+ *
+ * This method returns a list of ContainerDiscrepancyInfo objects representing
+ * the containers that are missing in either the Ozone Manager (OM) or the
+ * Storage Container Manager (SCM), based on the provided filter parameter.
+ * The returned list is paginated based on the provided limit and prevKey
+ * parameters.
+ *
+ * @param limit The maximum number of container discrepancies to return.
+ * @param prevKey The container ID after which the results are returned.
+ * @param missingIn The missing filter parameter to specify if it's
+ * "OM" or "SCM" missing containers to be returned.
+ */
@GET
@Path("/mismatch")
- public Response getContainerMisMatchInsights() {
+ public Response getContainerMisMatchInsights(
+ @DefaultValue(DEFAULT_FETCH_COUNT)
+ @QueryParam(RECON_QUERY_LIMIT) int limit,
+ @DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
+ @QueryParam(RECON_QUERY_PREVKEY) long prevKey,
+ @DefaultValue(DEFAULT_FILTER_FOR_MISSING_CONTAINERS)
+ @QueryParam(RECON_QUERY_FILTER) String missingIn) {
+ if (prevKey < 0 || limit < 0) {
+ // Send back an empty response
+ return Response.status(Response.Status.NOT_ACCEPTABLE).build();
+ }
+
List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList =
new ArrayList<>();
try {
Map<Long, ContainerMetadata> omContainers =
reconContainerMetadataManager.getContainers(-1, -1);
List<Long> scmNonDeletedContainers =
containerManager.getContainers().stream()
- .filter(containerInfo -> !(containerInfo.getState() ==
- HddsProtos.LifeCycleState.DELETED))
- .map(containerInfo -> containerInfo.getContainerID()).collect(
- Collectors.toList());
-
- // Filter list of container Ids which are present in OM but not in SCM.
- List<Map.Entry<Long, ContainerMetadata>> notSCMContainers =
- omContainers.entrySet().stream().filter(containerMetadataEntry ->
- !(scmNonDeletedContainers.contains(
- containerMetadataEntry.getKey())))
- .collect(
- Collectors.toList());
+ .filter(containerInfo -> containerInfo.getState() !=
+ HddsProtos.LifeCycleState.DELETED)
+ .map(containerInfo -> containerInfo.getContainerID())
+ .collect(Collectors.toList());
+ DataFilter dataFilter = DataFilter.fromValue(missingIn.toUpperCase());
+
+ switch (dataFilter) {
+
+ case SCM:
+ List<Map.Entry<Long, ContainerMetadata>> notSCMContainers =
+ omContainers.entrySet().stream()
+ .filter(
+ containerMetadataEntry -> !scmNonDeletedContainers.contains(
+ containerMetadataEntry.getKey()))
+ .collect(Collectors.toList());
+
+ if (prevKey > 0) {
+ int index = 0;
+ while (index < notSCMContainers.size() &&
+ notSCMContainers.get(index).getKey() <= prevKey) {
+ index++;
+ }
+ if (index < notSCMContainers.size()) {
+ notSCMContainers = notSCMContainers.subList(index,
+ Math.min(index + limit, notSCMContainers.size()));
+ } else {
+ notSCMContainers = Collections.emptyList();
+ }
+ } else {
+ notSCMContainers = notSCMContainers.subList(0,
+ Math.min(limit, notSCMContainers.size()));
+ }
- notSCMContainers.forEach(nonSCMContainer -> {
- ContainerDiscrepancyInfo containerDiscrepancyInfo =
- new ContainerDiscrepancyInfo();
- containerDiscrepancyInfo.setContainerID(nonSCMContainer.getKey());
- containerDiscrepancyInfo.setNumberOfKeys(
- nonSCMContainer.getValue().getNumberOfKeys());
- containerDiscrepancyInfo.setPipelines(nonSCMContainer.getValue()
- .getPipelines());
- containerDiscrepancyInfo.setExistsAt("OM");
- containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
- });
+ notSCMContainers.forEach(nonSCMContainer -> {
+ ContainerDiscrepancyInfo containerDiscrepancyInfo =
+ new ContainerDiscrepancyInfo();
+ containerDiscrepancyInfo.setContainerID(nonSCMContainer.getKey());
+ containerDiscrepancyInfo.setNumberOfKeys(
+ nonSCMContainer.getValue().getNumberOfKeys());
+ containerDiscrepancyInfo.setPipelines(
+ nonSCMContainer.getValue().getPipelines());
+ containerDiscrepancyInfo.setExistsAt("OM");
+ containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
+ });
+ break;
- // Filter list of container Ids which are present in SCM but not in OM.
- List<Long> nonOMContainers = scmNonDeletedContainers.stream()
- .filter(containerId -> !omContainers.containsKey(containerId))
- .collect(Collectors.toList());
+ case OM:
+ List<Long> nonOMContainers = scmNonDeletedContainers.stream()
+ .filter(containerId -> !omContainers.containsKey(containerId))
+ .collect(Collectors.toList());
- List<Pipeline> pipelines = new ArrayList<>();
- nonOMContainers.forEach(nonOMContainerId -> {
- ContainerDiscrepancyInfo containerDiscrepancyInfo =
- new ContainerDiscrepancyInfo();
- containerDiscrepancyInfo.setContainerID(nonOMContainerId);
- containerDiscrepancyInfo.setNumberOfKeys(0);
- PipelineID pipelineID = null;
- try {
- pipelineID = containerManager.getContainer(
- ContainerID.valueOf(nonOMContainerId))
- .getPipelineID();
-
- if (null != pipelineID) {
- pipelines.add(pipelineManager.getPipeline(pipelineID));
+ if (prevKey > 0) {
+ int index = 0;
+ while (index < nonOMContainers.size() &&
+ nonOMContainers.get(index) <= prevKey) {
+ index++;
+ }
+ if (index < nonOMContainers.size()) {
+ nonOMContainers = nonOMContainers.subList(index,
+ Math.min(index + limit, nonOMContainers.size()));
+ } else {
+ nonOMContainers = Collections.emptyList();
}
- } catch (ContainerNotFoundException e) {
- LOG.warn("Container {} not found in SCM: {}", nonOMContainerId, e);
- } catch (PipelineNotFoundException e) {
- LOG.debug("Pipeline not found for container: {} and pipelineId: {}",
- nonOMContainerId, pipelineID, e);
+ } else {
+ nonOMContainers = nonOMContainers.subList(0,
+ Math.min(limit, nonOMContainers.size()));
}
- containerDiscrepancyInfo.setPipelines(pipelines);
- containerDiscrepancyInfo.setExistsAt("SCM");
- containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
- });
+ List<Pipeline> pipelines = new ArrayList<>();
+ nonOMContainers.forEach(nonOMContainerId -> {
+ ContainerDiscrepancyInfo containerDiscrepancyInfo =
+ new ContainerDiscrepancyInfo();
+ containerDiscrepancyInfo.setContainerID(nonOMContainerId);
+ containerDiscrepancyInfo.setNumberOfKeys(0);
+ PipelineID pipelineID = null;
+ try {
+ pipelineID = containerManager.getContainer(
+ ContainerID.valueOf(nonOMContainerId)).getPipelineID();
+ if (pipelineID != null) {
+ pipelines.add(pipelineManager.getPipeline(pipelineID));
+ }
+ } catch (ContainerNotFoundException e) {
Review Comment:
Ideally container should not be present in SCM when it is not present in OM.
Discrepancy is only when if container is present in either of SCM or OM.
But in this place when SCM throws `ContainerNotFoundException` this means that this container has no discrepancy as this container is not present even in OM.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org