You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/02/08 14:25:07 UTC
[ozone] branch master updated: HDDS-6056. Recon /containers endpoint should return SCM container data instead of OM container data. (#4158)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5daca052cc HDDS-6056. Recon /containers endpoint should return SCM container data instead of OM container data. (#4158)
5daca052cc is described below
commit 5daca052ccc3e2d8ef88260eaaf9b369eac9e807
Author: Arafat2198 <98...@users.noreply.github.com>
AuthorDate: Wed Feb 8 19:55:00 2023 +0530
HDDS-6056. Recon /containers endpoint should return SCM container data instead of OM container data. (#4158)
---
.../ozone/recon/TestReconWithOzoneManager.java | 57 ------------------
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 43 +++++++++-----
.../ozone/recon/api/TestContainerEndpoint.java | 68 +++++++++++-----------
3 files changed, 61 insertions(+), 107 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
index 46b763b0e1..328750eb7b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
@@ -189,12 +189,6 @@ public class TestReconWithOzoneManager {
OmKeyInfo keyInfo1 =
metadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
- TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- omKeyValueTableIterator =
- metadataManager.getKeyTable(getBucketLayout()).iterator();
-
- long omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
-
// verify if OM has /vol0/bucket0/key0
Assert.assertEquals("vol0", keyInfo1.getVolumeName());
Assert.assertEquals("bucket0", keyInfo1.getBucketName());
@@ -203,21 +197,6 @@ public class TestReconWithOzoneManager {
cluster.getReconServer().getOzoneManagerServiceProvider();
impl.syncDataFromOM();
OzoneManagerSyncMetrics metrics = impl.getMetrics();
-
- // HTTP call to /api/containers
- String containerResponse = makeHttpCall(containerKeyServiceURL);
- long reconMetadataContainerCount =
- getReconContainerCount(containerResponse);
- // verify count of keys after full snapshot
- Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
-
- // verify if Recon Metadata captures vol0/bucket0/key0 info in container0
- LinkedTreeMap containerResponseMap = getContainerResponseMap(
- containerResponse, 0);
- Assert.assertEquals(0,
- (long)(double) containerResponseMap.get("ContainerID"));
- Assert.assertEquals(1,
- (long)(double) containerResponseMap.get("NumberOfKeys"));
// HTTP call to /api/task/status
long omLatestSeqNumber = ((RDBStore) metadataManager.getStore())
@@ -235,28 +214,10 @@ public class TestReconWithOzoneManager {
//add 4 keys to check for delta updates
addKeys(1, 5);
- omKeyValueTableIterator =
- metadataManager.getKeyTable(getBucketLayout()).iterator();
- omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
// update the next snapshot from om to verify delta updates
impl.syncDataFromOM();
- // HTTP call to /api/containers
- containerResponse = makeHttpCall(containerKeyServiceURL);
- reconMetadataContainerCount = getReconContainerCount(containerResponse);
-
- //verify count of keys
- Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
-
- //verify if Recon Metadata captures vol3/bucket3/key3 info in container3
- containerResponseMap = getContainerResponseMap(
- containerResponse, 3);
- Assert.assertEquals(3,
- (long)(double) containerResponseMap.get("ContainerID"));
- Assert.assertEquals(1,
- (long)(double) containerResponseMap.get("NumberOfKeys"));
-
// HTTP call to /api/task/status
omLatestSeqNumber = ((RDBStore) metadataManager.getStore())
.getDb().getLatestSequenceNumber();
@@ -282,28 +243,10 @@ public class TestReconWithOzoneManager {
//add 5 more keys to OM
addKeys(5, 10);
- omKeyValueTableIterator =
- metadataManager.getKeyTable(getBucketLayout()).iterator();
- omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
// get the next snapshot from om
impl.syncDataFromOM();
- // HTTP call to /api/containers
- containerResponse = makeHttpCall(containerKeyServiceURL);
- reconMetadataContainerCount = getReconContainerCount(containerResponse);
-
- // verify count of keys
- Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
-
- // verify if Recon Metadata captures vol7/bucket7/key7 info in container7
- containerResponseMap = getContainerResponseMap(
- containerResponse, 7);
- Assert.assertEquals(7,
- (long)(double) containerResponseMap.get("ContainerID"));
- Assert.assertEquals(1,
- (long)(double) containerResponseMap.get("NumberOfKeys"));
-
// HTTP call to /api/task/status
omLatestSeqNumber = ((RDBStore) metadataManager.getStore())
.getDb().getLatestSequenceNumber();
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index ce8cc70589..8059bb4c4d 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.recon.api;
import java.io.IOException;
import java.time.Instant;
+import java.util.Collection;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -49,13 +50,13 @@ import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
-import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
-import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -98,12 +99,16 @@ public class ContainerEndpoint {
}
/**
- * Return @{@link org.apache.hadoop.ozone.recon.api.types.ContainerMetadata}
+ * Return @{@link org.apache.hadoop.hdds.scm.container}
* for the containers starting from the given "prev-key" query param for the
* given "limit". The given "prev-key" is skipped from the results returned.
- *
- * @param limit max no. of containers to get.
* @param prevKey the containerID after which results are returned.
+ * start containerID, >=0,
+ * start searching at the head if 0.
+ * @param limit max no. of containers to get.
+ * count must be >= 0
+ * Usually the count will be replace with a very big
+ * value instead of being unlimited in case the db is very big.
* @return {@link Response}
*/
@GET
@@ -112,21 +117,29 @@ public class ContainerEndpoint {
int limit,
@DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
@QueryParam(RECON_QUERY_PREVKEY) long prevKey) {
- Map<Long, ContainerMetadata> containersMap;
- long containersCount;
- try {
- containersMap =
- reconContainerMetadataManager.getContainers(limit, prevKey);
- containersCount = reconContainerMetadataManager.getCountForContainers();
- } catch (IOException ioEx) {
- throw new WebApplicationException(ioEx,
- Response.Status.INTERNAL_SERVER_ERROR);
+ if (limit < 0 || prevKey < 0) {
+ // Send back an empty response
+ return Response.status(Response.Status.NOT_ACCEPTABLE).build();
}
+ long containersCount;
+ Collection<ContainerMetadata> containerMetaDataList =
+ containerManager.getContainers(ContainerID.valueOf(prevKey), limit)
+ .stream()
+ .map(container -> {
+ ContainerMetadata containerMetadata =
+ new ContainerMetadata(container.getContainerID());
+ containerMetadata.setNumberOfKeys(container.getNumberOfKeys());
+ return containerMetadata;
+ })
+ .collect(Collectors.toList());
+
+ containersCount = containerMetaDataList.size();
ContainersResponse containersResponse =
- new ContainersResponse(containersCount, containersMap.values());
+ new ContainersResponse(containersCount, containerMetaDataList);
return Response.ok(containersResponse).build();
}
+
/**
* Return @{@link org.apache.hadoop.ozone.recon.api.types.KeyMetadata} for
* all keys that belong to the container identified by the id param
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 5d23927b62..c383c92589 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandom
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -328,15 +329,17 @@ public class TestContainerEndpoint {
}
@Test
- public void testGetContainers() {
- Response response = containerEndpoint.getContainers(-1, 0L);
+ public void testGetContainers() throws IOException, TimeoutException {
+ putContainerInfos(5);
+
+ Response response = containerEndpoint.getContainers(10, 0L);
ContainersResponse responseObject =
(ContainersResponse) response.getEntity();
ContainersResponse.ContainersResponseData data =
responseObject.getContainersResponseData();
- assertEquals(2, data.getTotalCount());
+ assertEquals(5, data.getTotalCount());
List<ContainerMetadata> containers = new ArrayList<>(data.getContainers());
@@ -344,34 +347,41 @@ public class TestContainerEndpoint {
ContainerMetadata containerMetadata = iterator.next();
assertEquals(1L, containerMetadata.getContainerID());
- // Number of keys for CID:1 should be 3 because of two different versions
- // of key_two stored in CID:1
- assertEquals(3L, containerMetadata.getNumberOfKeys());
+ // Number of keys for CID:1
+ assertEquals(5L, containerMetadata.getNumberOfKeys());
containerMetadata = iterator.next();
assertEquals(2L, containerMetadata.getContainerID());
- assertEquals(2L, containerMetadata.getNumberOfKeys());
+ assertEquals(5L, containerMetadata.getNumberOfKeys());
// test if limit works as expected
- response = containerEndpoint.getContainers(1, 0L);
+ response = containerEndpoint.getContainers(2, 0L);
responseObject = (ContainersResponse) response.getEntity();
data = responseObject.getContainersResponseData();
containers = new ArrayList<>(data.getContainers());
- assertEquals(1, containers.size());
+ // The results will be limited to 2 containers only
+ assertEquals(2, containers.size());
assertEquals(2, data.getTotalCount());
}
@Test
- public void testGetContainersWithPrevKey() {
+ public void testGetContainersWithPrevKey()
+ throws IOException, TimeoutException {
+ putContainerInfos(5);
+
+ // Test the case where prevKey = 2 and limit = 5
+ Response response = containerEndpoint.getContainers(5, 2L);
- Response response = containerEndpoint.getContainers(1, 1L);
+ // Ensure that the response object is not null
+ assertNotNull(response);
ContainersResponse responseObject =
(ContainersResponse) response.getEntity();
ContainersResponse.ContainersResponseData data =
responseObject.getContainersResponseData();
- assertEquals(2, data.getTotalCount());
+ // Ensure that the total count of containers is 4
+ assertEquals(4, data.getTotalCount());
List<ContainerMetadata> containers = new ArrayList<>(data.getContainers());
@@ -379,33 +389,21 @@ public class TestContainerEndpoint {
ContainerMetadata containerMetadata = iterator.next();
- assertEquals(1, containers.size());
+ // Ensure that the containers list size is 4
+ assertEquals(4, containers.size());
+ // Ensure that the first container ID is 2
assertEquals(2L, containerMetadata.getContainerID());
- response = containerEndpoint.getContainers(-1, 0L);
- responseObject = (ContainersResponse) response.getEntity();
- data = responseObject.getContainersResponseData();
- containers = new ArrayList<>(data.getContainers());
- assertEquals(2, containers.size());
- assertEquals(2, data.getTotalCount());
- iterator = containers.iterator();
- containerMetadata = iterator.next();
- assertEquals(1L, containerMetadata.getContainerID());
-
// test for negative cases
- response = containerEndpoint.getContainers(-1, 5L);
+ response = containerEndpoint.getContainers(-1, 0L);
responseObject = (ContainersResponse) response.getEntity();
- data = responseObject.getContainersResponseData();
- containers = new ArrayList<>(data.getContainers());
- assertEquals(0, containers.size());
- assertEquals(2, data.getTotalCount());
+ // Ensure that the response object is null when limit is negative
+ assertNull(responseObject);
- response = containerEndpoint.getContainers(-1, -1L);
+ response = containerEndpoint.getContainers(10, -1L);
responseObject = (ContainersResponse) response.getEntity();
- data = responseObject.getContainersResponseData();
- containers = new ArrayList<>(data.getContainers());
- assertEquals(2, containers.size());
- assertEquals(2, data.getTotalCount());
+ // Ensure that the response object is null when prevKey is negative
+ assertNull(responseObject);
}
@Test
@@ -432,7 +430,7 @@ public class TestContainerEndpoint {
MissingContainerMetadata containerWithLimit =
responseWithLimitObject.getContainers().stream().findFirst()
.orElse(null);
- Assert.assertNotNull(containerWithLimit);
+ assertNotNull(containerWithLimit);
Collection<MissingContainerMetadata> recordsWithLimit
= responseWithLimitObject.getContainers();
@@ -448,7 +446,7 @@ public class TestContainerEndpoint {
assertEquals(5, responseObject.getTotalCount());
MissingContainerMetadata container =
responseObject.getContainers().stream().findFirst().orElse(null);
- Assert.assertNotNull(container);
+ assertNotNull(container);
assertEquals(containerID.getId(), container.getContainerID());
assertEquals(keyCount, container.getKeys());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org