You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/04/08 16:36:12 UTC
[hadoop] branch trunk updated: HDDS-1340. Add List Containers API
for Recon (#648)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 69e3745 HDDS-1340. Add List Containers API for Recon (#648)
69e3745 is described below
commit 69e3745b86aaa85e7d34dc41b5bbf2561c226f7a
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Mon Apr 8 09:36:06 2019 -0700
HDDS-1340. Add List Containers API for Recon (#648)
---
.../ozone/recon/api/ContainerKeyService.java | 23 +++++++++-
.../ozone/recon/api/types/ContainerMetadata.java | 33 ++++-----------
.../recon/spi/ContainerDBServiceProvider.java | 9 ++++
.../spi/impl/ContainerDBServiceProviderImpl.java | 30 ++++++++++++-
.../recon/WEB-INF}/web.xml | 0
.../ozone/recon/api/TestContainerKeyService.java | 49 ++++++++++++++++++++--
6 files changed, 112 insertions(+), 32 deletions(-)
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
index 822c2ea..35ae724 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
@@ -39,8 +39,8 @@ import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.recon.ReconServer;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -56,7 +56,8 @@ import org.slf4j.LoggerFactory;
@Produces(MediaType.APPLICATION_JSON)
public class ContainerKeyService {
- private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerKeyService.class);
@Inject
private ContainerDBServiceProvider containerDBServiceProvider;
@@ -65,6 +66,24 @@ public class ContainerKeyService {
private ReconOMMetadataManager omMetadataManager;
/**
+ * Return @{@link org.apache.hadoop.ozone.recon.api.types.ContainerMetadata}
+ * for all the containers.
+ *
+ * @return {@link Response}
+ */
+ @GET
+ public Response getContainers() {
+ Map<Long, ContainerMetadata> containersMap;
+ try {
+ containersMap = containerDBServiceProvider.getContainers();
+ } catch (IOException ioEx) {
+ throw new WebApplicationException(ioEx,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return Response.ok(containersMap.values()).build();
+ }
+
+ /**
* Return @{@link org.apache.hadoop.ozone.recon.api.types.KeyMetadata} for
* all keys that belong to the container identified by the id param.
*
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java
index 29af448..381f2ff 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java
@@ -27,32 +27,22 @@ import javax.xml.bind.annotation.XmlElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class ContainerMetadata {
- @XmlElement(name = "ContainerId")
- private long containerId;
-
- @XmlElement(name = "UserBytes")
- private long usedBytes;
+ @XmlElement(name = "ContainerID")
+ private long containerID;
@XmlElement(name = "NumberOfKeys")
private long numberOfKeys;
- @XmlElement(name = "Owner")
- private String owner;
-
- public long getContainerId() {
- return containerId;
- }
-
- public void setContainerId(long containerId) {
- this.containerId = containerId;
+ public ContainerMetadata(long containerID) {
+ this.containerID = containerID;
}
- public long getUsedBytes() {
- return usedBytes;
+ public long getContainerID() {
+ return containerID;
}
- public void setUsedBytes(long usedBytes) {
- this.usedBytes = usedBytes;
+ public void setContainerID(long containerID) {
+ this.containerID = containerID;
}
public long getNumberOfKeys() {
@@ -63,11 +53,4 @@ public class ContainerMetadata {
this.numberOfKeys = numberOfKeys;
}
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
}
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
index 322e19e..a1044ec 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
/**
* The Recon Container DB Service interface.
@@ -66,4 +67,12 @@ public interface ContainerDBServiceProvider {
*/
Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
throws IOException;
+
+ /**
+ * Get a Map of containerID, containerMetadata of all Containers.
+ *
+ * @return Map of containerID -> containerMetadata.
+ * @throws IOException
+ */
+ Map<Long, ContainerMetadata> getContainers() throws IOException;
}
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
index ade35f0..3a20e82 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
import javax.inject.Inject;
@@ -32,6 +33,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.Table;
@@ -144,7 +146,7 @@ public class ContainerDBServiceProviderImpl
while (containerIterator.hasNext()) {
KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
ContainerKeyPrefix containerKeyPrefix = keyValue.getKey();
- //The prefix seek only guarantees that the iterator's head will be
+ // The prefix seek only guarantees that the iterator's head will be
// positioned at the first prefix match. We still have to check the key
// prefix.
if (containerKeyPrefix.getContainerId() == containerId) {
@@ -163,4 +165,30 @@ public class ContainerDBServiceProviderImpl
return prefixes;
}
+ /**
+ * Iterate the DB to construct a Map of containerID -> containerMetadata.
+ *
+ * @return Map of containerID -> containerMetadata.
+ * @throws IOException
+ */
+ @Override
+ public Map<Long, ContainerMetadata> getContainers() throws IOException {
+ Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
+ TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
+ Integer>> containerIterator = containerKeyTable.iterator();
+ while (containerIterator.hasNext()) {
+ KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
+ Long containerID = keyValue.getKey().getContainerId();
+ Integer numberOfKeys = keyValue.getValue();
+
+ // initialize containerMetadata with 0 as number of keys.
+ containers.computeIfAbsent(containerID, ContainerMetadata::new);
+ // increment number of keys for the containerID
+ ContainerMetadata containerMetadata = containers.get(containerID);
+ containerMetadata.setNumberOfKeys(containerMetadata.getNumberOfKeys() +
+ numberOfKeys);
+ containers.put(containerID, containerMetadata);
+ }
+ return containers;
+ }
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml b/hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/WEB-INF/web.xml
similarity index 100%
rename from hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml
rename to hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/WEB-INF/web.xml
diff --git a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
index ad2d6a2..bd7ea84 100644
--- a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
+++ b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
@@ -112,10 +113,6 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
});
containerDbServiceProvider = injector.getInstance(
ContainerDBServiceProvider.class);
- }
-
- @Test
- public void testGetKeysForContainer() throws Exception {
//Write Data to OM
Pipeline pipeline = getRandomPipeline();
@@ -162,6 +159,25 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
writeDataToOm(omMetadataManager,
"key_two", "bucketOne", "sampleVol", infoGroups);
+ List<OmKeyLocationInfo> omKeyLocationInfoList2 = new ArrayList<>();
+ BlockID blockID5 = new BlockID(2, 2);
+ OmKeyLocationInfo omKeyLocationInfo5 = getOmKeyLocationInfo(blockID5,
+ pipeline);
+ omKeyLocationInfoList2.add(omKeyLocationInfo5);
+
+ BlockID blockID6 = new BlockID(2, 3);
+ OmKeyLocationInfo omKeyLocationInfo6 = getOmKeyLocationInfo(blockID6,
+ pipeline);
+ omKeyLocationInfoList2.add(omKeyLocationInfo6);
+
+ OmKeyLocationInfoGroup omKeyLocationInfoGroup2 = new
+ OmKeyLocationInfoGroup(0, omKeyLocationInfoList2);
+
+ //key = key_three, Blocks = [ {CID = 2, LID = 2}, {CID = 2, LID = 3} ]
+ writeDataToOm(omMetadataManager,
+ "key_three", "bucketOne", "sampleVol",
+ Collections.singletonList(omKeyLocationInfoGroup2));
+
//Take snapshot of OM DB and copy over to Recon OM DB.
DBCheckpoint checkpoint = omMetadataManager.getStore()
.getCheckpoint(true);
@@ -176,6 +192,10 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
ContainerKeyMapperTask containerKeyMapperTask = new ContainerKeyMapperTask(
ozoneManagerServiceProvider, containerDbServiceProvider);
containerKeyMapperTask.run();
+ }
+
+ @Test
+ public void testGetKeysForContainer() {
Response response = containerKeyService.getKeysForContainer(1L);
@@ -208,6 +228,27 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
assertTrue(keyMetadataList.isEmpty());
}
+ @Test
+ public void testGetContainers() {
+
+ Response response = containerKeyService.getContainers();
+
+ List<ContainerMetadata> containers = new ArrayList<>(
+ (Collection<ContainerMetadata>) response.getEntity());
+
+ assertTrue(containers.size() == 2);
+
+ Iterator<ContainerMetadata> iterator = containers.iterator();
+
+ ContainerMetadata containerMetadata = iterator.next();
+ assertTrue(containerMetadata.getContainerID() == 1L);
+ assertTrue(containerMetadata.getNumberOfKeys() == 3L);
+
+ containerMetadata = iterator.next();
+ assertTrue(containerMetadata.getContainerID() == 2L);
+ assertTrue(containerMetadata.getNumberOfKeys() == 2L);
+ }
+
/**
* Get Test OzoneConfiguration instance.
* @return OzoneConfiguration
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org