You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/05/23 17:12:59 UTC
[11/18] hadoop git commit: HDDS-85. Send Container State Info while
sending the container report from Datanode to SCM. Contributed by Shashikant
Banerjee.
HDDS-85. Send Container State Info while sending the container report from Datanode to SCM. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fed2bef6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fed2bef6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fed2bef6
Branch: refs/heads/HDDS-48
Commit: fed2bef647d9a15fe020ad5d3bb89fcb77ed30e6
Parents: 745f203
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Wed May 23 14:15:35 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Wed May 23 14:15:35 2018 +0530
----------------------------------------------------------------------
.../main/proto/DatanodeContainerProtocol.proto | 1 +
.../container/common/helpers/ContainerData.java | 8 ++++
.../common/impl/ContainerManagerImpl.java | 45 ++++++++++++++++++--
.../common/interfaces/ContainerManager.java | 2 +-
.../commandhandler/ContainerReportHandler.java | 4 +-
.../container/ozoneimpl/OzoneContainer.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
7 files changed, 57 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 1138297..53da18a 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -131,6 +131,7 @@ enum Result {
UNCLOSED_CONTAINER_IO = 25;
DELETE_ON_OPEN_CONTAINER = 26;
CLOSED_CONTAINER_RETRY = 27;
+ INVALID_CONTAINER_STATE = 28;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
index 14ee33a..d1746f2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
@@ -340,6 +340,14 @@ public class ContainerData {
}
/**
+ * checks if the container is closed.
+ * @return - boolean
+ */
+ public synchronized boolean isClosed() {
+ return ContainerLifeCycleState.CLOSED == state;
+ }
+
+ /**
* Marks this container as closed.
*/
public synchronized void closeContainer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index faee5d0..9355364 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -100,6 +102,8 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.UNCLOSED_CONTAINER_IO;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.UNSUPPORTED_REQUEST;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
+ Result.INVALID_CONTAINER_STATE;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
/**
@@ -707,6 +711,39 @@ public class ContainerManagerImpl implements ContainerManager {
}
/**
+ * Returns LifeCycle State of the container
+ * @param containerID - Id of the container
+ * @return LifeCycle State of the container
+ * @throws StorageContainerException
+ */
+ private HddsProtos.LifeCycleState getState(long containerID)
+ throws StorageContainerException {
+ LifeCycleState state;
+ final ContainerData data = containerMap.get(containerID);
+ if (data == null) {
+ throw new StorageContainerException(
+ "Container status not found: " + containerID, CONTAINER_NOT_FOUND);
+ }
+ switch (data.getState()) {
+ case OPEN:
+ state = LifeCycleState.OPEN;
+ break;
+ case CLOSING:
+ state = LifeCycleState.CLOSING;
+ break;
+ case CLOSED:
+ state = LifeCycleState.CLOSED;
+ break;
+ default:
+ throw new StorageContainerException(
+ "Invalid Container state found: " + containerID,
+ INVALID_CONTAINER_STATE);
+ }
+
+ return state;
+ }
+
+ /**
* Supports clean shutdown of container.
*
* @throws IOException
@@ -835,14 +872,14 @@ public class ContainerManagerImpl implements ContainerManager {
* @throws IOException
*/
@Override
- public List<ContainerData> getContainerReports() throws IOException {
+ public List<ContainerData> getClosedContainerReports() throws IOException {
LOG.debug("Starting container report iteration.");
// No need for locking since containerMap is a ConcurrentSkipListMap
// And we can never get the exact state since close might happen
// after we iterate a point.
return containerMap.entrySet().stream()
.filter(containerData ->
- !containerData.getValue().isOpen())
+ containerData.getValue().isClosed())
.map(containerData -> containerData.getValue())
.collect(Collectors.toList());
}
@@ -870,6 +907,7 @@ public class ContainerManagerImpl implements ContainerManager {
.setType(ContainerReportsRequestProto.reportType.fullReport);
for (ContainerData container: containers) {
+ long containerId = container.getContainerID();
StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
ciBuilder.setContainerID(container.getContainerID())
@@ -879,7 +917,8 @@ public class ContainerManagerImpl implements ContainerManager {
.setReadCount(container.getReadCount())
.setWriteCount(container.getWriteCount())
.setReadBytes(container.getReadBytes())
- .setWriteBytes(container.getWriteBytes());
+ .setWriteBytes(container.getWriteBytes())
+ .setState(getState(containerId));
crBuilder.addReports(ciBuilder.build());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
index 3a1a73d..ba70953 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
@@ -185,7 +185,7 @@ public interface ContainerManager extends RwLock {
* @return List of all closed containers.
* @throws IOException
*/
- List<ContainerData> getContainerReports() throws IOException;
+ List<ContainerData> getClosedContainerReports() throws IOException;
/**
* Increase pending deletion blocks count number of specified container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
index ba6b418..fbea290 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
@@ -63,13 +63,13 @@ public class ContainerReportHandler implements CommandHandler {
invocationCount++;
long startTime = Time.monotonicNow();
try {
- ContainerReportsRequestProto contianerReport =
+ ContainerReportsRequestProto containerReport =
container.getContainerReport();
// TODO : We send this report to all SCMs.Check if it is enough only to
// send to the leader once we have RAFT enabled SCMs.
for (EndpointStateMachine endPoint : connectionManager.getValues()) {
- endPoint.getEndPoint().sendContainerReport(contianerReport);
+ endPoint.getEndPoint().sendContainerReport(containerReport);
}
} catch (IOException ex) {
LOG.error("Unable to process the Container Report command.", ex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index b497cdc..6758479 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -265,8 +265,8 @@ public class OzoneContainer {
* @return - List of closed containers.
* @throws IOException
*/
- public List<ContainerData> getContainerReports() throws IOException {
- return this.manager.getContainerReports();
+ public List<ContainerData> getClosedContainerReports() throws IOException {
+ return this.manager.getClosedContainerReports();
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 89ee673..4975fd3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -307,7 +307,7 @@ public class TestContainerPersistence {
}
// The container report only returns reports of closed containers.
- List<ContainerData> reports = containerManager.getContainerReports();
+ List<ContainerData> reports = containerManager.getClosedContainerReports();
Assert.assertEquals(4, reports.size());
for(ContainerData report : reports) {
long actualContainerID = report.getContainerID();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org