You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/12/19 02:57:17 UTC
hadoop git commit: HDFS-12795. Ozone: SCM: Support for Container
LifeCycleState PENDING_CLOSE and LifeCycleEvent FULL_CONTAINER. Contributed
by Nandakumar.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 cadb1aae8 -> 5d17ead23
HDFS-12795. Ozone: SCM: Support for Container LifeCycleState PENDING_CLOSE and LifeCycleEvent FULL_CONTAINER. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d17ead2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d17ead2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d17ead2
Branch: refs/heads/HDFS-7240
Commit: 5d17ead237e4171192e7736bca79e45fc4b06888
Parents: cadb1aa
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Dec 18 18:48:12 2017 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Dec 18 18:48:12 2017 -0800
----------------------------------------------------------------------
.../src/main/proto/Ozone.proto | 18 +++++----
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 +++++-
.../ozone/scm/container/ContainerMapping.java | 40 ++++++++++++++------
.../scm/container/ContainerStateManager.java | 9 ++++-
.../scm/container/TestContainerMapping.java | 28 ++++++++++++++
5 files changed, 85 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d17ead2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
index cf5a1e9..9be36dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
@@ -104,19 +104,21 @@ enum LifeCycleState {
ALLOCATED = 1;
CREATING = 2; // Used for container allocated/created by different client.
OPEN =3; // Mostly an update to SCM via HB or client call.
- CLOSED = 4; // !!State after this has not been used yet.
- DELETING = 5;
- DELETED = 6; // object is deleted.
+ PENDING_CLOSE = 4;
+ CLOSED = 5; // !!State after this has not been used yet.
+ DELETING = 6;
+ DELETED = 7; // object is deleted.
}
enum LifeCycleEvent {
BEGIN_CREATE = 1; // A request to client to create this object
COMPLETE_CREATE = 2;
- CLOSE = 3; // !!Event after this has not been used yet.
- UPDATE = 4;
- TIMEOUT = 5; // creation has timed out from SCM's View.
- DELETE = 6;
- CLEANUP = 7;
+ FULL_CONTAINER = 3;
+ CLOSE = 4; // !!Event after this has not been used yet.
+ UPDATE = 5;
+ TIMEOUT = 6; // creation has timed out from SCM's View.
+ DELETE = 7;
+ CLEANUP = 8;
}
message SCMContainerInfo {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d17ead2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d4375cd..46e8966 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -978,6 +978,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaInfo newReplicaInfo = targetVolume.moveBlockToTmpLocation(block,
replicaInfo, smallBufferSize, conf);
+ // Latch here --> wait for the signal.
+
// Finalize the copied files
newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
try (AutoCloseableLock lock = datasetLock.acquire()) {
@@ -1671,8 +1673,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
assert newReplicaInfo.getState() == ReplicaState.FINALIZED
: "Replica should be finalized";
- volumeMap.add(bpid, newReplicaInfo);
- return newReplicaInfo;
+ if(volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp() <=
+ newReplicaInfo.getGenerationStamp()) {
+ volumeMap.add(bpid, newReplicaInfo);
+ return newReplicaInfo;
+ } else {
+ throw new IOException("Generation Stamp should be monotonically " +
+ "increased. That assumption is violated here.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d17ead2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
index 2f2f55a..2882959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
@@ -56,6 +56,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB;
+import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes.FAILED_TO_CHANGE_CONTAINER_STATE;
/**
* Mapping class contains the mapping from a name to a pipeline mapping. This
@@ -296,8 +297,8 @@ public class ContainerMapping implements Mapping {
.parseFrom(containerBytes));
Preconditions.checkNotNull(containerInfo);
-
- if (event == OzoneProtos.LifeCycleEvent.BEGIN_CREATE) {
+ switch (event) {
+ case BEGIN_CREATE:
// Acquire lease on container
Lease<ContainerInfo> containerLease =
containerLeaseManager.acquire(containerInfo);
@@ -307,10 +308,30 @@ public class ContainerMapping implements Mapping {
OzoneProtos.LifeCycleEvent.TIMEOUT);
return null;
});
- } else if (event == OzoneProtos.LifeCycleEvent.COMPLETE_CREATE) {
+ break;
+ case COMPLETE_CREATE:
// Release the lease on container
containerLeaseManager.release(containerInfo);
+ break;
+ case TIMEOUT:
+ break;
+ case CLEANUP:
+ break;
+ case FULL_CONTAINER:
+ break;
+ case CLOSE:
+ break;
+ case UPDATE:
+ break;
+ case DELETE:
+ break;
+ default:
+ throw new SCMException("Unsupported container LifeCycleEvent.",
+ FAILED_TO_CHANGE_CONTAINER_STATE);
}
+ // If the below updateContainerState call fails, we should revert the
+ // changes made in switch case.
+ // Like releasing the lease in case of BEGIN_CREATE.
ContainerInfo updatedContainer = containerStateManager
.updateContainerState(containerInfo, event);
containerStore.put(dbKey, updatedContainer.getProtobuf().toByteArray());
@@ -383,16 +404,13 @@ public class ContainerMapping implements Mapping {
// Close container implementation can decide on how to maintain
// list of containers to be closed, this is the place where we
// have to add the containers to that list.
- ContainerInfo updatedContainer =
- containerStateManager.updateContainerState(
- ContainerInfo.fromProtobuf(newContainerInfo),
- OzoneProtos.LifeCycleEvent.CLOSE);
- if (updatedContainer.getState() !=
- OzoneProtos.LifeCycleState.CLOSED) {
+ OzoneProtos.LifeCycleState state = updateContainerState(
+ ContainerInfo.fromProtobuf(newContainerInfo).getContainerName(),
+ OzoneProtos.LifeCycleEvent.FULL_CONTAINER);
+ if (state != OzoneProtos.LifeCycleState.PENDING_CLOSE) {
LOG.error("Failed to close container {}, reason : Not able to " +
"update container state, current container state: {}." +
- "in state {}", containerInfo.getContainerName(),
- updatedContainer.getState());
+ "in state {}", containerInfo.getContainerName(), state);
}
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d17ead2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
index c22bbcc..57790d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
@@ -251,11 +251,14 @@ public class ContainerStateManager implements Closeable {
// DELETING----------------->[DELETED]
// (CLEANUP)
// SCM Open/Close State Machine
- // States: OPEN------------------>PENDING_CLOSE---------->[CLOSE]
+ // States: OPEN------------------>PENDING_CLOSE---------->[CLOSED]
// Events: (FULL_CONTAINER) (CLOSE)
// Delete State Machine
// States: OPEN------------------>DELETING------------------>[DELETED]
// Events: (DELETE) (CLEANUP)
+
+ // Should we allow DELETING of OPEN containers? we can always have
+ // OPEN--------->PENDING_CLOSE----->CLOSE---->DELETING---->[DELETED]
private void initializeStateMachine() {
stateMachine.addTransition(LifeCycleState.ALLOCATED,
LifeCycleState.CREATING,
@@ -266,6 +269,10 @@ public class ContainerStateManager implements Closeable {
LifeCycleEvent.COMPLETE_CREATE);
stateMachine.addTransition(LifeCycleState.OPEN,
+ LifeCycleState.PENDING_CLOSE,
+ LifeCycleEvent.FULL_CONTAINER);
+
+ stateMachine.addTransition(LifeCycleState.PENDING_CLOSE,
LifeCycleState.CLOSED,
LifeCycleEvent.CLOSE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d17ead2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
index c21ea1b..25cb75e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
@@ -264,6 +264,34 @@ public class TestContainerMapping {
ContainerInfo updatedContainer = mapping.getContainer(containerName);
Assert.assertEquals(500000000L, updatedContainer.getNumberOfKeys());
Assert.assertEquals(5368705120L, updatedContainer.getUsedBytes());
+ List<ContainerInfo> pendingCloseContainers = mapping.getStateManager()
+ .getMatchingContainers(
+ OzoneProtos.Owner.OZONE,
+ xceiverClientManager.getType(),
+ xceiverClientManager.getFactor(),
+ OzoneProtos.LifeCycleState.PENDING_CLOSE);
+ Assert.assertTrue(pendingCloseContainers.stream().map(
+ container -> container.getContainerName()).collect(
+ Collectors.toList()).contains(containerName));
+ }
+
+ @Test
+ public void testCloseContainer() throws IOException {
+ String containerName = UUID.randomUUID().toString();
+ createContainer(containerName);
+ mapping.updateContainerState(containerName,
+ OzoneProtos.LifeCycleEvent.FULL_CONTAINER);
+ List<ContainerInfo> pendingCloseContainers = mapping.getStateManager()
+ .getMatchingContainers(
+ OzoneProtos.Owner.OZONE,
+ xceiverClientManager.getType(),
+ xceiverClientManager.getFactor(),
+ OzoneProtos.LifeCycleState.PENDING_CLOSE);
+ Assert.assertTrue(pendingCloseContainers.stream().map(
+ container -> container.getContainerName()).collect(
+ Collectors.toList()).contains(containerName));
+ mapping.updateContainerState(containerName,
+ OzoneProtos.LifeCycleEvent.CLOSE);
List<ContainerInfo> closeContainers = mapping.getStateManager()
.getMatchingContainers(
OzoneProtos.Owner.OZONE,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org