You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/07/06 22:02:57 UTC
hadoop git commit: HDDS-211. Add a create container Lock. Contributed
by Bharat Viswanadham
Repository: hadoop
Updated Branches:
refs/heads/HDDS-48 7dcf5878a -> e899c4cc0
HDDS-211. Add a create container Lock. Contributed by Bharat Viswanadham
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e899c4cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e899c4cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e899c4cc
Branch: refs/heads/HDDS-48
Commit: e899c4cc01df3b657893314adcdbb411dfbf3764
Parents: 7dcf587
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Fri Jul 6 15:02:40 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Fri Jul 6 15:02:40 2018 -0700
----------------------------------------------------------------------
.../container/common/impl/HddsDispatcher.java | 4 +++-
.../container/common/interfaces/Handler.java | 8 +++-----
.../container/keyvalue/KeyValueHandler.java | 20 +++++++++++++++++---
.../genesis/BenchMarkDatanodeDispatcher.java | 7 -------
4 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e899c4cc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 25700f9..f0c2aa9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -119,8 +119,10 @@ public class HddsDispatcher implements ContainerDispatcher {
responseProto = handler.handle(msg, container);
if (responseProto != null) {
metrics.incContainerOpsLatencies(cmdType, System.nanoTime() - startTime);
+ return responseProto;
+ } else {
+ return ContainerUtils.unsupportedRequest(msg);
}
- return responseProto;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e899c4cc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 2725f59..15eed4f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
* Dispatcher sends ContainerCommandRequests to Handler. Each Container Type
* should have an implementation for Handler.
*/
-public class Handler {
+public abstract class Handler {
protected final Configuration conf;
protected final ContainerSet containerSet;
@@ -64,10 +64,8 @@ public class Handler {
}
}
- public ContainerCommandResponseProto handle(
- ContainerCommandRequestProto msg, Container container) {
- return null;
- }
+ public abstract ContainerCommandResponseProto handle(
+ ContainerCommandRequestProto msg, Container container);
public void setScmID(String scmId) {
this.scmID = scmId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e899c4cc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 110d0b4..3806ed6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
+import org.apache.hadoop.util.AutoCloseableLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +71,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.CONTAINER_INTERNAL_ERROR;
@@ -102,6 +104,7 @@ public class KeyValueHandler extends Handler {
private final ChunkManager chunkManager;
private VolumeChoosingPolicy volumeChoosingPolicy;
private final int maxContainerSizeGB;
+ private final AutoCloseableLock handlerLock;
public KeyValueHandler(Configuration config, ContainerSet contSet,
@@ -115,6 +118,9 @@ public class KeyValueHandler extends Handler {
maxContainerSizeGB = config.getInt(ScmConfigKeys
.OZONE_SCM_CONTAINER_SIZE_GB, ScmConfigKeys
.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
+ // this handler lock is used for synchronizing createContainer Requests,
+ // so using a fair lock here.
+ handlerLock = new AutoCloseableLock(new ReentrantLock(true));
}
@Override
@@ -159,7 +165,6 @@ public class KeyValueHandler extends Handler {
case GetSmallFile:
return handleGetSmallFile(request, kvContainer);
}
-
return null;
}
@@ -204,10 +209,19 @@ public class KeyValueHandler extends Handler {
newContainerData, conf);
try {
- newContainer.create(volumeSet, volumeChoosingPolicy, scmID);
- containerSet.addContainer(newContainer);
+ handlerLock.acquire();
+ if (containerSet.getContainer(containerID) == null) {
+ newContainer.create(volumeSet, volumeChoosingPolicy, scmID);
+ containerSet.addContainer(newContainer);
+ } else {
+ throw new StorageContainerException("Container already exists with " +
+ "container Id " + containerID, ContainerProtos.Result
+ .CONTAINER_EXISTS);
+ }
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
+ } finally {
+ handlerLock.release();
}
return ContainerUtils.getSuccessResponse(request);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e899c4cc/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 331e3ed..93e7ef1 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -28,7 +28,6 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.util.Time;
@@ -42,13 +41,11 @@ import org.openjdk.jmh.annotations.TearDown;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -106,10 +103,6 @@ public class BenchMarkDatanodeDispatcher {
// data directory
conf.set("dfs.datanode.data.dir", baseDir + File.separator + "data");
- // metadata directory
- StorageLocation metadataDir = StorageLocation.parse(
- baseDir + File.separator + CONTAINER_ROOT_PREFIX);
-
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(datanodeUuid, conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org