You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by na...@apache.org on 2018/03/06 03:12:01 UTC
hadoop git commit: HDFS-13008. Ozone: Add DN container open/close
state to container report. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 61516762b -> 4c03c5845
HDFS-13008. Ozone: Add DN container open/close state to container report. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c03c584
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c03c584
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c03c584
Branch: refs/heads/HDFS-7240
Commit: 4c03c58452aee728941931d305168ba96349e4c8
Parents: 6151676
Author: Nanda kumar <na...@apache.org>
Authored: Tue Mar 6 08:41:33 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Tue Mar 6 08:41:33 2018 +0530
----------------------------------------------------------------------
.../main/proto/DatanodeContainerProtocol.proto | 5 +-
.../container/common/helpers/ContainerData.java | 55 +++++---
.../common/impl/ContainerManagerImpl.java | 3 +-
.../ozone/scm/StorageContainerManager.java | 3 +-
.../scm/cli/container/InfoContainerHandler.java | 5 +-
.../StorageContainerDatanodeProtocol.proto | 1 +
.../common/TestBlockDeletingService.java | 17 ++-
.../TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 37 ++---
.../ozone/ksm/TestContainerReportWithKeys.java | 138 +++++++++++++++++++
10 files changed, 224 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
index 0890201..4e79827 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
@@ -201,10 +201,13 @@ message ContainerData {
repeated KeyValue metadata = 2;
optional string dbPath = 3;
optional string containerPath = 4;
- optional bool open = 5 [default = true];
optional string hash = 6;
optional int64 bytesUsed = 7;
optional int64 size = 8;
+ optional int64 keyCount = 9;
+ //TODO: change required after we switch container ID from string to long
+ optional int64 containerID = 10;
+ optional LifeCycleState state = 11 [default = OPEN];
}
message ContainerMeta {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
index 58853ed..d0886e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
@@ -45,24 +45,26 @@ public class ContainerData {
private String dbPath; // Path to Level DB Store.
// Path to Physical file system where container and checksum are stored.
private String containerFilePath;
- private boolean open;
private String hash;
private AtomicLong bytesUsed;
private long maxSize;
+ private Long containerID;
+ private OzoneProtos.LifeCycleState state;
/**
* Constructs a ContainerData Object.
*
* @param containerName - Name
*/
- public ContainerData(String containerName, Configuration conf) {
+ public ContainerData(String containerName, Long containerID,
+ Configuration conf) {
this.metadata = new TreeMap<>();
this.containerName = containerName;
- this.open = true;
this.maxSize = conf.getLong(ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_KEY,
ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT) * OzoneConsts.GB;
this.bytesUsed = new AtomicLong(0L);
-
+ this.containerID = containerID;
+ this.state = OzoneProtos.LifeCycleState.OPEN;
}
/**
@@ -74,7 +76,8 @@ public class ContainerData {
public static ContainerData getFromProtBuf(
ContainerProtos.ContainerData protoData, Configuration conf)
throws IOException {
- ContainerData data = new ContainerData(protoData.getName(), conf);
+ ContainerData data = new ContainerData(protoData.getName(),
+ protoData.getContainerID(), conf);
for (int x = 0; x < protoData.getMetadataCount(); x++) {
data.addMetadata(protoData.getMetadata(x).getKey(),
protoData.getMetadata(x).getValue());
@@ -88,10 +91,8 @@ public class ContainerData {
data.setDBPath(protoData.getDbPath());
}
- if (protoData.hasOpen()) {
- data.setOpen(protoData.getOpen());
- } else {
- data.setOpen(true);
+ if (protoData.hasState()) {
+ data.setState(protoData.getState());
}
if(protoData.hasHash()) {
@@ -117,6 +118,7 @@ public class ContainerData {
ContainerProtos.ContainerData.Builder builder = ContainerProtos
.ContainerData.newBuilder();
builder.setName(this.getContainerName());
+ builder.setContainerID(this.getContainerID());
if (this.getDBPath() != null) {
builder.setDbPath(this.getDBPath());
@@ -130,7 +132,7 @@ public class ContainerData {
builder.setContainerPath(this.getContainerPath());
}
- builder.setOpen(this.isOpen());
+ builder.setState(this.getState());
for (Map.Entry<String, String> entry : metadata.entrySet()) {
OzoneProtos.KeyValue.Builder keyValBuilder =
@@ -143,6 +145,10 @@ public class ContainerData {
builder.setBytesUsed(this.getBytesUsed());
}
+ if (this.getKeyCount() >= 0) {
+ builder.setKeyCount(this.getKeyCount());
+ }
+
if (this.getMaxSize() >= 0) {
builder.setSize(this.getMaxSize());
}
@@ -246,18 +252,35 @@ public class ContainerData {
}
/**
+ * Get container ID.
+ * @return - container ID.
+ */
+ public synchronized Long getContainerID() {
+ return containerID;
+ }
+
+ public synchronized void setState(OzoneProtos.LifeCycleState state) {
+ this.state = state;
+ }
+
+ public synchronized OzoneProtos.LifeCycleState getState() {
+ return this.state;
+ }
+
+ /**
* checks if the container is open.
* @return - boolean
*/
public synchronized boolean isOpen() {
- return open;
+ return OzoneProtos.LifeCycleState.OPEN == state;
}
/**
* Marks this container as closed.
*/
public synchronized void closeContainer() {
- setOpen(false);
+ // TODO: closed or closing here
+ setState(OzoneProtos.LifeCycleState.CLOSED);
// Some thing brain dead for now. name + Time stamp of when we get the close
// container message.
@@ -277,14 +300,6 @@ public class ContainerData {
this.hash = hash;
}
- /**
- * Sets the open or closed values.
- * @param open
- */
- public synchronized void setOpen(boolean open) {
- this.open = open;
- }
-
public void setMaxSize(long maxSize) {
this.maxSize = maxSize;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index f701900..62fcd66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -896,7 +896,8 @@ public class ContainerManagerImpl implements ContainerManager {
.setReadCount(container.getReadCount())
.setWriteCount(container.getWriteCount())
.setReadBytes(container.getReadBytes())
- .setWriteBytes(container.getWriteBytes());
+ .setWriteBytes(container.getWriteBytes())
+ .setContainerID(container.getContainer().getContainerID());
if (container.getContainer().getHash() != null) {
ciBuilder.setFinalhash(container.getContainer().getHash());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
index 4245fe0..3eb6dc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
@@ -618,7 +618,8 @@ public class StorageContainerManager extends ServiceRuntimeInfoImpl
}
@VisibleForTesting
- ContainerInfo getContainerInfo(String containerName) throws IOException {
+ public ContainerInfo getContainerInfo(String containerName)
+ throws IOException {
return scmContainerManager.getContainer(containerName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
index fc9ef0c..f54a2b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
import org.apache.hadoop.scm.client.ScmClient;
import org.apache.hadoop.scm.container.common.helpers.Pipeline;
@@ -75,7 +76,9 @@ public class InfoContainerHandler extends OzoneCommandHandler {
// Print container report info.
logOut("Container Name: %s",
containerData.getName());
- String openStatus = containerData.getOpen() ? "OPEN" : "CLOSED";
+ String openStatus =
+ containerData.getState() == OzoneProtos.LifeCycleState.OPEN ? "OPEN" :
+ "CLOSED";
logOut("Container State: %s", openStatus);
if (!containerData.getHash().isEmpty()) {
logOut("Container Hash: %s", containerData.getHash());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
index 6f2717c..7306b81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -104,6 +104,7 @@ message ContainerInfo {
optional int64 readBytes = 8;
optional int64 writeBytes = 9;
required int64 containerID = 10;
+ optional hadoop.hdfs.ozone.LifeCycleState state = 11;
}
// The deleted blocks which are stored in deletedBlock.db of scm.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index c00f828..9f1a90e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -128,7 +128,7 @@ public class TestBlockDeletingService {
int numOfChunksPerBlock, File chunkDir) throws IOException {
for (int x = 0; x < numOfContainers; x++) {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, new Long(x), conf);
mgr.createContainer(createSingleNodePipeline(containerName), data);
data = mgr.readContainer(containerName);
MetadataStore metadata = KeyUtils.getDB(data, conf);
@@ -216,6 +216,7 @@ public class TestBlockDeletingService {
Assert.assertEquals(0, getUnderDeletionBlocksCount(meta));
svc.shutdown();
+ shutdownContainerMangaer(containerManager);
}
@Test
@@ -244,6 +245,7 @@ public class TestBlockDeletingService {
// Shutdown service and verify all threads are stopped
service.shutdown();
GenericTestUtils.waitFor(() -> service.getThreadCount() == 0, 100, 1000);
+ shutdownContainerMangaer(containerManager);
}
@Test
@@ -302,6 +304,7 @@ public class TestBlockDeletingService {
Assert.assertTrue(!newLog.getOutput().contains(
"Background task executes timed out, retrying in next interval"));
svc.shutdown();
+ shutdownContainerMangaer(containerManager);
}
@Test(timeout = 30000)
@@ -333,6 +336,7 @@ public class TestBlockDeletingService {
Assert.assertEquals(10, chunksDir.listFiles().length);
} finally {
service.shutdown();
+ shutdownContainerMangaer(containerManager);
}
}
@@ -376,6 +380,17 @@ public class TestBlockDeletingService {
Assert.assertEquals(0, chunksDir.listFiles().length);
} finally {
service.shutdown();
+ shutdownContainerMangaer(containerManager);
+ }
+ }
+
+ private void shutdownContainerMangaer(ContainerManager mgr)
+ throws IOException {
+ mgr.writeLock();
+ try {
+ mgr.shutdown();
+ } finally {
+ mgr.writeUnlock();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index b2a61ab..c6964a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -94,7 +94,7 @@ public class TestContainerDeletionChoosingPolicy {
int numContainers = 10;
for (int i = 0; i < numContainers; i++) {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, new Long(i), conf);
containerManager.createContainer(createSingleNodePipeline(containerName),
data);
Assert.assertTrue(
@@ -144,7 +144,7 @@ public class TestContainerDeletionChoosingPolicy {
// create [numContainers + 1] containers
for (int i = 0; i <= numContainers; i++) {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, new Long(i), conf);
containerManager.createContainer(createSingleNodePipeline(containerName),
data);
Assert.assertTrue(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 72cfa03..c993122 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -101,6 +101,7 @@ public class TestContainerPersistence {
private static KeyManagerImpl keyManager;
private static OzoneConfiguration conf;
private static List<StorageLocation> pathLists = new LinkedList<>();
+ private Long containerID = 8888L;;
@BeforeClass
public static void init() throws Throwable {
@@ -180,7 +181,7 @@ public class TestContainerPersistence {
public void testCreateContainer() throws Exception {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName),
@@ -216,7 +217,7 @@ public class TestContainerPersistence {
public void testCreateDuplicateContainer() throws Exception {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName),
@@ -236,14 +237,14 @@ public class TestContainerPersistence {
String containerName2 = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName1, conf);
+ ContainerData data = new ContainerData(containerName1, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName1),
data);
containerManager.closeContainer(containerName1);
- data = new ContainerData(containerName2, conf);
+ data = new ContainerData(containerName2, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName2),
@@ -263,7 +264,7 @@ public class TestContainerPersistence {
// Let us make sure that we are able to re-use a container name after
// delete.
- data = new ContainerData(containerName1, conf);
+ data = new ContainerData(containerName1, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName1),
@@ -301,7 +302,8 @@ public class TestContainerPersistence {
for (int i = 0; i < count; i++) {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, containerID++,
+ conf);
containerManager.createContainer(createSingleNodePipeline(containerName),
data);
@@ -337,8 +339,8 @@ public class TestContainerPersistence {
Map<String, ContainerData> testMap = new HashMap<>();
for (int x = 0; x < count; x++) {
String containerName = OzoneUtils.getRequestID();
-
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, containerID++,
+ conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(containerName),
@@ -373,7 +375,7 @@ public class TestContainerPersistence {
final int datalen = 1024;
Pipeline newPipeline =
new Pipeline(containerName, pipeline.getPipelineChannel());
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner", "bilbo");
if(!containerManager.getContainerMap()
@@ -421,7 +423,7 @@ public class TestContainerPersistence {
Pipeline pipeline = createSingleNodePipeline(containerName);
Map<String, ChunkInfo> fileHashMap = new HashMap<>();
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner)", "bilbo");
containerManager.createContainer(pipeline, cData);
@@ -484,7 +486,7 @@ public class TestContainerPersistence {
String keyName = OzoneUtils.getRequestID();
Pipeline pipeline = createSingleNodePipeline(containerName);
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner)", "bilbo");
containerManager.createContainer(pipeline, cData);
@@ -518,7 +520,7 @@ public class TestContainerPersistence {
String keyName = OzoneUtils.getRequestID();
Pipeline pipeline = createSingleNodePipeline(containerName);
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner)", "bilbo");
containerManager.createContainer(pipeline, cData);
@@ -560,7 +562,7 @@ public class TestContainerPersistence {
String keyName = OzoneUtils.getRequestID();
Pipeline pipeline = createSingleNodePipeline(containerName);
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner)", "bilbo");
containerManager.createContainer(pipeline, cData);
@@ -598,7 +600,7 @@ public class TestContainerPersistence {
String keyName = OzoneUtils.getRequestID();
Pipeline pipeline = createSingleNodePipeline(containerName);
- ContainerData cData = new ContainerData(containerName, conf);
+ ContainerData cData = new ContainerData(containerName, containerID++, conf);
cData.addMetadata("VOLUME", "shire");
cData.addMetadata("owner)", "bilbo");
containerManager.createContainer(pipeline, cData);
@@ -744,7 +746,7 @@ public class TestContainerPersistence {
@Test
public void testUpdateContainer() throws IOException {
String containerName = OzoneUtils.getRequestID();
- ContainerData data = new ContainerData(containerName, conf);
+ ContainerData data = new ContainerData(containerName, containerID++, conf);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner", "bilbo");
@@ -755,7 +757,8 @@ public class TestContainerPersistence {
File orgContainerFile = containerManager.getContainerFile(data);
Assert.assertTrue(orgContainerFile.exists());
- ContainerData newData = new ContainerData(containerName, conf);
+ ContainerData newData = new ContainerData(containerName, containerID++,
+ conf);
newData.addMetadata("VOLUME", "shire_new");
newData.addMetadata("owner", "bilbo_new");
@@ -807,7 +810,7 @@ public class TestContainerPersistence {
}
// Update with force flag, it should be success.
- newData = new ContainerData(containerName, conf);
+ newData = new ContainerData(containerName, containerID++, conf);
newData.addMetadata("VOLUME", "shire_new_1");
newData.addMetadata("owner", "bilbo_new_1");
containerManager.updateContainer(createSingleNodePipeline(containerName),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c03c584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
new file mode 100644
index 0000000..a5320b8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.ksm;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
+import org.apache.hadoop.ozone.scm.StorageContainerManager;
+import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * This class tests container report with DN container state info.
+ */
+public class TestContainerReportWithKeys {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestContainerReportWithKeys.class);
+ private static MiniOzoneClassicCluster cluster = null;
+ private static OzoneConfiguration conf;
+ private static StorageContainerManager scm;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
+ OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
+ cluster = new MiniOzoneClassicCluster.Builder(conf)
+ .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+ scm = cluster.getStorageContainerManager();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testContainerReportKeyWrite() throws Exception {
+ final String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ final String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ final String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ final int keySize = 100;
+
+ OzoneClient client = OzoneClientFactory.getClient(conf);
+ ObjectStore objectStore = client.getObjectStore();
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+ OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName)
+ .createKey(keyName, keySize, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ String dataString = RandomStringUtils.randomAlphabetic(keySize);
+ key.write(dataString.getBytes());
+ key.close();
+
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setType(OzoneProtos.ReplicationType.STAND_ALONE)
+ .setFactor(OzoneProtos.ReplicationFactor.ONE).setDataSize(keySize)
+ .build();
+
+
+ KsmKeyLocationInfo keyInfo =
+ cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
+ .get(0).getBlocksLatestVersionOnly().get(0);
+
+ ContainerData cd = getContainerData(cluster, keyInfo.getContainerName());
+
+ LOG.info("DN Container Data: keyCount: {} used: {} ",
+ cd.getKeyCount(), cd.getBytesUsed());
+
+ ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerName());
+
+ LOG.info("SCM Container Info keyCount: {} usedBytes: {}",
+ cinfo.getNumberOfKeys(), cinfo.getUsedBytes());
+ }
+
+
+ private static ContainerData getContainerData(MiniOzoneClassicCluster clus,
+ String containerName) {
+ ContainerData containerData = null;
+ try {
+ containerData = clus.getDataNodes().get(0).getOzoneContainerManager()
+ .getContainerManager().readContainer(containerName);
+ } catch (StorageContainerException e) {
+ throw new AssertionError(e);
+ }
+ return containerData;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org