You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/08/08 20:11:48 UTC
[07/50] [abbrv] hadoop git commit: HDDS-312. Add blockIterator to
Container. Contributed by Bharat Viswanadham.
HDDS-312. Add blockIterator to Container. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40ab8ee5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40ab8ee5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40ab8ee5
Branch: refs/heads/HDFS-12943
Commit: 40ab8ee597d730fa2a8a386ef25b0dbecd4e839c
Parents: 889df6f
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Aug 2 16:48:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Aug 2 16:48:21 2018 -0700
----------------------------------------------------------------------
.../container/common/interfaces/Container.java | 8 +++
.../container/keyvalue/KeyValueContainer.java | 8 ++-
.../keyvalue/TestKeyValueContainer.java | 58 ++++++++++++++++++--
3 files changed, 69 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab8ee5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
index fc91920..a7077d9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import java.io.File;
+import java.io.IOException;
import java.util.Map;
@@ -103,4 +104,11 @@ public interface Container extends RwLock {
*/
void updateDeleteTransactionId(long deleteTransactionId);
+ /**
+ * Returns blockIterator for the container.
+ * @return BlockIterator
+ * @throws IOException
+ */
+ BlockIterator blockIterator() throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab8ee5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index d0e77d2..353fe4f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -346,6 +346,12 @@ public class KeyValueContainer implements Container {
containerData.updateDeleteTransactionId(deleteTransactionId);
}
+ @Override
+ public KeyValueBlockIterator blockIterator() throws IOException{
+ return new KeyValueBlockIterator(containerData.getContainerID(), new File(
+ containerData.getContainerPath()));
+ }
+
/**
* Acquire read lock.
*/
@@ -420,7 +426,7 @@ public class KeyValueContainer implements Container {
}
/**
- * Returns container DB file
+ * Returns container DB file.
* @return
*/
public File getContainerDBFile() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab8ee5/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 35772ff..37c7f8a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -18,22 +18,26 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.helpers
- .KeyValueContainerLocationUtil;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.utils.MetadataStore;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -46,6 +50,8 @@ import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.List;
+import java.util.LinkedList;
import java.util.UUID;
import static org.apache.ratis.util.Preconditions.assertTrue;
@@ -93,6 +99,50 @@ public class TestKeyValueContainer {
}
@Test
+ public void testBlockIterator() throws Exception{
+ keyValueContainerData = new KeyValueContainerData(100L, 1);
+ keyValueContainer = new KeyValueContainer(
+ keyValueContainerData, conf);
+ keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
+ KeyValueBlockIterator blockIterator = keyValueContainer.blockIterator();
+ //As no blocks created, hasNext should return false.
+ assertFalse(blockIterator.hasNext());
+ int blockCount = 10;
+ addBlocks(blockCount);
+ blockIterator = keyValueContainer.blockIterator();
+ assertTrue(blockIterator.hasNext());
+ KeyData keyData;
+ int blockCounter = 0;
+ while(blockIterator.hasNext()) {
+ keyData = blockIterator.nextBlock();
+ assertEquals(blockCounter++, keyData.getBlockID().getLocalID());
+ }
+ assertEquals(blockCount, blockCounter);
+ }
+
+ private void addBlocks(int count) throws Exception {
+ long containerId = keyValueContainerData.getContainerID();
+
+ MetadataStore metadataStore = KeyUtils.getDB(keyValueContainer
+ .getContainerData(), conf);
+ for (int i=0; i < count; i++) {
+ // Creating KeyData
+ BlockID blockID = new BlockID(containerId, i);
+ KeyData keyData = new KeyData(blockID);
+ keyData.addMetadata("VOLUME", "ozone");
+ keyData.addMetadata("OWNER", "hdfs");
+ List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
+ ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
+ .getLocalID(), 0), 0, 1024);
+ chunkList.add(info.getProtoBufMessage());
+ keyData.setChunks(chunkList);
+ metadataStore.put(Longs.toByteArray(blockID.getLocalID()), keyData
+ .getProtoBufMessage().toByteArray());
+ }
+
+ }
+
+ @Test
public void testCreateContainer() throws Exception {
// Create Container.
@@ -113,8 +163,8 @@ public class TestKeyValueContainer {
//Check whether container file and container db file exists or not.
assertTrue(keyValueContainer.getContainerFile().exists(),
".Container File does not exist");
- assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container DB does " +
- "not exist");
+ assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container " +
+ "DB does not exist");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org