You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/08/31 16:10:29 UTC
[03/47] hadoop git commit: HDDS-374. Support to configure container
size in units lesser than GB. Contributed by Nanda kumar.
HDDS-374. Support to configure container size in units lesser than GB. Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12b2f362
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12b2f362
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12b2f362
Branch: refs/heads/HDFS-12943
Commit: 12b2f362cc9a370904da724a68ba015cd3a99eff
Parents: 91836f0
Author: Nanda kumar <na...@apache.org>
Authored: Mon Aug 27 18:29:32 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Mon Aug 27 18:29:32 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 2 +-
.../container/common/impl/ContainerData.java | 24 ++++++++++----------
.../common/impl/ContainerDataYaml.java | 5 ++--
.../container/common/impl/HddsDispatcher.java | 6 +----
.../container/keyvalue/KeyValueContainer.java | 2 +-
.../keyvalue/KeyValueContainerData.java | 10 ++++----
.../container/keyvalue/KeyValueHandler.java | 15 ++++++------
.../common/TestKeyValueContainerData.java | 5 ++--
.../common/impl/TestContainerDataYaml.java | 7 +++---
.../container/common/impl/TestContainerSet.java | 7 ++++--
.../common/impl/TestHddsDispatcher.java | 3 ++-
.../keyvalue/TestChunkManagerImpl.java | 4 +++-
.../container/keyvalue/TestKeyManagerImpl.java | 4 +++-
.../keyvalue/TestKeyValueBlockIterator.java | 4 +++-
.../keyvalue/TestKeyValueContainer.java | 13 +++++++----
.../container/keyvalue/TestKeyValueHandler.java | 4 +++-
.../container/ozoneimpl/TestOzoneContainer.java | 5 ++--
.../test/resources/additionalfields.container | 4 ++--
.../test/resources/incorrect.checksum.container | 2 +-
.../src/test/resources/incorrect.container | 2 +-
.../ozone/container/ContainerTestHelper.java | 4 +++-
.../common/TestBlockDeletingService.java | 2 +-
.../TestContainerDeletionChoosingPolicy.java | 8 +++----
.../common/impl/TestContainerPersistence.java | 3 +--
24 files changed, 79 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f912f02..320a3ed 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -185,7 +185,7 @@ public final class OzoneConsts {
public static final String CONTAINER_TYPE = "containerType";
public static final String STATE = "state";
public static final String METADATA = "metadata";
- public static final String MAX_SIZE_GB = "maxSizeGB";
+ public static final String MAX_SIZE = "maxSize";
public static final String METADATA_PATH = "metadataPath";
public static final String CHUNKS_PATH = "chunksPath";
public static final String CONTAINER_DB_TYPE = "containerDBType";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index afd1407..efea20b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -40,7 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.CHECKSUM;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ID;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE;
import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION;
-import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE_GB;
+import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.METADATA;
import static org.apache.hadoop.ozone.OzoneConsts.STATE;
@@ -67,7 +67,7 @@ public abstract class ContainerData {
// State of the Container
private ContainerLifeCycleState state;
- private final int maxSizeGB;
+ private final long maxSize;
/** parameters for read/write statistics on the container. **/
private final AtomicLong readBytes;
@@ -92,16 +92,16 @@ public abstract class ContainerData {
LAYOUTVERSION,
STATE,
METADATA,
- MAX_SIZE_GB,
+ MAX_SIZE,
CHECKSUM));
/**
* Creates a ContainerData Object, which holds metadata of the container.
* @param type - ContainerType
* @param containerId - ContainerId
- * @param size - container maximum size
+ * @param size - container maximum size in bytes
*/
- protected ContainerData(ContainerType type, long containerId, int size) {
+ protected ContainerData(ContainerType type, long containerId, long size) {
this(type, containerId,
ChunkLayOutVersion.getLatestVersion().getVersion(), size);
}
@@ -111,10 +111,10 @@ public abstract class ContainerData {
* @param type - ContainerType
* @param containerId - ContainerId
* @param layOutVersion - Container layOutVersion
- * @param size - Container maximum size in GB
+ * @param size - Container maximum size in bytes
*/
protected ContainerData(ContainerType type, long containerId,
- int layOutVersion, int size) {
+ int layOutVersion, long size) {
Preconditions.checkNotNull(type);
this.containerType = type;
@@ -128,7 +128,7 @@ public abstract class ContainerData {
this.writeBytes = new AtomicLong(0L);
this.bytesUsed = new AtomicLong(0L);
this.keyCount = new AtomicLong(0L);
- this.maxSizeGB = size;
+ this.maxSize = size;
setChecksumTo0ByteArray();
}
@@ -171,11 +171,11 @@ public abstract class ContainerData {
}
/**
- * Return's maximum size of the container in GB.
- * @return maxSizeGB
+ * Return's maximum size of the container in bytes.
+ * @return maxSize in bytes
*/
- public int getMaxSizeGB() {
- return maxSizeGB;
+ public long getMaxSize() {
+ return maxSize;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
index ec6d642..65262d4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
@@ -236,12 +236,11 @@ public final class ContainerDataYaml {
long layOutVersion = (long) nodes.get(OzoneConsts.LAYOUTVERSION);
int lv = (int) layOutVersion;
- long size = (long) nodes.get(OzoneConsts.MAX_SIZE_GB);
- int maxSize = (int) size;
+ long size = (long) nodes.get(OzoneConsts.MAX_SIZE);
//When a new field is added, it needs to be added here.
KeyValueContainerData kvData = new KeyValueContainerData(
- (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, maxSize);
+ (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size);
kvData.setContainerDBType((String)nodes.get(
OzoneConsts.CONTAINER_DB_TYPE));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index d92eb17..ebc2628 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -21,12 +21,8 @@ package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -166,7 +162,7 @@ public class HddsDispatcher implements ContainerDispatcher {
if (isOpen) {
ContainerData containerData = container.getContainerData();
double containerUsedPercentage = 1.0f * containerData.getBytesUsed() /
- StorageUnit.GB.toBytes(containerData.getMaxSizeGB());
+ containerData.getMaxSize();
if (containerUsedPercentage >= containerCloseThreshold) {
ContainerAction action = ContainerAction.newBuilder()
.setContainerID(containerData.getContainerID())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 8108a11..f60c5e9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -109,7 +109,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
File containerMetaDataPath = null;
//acquiring volumeset lock and container lock
volumeSet.acquireLock();
- long maxSize = (containerData.getMaxSizeGB() * 1024L * 1024L * 1024L);
+ long maxSize = containerData.getMaxSize();
try {
HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet
.getVolumesList(), maxSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index e4cb5f3..7ffdbf5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -84,9 +84,9 @@ public class KeyValueContainerData extends ContainerData {
/**
* Constructs KeyValueContainerData object.
* @param id - ContainerId
- * @param size - maximum size in GB of the container
+ * @param size - maximum size of the container in bytes
*/
- public KeyValueContainerData(long id, int size) {
+ public KeyValueContainerData(long id, long size) {
super(ContainerProtos.ContainerType.KeyValueContainer, id, size);
this.numPendingDeletionBlocks = new AtomicInteger(0);
this.deleteTransactionId = 0;
@@ -96,9 +96,9 @@ public class KeyValueContainerData extends ContainerData {
* Constructs KeyValueContainerData object.
* @param id - ContainerId
* @param layOutVersion
- * @param size - maximum size in GB of the container
+ * @param size - maximum size of the container in bytes
*/
- public KeyValueContainerData(long id, int layOutVersion, int size) {
+ public KeyValueContainerData(long id, int layOutVersion, long size) {
super(ContainerProtos.ContainerType.KeyValueContainer, id, layOutVersion,
size);
this.numPendingDeletionBlocks = new AtomicInteger(0);
@@ -272,7 +272,7 @@ public class KeyValueContainerData extends ContainerData {
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
KeyValueContainerData data = new KeyValueContainerData(
protoData.getContainerID(),
- (int)storageSize.getUnit().toBytes(storageSize.getValue()));
+ (long)storageSize.getUnit().toBytes(storageSize.getValue()));
for (int x = 0; x < protoData.getMetadataCount(); x++) {
data.addMetadata(protoData.getMetadata(x).getKey(),
protoData.getMetadata(x).getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 8409561..e254833 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -81,8 +81,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.CLOSED_CONTAINER_RETRY;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.CONTAINER_INTERNAL_ERROR;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.CLOSED_CONTAINER_IO;
@@ -124,8 +122,8 @@ public class KeyValueHandler extends Handler {
private final KeyManager keyManager;
private final ChunkManager chunkManager;
private final BlockDeletingService blockDeletingService;
- private VolumeChoosingPolicy volumeChoosingPolicy;
- private final int maxContainerSizeGB;
+ private final VolumeChoosingPolicy volumeChoosingPolicy;
+ private final long maxContainerSize;
private final AutoCloseableLock handlerLock;
private final OpenContainerBlockMap openContainerBlockMap;
@@ -150,9 +148,9 @@ public class KeyValueHandler extends Handler {
volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass(
HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
.class, VolumeChoosingPolicy.class), conf);
- maxContainerSizeGB = (int)config.getStorageSize(
+ maxContainerSize = (long)config.getStorageSize(
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB);
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
// this handler lock is used for synchronizing createContainer Requests,
// so using a fair lock here.
handlerLock = new AutoCloseableLock(new ReentrantLock(true));
@@ -215,8 +213,9 @@ public class KeyValueHandler extends Handler {
return handleGetSmallFile(request, kvContainer);
case GetCommittedBlockLength:
return handleGetCommittedBlockLength(request, kvContainer);
+ default:
+ return null;
}
- return null;
}
@VisibleForTesting
@@ -247,7 +246,7 @@ public class KeyValueHandler extends Handler {
long containerID = request.getContainerID();
KeyValueContainerData newContainerData = new KeyValueContainerData(
- containerID, maxContainerSizeGB);
+ containerID, maxContainerSize);
// TODO: Add support to add metadataList to ContainerData. Add metadata
// to container during creation.
KeyValueContainer newContainer = new KeyValueContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 12ce163..f991520 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.junit.Test;
@@ -31,7 +32,7 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class TestKeyValueContainerData {
- private static final int MAXSIZE = 5;
+ private static final long MAXSIZE = (long) StorageUnit.GB.toBytes(5);
@Test
public void testKeyValueData() {
long containerId = 1L;
@@ -58,7 +59,7 @@ public class TestKeyValueContainerData {
assertEquals(val.get(), kvData.getWriteCount());
assertEquals(val.get(), kvData.getKeyCount());
assertEquals(val.get(), kvData.getNumPendingDeletionBlocks());
- assertEquals(MAXSIZE, kvData.getMaxSizeGB());
+ assertEquals(MAXSIZE, kvData.getMaxSize());
kvData.setState(state);
kvData.setContainerDBType(containerDBType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index fd51db3..c7b9e0a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.impl;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -42,7 +43,7 @@ public class TestContainerDataYaml {
private static String testRoot = new FileSystemTestHelper().getTestRootDir();
- private static final int MAXSIZE = 5;
+ private static final long MAXSIZE = (long) StorageUnit.GB.toBytes(5);
/**
* Creates a .container file. cleanup() should be called at the end of the
@@ -94,7 +95,7 @@ public class TestContainerDataYaml {
.getState());
assertEquals(1, kvData.getLayOutVersion());
assertEquals(0, kvData.getMetadata().size());
- assertEquals(MAXSIZE, kvData.getMaxSizeGB());
+ assertEquals(MAXSIZE, kvData.getMaxSize());
// Update ContainerData.
kvData.addMetadata("VOLUME", "hdfs");
@@ -122,7 +123,7 @@ public class TestContainerDataYaml {
assertEquals(2, kvData.getMetadata().size());
assertEquals("hdfs", kvData.getMetadata().get("VOLUME"));
assertEquals("ozone", kvData.getMetadata().get("OWNER"));
- assertEquals(MAXSIZE, kvData.getMaxSizeGB());
+ assertEquals(MAXSIZE, kvData.getMaxSize());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index ae670e0..af322ea 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.container.common.impl;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto
@@ -53,7 +54,8 @@ public class TestContainerSet {
ContainerProtos.ContainerLifeCycleState state = ContainerProtos
.ContainerLifeCycleState.CLOSED;
- KeyValueContainerData kvData = new KeyValueContainerData(containerId, 5);
+ KeyValueContainerData kvData = new KeyValueContainerData(containerId,
+ (long) StorageUnit.GB.toBytes(5));
kvData.setState(state);
KeyValueContainer keyValueContainer = new KeyValueContainer(kvData, new
OzoneConfiguration());
@@ -163,7 +165,8 @@ public class TestContainerSet {
private ContainerSet createContainerSet() throws StorageContainerException {
ContainerSet containerSet = new ContainerSet();
for (int i=0; i<10; i++) {
- KeyValueContainerData kvData = new KeyValueContainerData(i, 5);
+ KeyValueContainerData kvData = new KeyValueContainerData(i,
+ (long) StorageUnit.GB.toBytes(5));
if (i%2 == 0) {
kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index b107782..fc622b2 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -70,7 +70,8 @@ public class TestHddsDispatcher {
ContainerSet containerSet = new ContainerSet();
VolumeSet volumeSet = new VolumeSet(dd.getUuidString(), conf);
StateContext context = Mockito.mock(StateContext.class);
- KeyValueContainerData containerData = new KeyValueContainerData(1L, 1);
+ KeyValueContainerData containerData = new KeyValueContainerData(1L,
+ (long) StorageUnit.GB.toBytes(1));
Container container = new KeyValueContainer(containerData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 760d873..9664052 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -79,7 +80,8 @@ public class TestChunkManagerImpl {
Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
.thenReturn(hddsVolume);
- keyValueContainerData = new KeyValueContainerData(1L, 5);
+ keyValueContainerData = new KeyValueContainerData(1L,
+ (long) StorageUnit.GB.toBytes(5));
keyValueContainer = new KeyValueContainer(keyValueContainerData, config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
index a90cf80..b05dbca 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -79,7 +80,8 @@ public class TestKeyManagerImpl {
Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
.thenReturn(hddsVolume);
- keyValueContainerData = new KeyValueContainerData(1L, 5);
+ keyValueContainerData = new KeyValueContainerData(1L,
+ (long) StorageUnit.GB.toBytes(5));
keyValueContainer = new KeyValueContainer(
keyValueContainerData, config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index ba57c3f..f1fe88e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue;
import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -244,7 +245,8 @@ public class TestKeyValueBlockIterator {
private void createContainerWithBlocks(long containerId, int
normalBlocks, int deletedBlocks) throws
Exception {
- containerData = new KeyValueContainerData(containerId, 1);
+ containerData = new KeyValueContainerData(containerId,
+ (long) StorageUnit.GB.toBytes(1));
container = new KeyValueContainer(containerData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID
.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 7359868..f84ba7d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.container.keyvalue;
import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -94,7 +95,8 @@ public class TestKeyValueContainer {
Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
.thenReturn(hddsVolume);
- keyValueContainerData = new KeyValueContainerData(1L, 5);
+ keyValueContainerData = new KeyValueContainerData(1L,
+ (long) StorageUnit.GB.toBytes(5));
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
@@ -103,7 +105,8 @@ public class TestKeyValueContainer {
@Test
public void testBlockIterator() throws Exception{
- keyValueContainerData = new KeyValueContainerData(100L, 1);
+ keyValueContainerData = new KeyValueContainerData(100L,
+ (long) StorageUnit.GB.toBytes(1));
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
@@ -213,7 +216,7 @@ public class TestKeyValueContainer {
//create a new one
KeyValueContainerData containerData =
new KeyValueContainerData(containerId, 1,
- keyValueContainerData.getMaxSizeGB());
+ keyValueContainerData.getMaxSize());
KeyValueContainer container = new KeyValueContainer(containerData, conf);
HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet
@@ -234,8 +237,8 @@ public class TestKeyValueContainer {
containerData.getKeyCount());
Assert.assertEquals(keyValueContainerData.getLayOutVersion(),
containerData.getLayOutVersion());
- Assert.assertEquals(keyValueContainerData.getMaxSizeGB(),
- containerData.getMaxSizeGB());
+ Assert.assertEquals(keyValueContainerData.getMaxSize(),
+ containerData.getMaxSize());
Assert.assertEquals(keyValueContainerData.getBytesUsed(),
containerData.getBytesUsed());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index ce12e1f..d91bbf7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -262,7 +263,8 @@ public class TestKeyValueHandler {
public void testCloseInvalidContainer() {
long containerID = 1234L;
Configuration conf = new Configuration();
- KeyValueContainerData kvData = new KeyValueContainerData(containerID, 1);
+ KeyValueContainerData kvData = new KeyValueContainerData(containerID,
+ (long) StorageUnit.GB.toBytes(1));
KeyValueContainer container = new KeyValueContainer(kvData, conf);
kvData.setState(ContainerProtos.ContainerLifeCycleState.INVALID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 19ec6a2..173a8b2 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@ -77,7 +77,8 @@ public class TestOzoneContainer {
// Add containers to disk
for (int i=0; i<10; i++) {
- keyValueContainerData = new KeyValueContainerData(i, 1);
+ keyValueContainerData = new KeyValueContainerData(i,
+ (long) StorageUnit.GB.toBytes(1));
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/additionalfields.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/additionalfields.container b/hadoop-hdds/container-service/src/test/resources/additionalfields.container
index 73cf5f3..fff5304 100644
--- a/hadoop-hdds/container-service/src/test/resources/additionalfields.container
+++ b/hadoop-hdds/container-service/src/test/resources/additionalfields.container
@@ -5,8 +5,8 @@ containerID: 9223372036854775807
containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1
layOutVersion: 1
-maxSizeGB: 5
+maxSize: 5368709120
metadata: {OWNER: ozone, VOLUME: hdfs}
state: CLOSED
aclEnabled: true
-checksum: 1bbff32aeaa8fadc0b80c5c1e0597036e96acd8ae4bddbed188a2162762251a2
\ No newline at end of file
+checksum: c5b5373b8755c4e7199478dcaded9d996f9aca089704e08950259cdb0f290680
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
index feeeadc..d06ba57 100644
--- a/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
+++ b/hadoop-hdds/container-service/src/test/resources/incorrect.checksum.container
@@ -5,7 +5,7 @@ containerID: 9223372036854775807
containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1
layOutVersion: 1
-maxSizeGB: 5
+maxSize: 5368709120
metadata: {OWNER: ozone, VOLUME: hdfs}
state: OPEN
checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-hdds/container-service/src/test/resources/incorrect.container
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.container b/hadoop-hdds/container-service/src/test/resources/incorrect.container
index 8aeb30c..0053ab2 100644
--- a/hadoop-hdds/container-service/src/test/resources/incorrect.container
+++ b/hadoop-hdds/container-service/src/test/resources/incorrect.container
@@ -5,7 +5,7 @@ containerID: 9223372036854775807
containerType: KeyValueContainer
metadataPath: /hdds/current/aed-fg4-hji-jkl/containerDir0/1
layOutVersion: 1
-maxSizeGB: 5
+maxSize: 5368709120
metadata: {OWNER: ozone, VOLUME: hdfs}
state: INVALID
checksum: 08bc9d390f9183aeed3cf33c789e2a07310bba60f3cf55941caccc939db8670f
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index ff68863..ca92110 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.container;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.apache.commons.codec.binary.Hex;
@@ -59,7 +60,8 @@ public final class ContainerTestHelper {
ContainerTestHelper.class);
private static Random r = new Random();
- public static final int CONTAINER_MAX_SIZE_GB = 1;
+ public static final long CONTAINER_MAX_SIZE =
+ (long) StorageUnit.GB.toBytes(1);
/**
* Never constructed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index dcf4022..25c8c6b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -110,7 +110,7 @@ public class TestBlockDeletingService {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
long containerID = ContainerTestHelper.getTestContainerID();
KeyValueContainerData data = new KeyValueContainerData(containerID,
- ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
+ ContainerTestHelper.CONTAINER_MAX_SIZE);
Container container = new KeyValueContainer(data, conf);
container.create(new VolumeSet(scmId, clusterID, conf),
new RoundRobinVolumeChoosingPolicy(), scmId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index b2e4c9a..0d46ecf 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -75,8 +75,8 @@ public class TestContainerDeletionChoosingPolicy {
int numContainers = 10;
for (int i = 0; i < numContainers; i++) {
- KeyValueContainerData data = new KeyValueContainerData(new Long(i),
- ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
+ KeyValueContainerData data = new KeyValueContainerData(i,
+ ContainerTestHelper.CONTAINER_MAX_SIZE);
KeyValueContainer container = new KeyValueContainer(data, conf);
containerSet.addContainer(container);
Assert.assertTrue(
@@ -128,8 +128,8 @@ public class TestContainerDeletionChoosingPolicy {
for (int i = 0; i <= numContainers; i++) {
long containerId = RandomUtils.nextLong();
KeyValueContainerData data =
- new KeyValueContainerData(new Long(containerId),
- ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
+ new KeyValueContainerData(containerId,
+ ContainerTestHelper.CONTAINER_MAX_SIZE);
if (i != numContainers) {
int deletionBlocks = random.nextInt(numContainers) + 1;
data.incrPendingDeletionBlocks(deletionBlocks);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b2f362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 016b94c..8e11a97 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.collect.Maps;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto
@@ -193,7 +192,7 @@ public class TestContainerPersistence {
private Container addContainer(ContainerSet containerSet, long containerID)
throws IOException {
KeyValueContainerData data = new KeyValueContainerData(containerID,
- ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
+ ContainerTestHelper.CONTAINER_MAX_SIZE);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
KeyValueContainer container = new KeyValueContainer(data, conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org