You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ma...@apache.org on 2022/03/21 02:20:55 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
This is an automated email from the ASF dual-hosted git repository.
markgui pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new af48c53 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
af48c53 is described below
commit af48c537a9aba6d2b6dd544766e4d66365b427a2
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Mon Mar 21 10:20:29 2022 +0800
HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +++++++++++++++++-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 +-
.../om/request/key/TestOMKeyCreateRequest.java | 81 +++++++++++++++++++---
.../ozone/om/request/key/TestOMKeyRequest.java | 21 +++---
4 files changed, 155 insertions(+), 22 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index c1256bc..6e271fc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -102,6 +102,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
@@ -190,7 +191,7 @@ public abstract class TestOzoneRpcClientAbstract {
// for testZReadKeyWithUnhealthyContainerReplica.
conf.set("ozone.scm.stale.node.interval", "10s");
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
+ .setNumDatanodes(5)
.setTotalPipelineNumLimit(10)
.setScmId(scmId)
.setClusterId(clusterId)
@@ -1012,6 +1013,71 @@ public abstract class TestOzoneRpcClientAbstract {
store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
}
+ // TODO: testBucketQuota overlaps with testBucketUsedBytes,
+ // do cleanup when EC branch gets merged into master.
+ @Test
+ public void testBucketQuota() throws IOException {
+ int blockSize = (int) ozoneManager.getConfiguration().getStorageSize(
+ OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+
+ ReplicationConfig[] repConfigs = new ReplicationConfig[]{
+ ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
+ ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
+ new ECReplicationConfig("rs-3-2-1024k"),
+ };
+
+ for (ReplicationConfig repConfig : repConfigs) {
+ for (int i = 0; i <= repConfig.getRequiredNodes(); i++) {
+ bucketQuotaTestHelper(i * blockSize, repConfig);
+ bucketQuotaTestHelper(i * blockSize + 1, repConfig);
+ }
+ }
+ }
+
+ private void bucketQuotaTestHelper(int keyLength, ReplicationConfig repConfig)
+ throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+ long blockSize = (long) ozoneManager.getConfiguration().getStorageSize(
+ OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ byte[] value = new byte[keyLength];
+ int dataGroupSize = repConfig instanceof ECReplicationConfig ?
+ ((ECReplicationConfig) repConfig).getData() : 1;
+ long preAllocatedBlocks = Math.min(ozoneManager.getPreallocateBlocksMax(),
+ (keyLength - 1) / (blockSize * dataGroupSize) + 1);
+ long preAllocatedSpace = preAllocatedBlocks * blockSize
+ * repConfig.getRequiredNodes();
+ long keyQuota = QuotaUtil.getReplicatedSize(keyLength, repConfig);
+
+ OzoneOutputStream out = bucket.createKey(keyName, keyLength,
+ repConfig, new HashMap<>());
+ Assert.assertEquals(preAllocatedSpace,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+ out.write(value);
+ out.close();
+ Assert.assertEquals(keyQuota,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ out = bucket.createKey(keyName, keyLength, repConfig, new HashMap<>());
+ Assert.assertEquals(keyQuota + preAllocatedSpace,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+ out.write(value);
+ out.close();
+ Assert.assertEquals(keyQuota,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ bucket.deleteKey(keyName);
+ Assert.assertEquals(0L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+ }
+
@Test
public void testVolumeUsedNamespace() throws IOException {
String volumeName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 629e501..cb69f27 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -32,6 +32,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -129,8 +130,10 @@ public abstract class OMKeyRequest extends OMClientRequest {
ReplicationConfig replicationConfig, ExcludeList excludeList,
long requestedSize, long scmBlockSize, int preallocateBlocksMax,
boolean grpcBlockTokenEnabled, String omID) throws IOException {
- int numBlocks = Math.min((int) ((requestedSize - 1) / scmBlockSize + 1),
- preallocateBlocksMax);
+ int dataGroupSize = replicationConfig instanceof ECReplicationConfig
+ ? ((ECReplicationConfig) replicationConfig).getData() : 1;
+ int numBlocks = (int) Math.min(preallocateBlocksMax,
+ (requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1);
List<OmKeyLocationInfo> locationInfos = new ArrayList<>(numBlocks);
String remoteUser = getRemoteUser().getShortUserName();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index 5bc9d45..6823a92 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@ -24,6 +24,8 @@ import java.nio.file.Paths;
import java.util.List;
import java.util.UUID;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -44,6 +46,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB;
@@ -59,12 +62,33 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
@Test
public void testPreExecuteWithNormalKey() throws Exception {
- doPreExecute(createKeyRequest(false, 0));
+ ReplicationConfig ratis3Config =
+ ReplicationConfig.fromProtoTypeAndFactor(RATIS, THREE);
+ preExecuteTest(false, 0, ratis3Config);
+ }
+
+ @Test
+ public void testPreExecuteWithECKey() throws Exception {
+ ReplicationConfig ec3Plus2Config = new ECReplicationConfig("rs-3-2-1024k");
+ preExecuteTest(false, 0, ec3Plus2Config);
}
@Test
public void testPreExecuteWithMultipartKey() throws Exception {
- doPreExecute(createKeyRequest(true, 1));
+ ReplicationConfig ratis3Config =
+ ReplicationConfig.fromProtoTypeAndFactor(RATIS, THREE);
+ preExecuteTest(true, 1, ratis3Config);
+ }
+
+ private void preExecuteTest(boolean isMultipartKey, int partNumber,
+ ReplicationConfig repConfig) throws Exception {
+ long scmBlockSize = ozoneManager.getScmBlockSize();
+ for (int i = 0; i <= repConfig.getRequiredNodes(); i++) {
+ doPreExecute(createKeyRequest(isMultipartKey, partNumber,
+ scmBlockSize * i, repConfig));
+ doPreExecute(createKeyRequest(isMultipartKey, partNumber,
+ scmBlockSize * i + 1, repConfig));
+ }
}
@@ -163,10 +187,13 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
omMetadataManager.getOpenKeyTable(bucketLayout).get(openKey);
Assert.assertNotNull(omKeyInfo);
+ Assert.assertNotNull(omKeyInfo.getLatestVersionLocations());
+ // As our data size is 100, and scmBlockSize is default to 1000, so we
+ // shall have only one block.
List<OmKeyLocationInfo> omKeyLocationInfoList =
omKeyInfo.getLatestVersionLocations().getLocationList();
- Assert.assertTrue(omKeyLocationInfoList.size() == 1);
+ Assert.assertEquals(1, omKeyLocationInfoList.size());
OmKeyLocationInfo omKeyLocationInfo = omKeyLocationInfoList.get(0);
@@ -342,6 +369,12 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
modifiedOmRequest.getCreateKeyRequest();
KeyArgs keyArgs = createKeyRequest.getKeyArgs();
+ int dataGroupSize = keyArgs.hasEcReplicationConfig() ?
+ keyArgs.getEcReplicationConfig().getData() : 1;
+ long blockSize = ozoneManager.getScmBlockSize();
+ long preAllocatedBlocks = Math.min(ozoneManager.getPreallocateBlocksMax(),
+ (keyArgs.getDataSize() - 1) / (blockSize * dataGroupSize) + 1);
+
// Time should be set
Assert.assertTrue(keyArgs.getModificationTime() > 0);
@@ -354,12 +387,10 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
if (!originalOMRequest.getCreateKeyRequest().getKeyArgs()
.getIsMultipartKey()) {
- // As our data size is 100, and scmBlockSize is default to 1000, so we
- // shall have only one block.
- List< OzoneManagerProtocolProtos.KeyLocation> keyLocations =
+ List<OzoneManagerProtocolProtos.KeyLocation> keyLocations =
keyArgs.getKeyLocationsList();
// KeyLocation should be set.
- Assert.assertTrue(keyLocations.size() == 1);
+ Assert.assertEquals(preAllocatedBlocks, keyLocations.size());
Assert.assertEquals(CONTAINER_ID,
keyLocations.get(0).getBlockID().getContainerBlockID()
.getContainerID());
@@ -373,7 +404,7 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
Assert.assertEquals(scmBlockSize, keyLocations.get(0).getLength());
} else {
// We don't create blocks for multipart key in createKey preExecute.
- Assert.assertTrue(keyArgs.getKeyLocationsList().size() == 0);
+ Assert.assertEquals(0, keyArgs.getKeyLocationsList().size());
}
return modifiedOmRequest;
@@ -414,6 +445,36 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
.setCreateKeyRequest(createKeyRequest).build();
}
+ private OMRequest createKeyRequest(boolean isMultipartKey, int partNumber,
+ long keyLength, ReplicationConfig repConfig) {
+
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName).setBucketName(bucketName)
+ .setKeyName(keyName).setIsMultipartKey(isMultipartKey)
+ .setType(repConfig.getReplicationType())
+ .setLatestVersionLocation(true)
+ .setDataSize(keyLength);
+
+ if (repConfig.getReplicationType() == EC) {
+ keyArgs.setEcReplicationConfig(
+ ((ECReplicationConfig) repConfig).toProto());
+ } else {
+ keyArgs.setFactor(ReplicationConfig.getLegacyFactor(repConfig));
+ }
+
+ if (isMultipartKey) {
+ keyArgs.setMultipartNumber(partNumber);
+ }
+
+ OzoneManagerProtocolProtos.CreateKeyRequest createKeyRequest =
+ CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build();
+
+ return OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey)
+ .setClientId(UUID.randomUUID().toString())
+ .setCreateKeyRequest(createKeyRequest).build();
+ }
+
@Test
public void testKeyCreateWithFileSystemPathsEnabled() throws Exception {
@@ -427,11 +488,11 @@ public class TestOMKeyCreateRequest extends TestOMKeyRequest {
omMetadataManager);
- keyName = "dir1/dir2/dir3/file1";
+ String keyName = "dir1/dir2/dir3/file1";
createAndCheck(keyName);
// Key with leading '/'.
- String keyName = "/a/b/c/file1";
+ keyName = "/a/b/c/file1";
createAndCheck(keyName);
// Commit openKey entry.
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index 66fdf3a..de0eba9 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -168,18 +168,21 @@ public class TestOMKeyRequest {
.setNodes(new ArrayList<>())
.build();
- AllocatedBlock allocatedBlock =
- new AllocatedBlock.Builder()
- .setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID))
- .setPipeline(pipeline).build();
-
- List<AllocatedBlock> allocatedBlocks = new ArrayList<>();
-
- allocatedBlocks.add(allocatedBlock);
+ AllocatedBlock.Builder blockBuilder = new AllocatedBlock.Builder()
+ .setPipeline(pipeline);
when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(),
any(ReplicationConfig.class),
- anyString(), any(ExcludeList.class))).thenReturn(allocatedBlocks);
+ anyString(), any(ExcludeList.class))).thenAnswer(invocation -> {
+ int num = invocation.getArgument(1);
+ List<AllocatedBlock> allocatedBlocks = new ArrayList<>(num);
+ for (int i = 0; i < num; i++) {
+ blockBuilder.setContainerBlockID(
+ new ContainerBlockID(CONTAINER_ID + i, LOCAL_ID + i));
+ allocatedBlocks.add(blockBuilder.build());
+ }
+ return allocatedBlocks;
+ });
volumeName = UUID.randomUUID().toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org