You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/02/07 11:17:08 UTC
[ozone] branch master updated: HDDS-6444. Apply HDDS-5875 to OMKeyCommitRequestWithFSO (#3250)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 07629324c0 HDDS-6444. Apply HDDS-5875 to OMKeyCommitRequestWithFSO (#3250)
07629324c0 is described below
commit 07629324c0641456e12887e267205e9129a97895
Author: Jyotinder Singh <jy...@gmail.com>
AuthorDate: Tue Feb 7 16:47:01 2023 +0530
HDDS-6444. Apply HDDS-5875 to OMKeyCommitRequestWithFSO (#3250)
---
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 55 ++++++++++++++++++++--
.../ozone/om/request/key/OMKeyCommitRequest.java | 35 +++++++++-----
.../om/request/key/OMKeyCommitRequestWithFSO.java | 8 +---
3 files changed, 76 insertions(+), 22 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
index a01719cf52..8190b88b38 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -28,6 +28,10 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -38,14 +42,17 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoVolumeRequest;
@@ -151,14 +158,23 @@ public class TestSecureOzoneRpcClient extends TestOzoneRpcClient {
* */
@Test
public void testPutKeySuccessWithBlockToken() throws Exception {
+ testPutKeySuccessWithBlockTokenWithBucketLayout(BucketLayout.OBJECT_STORE);
+ testPutKeySuccessWithBlockTokenWithBucketLayout(
+ BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ }
+
+ private void testPutKeySuccessWithBlockTokenWithBucketLayout(
+ BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
Instant testStartTime = Instant.now();
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
String value = "sample value";
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
- volume.createBucket(bucketName);
+ volume.createBucket(bucketName,
+ new BucketArgs.Builder().setBucketLayout(bucketLayout).build());
OzoneBucket bucket = volume.getBucket(bucketName);
for (int i = 0; i < 10; i++) {
@@ -182,6 +198,30 @@ public class TestSecureOzoneRpcClient extends TestOzoneRpcClient {
is.read(fileContent);
}
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ String bucketId = String.valueOf(
+ omMetadataManager.getBucketTable().get(bucketKey).getObjectID());
+ String keyPrefix =
+ bucketLayout.isFileSystemOptimized() ? bucketId : bucketKey;
+ Table table = omMetadataManager.getKeyTable(bucketLayout);
+
+ // Check table entry.
+ try (
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ keyIterator = table.iterator()) {
+ Table.KeyValue<String, OmKeyInfo> kv =
+ keyIterator.seek(keyPrefix + "/" + keyName);
+
+ CacheValue<OmKeyInfo> cacheValue =
+ table.getCacheValue(new CacheKey(kv.getKey()));
+ if (cacheValue != null) {
+ assertTokenIsNull(cacheValue.getCacheValue());
+ } else {
+ assertTokenIsNull(kv.getValue());
+ }
+ }
+
+
Assert.assertTrue(verifyRatisReplication(volumeName, bucketName,
keyName, ReplicationType.RATIS,
ReplicationFactor.ONE));
@@ -191,6 +231,15 @@ public class TestSecureOzoneRpcClient extends TestOzoneRpcClient {
}
}
+ private void assertTokenIsNull(OmKeyInfo value) {
+ value.getKeyLocationVersions()
+ .forEach(
+ keyLocationInfoGroup -> keyLocationInfoGroup.getLocationList()
+ .forEach(
+ keyLocationInfo -> Assert.assertNull(keyLocationInfo
+ .getToken())));
+ }
+
/**
* Tests failure in following operations when grpc block token is
* not present.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 97dfe97957..45a9e95078 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -160,19 +161,8 @@ public class OMKeyCommitRequest extends OMKeyRequest {
String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, commitKeyRequest.getClientID());
- List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
- for (KeyLocation keyLocation : commitKeyArgs.getKeyLocationsList()) {
- OmKeyLocationInfo locationInfo =
- OmKeyLocationInfo.getFromProtobuf(keyLocation);
-
- // Strip out tokens before adding to cache.
- // This way during listStatus token information does not pass on to
- // client when returning from cache.
- if (ozoneManager.isGrpcBlockTokenEnabled()) {
- locationInfo.setToken(null);
- }
- locationInfoList.add(locationInfo);
- }
+ List<OmKeyLocationInfo>
+ locationInfoList = getOmKeyLocationInfos(ozoneManager, commitKeyArgs);
bucketLockAcquired =
omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
@@ -310,6 +300,25 @@ public class OMKeyCommitRequest extends OMKeyRequest {
return omClientResponse;
}
+ @NotNull
+ protected List<OmKeyLocationInfo> getOmKeyLocationInfos(
+ OzoneManager ozoneManager, KeyArgs commitKeyArgs) {
+ List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
+ for (KeyLocation keyLocation : commitKeyArgs.getKeyLocationsList()) {
+ OmKeyLocationInfo locationInfo =
+ OmKeyLocationInfo.getFromProtobuf(keyLocation);
+
+ // Strip out tokens before adding to cache.
+ // This way during listStatus token information does not pass on to
+ // client when returning from cache.
+ if (ozoneManager.isGrpcBlockTokenEnabled()) {
+ locationInfo.setToken(null);
+ }
+ locationInfoList.add(locationInfo);
+ }
+ return locationInfoList;
+ }
+
/**
* Process result of om request execution.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index f3e3043630..3a3473943a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponseWithFSO;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -47,7 +46,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -118,10 +116,8 @@ public class OMKeyCommitRequestWithFSO extends OMKeyCommitRequest {
Iterator<Path> pathComponents = Paths.get(keyName).iterator();
String dbOpenFileKey = null;
- List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
- for (KeyLocation keyLocation : commitKeyArgs.getKeyLocationsList()) {
- locationInfoList.add(OmKeyLocationInfo.getFromProtobuf(keyLocation));
- }
+ List<OmKeyLocationInfo>
+ locationInfoList = getOmKeyLocationInfos(ozoneManager, commitKeyArgs);
bucketLockAcquired =
omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org