You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by bh...@apache.org on 2020/08/11 18:03:56 UTC
[hadoop-ozone] branch HDDS-4017 created (now 4501dcc)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a change to branch HDDS-4017
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
at 4501dcc HDDS-4012. Normalize Keypath for lookupKey.
This branch includes the following new commits:
new 4501dcc HDDS-4012. Normalize Keypath for lookupKey.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: HDDS-4012. Normalize Keypath for lookupKey.
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch HDDS-4017
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4501dcc9d692b122a69f0a2fbe886663a8a23dbc
Author: Bharat Viswanadham <bv...@cloudera.com>
AuthorDate: Thu Jul 30 15:36:07 2020 -0700
HDDS-4012. Normalize Keypath for lookupKey.
---
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 40 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 11 +++++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1 -
.../hadoop/ozone/om/request/OMClientRequest.java | 2 +-
4 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
index c4e5435..476c1fc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -27,6 +28,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.junit.Assert;
@@ -209,6 +211,44 @@ public class TestOzoneFSWithObjectStoreCreate {
}
+ @Test
+ public void testReadWithNotNormalizedPath() throws Exception {
+ OzoneVolume ozoneVolume =
+ cluster.getRpcClient().getObjectStore().getVolume(volumeName);
+
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
+
+ String key = "/dir1///dir2/file1/";
+
+ int length = 10;
+ byte[] input = new byte[length];
+ Arrays.fill(input, (byte)96);
+ String inputString = new String(input);
+
+ OzoneOutputStream ozoneOutputStream =
+ ozoneBucket.createKey(key, length);
+
+ ozoneOutputStream.write(input);
+ ozoneOutputStream.write(input, 0, 10);
+ ozoneOutputStream.close();
+
+ // Read the key with given key name.
+ OzoneInputStream ozoneInputStream = ozoneBucket.readKey(key);
+ byte[] read = new byte[length];
+ ozoneInputStream.read(read, 0, length);
+ ozoneInputStream.close();
+
+ Assert.assertEquals(inputString, new String(read));
+
+ // Read using filesystem.
+ FSDataInputStream fsDataInputStream = o3fs.open(new Path(key));
+ read = new byte[length];
+ fsDataInputStream.read(read, 0, length);
+ ozoneInputStream.close();
+
+ Assert.assertEquals(inputString, new String(read));
+ }
+
private void checkPath(Path path) {
try {
o3fs.getFileStatus(path);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 66ddeb6..4027358 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -165,6 +166,8 @@ public class KeyManagerImpl implements KeyManager {
private final KeyProviderCryptoExtension kmsProvider;
private final PrefixManager prefixManager;
+ private final boolean enableFileSystemPaths;
+
@VisibleForTesting
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
@@ -208,6 +211,9 @@ public class KeyManagerImpl implements KeyManager {
this.listTrashKeysMax = conf.getInt(
OZONE_CLIENT_LIST_TRASH_KEYS_MAX,
OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT);
+ this.enableFileSystemPaths =
+ conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
+ OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
this.ozoneManager = om;
this.omId = omId;
@@ -645,7 +651,10 @@ public class KeyManagerImpl implements KeyManager {
Preconditions.checkNotNull(args);
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
+ String keyName = OMClientRequest.validateAndNormalizeKey(
+ enableFileSystemPaths, args.getKeyName());
+
+
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
OmKeyInfo value = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 13e47a7..f70bf33 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -2226,7 +2226,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
Map<String, String> auditMap = bucket.audit(args.toAuditMap());
args = bucket.update(args);
-
try {
metrics.incNumKeyLookups();
return keyManager.lookupKey(args, getClientAddress());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
index 0fa9ca1..f0b8cf5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
@@ -295,7 +295,7 @@ public abstract class OMClientRequest implements RequestAuditor {
.normalize().getPath();
}
if (!keyName.equals(normalizedKeyName)) {
- LOG.debug("Normalized key {} to {} ", keyName,
+ LOG.info("Normalized key {} to {} ", keyName,
normalizedKeyName.substring(1));
}
return isValidKeyPath(normalizedKeyName.substring(1));
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org