You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ck...@apache.org on 2022/10/26 03:39:37 UTC
[ozone] branch master updated: Revert "HDDS-7253. Fix exception when '/' in key name (#3774)"
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 561788ebe7 Revert "HDDS-7253. Fix exception when '/' in key name (#3774)"
561788ebe7 is described below
commit 561788ebe7da445aba93356c61e018f74c9cde52
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Wed Oct 26 11:31:28 2022 +0800
Revert "HDDS-7253. Fix exception when '/' in key name (#3774)"
This reverts commit b9a47f6bbf72605d5391cafc8d382190f822b375.
---
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 81 ++--------------------
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 70 ++-----------------
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 68 ++++--------------
3 files changed, 20 insertions(+), 199 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 1f688029b6..f3e8cf10be 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -153,7 +153,6 @@ public class TestOzoneFileSystem {
private static OzoneManagerProtocol writeClient;
private static FileSystem fs;
private static OzoneFileSystem o3fs;
- private static OzoneBucket ozoneBucket;
private static String volumeName;
private static String bucketName;
private static Trash trash;
@@ -180,9 +179,10 @@ public class TestOzoneFileSystem {
writeClient = cluster.getRpcClient().getObjectStore()
.getClientProxy().getOzoneManagerClient();
// create a volume and a bucket to be used by OzoneFileSystem
- ozoneBucket = TestDataUtil.createVolumeAndBucket(cluster, bucketLayout);
- volumeName = ozoneBucket.getVolumeName();
- bucketName = ozoneBucket.getName();
+ OzoneBucket bucket =
+ TestDataUtil.createVolumeAndBucket(cluster, bucketLayout);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
String rootPath = String.format("%s://%s.%s/",
OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
@@ -334,30 +334,6 @@ public class TestOzoneFileSystem {
assertTrue("Shouldn't send error if dir exists", status);
}
- @Test
- public void testMakeDirsWithAnFakeDirectory() throws Exception {
- /*
- * Op 1. commit a key -> "dir1/dir2/key1"
- * Op 2. create dir -> "dir1/testDir", the dir1 is a fake dir,
- * "dir1/testDir" can be created normal
- */
-
- String fakeGrandpaKey = "dir1";
- String fakeParentKey = fakeGrandpaKey + "/dir2";
- String fullKeyName = fakeParentKey + "/key1";
- TestDataUtil.createKey(ozoneBucket, fullKeyName, "");
-
- // /dir1/dir2 should not exist
- assertFalse(fs.exists(new Path(fakeParentKey)));
-
- // /dir1/dir2/key2 should be created because has a fake parent directory
- Path subdir = new Path(fakeParentKey, "key2");
- assertTrue(fs.mkdirs(subdir));
- // the intermediate directories /dir1 and /dir1/dir2 will be created too
- assertTrue(fs.exists(new Path(fakeGrandpaKey)));
- assertTrue(fs.exists(new Path(fakeParentKey)));
- }
-
@Test
public void testCreateWithInvalidPaths() throws Exception {
// Test for path with ..
@@ -751,37 +727,6 @@ public class TestOzoneFileSystem {
}
}
- @Test
- public void testListStatusOnKeyNameContainDelimiter() throws Exception {
- /*
- * op1: create a key -> "dir1/dir2/key1"
- * op2: `ls /` child dir "/dir1/" will be return
- * op2: `ls /dir1` child dir "/dir1/dir2/" will be return
- * op3: `ls /dir1/dir2` file "/dir1/dir2/key" will be return
- *
- * the "/dir1", "/dir1/dir2/" are fake directory
- * */
- String keyName = "dir1/dir2/key1";
- TestDataUtil.createKey(ozoneBucket, keyName, "");
- FileStatus[] fileStatuses;
-
- fileStatuses = fs.listStatus(new Path("/"));
- assertEquals(1, fileStatuses.length);
- assertEquals("/dir1", fileStatuses[0].getPath().toUri().getPath());
- assertTrue(fileStatuses[0].isDirectory());
-
- fileStatuses = fs.listStatus(new Path("/dir1"));
- assertEquals(1, fileStatuses.length);
- assertEquals("/dir1/dir2", fileStatuses[0].getPath().toUri().getPath());
- assertTrue(fileStatuses[0].isDirectory());
-
- fileStatuses = fs.listStatus(new Path("/dir1/dir2"));
- assertEquals(1, fileStatuses.length);
- assertEquals("/dir1/dir2/key1",
- fileStatuses[0].getPath().toUri().getPath());
- assertTrue(fileStatuses[0].isFile());
- }
-
/**
* Cleanup files and directories.
*
@@ -1328,24 +1273,6 @@ public class TestOzoneFileSystem {
"file1")));
}
- @Test
- public void testRenameContainDelimiterFile() throws Exception {
- String fakeGrandpaKey = "dir1";
- String fakeParentKey = fakeGrandpaKey + "/dir2";
- String sourceKeyName = fakeParentKey + "/key1";
- String targetKeyName = fakeParentKey + "/key2";
- TestDataUtil.createKey(ozoneBucket, sourceKeyName, "");
-
- Path sourcePath = new Path(fs.getUri().toString() + "/" + sourceKeyName);
- Path targetPath = new Path(fs.getUri().toString() + "/" + targetKeyName);
- assertTrue(fs.rename(sourcePath, targetPath));
- assertFalse(fs.exists(sourcePath));
- assertTrue(fs.exists(targetPath));
- // intermediate directories will not be created
- assertFalse(fs.exists(new Path(fakeGrandpaKey)));
- assertFalse(fs.exists(new Path(fakeParentKey)));
- }
-
/**
* Fails if the (a) parent of dst does not exist or (b) parent is a file.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index e30d27fc74..492173b71c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -800,20 +800,19 @@ public class TestKeyManagerImpl {
List<OmKeyLocationInfo> locationList =
keySession.getKeyInfo().getLatestVersionLocations().getLocationList();
Assert.assertEquals(1, locationList.size());
- long containerID = locationList.get(0).getContainerID();
locationInfoList.add(
new OmKeyLocationInfo.Builder().setPipeline(pipeline)
- .setBlockID(new BlockID(containerID,
+ .setBlockID(new BlockID(locationList.get(0).getContainerID(),
locationList.get(0).getLocalID())).build());
keyArgs.setLocationInfoList(locationInfoList);
writeClient.commitKey(keyArgs, keySession.getId());
- ContainerInfo containerInfo = new ContainerInfo.Builder()
- .setContainerID(containerID).setPipelineID(pipeline.getId()).build();
+ ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(1L)
+ .setPipelineID(pipeline.getId()).build();
List<ContainerWithPipeline> containerWithPipelines = Arrays.asList(
new ContainerWithPipeline(containerInfo, pipeline));
when(mockScmContainerClient.getContainerWithPipelineBatch(
- Arrays.asList(containerID))).thenReturn(containerWithPipelines);
+ Arrays.asList(1L))).thenReturn(containerWithPipelines);
OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
Assert.assertEquals(key.getKeyName(), keyName);
@@ -1274,67 +1273,6 @@ public class TestKeyManagerImpl {
}
}
- @Test
- public void testGetFileStatus() throws IOException {
- // create a key
- String keyName = RandomStringUtils.randomAlphabetic(5);
- OmKeyArgs keyArgs = createBuilder()
- .setKeyName(keyName)
- .setLatestVersionLocation(true)
- .build();
- writeClient.createFile(keyArgs, false, false);
- OpenKeySession keySession = writeClient.createFile(keyArgs, true, true);
- keyArgs.setLocationInfoList(
- keySession.getKeyInfo().getLatestVersionLocations().getLocationList());
- writeClient.commitKey(keyArgs, keySession.getId());
- OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs);
- Assert.assertEquals(keyName, ozoneFileStatus.getKeyInfo().getFileName());
- }
-
- @Test
- public void testGetFileStatusWithFakeDir() throws IOException {
- String parentDir = "dir1";
- String key = "key1";
- String fullKeyName = parentDir + OZONE_URI_DELIMITER + key;
- OzoneFileStatus ozoneFileStatus;
-
- // create a key "dir1/key1"
- OmKeyArgs keyArgs = createBuilder().setKeyName(fullKeyName).build();
- OpenKeySession keySession = writeClient.openKey(keyArgs);
- keyArgs.setLocationInfoList(
- keySession.getKeyInfo().getLatestVersionLocations().getLocationList());
- writeClient.commitKey(keyArgs, keySession.getId());
-
- // verify
- String keyArg;
- keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, parentDir);
- Assert.assertNull(
- metadataManager.getKeyTable(getDefaultBucketLayout()).get(keyArg));
- keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, fullKeyName);
- Assert.assertNotNull(metadataManager.getKeyTable(getDefaultBucketLayout())
- .get(keyArg));
-
- // get a non-existing "dir1", since the key is prefixed "dir1/key1",
- // a fake "/dir1" will be returned
- keyArgs = createBuilder().setKeyName(parentDir).build();
- ozoneFileStatus = keyManager.getFileStatus(keyArgs);
- Assert.assertEquals(parentDir, ozoneFileStatus.getKeyInfo().getFileName());
- Assert.assertTrue(ozoneFileStatus.isDirectory());
-
- // get a non-existing "dir", since the key is not prefixed "dir1/key1",
- // a `OMException` will be thrown
- keyArgs = createBuilder().setKeyName("dir").build();
- OmKeyArgs finalKeyArgs = keyArgs;
- Assert.assertThrows(OMException.class, () -> keyManager.getFileStatus(
- finalKeyArgs));
-
- // get a file "dir1/key1"
- keyArgs = createBuilder().setKeyName(fullKeyName).build();
- ozoneFileStatus = keyManager.getFileStatus(keyArgs);
- Assert.assertEquals(key, ozoneFileStatus.getKeyInfo().getFileName());
- Assert.assertTrue(ozoneFileStatus.isFile());
- }
-
@Test
public void testRefreshPipeline() throws Exception {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 483a2e88b0..dd66c33a1e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1209,8 +1209,6 @@ public class KeyManagerImpl implements KeyManager {
final String keyName = args.getKeyName();
OmKeyInfo fileKeyInfo = null;
- OmKeyInfo dirKeyInfo = null;
- OmKeyInfo fakeDirKeyInfo = null;
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
try {
@@ -1223,27 +1221,28 @@ public class KeyManagerImpl implements KeyManager {
// Check if the key is a file.
String fileKeyBytes = metadataManager.getOzoneKey(
volumeName, bucketName, keyName);
- BucketLayout layout =
- getBucketLayout(metadataManager, volumeName, bucketName);
- fileKeyInfo = metadataManager.getKeyTable(layout).get(fileKeyBytes);
- String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
+ fileKeyInfo = metadataManager
+ .getKeyTable(getBucketLayout(metadataManager, volumeName, bucketName))
+ .get(fileKeyBytes);
// Check if the key is a directory.
if (fileKeyInfo == null) {
+ String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
String dirKeyBytes = metadataManager.getOzoneKey(
volumeName, bucketName, dirKey);
- dirKeyInfo = metadataManager.getKeyTable(layout).get(dirKeyBytes);
- if (dirKeyInfo == null) {
- fakeDirKeyInfo =
- createFakeDirIfShould(volumeName, bucketName, keyName, layout);
+ OmKeyInfo dirKeyInfo = metadataManager.getKeyTable(
+ getBucketLayout(metadataManager, volumeName, bucketName))
+ .get(dirKeyBytes);
+ if (dirKeyInfo != null) {
+ return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true);
}
}
} finally {
metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
bucketName);
+
+ // if the key is a file then do refresh pipeline info in OM by asking SCM
if (fileKeyInfo != null) {
- // if the key is a file
- // then do refresh pipeline info in OM by asking SCM
if (args.getLatestVersionLocation()) {
slimLocationVersion(fileKeyInfo);
}
@@ -1258,21 +1257,10 @@ public class KeyManagerImpl implements KeyManager {
sortDatanodes(clientAddress, fileKeyInfo);
}
}
+ return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false);
}
}
- if (fileKeyInfo != null) {
- return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false);
- }
-
- if (dirKeyInfo != null) {
- return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true);
- }
-
- if (fakeDirKeyInfo != null) {
- return new OzoneFileStatus(fakeDirKeyInfo, scmBlockSize, true);
- }
-
// Key is not found, throws exception
if (LOG.isDebugEnabled()) {
LOG.debug("Unable to get file status for the key: volume: {}, bucket:" +
@@ -1284,37 +1272,6 @@ public class KeyManagerImpl implements KeyManager {
FILE_NOT_FOUND);
}
- /**
- * Create a fake directory if the key is a path prefix,
- * otherwise returns null.
- * Some keys may contain '/' Ozone will treat '/' as directory separator
- * such as : key name is 'a/b/c', 'a' and 'b' may not really exist,
- * but Ozone treats 'a' and 'b' as a directory.
- * we need create a fake directory 'a' or 'a/b'
- *
- * @return OmKeyInfo if the key is a path prefix, otherwise returns null.
- */
- private OmKeyInfo createFakeDirIfShould(String volume, String bucket,
- String keyName, BucketLayout layout) throws IOException {
- OmKeyInfo fakeDirKeyInfo = null;
- String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
- String fileKeyBytes = metadataManager.getOzoneKey(volume, bucket, keyName);
- Table.KeyValue<String, OmKeyInfo> keyValue =
- metadataManager.getKeyTable(layout).iterator().seek(fileKeyBytes);
-
- if (keyValue != null) {
- Path fullPath = Paths.get(keyValue.getValue().getKeyName());
- Path subPath = Paths.get(dirKey);
- OmKeyInfo omKeyInfo = keyValue.getValue();
- if (fullPath.startsWith(subPath)) {
- // create fake directory
- fakeDirKeyInfo = createDirectoryKey(omKeyInfo, dirKey);
- }
- }
-
- return fakeDirKeyInfo;
- }
-
private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args,
String clientAddress, boolean skipFileNotFoundError) throws IOException {
@@ -1392,7 +1349,6 @@ public class KeyManagerImpl implements KeyManager {
.setVolumeName(keyInfo.getVolumeName())
.setBucketName(keyInfo.getBucketName())
.setKeyName(dir)
- .setFileName(OzoneFSUtils.getFileName(keyName))
.setOmKeyLocationInfos(Collections.singletonList(
new OmKeyLocationInfoGroup(0, new ArrayList<>())))
.setCreationTime(Time.now())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org