You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by xy...@apache.org on 2020/02/07 00:51:23 UTC
[hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2979. Implement
ofs://: Fix getFileStatus for mkdir volume (#528)
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
new 3e4782d HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
3e4782d is described below
commit 3e4782d3aaeb5b4d4b7429f39a8ce867d3481be4
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Thu Feb 6 16:51:17 2020 -0800
HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 138 +++++++++++----------
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 44 ++++++-
2 files changed, 116 insertions(+), 66 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 97d739b..97f840f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -25,17 +25,18 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -48,17 +49,10 @@ import java.util.TreeSet;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
* Ozone file system tests that are not covered by contract tests.
- *
- * TODO: Refactor this and TestOzoneFileSystem to eliminate most
- * code duplication.
+ * TODO: Refactor this and TestOzoneFileSystem later to reduce code duplication.
*/
public class TestRootedOzoneFileSystem {
@@ -66,19 +60,13 @@ public class TestRootedOzoneFileSystem {
public Timeout globalTimeout = new Timeout(300_000);
private static MiniOzoneCluster cluster = null;
-
private static FileSystem fs;
private static RootedOzoneFileSystem ofs;
-
private static ObjectStore objectStore;
private String volumeName;
private String bucketName;
-
- private String rootPath;
-
// Store path commonly used by tests that test functionality within a bucket
- private String testBucketStr;
private Path testBucketPath;
@Before
@@ -94,11 +82,11 @@ public class TestRootedOzoneFileSystem {
OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
volumeName = bucket.getVolumeName();
bucketName = bucket.getName();
- testBucketStr = "/" + volumeName + "/" + bucketName;
+ String testBucketStr = "/" + volumeName + "/" + bucketName;
testBucketPath = new Path(testBucketStr);
- rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
- conf.get(OZONE_OM_ADDRESS_KEY));
+ String rootPath = String.format("%s://%s/",
+ OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY));
// Set the fs.defaultFS and start the filesystem
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
@@ -123,7 +111,7 @@ public class TestRootedOzoneFileSystem {
// Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
// hence this workaround.
conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
- assertEquals(
+ Assert.assertEquals(
FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
RootedOzoneFileSystem.class);
}
@@ -138,7 +126,7 @@ public class TestRootedOzoneFileSystem {
OzoneKeyDetails key = getKey(child, false);
OFSPath childOFSPath = new OFSPath(child);
- assertEquals(key.getName(), childOFSPath.getKeyName());
+ Assert.assertEquals(key.getName(), childOFSPath.getKeyName());
// Creating a child should not add parent keys to the bucket
try {
@@ -148,9 +136,11 @@ public class TestRootedOzoneFileSystem {
}
// List status on the parent should show the child file
- assertEquals("List status of parent should include the 1 child file", 1L,
- (long)fs.listStatus(parent).length);
- assertTrue("Parent directory does not appear to be a directory",
+ Assert.assertEquals(
+ "List status of parent should include the 1 child file",
+ 1L, fs.listStatus(parent).length);
+ Assert.assertTrue(
+ "Parent directory does not appear to be a directory",
fs.getFileStatus(parent).isDirectory());
}
@@ -171,17 +161,17 @@ public class TestRootedOzoneFileSystem {
}
// Delete the child key
- assertTrue(fs.delete(child, false));
+ Assert.assertTrue(fs.delete(child, false));
// Deleting the only child should create the parent dir key if it does
// not exist
OFSPath parentOFSPath = new OFSPath(parent);
String parentKey = parentOFSPath.getKeyName() + "/";
OzoneKeyDetails parentKeyInfo = getKey(parent, true);
- assertEquals(parentKey, parentKeyInfo.getName());
+ Assert.assertEquals(parentKey, parentKeyInfo.getName());
// Recursive delete with DeleteIterator
- assertTrue(fs.delete(grandparent, true));
+ Assert.assertTrue(fs.delete(grandparent, true));
}
@Test
@@ -191,21 +181,22 @@ public class TestRootedOzoneFileSystem {
Path file2 = new Path(parent, "key2");
FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
- assertEquals("Should be empty", 0, fileStatuses.length);
+ Assert.assertEquals("Should be empty", 0, fileStatuses.length);
ContractTestUtils.touch(fs, file1);
ContractTestUtils.touch(fs, file2);
fileStatuses = ofs.listStatus(testBucketPath);
- assertEquals("Should have created parent",
+ Assert.assertEquals("Should have created parent",
1, fileStatuses.length);
- assertEquals("Parent path doesn't match",
+ Assert.assertEquals("Parent path doesn't match",
fileStatuses[0].getPath().toUri().getPath(), parent.toString());
// ListStatus on a directory should return all subdirs along with
// files, even if there exists a file and sub-dir with the same name.
fileStatuses = ofs.listStatus(parent);
- assertEquals("FileStatus did not return all children of the directory",
+ Assert.assertEquals(
+ "FileStatus did not return all children of the directory",
2, fileStatuses.length);
// ListStatus should return only the immediate children of a directory.
@@ -214,7 +205,8 @@ public class TestRootedOzoneFileSystem {
ContractTestUtils.touch(fs, file3);
ContractTestUtils.touch(fs, file4);
fileStatuses = ofs.listStatus(parent);
- assertEquals("FileStatus did not return all children of the directory",
+ Assert.assertEquals(
+ "FileStatus did not return all children of the directory",
3, fileStatuses.length);
}
@@ -264,23 +256,26 @@ public class TestRootedOzoneFileSystem {
OFSPath ofsPathDir1 = new OFSPath(dir12);
String key = ofsPathDir1.getKeyName() + "/";
OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
- assertEquals(key, ozoneKeyDetails.getName());
+ Assert.assertEquals(key, ozoneKeyDetails.getName());
// Verify that directories are created.
FileStatus[] fileStatuses = ofs.listStatus(root);
- assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
- assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+ Assert.assertEquals(
+ fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ Assert.assertEquals(
+ fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
fileStatuses = ofs.listStatus(dir1);
- assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ Assert.assertEquals(
+ fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
fileStatuses = ofs.listStatus(dir12);
- assertEquals(fileStatuses.length, 0);
+ Assert.assertEquals(fileStatuses.length, 0);
fileStatuses = ofs.listStatus(dir2);
- assertEquals(fileStatuses.length, 0);
+ Assert.assertEquals(fileStatuses.length, 0);
}
/**
- * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+ * OFS: Test mkdir on a volume and bucket that doesn't exist.
*/
@Test
public void testMkdirNonExistentVolumeBucket() throws Exception {
@@ -294,20 +289,20 @@ public class TestRootedOzoneFileSystem {
Iterator<? extends OzoneVolume> iterVol =
objectStore.listVolumesByUser(null, volumeNameLocal, null);
OzoneVolume ozoneVolume = iterVol.next();
- assertNotNull(ozoneVolume);
- assertEquals(volumeNameLocal, ozoneVolume.getName());
+ Assert.assertNotNull(ozoneVolume);
+ Assert.assertEquals(volumeNameLocal, ozoneVolume.getName());
Iterator<? extends OzoneBucket> iterBuc =
ozoneVolume.listBuckets("bucket-");
OzoneBucket ozoneBucket = iterBuc.next();
- assertNotNull(ozoneBucket);
- assertEquals(bucketNameLocal, ozoneBucket.getName());
+ Assert.assertNotNull(ozoneBucket);
+ Assert.assertEquals(bucketNameLocal, ozoneBucket.getName());
// TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
}
/**
- * OFS: Tests mkdir on a volume that doesn't exist.
+ * OFS: Test mkdir on a volume that doesn't exist.
*/
@Test
public void testMkdirNonExistentVolume() throws Exception {
@@ -319,17 +314,31 @@ public class TestRootedOzoneFileSystem {
Iterator<? extends OzoneVolume> iterVol =
objectStore.listVolumesByUser(null, volumeNameLocal, null);
OzoneVolume ozoneVolume = iterVol.next();
- assertNotNull(ozoneVolume);
- assertEquals(volumeNameLocal, ozoneVolume.getName());
+ Assert.assertNotNull(ozoneVolume);
+ Assert.assertEquals(volumeNameLocal, ozoneVolume.getName());
// TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
}
/**
- * Tests listStatus operation in a bucket.
+ * OFS: Test getFileStatus on root.
*/
@Test
- public void testListStatusOnRoot() throws Exception {
+ public void testGetFileStatusRoot() throws Exception {
+ Path root = new Path("/");
+ FileStatus fileStatus = fs.getFileStatus(root);
+ Assert.assertNotNull(fileStatus);
+ Assert.assertNull(fileStatus.getPath());
+ Assert.assertTrue(fileStatus.isDirectory());
+ Assert.assertEquals(
+ new FsPermission((short)00755), fileStatus.getPermission());
+ }
+
+ /**
+ * Test listStatus operation in a bucket.
+ */
+ @Test
+ public void testListStatusInBucket() throws Exception {
Path root = new Path("/" + volumeName + "/" + bucketName);
Path dir1 = new Path(root, "dir1");
Path dir12 = new Path(dir1, "dir12");
@@ -341,14 +350,15 @@ public class TestRootedOzoneFileSystem {
// exist) and dir2 only. dir12 is not an immediate child of root and
// hence should not be listed.
FileStatus[] fileStatuses = ofs.listStatus(root);
- assertEquals("FileStatus should return only the immediate children", 2,
- fileStatuses.length);
+ Assert.assertEquals(
+ "FileStatus should return only the immediate children",
+ 2, fileStatuses.length);
// Verify that dir12 is not included in the result of the listStatus on root
String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
- assertFalse(fileStatus1.equals(dir12.toString()));
- assertFalse(fileStatus2.equals(dir12.toString()));
+ Assert.assertNotEquals(fileStatus1, dir12.toString());
+ Assert.assertNotEquals(fileStatus2, dir12.toString());
}
/**
@@ -366,12 +376,12 @@ public class TestRootedOzoneFileSystem {
}
FileStatus[] fileStatuses = ofs.listStatus(root);
- assertEquals(
+ Assert.assertEquals(
"Total directories listed do not match the existing directories",
numDirs, fileStatuses.length);
for (int i=0; i < numDirs; i++) {
- assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
+ Assert.assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
}
}
@@ -401,16 +411,17 @@ public class TestRootedOzoneFileSystem {
fs.mkdirs(dir2);
FileStatus[] fileStatuses = ofs.listStatus(dir1);
- assertEquals("FileStatus should return only the immediate children", 2,
- fileStatuses.length);
+ Assert.assertEquals(
+ "FileStatus should return only the immediate children",
+ 2, fileStatuses.length);
// Verify that the two children of /dir1 returned by listStatus operation
// are /dir1/dir11 and /dir1/dir12.
String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
- assertTrue(fileStatus1.equals(dir11.toString()) ||
+ Assert.assertTrue(fileStatus1.equals(dir11.toString()) ||
fileStatus1.equals(dir12.toString()));
- assertTrue(fileStatus2.equals(dir11.toString()) ||
+ Assert.assertTrue(fileStatus2.equals(dir11.toString()) ||
fileStatus2.equals(dir12.toString()));
}
@@ -427,15 +438,15 @@ public class TestRootedOzoneFileSystem {
fs.mkdirs(target);
fs.mkdirs(leafInsideInterimPath);
- assertTrue(fs.rename(leafInsideInterimPath, leafInTarget));
+ Assert.assertTrue(fs.rename(leafInsideInterimPath, leafInTarget));
// after rename listStatus for interimPath should succeed and
// interimPath should have no children
FileStatus[] statuses = fs.listStatus(interimPath);
- assertNotNull("liststatus returns a null array", statuses);
- assertEquals("Statuses array is not empty", 0, statuses.length);
+ Assert.assertNotNull("liststatus returns a null array", statuses);
+ Assert.assertEquals("Statuses array is not empty", 0, statuses.length);
FileStatus fileStatus = fs.getFileStatus(interimPath);
- assertEquals("FileStatus does not point to interimPath",
+ Assert.assertEquals("FileStatus does not point to interimPath",
interimPath.getName(), fileStatus.getPath().getName());
}
@@ -458,14 +469,15 @@ public class TestRootedOzoneFileSystem {
Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
try {
fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
- fail("Should have thrown exception when renaming to a different bucket");
+ Assert.fail(
+ "Should have thrown exception when renaming to a different bucket");
} catch (IOException ignored) {
// Test passed. Exception thrown as expected.
}
}
private OzoneKeyDetails getKey(Path keyPath, boolean isDirectory)
- throws IOException, OzoneClientException {
+ throws IOException {
String key = ofs.pathToKey(keyPath);
if (isDirectory) {
key = key + "/";
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 36559d2..a1871fe 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
@@ -80,8 +81,8 @@ public class BasicRootedOzoneClientAdapterImpl
LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
private OzoneClient ozoneClient;
- private ClientProtocol proxy;
private ObjectStore objectStore;
+ private ClientProtocol proxy;
private ReplicationType replicationType;
private ReplicationFactor replicationFactor;
private boolean securityEnabled;
@@ -196,6 +197,14 @@ public class BasicRootedOzoneClientAdapterImpl
*/
private OzoneBucket getBucket(String volumeStr, String bucketStr,
boolean createIfNotExist) throws IOException {
+ Preconditions.checkNotNull(volumeStr);
+ Preconditions.checkNotNull(bucketStr);
+
+ if (bucketStr.isEmpty()) {
+ // throw FileNotFoundException in this case to make Hadoop common happy
+ throw new FileNotFoundException(
+ "getBucket: Invalid argument: given bucket string is empty.");
+ }
OzoneBucket bucket;
try {
@@ -348,11 +357,11 @@ public class BasicRootedOzoneClientAdapterImpl
LOG.trace("creating dir for path: {}", pathStr);
incrementCounter(Statistic.OBJECTS_CREATED);
OFSPath ofsPath = new OFSPath(pathStr);
- if (ofsPath.getVolumeName().length() == 0) {
+ if (ofsPath.getVolumeName().isEmpty()) {
// Volume name unspecified, invalid param, return failure
return false;
}
- if (ofsPath.getBucketName().length() == 0) {
+ if (ofsPath.getBucketName().isEmpty()) {
// Create volume only
objectStore.createVolume(ofsPath.getVolumeName());
return true;
@@ -430,6 +439,12 @@ public class BasicRootedOzoneClientAdapterImpl
incrementCounter(Statistic.OBJECTS_QUERY);
OFSPath ofsPath = new OFSPath(path);
String key = ofsPath.getKeyName();
+ // getFileStatus is called for root
+ if (ofsPath.getVolumeName().isEmpty() &&
+ ofsPath.getBucketName().isEmpty()) {
+ // Generate a FileStatusAdapter for root
+ return rootFileStatusAdapter();
+ }
try {
OzoneBucket bucket = getBucket(ofsPath, false);
OzoneFileStatus status = bucket.getFileStatus(key);
@@ -648,4 +663,27 @@ public class BasicRootedOzoneClientAdapterImpl
status.getPath()
);
}
+
+ /**
+ * Generate a FileStatusAdapter for OFS root.
+ * @return FileStatusAdapter for root.
+ */
+ private static FileStatusAdapter rootFileStatusAdapter() {
+ // Note that most fields are mimicked from HDFS FileStatus for root,
+ // except modification time, permission, owner and group.
+ // TODO: Revisit the return value.
+ return new FileStatusAdapter(
+ 0L,
+ null,
+ true,
+ (short)0,
+ 0L,
+ 0L,
+ 0L,
+ (short)00755,
+ null,
+ null,
+ null
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org