You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by xy...@apache.org on 2020/02/25 05:59:53 UTC
[hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2928. Implement
ofs://: listStatus (#547)
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
new 7c35d76 HDDS-2928. Implement ofs://: listStatus (#547)
7c35d76 is described below
commit 7c35d76b2ec22dbd869ec7332e1c333567821e11
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Mon Feb 24 21:59:44 2020 -0800
HDDS-2928. Implement ofs://: listStatus (#547)
---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 142 +++++++++++++++++--
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 154 +++++++++++++++++----
.../fs/ozone/BasicRootedOzoneFileSystem.java | 6 +-
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 16 +++
4 files changed, 280 insertions(+), 38 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 97f840f..033efdb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -43,11 +43,15 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import java.util.stream.Collectors;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
/**
@@ -68,6 +72,7 @@ public class TestRootedOzoneFileSystem {
private String bucketName;
// Store path commonly used by tests that test functionality within a bucket
private Path testBucketPath;
+ private String rootPath;
@Before
public void init() throws Exception {
@@ -82,10 +87,11 @@ public class TestRootedOzoneFileSystem {
OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
volumeName = bucket.getVolumeName();
bucketName = bucket.getName();
- String testBucketStr = "/" + volumeName + "/" + bucketName;
+ String testBucketStr =
+ OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName;
testBucketPath = new Path(testBucketStr);
- String rootPath = String.format("%s://%s/",
+ rootPath = String.format("%s://%s/",
OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY));
// Set the fs.defaultFS and start the filesystem
@@ -213,7 +219,7 @@ public class TestRootedOzoneFileSystem {
/**
* OFS: Helper function for tests. Return a volume name that doesn't exist.
*/
- private String getRandomNonExistVolumeName() throws Exception {
+ private String getRandomNonExistVolumeName() throws IOException {
final int numDigit = 5;
long retriesLeft = Math.round(Math.pow(10, 5));
String name = null;
@@ -230,7 +236,7 @@ public class TestRootedOzoneFileSystem {
}
}
if (retriesLeft <= 0) {
- throw new Exception(
+ Assert.fail(
"Failed to generate random volume name that doesn't exist already.");
}
return name;
@@ -328,10 +334,10 @@ public class TestRootedOzoneFileSystem {
Path root = new Path("/");
FileStatus fileStatus = fs.getFileStatus(root);
Assert.assertNotNull(fileStatus);
- Assert.assertNull(fileStatus.getPath());
+ Assert.assertEquals(new Path(rootPath), fileStatus.getPath());
Assert.assertTrue(fileStatus.isDirectory());
- Assert.assertEquals(
- new FsPermission((short)00755), fileStatus.getPermission());
+ Assert.assertEquals(FsPermission.getDirDefault(),
+ fileStatus.getPermission());
}
/**
@@ -465,7 +471,8 @@ public class TestRootedOzoneFileSystem {
fs.mkdirs(leafInsideInterimPath);
// Attempt to rename the key to a different bucket
- Path bucket2 = new Path("/" + volumeName + "/" + bucketName + "test");
+ Path bucket2 = new Path(OZONE_URI_DELIMITER + volumeName +
+ OZONE_URI_DELIMITER + bucketName + "test");
Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
try {
fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
@@ -480,7 +487,7 @@ public class TestRootedOzoneFileSystem {
throws IOException {
String key = ofs.pathToKey(keyPath);
if (isDirectory) {
- key = key + "/";
+ key = key + OZONE_URI_DELIMITER;
}
OFSPath ofsPath = new OFSPath(key);
String keyInBucket = ofsPath.getKeyName();
@@ -492,4 +499,121 @@ public class TestRootedOzoneFileSystem {
GenericTestUtils.assertExceptionContains("KEY_NOT_FOUND", ex);
}
+ /**
+ * Helper function for testListStatusRootAndVolume*.
+ */
+ private Path createRandomVolumeBucketWithDirs() throws IOException {
+ String volume1 = getRandomNonExistVolumeName();
+ String bucket1 = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path bucketPath1 = new Path(
+ OZONE_URI_DELIMITER + volume1 + OZONE_URI_DELIMITER + bucket1);
+
+ Path dir1 = new Path(bucketPath1, "dir1");
+ fs.mkdirs(dir1); // Intentionally creating this "in-the-middle" dir key
+ Path subdir1 = new Path(dir1, "subdir1");
+ fs.mkdirs(subdir1);
+ Path dir2 = new Path(bucketPath1, "dir2");
+ fs.mkdirs(dir2);
+
+ return bucketPath1;
+ }
+
+ /**
+ * OFS: Test non-recursive listStatus on root and volume.
+ */
+ @Test
+ public void testListStatusRootAndVolumeNonRecursive() throws Exception {
+ Path bucketPath1 = createRandomVolumeBucketWithDirs();
+ createRandomVolumeBucketWithDirs();
+ // listStatus("/volume/bucket")
+ FileStatus[] fileStatusBucket = ofs.listStatus(bucketPath1);
+ Assert.assertEquals(2, fileStatusBucket.length);
+ // listStatus("/volume")
+ Path volume = new Path(
+ OZONE_URI_DELIMITER + new OFSPath(bucketPath1).getVolumeName());
+ FileStatus[] fileStatusVolume = ofs.listStatus(volume);
+ Assert.assertEquals(1, fileStatusVolume.length);
+ // listStatus("/")
+ Path root = new Path(OZONE_URI_DELIMITER);
+ FileStatus[] fileStatusRoot = ofs.listStatus(root);
+ Assert.assertEquals(2, fileStatusRoot.length);
+ }
+
+ /**
+ * Helper function to do FileSystem#listStatus recursively.
+ * Simulate what FsShell does, using DFS.
+ */
+ private void listStatusRecursiveHelper(Path curPath, List<FileStatus> result)
+ throws IOException {
+ FileStatus[] startList = ofs.listStatus(curPath);
+ for (FileStatus fileStatus : startList) {
+ result.add(fileStatus);
+ if (fileStatus.isDirectory()) {
+ Path nextPath = fileStatus.getPath();
+ listStatusRecursiveHelper(nextPath, result);
+ }
+ }
+ }
+
+ /**
+ * Helper function to call adapter impl for listStatus (with recursive).
+ */
+ private List<FileStatus> listStatusCallAdapterHelper(String pathStr)
+ throws IOException {
+ // FileSystem interface does not support recursive listStatus, use adapter
+ BasicRootedOzoneClientAdapterImpl adapter =
+ (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter();
+ return adapter.listStatus(pathStr, true, "", 1000,
+ ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername())
+ .stream().map(ofs::convertFileStatus).collect(Collectors.toList());
+ }
+
+ /**
+ * Helper function to compare recursive listStatus results from adapter
+ * and (simulated) FileSystem.
+ */
+ private void listStatusCheckHelper(Path path) throws IOException {
+ // Get recursive listStatus result directly from adapter impl
+ List<FileStatus> statusesFromAdapter =
+ listStatusCallAdapterHelper(path.toString());
+ // Get recursive listStatus result with FileSystem API by simulating FsShell
+ List<FileStatus> statusesFromFS = new ArrayList<>();
+ listStatusRecursiveHelper(path, statusesFromFS);
+ // Compare. The results would be in the same order due to assumptions:
+ // 1. They are both using DFS internally;
+ // 2. They both return ordered results.
+ Assert.assertEquals(statusesFromAdapter.size(), statusesFromFS.size());
+ final int n = statusesFromFS.size();
+ for (int i = 0; i < n; i++) {
+ FileStatus statusFromAdapter = statusesFromAdapter.get(i);
+ FileStatus statusFromFS = statusesFromFS.get(i);
+ Assert.assertEquals(statusFromAdapter.getPath(), statusFromFS.getPath());
+ Assert.assertEquals(statusFromAdapter.getLen(), statusFromFS.getLen());
+ Assert.assertEquals(statusFromAdapter.isDirectory(),
+ statusFromFS.isDirectory());
+ // TODO: When HDDS-3054 is in, uncomment the lines below.
+ // As of now the modification time almost certainly won't match.
+// Assert.assertEquals(statusFromAdapter.getModificationTime(),
+// statusFromFS.getModificationTime());
+ }
+ }
+
+ /**
+ * OFS: Test recursive listStatus on root and volume.
+ */
+ @Test
+ public void testListStatusRootAndVolumeRecursive() throws Exception {
+ Path bucketPath1 = createRandomVolumeBucketWithDirs();
+ createRandomVolumeBucketWithDirs();
+ // listStatus("/volume/bucket")
+ listStatusCheckHelper(bucketPath1);
+ // listStatus("/volume")
+ Path volume = new Path(
+ OZONE_URI_DELIMITER + new OFSPath(bucketPath1).getVolumeName());
+ listStatusCheckHelper(volume);
+ // listStatus("/")
+ Path root = new Path(OZONE_URI_DELIMITER);
+ listStatusCheckHelper(root);
+ }
+
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index a1871fe..69a350c 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -58,6 +59,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.BUCKET_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
@@ -434,16 +436,16 @@ public class BasicRootedOzoneClientAdapterImpl
}
public FileStatusAdapter getFileStatus(String path, URI uri,
- Path qualifiedPath, String userName)
- throws IOException {
+ Path qualifiedPath, String userName) throws IOException {
incrementCounter(Statistic.OBJECTS_QUERY);
OFSPath ofsPath = new OFSPath(path);
String key = ofsPath.getKeyName();
- // getFileStatus is called for root
- if (ofsPath.getVolumeName().isEmpty() &&
- ofsPath.getBucketName().isEmpty()) {
- // Generate a FileStatusAdapter for root
- return rootFileStatusAdapter();
+ if (ofsPath.isRoot()) {
+ return getFileStatusAdapterForRoot(uri);
+ }
+ if (ofsPath.isVolume()) {
+ OzoneVolume volume = objectStore.getVolume(ofsPath.getVolumeName());
+ return getFileStatusAdapterForVolume(volume, uri);
}
try {
OzoneBucket bucket = getBucket(ofsPath, false);
@@ -452,11 +454,9 @@ public class BasicRootedOzoneClientAdapterImpl
// BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
makeQualified(status, uri, qualifiedPath, userName);
return toFileStatusAdapter(status);
-
} catch (OMException e) {
if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
- throw new
- FileNotFoundException(key + ": No such file or directory!");
+ throw new FileNotFoundException(key + ": No such file or directory!");
}
throw e;
}
@@ -487,6 +487,59 @@ public class BasicRootedOzoneClientAdapterImpl
}
/**
+ * Helper for OFS listStatus on root.
+ */
+ private List<FileStatusAdapter> listStatusRoot(
+ boolean recursive, String startPath, long numEntries,
+ URI uri, Path workingDir, String username) throws IOException {
+
+ OFSPath ofsStartPath = new OFSPath(startPath);
+ // list volumes
+ Iterator<? extends OzoneVolume> iter = objectStore.listVolumesByUser(
+ username, ofsStartPath.getVolumeName(), null);
+ List<FileStatusAdapter> res = new ArrayList<>();
+ // TODO: Test continuation
+ while (iter.hasNext() && res.size() <= numEntries) {
+ OzoneVolume volume = iter.next();
+ res.add(getFileStatusAdapterForVolume(volume, uri));
+ if (recursive) {
+ String pathStrNextVolume = volume.getName();
+ // TODO: Check startPath
+ res.addAll(listStatus(pathStrNextVolume, recursive, startPath,
+ numEntries - res.size(), uri, workingDir, username));
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Helper for OFS listStatus on a volume.
+ */
+ private List<FileStatusAdapter> listStatusVolume(String volumeStr,
+ boolean recursive, String startPath, long numEntries,
+ URI uri, Path workingDir, String username) throws IOException {
+
+ OFSPath ofsStartPath = new OFSPath(startPath);
+ // list buckets in the volume
+ OzoneVolume volume = objectStore.getVolume(volumeStr);
+ Iterator<? extends OzoneBucket> iter =
+ volume.listBuckets(null, ofsStartPath.getBucketName());
+ List<FileStatusAdapter> res = new ArrayList<>();
+ // TODO: Test continuation
+ while (iter.hasNext() && res.size() <= numEntries) {
+ OzoneBucket bucket = iter.next();
+ res.add(getFileStatusAdapterForBucket(bucket, uri, username));
+ if (recursive) {
+ String pathStrNext = volumeStr + OZONE_URI_DELIMITER + bucket.getName();
+ // TODO: Check startPath
+ res.addAll(listStatus(pathStrNext, recursive, startPath,
+ numEntries - res.size(), uri, workingDir, username));
+ }
+ }
+ return res;
+ }
+
+ /**
* OFS listStatus implementation.
*
* @param pathStr Path for the listStatus to operate on.
@@ -511,9 +564,18 @@ public class BasicRootedOzoneClientAdapterImpl
incrementCounter(Statistic.OBJECTS_LIST);
OFSPath ofsPath = new OFSPath(pathStr);
- // TODO: Subject to change in HDDS-2928.
- String keyName = ofsPath.getKeyName();
+ if (ofsPath.isRoot()) {
+ return listStatusRoot(
+ recursive, startPath, numEntries, uri, workingDir, username);
+ }
OFSPath ofsStartPath = new OFSPath(startPath);
+ if (ofsPath.isVolume()) {
+ String startBucket = ofsStartPath.getBucketName();
+ return listStatusVolume(ofsPath.getVolumeName(),
+ recursive, startBucket, numEntries, uri, workingDir, username);
+ }
+
+ String keyName = ofsPath.getKeyName();
// Internally we need startKey to be passed into bucket.listStatus
String startKey = ofsStartPath.getKeyName();
try {
@@ -665,25 +727,65 @@ public class BasicRootedOzoneClientAdapterImpl
}
/**
+ * Generate a FileStatusAdapter for a volume.
+ * @param ozoneVolume OzoneVolume object
+ * @param uri Full URI to OFS root.
+ * @return FileStatusAdapter for a volume.
+ */
+ private static FileStatusAdapter getFileStatusAdapterForVolume(
+ OzoneVolume ozoneVolume, URI uri) {
+ String pathStr = uri.toString() +
+ OZONE_URI_DELIMITER + ozoneVolume.getName();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatusAdapterForVolume: ozoneVolume={}, pathStr={}",
+ ozoneVolume.getName(), pathStr);
+ }
+ Path path = new Path(pathStr);
+ return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+ ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L,
+ FsPermission.getDirDefault().toShort(),
+ // TODO: Revisit owner and admin
+ ozoneVolume.getOwner(), ozoneVolume.getAdmin(), path
+ );
+ }
+
+ /**
+ * Generate a FileStatusAdapter for a bucket.
+ * @param ozoneBucket OzoneBucket object.
+ * @param uri Full URI to OFS root.
+ * @return FileStatusAdapter for a bucket.
+ */
+ private static FileStatusAdapter getFileStatusAdapterForBucket(
+ OzoneBucket ozoneBucket, URI uri, String username) {
+ String pathStr = uri.toString() +
+ OZONE_URI_DELIMITER + ozoneBucket.getVolumeName() +
+ OZONE_URI_DELIMITER + ozoneBucket.getName();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatusAdapterForBucket: ozoneBucket={}, pathStr={}, "
+ + "username={}", ozoneBucket.getVolumeName() + OZONE_URI_DELIMITER
+ + ozoneBucket.getName(), pathStr, username);
+ }
+ Path path = new Path(pathStr);
+ return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+ ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L,
+ FsPermission.getDirDefault().toShort(), // TODO: derive from ACLs later
+ // TODO: revisit owner and group
+ username, username, path);
+ }
+
+ /**
* Generate a FileStatusAdapter for OFS root.
+ * @param uri Full URI to OFS root.
* @return FileStatusAdapter for root.
*/
- private static FileStatusAdapter rootFileStatusAdapter() {
+ private static FileStatusAdapter getFileStatusAdapterForRoot(URI uri) {
// Note that most fields are mimicked from HDFS FileStatus for root,
// except modification time, permission, owner and group.
- // TODO: Revisit the return value.
- return new FileStatusAdapter(
- 0L,
- null,
- true,
- (short)0,
- 0L,
- 0L,
- 0L,
- (short)00755,
- null,
- null,
- null
+ Path path = new Path(uri.toString() + OZONE_URI_DELIMITER);
+ return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+ System.currentTimeMillis(), 0L,
+ FsPermission.getDirDefault().toShort(),
+ null, null, null
);
}
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 35256c8..9274ffc 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.ozone;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -763,9 +764,8 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
return true;
}
- private FileStatus convertFileStatus(
- FileStatusAdapter fileStatusAdapter) {
-
+ @VisibleForTesting
+ FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
Path symLink = null;
try {
fileStatusAdapter.getSymlink();
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index 748d0b4..02b8e02 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -142,4 +142,20 @@ class OFSPath {
public boolean isInSameBucketAs(OFSPath p2) {
return isInSameBucketAsInternal(this, p2);
}
+
+ /**
+ * If both volume and bucket names are empty, the given path is root.
+ * i.e. /
+ */
+ public boolean isRoot() {
+ return this.getVolumeName().isEmpty() && this.getBucketName().isEmpty();
+ }
+
+ /**
+ * If bucket name is empty but volume name is not, the given path is volume.
+ * e.g. /volume1
+ */
+ public boolean isVolume() {
+ return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org