You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2020/06/02 14:53:35 UTC
[hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-3574. Implement
ofs://: Override getTrashRoot (#941)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
new 28540cb HDDS-3574. Implement ofs://: Override getTrashRoot (#941)
28540cb is described below
commit 28540cb1cc1f27c43f727b797b408125e21a9abb
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Tue Jun 2 07:53:21 2020 -0700
HDDS-3574. Implement ofs://: Override getTrashRoot (#941)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 88 ++++++++++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +++
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 130 ++++++++++++++++-----
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 36 ++++--
4 files changed, 228 insertions(+), 39 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index caf9c69..6e1540c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.ozone.shell;
import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ozone.OFSPath;
+import org.apache.hadoop.fs.ozone.OzoneFsShell;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
@@ -29,6 +34,7 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.shell.s3.S3Shell;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -48,11 +54,15 @@ import picocli.CommandLine.RunLast;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.junit.Assert.fail;
/**
@@ -442,6 +452,84 @@ public class TestOzoneShellHA {
Assert.assertEquals(0, getNumOfBuckets("bucket"));
}
+ /**
+ * Helper function to retrieve Ozone client configuration for trash testing.
+ * @param hostPrefix Scheme + Authority. e.g. ofs://om-service-test1
+ * @param configuration Server config to generate client config from.
+ * @return Config added with fs.ofs.impl, fs.defaultFS and fs.trash.interval.
+ */
+ private OzoneConfiguration getClientConfForOFS(
+ String hostPrefix, OzoneConfiguration configuration) {
+
+ OzoneConfiguration clientConf = new OzoneConfiguration(configuration);
+ clientConf.set("fs.ofs.impl",
+ "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix);
+ clientConf.setInt(FS_TRASH_INTERVAL_KEY, 60);
+ return clientConf;
+ }
+
+ @Test
+ public void testDeleteToTrashOrSkipTrash() throws Exception {
+ final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
+ OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ OzoneFsShell shell = new OzoneFsShell(clientConf);
+ FileSystem fs = FileSystem.get(clientConf);
+ final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1";
+ // Note: CURRENT is also privately defined in TrashPolicyDefault
+ final Path trashCurrent = new Path("Current");
+
+ final String strKey1 = strDir1 + "/key1";
+ final Path pathKey1 = new Path(strKey1);
+ final Path trashPathKey1 = Path.mergePaths(new Path(
+ new OFSPath(strKey1).getTrashRoot(), trashCurrent), pathKey1);
+
+ final String strKey2 = strDir1 + "/key2";
+ final Path pathKey2 = new Path(strKey2);
+ final Path trashPathKey2 = Path.mergePaths(new Path(
+ new OFSPath(strKey2).getTrashRoot(), trashCurrent), pathKey2);
+
+ int res;
+ try {
+ res = ToolRunner.run(shell, new String[]{"-mkdir", "-p", strDir1});
+ Assert.assertEquals(0, res);
+
+ // Check delete to trash behavior
+ res = ToolRunner.run(shell, new String[]{"-touch", strKey1});
+ Assert.assertEquals(0, res);
+ // Verify key1 creation
+ FileStatus statusPathKey1 = fs.getFileStatus(pathKey1);
+ Assert.assertEquals(strKey1, statusPathKey1.getPath().toString());
+ // rm without -skipTrash. since trash interval > 0, should moved to trash
+ res = ToolRunner.run(shell, new String[]{"-rm", strKey1});
+ Assert.assertEquals(0, res);
+ // Verify that the file is moved to the correct trash location
+ FileStatus statusTrashPathKey1 = fs.getFileStatus(trashPathKey1);
+ // It'd be more meaningful if we actually write some content to the file
+ Assert.assertEquals(
+ statusPathKey1.getLen(), statusTrashPathKey1.getLen());
+ Assert.assertEquals(
+ fs.getFileChecksum(pathKey1), fs.getFileChecksum(trashPathKey1));
+
+ // Check delete skip trash behavior
+ res = ToolRunner.run(shell, new String[]{"-touch", strKey2});
+ Assert.assertEquals(0, res);
+ // Verify key2 creation
+ FileStatus statusPathKey2 = fs.getFileStatus(pathKey2);
+ Assert.assertEquals(strKey2, statusPathKey2.getPath().toString());
+ // rm with -skipTrash
+ res = ToolRunner.run(shell, new String[]{"-rm", "-skipTrash", strKey2});
+ Assert.assertEquals(0, res);
+ // Verify that the file is NOT moved to the trash location
+ try {
+ fs.getFileStatus(trashPathKey2);
+ Assert.fail("getFileStatus on non-existent should throw.");
+ } catch (FileNotFoundException ignored) {
+ }
+ } finally {
+ shell.close();
+ }
+ }
@Test
public void testS3PathCommand() throws Exception {
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index fd6df55..dbf0074 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -659,6 +659,19 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
}
/**
+ * Get the root directory of Trash for a path in OFS.
+ * Returns /<volumename>/<bucketname>/.Trash/<username>
+ * Caller appends either Current or checkpoint timestamp for trash destination
+ * @param path the trash root of the path to be determined.
+ * @return trash root
+ */
+ @Override
+ public Path getTrashRoot(Path path) {
+ OFSPath ofsPath = new OFSPath(path);
+ return ofsPath.getTrashRoot();
+ }
+
+ /**
* Creates a directory. Directory is represented using a key with no value.
*
* @param path directory path to be created
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index 88f89fc..f602833 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -34,6 +34,8 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.StringTokenizer;
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
/**
@@ -41,18 +43,19 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-class OFSPath {
+public class OFSPath {
+ private String authority = "";
/**
* Here is a table illustrating what each name variable is given an input path
* Assuming /tmp is mounted to /tempVol/tempBucket
* (empty) = empty string "".
*
- * Path volumeName bucketName mountName keyName
+ * Path volumeName bucketName mountName keyName
* --------------------------------------------------------------------------
- * /vol1/buc2/dir3/key4 vol1 buc2 (empty) dir3/key4
- * /vol1/buc2 vol1 buc2 (empty) (empty)
- * /vol1 vol1 (empty) (empty) (empty)
- * /tmp/dir3/key4 tmp <username> tmp dir3/key4
+ * /vol1/buc2/dir3/key4 vol1 buc2 (empty) dir3/key4
+ * /vol1/buc2 vol1 buc2 (empty) (empty)
+ * /vol1 vol1 (empty) (empty) (empty)
+ * /tmp/dir3/key4 tmp md5(<username>) tmp dir3/key4
*
* Note the leading '/' doesn't matter.
*/
@@ -65,37 +68,38 @@ class OFSPath {
@VisibleForTesting
static final String OFS_MOUNT_TMP_VOLUMENAME = "tmp";
- OFSPath(Path path) {
- String pathStr = path.toUri().getPath();
- initOFSPath(pathStr);
+ public OFSPath(Path path) {
+ initOFSPath(path.toUri());
}
- OFSPath(String pathStr) {
- initOFSPath(pathStr);
+ public OFSPath(String pathStr) {
+ try {
+ initOFSPath(new URI(pathStr));
+ } catch (URISyntaxException ex) {
+ throw new RuntimeException(ex);
+ }
}
- private void initOFSPath(String pathStr) {
- // pathStr should not have authority
- try {
- URI uri = new URI(pathStr);
- String authority = uri.getAuthority();
- if (authority != null && !authority.isEmpty()) {
- throw new ParseException("Invalid path " + pathStr +
- ". Shouldn't contain authority.");
+ private void initOFSPath(URI uri) {
+ // Scheme is case-insensitive
+ String scheme = uri.getScheme();
+ if (scheme != null) {
+ if (!scheme.toLowerCase().equals(OZONE_OFS_URI_SCHEME)) {
+ throw new ParseException("Can't parse schemes other than ofs://.");
}
- } catch (URISyntaxException ex) {
- throw new ParseException("Failed to parse path " + pathStr + " as URI.");
}
- // tokenize
+ // authority could be empty
+ authority = uri.getAuthority() == null ? "" : uri.getAuthority();
+ String pathStr = uri.getPath();
StringTokenizer token = new StringTokenizer(pathStr, OZONE_URI_DELIMITER);
int numToken = token.countTokens();
+
if (numToken > 0) {
String firstToken = token.nextToken();
- // TODO: Compare a keyword list instead for future expansion.
+ // TODO: Compare a list of mounts in the future.
if (firstToken.equals(OFS_MOUNT_NAME_TMP)) {
mountName = firstToken;
- // TODO: In the future, may retrieve volume and bucket from
- // UserVolumeInfo on the server side. TBD.
+ // TODO: Make this configurable in the future.
volumeName = OFS_MOUNT_TMP_VOLUMENAME;
try {
bucketName = getTempMountBucketNameOfCurrentUser();
@@ -119,6 +123,10 @@ class OFSPath {
}
}
+ public String getAuthority() {
+ return authority;
+ }
+
public String getVolumeName() {
return volumeName;
}
@@ -137,6 +145,39 @@ class OFSPath {
}
/**
+ * Return the reconstructed path string.
+ * Directories including volumes and buckets will have a trailing '/'.
+ */
+ @Override
+ public String toString() {
+ Preconditions.checkNotNull(authority);
+ StringBuilder sb = new StringBuilder();
+ if (!isMount()) {
+ sb.append(volumeName);
+ sb.append(OZONE_URI_DELIMITER);
+ if (!bucketName.isEmpty()) {
+ sb.append(bucketName);
+ sb.append(OZONE_URI_DELIMITER);
+ }
+ } else {
+ sb.append(mountName);
+ sb.append(OZONE_URI_DELIMITER);
+ }
+ if (!keyName.isEmpty()) {
+ sb.append(keyName);
+ }
+ if (authority.isEmpty()) {
+ sb.insert(0, OZONE_URI_DELIMITER);
+ return sb.toString();
+ } else {
+ final Path pathWithSchemeAuthority = new Path(
+ OZONE_OFS_URI_SCHEME, authority, OZONE_URI_DELIMITER);
+ sb.insert(0, pathWithSchemeAuthority.toString());
+ return sb.toString();
+ }
+ }
+
+ /**
* Get the volume & bucket or mount name (non-key path).
* @return String of path excluding key in bucket.
*/
@@ -176,15 +217,15 @@ class OFSPath {
/**
* If both volume and bucket names are empty, the given path is root.
- * i.e. /
+ * i.e. / is root.
*/
public boolean isRoot() {
return this.getVolumeName().isEmpty() && this.getBucketName().isEmpty();
}
/**
- * If bucket name is empty but volume name is not, the given path is volume.
- * e.g. /volume1
+ * If bucket name is empty but volume name is not, the given path is a volume.
+ * e.g. /volume1 is a volume.
*/
public boolean isVolume() {
return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
@@ -192,8 +233,8 @@ class OFSPath {
/**
* If key name is empty but volume and bucket names are not, the given path
- * it bucket.
- * e.g. /volume1/bucket2
+ * is a bucket.
+ * e.g. /volume1/bucket2 is a bucket.
*/
public boolean isBucket() {
return this.getKeyName().isEmpty() &&
@@ -201,6 +242,14 @@ class OFSPath {
!this.getVolumeName().isEmpty();
}
+ /**
+ * If key name is not empty, the given path is a key.
+ * e.g. /volume1/bucket2/key3 is a key.
+ */
+ public boolean isKey() {
+ return !this.getKeyName().isEmpty();
+ }
+
private static String md5Hex(String input) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
@@ -238,4 +287,25 @@ class OFSPath {
String username = UserGroupInformation.getCurrentUser().getUserName();
return getTempMountBucketName(username);
}
+
+ /**
+ * Return trash root for the given path.
+ * @return trash root for the given path.
+ */
+ public Path getTrashRoot() {
+ if (!this.isKey()) {
+ throw new RuntimeException("Volume or bucket doesn't have trash root.");
+ }
+ try {
+ String username = UserGroupInformation.getCurrentUser().getUserName();
+ final Path pathRoot = new Path(
+ OZONE_OFS_URI_SCHEME, authority, OZONE_URI_DELIMITER);
+ final Path pathToVolume = new Path(pathRoot, volumeName);
+ final Path pathToBucket = new Path(pathToVolume, bucketName);
+ final Path pathToTrash = new Path(pathToBucket, TRASH_PREFIX);
+ return new Path(pathToTrash, username);
+ } catch (IOException ex) {
+ throw new RuntimeException("getTrashRoot failed.", ex);
+ }
+ }
}
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
index c46c09f..afdeb51 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
@@ -31,56 +31,67 @@ public class TestOFSPath {
public void testParsingVolumeBucketWithKey() {
// Two most common cases: file key and dir key inside a bucket
OFSPath ofsPath = new OFSPath("/volume1/bucket2/dir3/key4");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("bucket2", ofsPath.getBucketName());
Assert.assertEquals("dir3/key4", ofsPath.getKeyName());
Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/bucket2/dir3/key4", ofsPath.toString());
// The ending '/' matters for key inside a bucket, indicating directory
ofsPath = new OFSPath("/volume1/bucket2/dir3/dir5/");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("bucket2", ofsPath.getBucketName());
// Check the key must end with '/' (dir5 is a directory)
Assert.assertEquals("dir3/dir5/", ofsPath.getKeyName());
Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/bucket2/dir3/dir5/", ofsPath.toString());
}
@Test
public void testParsingVolumeBucketOnly() {
// Volume and bucket only
OFSPath ofsPath = new OFSPath("/volume1/bucket2/");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("bucket2", ofsPath.getBucketName());
Assert.assertEquals("", ofsPath.getMountName());
Assert.assertEquals("", ofsPath.getKeyName());
Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/bucket2/", ofsPath.toString());
- // The ending '/' shouldn't for buckets
+ // The trailing '/' doesn't matter when parsing a bucket path
ofsPath = new OFSPath("/volume1/bucket2");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("bucket2", ofsPath.getBucketName());
Assert.assertEquals("", ofsPath.getMountName());
Assert.assertEquals("", ofsPath.getKeyName());
Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/bucket2/", ofsPath.toString());
}
@Test
public void testParsingVolumeOnly() {
// Volume only
OFSPath ofsPath = new OFSPath("/volume1/");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("", ofsPath.getBucketName());
Assert.assertEquals("", ofsPath.getMountName());
Assert.assertEquals("", ofsPath.getKeyName());
Assert.assertEquals("/volume1/", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/", ofsPath.toString());
- // Ending '/' shouldn't matter
+ // The trailing '/' doesn't matter when parsing a volume path
ofsPath = new OFSPath("/volume1");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals("volume1", ofsPath.getVolumeName());
Assert.assertEquals("", ofsPath.getBucketName());
Assert.assertEquals("", ofsPath.getMountName());
@@ -90,17 +101,20 @@ public class TestOFSPath {
// The behavior might change in the future.
Assert.assertEquals("/volume1/", ofsPath.getNonKeyPath());
Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("/volume1/", ofsPath.toString());
}
@Test
public void testParsingWithAuthority() {
- try {
- new OFSPath("ofs://svc1/volume1/bucket1/dir1/");
- Assert.fail(
- "Should have thrown exception when parsing path with authority.");
- } catch (Exception ignored) {
- // Test pass
- }
+ OFSPath ofsPath = new OFSPath("ofs://svc1:9876/volume1/bucket2/dir3/");
+ Assert.assertEquals("svc1:9876", ofsPath.getAuthority());
+ Assert.assertEquals("volume1", ofsPath.getVolumeName());
+ Assert.assertEquals("bucket2", ofsPath.getBucketName());
+ Assert.assertEquals("dir3/", ofsPath.getKeyName());
+ Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath());
+ Assert.assertFalse(ofsPath.isMount());
+ Assert.assertEquals("ofs://svc1:9876/volume1/bucket2/dir3/",
+ ofsPath.toString());
}
@Test
@@ -115,6 +129,7 @@ public class TestOFSPath {
}
// Mount only
OFSPath ofsPath = new OFSPath("/tmp/");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals(
OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
Assert.assertEquals(bucketName, ofsPath.getBucketName());
@@ -122,9 +137,11 @@ public class TestOFSPath {
Assert.assertEquals("", ofsPath.getKeyName());
Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());
Assert.assertTrue(ofsPath.isMount());
+ Assert.assertEquals("/tmp/", ofsPath.toString());
// Mount with key
ofsPath = new OFSPath("/tmp/key1");
+ Assert.assertEquals("", ofsPath.getAuthority());
Assert.assertEquals(
OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
Assert.assertEquals(bucketName, ofsPath.getBucketName());
@@ -132,5 +149,6 @@ public class TestOFSPath {
Assert.assertEquals("key1", ofsPath.getKeyName());
Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());
Assert.assertTrue(ofsPath.isMount());
+ Assert.assertEquals("/tmp/key1", ofsPath.toString());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org