You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/22 23:11:41 UTC
[47/50] [abbrv] hadoop git commit: HDFS-11145. Implement
getTrashRoot() for ViewFileSystem. (Manoj Govindassamy via lei)
HDFS-11145. Implement getTrashRoot() for ViewFileSystem. (Manoj Govindassamy via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a09e597
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a09e597
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a09e597
Branch: refs/heads/HADOOP-13345
Commit: 3a09e5970db7e6db4cc47f18dafc1daf0ff2538b
Parents: beb70fe
Author: Lei Xu <le...@apache.org>
Authored: Tue Nov 22 13:50:29 2016 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Tue Nov 22 13:52:09 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 34 ++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 68 ++++++++++++++++++++
.../fs/viewfs/TestViewFileSystemHdfs.java | 57 ++++++++++++++++
3 files changed, 159 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a09e597/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index e2ebe1b..76606e9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
@@ -800,6 +801,39 @@ public class ViewFileSystem extends FileSystem {
}
/**
+ * Get the trash root directory for current user when the path
+ * specified is deleted.
+ *
+ * @param path the trash root of the path to be determined.
+ * @return the trash root path.
+ */
+ @Override
+ public Path getTrashRoot(Path path) {
+ try {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ return res.targetFileSystem.getTrashRoot(res.remainingPath);
+ } catch (Exception e) {
+ throw new NotInMountpointException(path, "getTrashRoot");
+ }
+ }
+
+ /**
+ * Get all the trash roots for current user or all users.
+ *
+ * @param allUsers return trash roots for all users if true.
+ * @return all Trash root directories.
+ */
+ @Override
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) {
+ List<FileStatus> trashRoots = new ArrayList<>();
+ for (FileSystem fs : getChildFileSystems()) {
+ trashRoots.addAll(fs.getTrashRoots(allUsers));
+ }
+ return trashRoots;
+ }
+
+ /**
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
* It is a read only mount tables and create, mkdir or delete operations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a09e597/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 7393af6..12c310b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -952,4 +953,71 @@ abstract public class ViewFileSystemBaseTest {
}
}
+ @Test
+ public void testTrashRoot() throws IOException {
+
+ Path mountDataRootPath = new Path("/data");
+ Path fsTargetFilePath = new Path("debug.log");
+ Path mountDataFilePath = new Path(mountDataRootPath, fsTargetFilePath);
+ Path mountDataNonExistingFilePath = new Path(mountDataRootPath, "no.log");
+ fileSystemTestHelper.createFile(fsTarget, fsTargetFilePath);
+
+ // Get Trash roots for paths via ViewFileSystem handle
+ Path mountDataRootTrashPath = fsView.getTrashRoot(mountDataRootPath);
+ Path mountDataFileTrashPath = fsView.getTrashRoot(mountDataFilePath);
+
+ // Get Trash roots for the same set of paths via the mounted filesystem
+ Path fsTargetRootTrashRoot = fsTarget.getTrashRoot(mountDataRootPath);
+ Path fsTargetFileTrashPath = fsTarget.getTrashRoot(mountDataFilePath);
+
+ // Verify if Trash roots from ViewFileSystem matches that of the ones
+ // from the target mounted FileSystem.
+ assertEquals(mountDataRootTrashPath.toUri().getPath(),
+ fsTargetRootTrashRoot.toUri().getPath());
+ assertEquals(mountDataFileTrashPath.toUri().getPath(),
+ fsTargetFileTrashPath.toUri().getPath());
+ assertEquals(mountDataRootTrashPath.toUri().getPath(),
+ mountDataFileTrashPath.toUri().getPath());
+
+
+ // Verify trash root for an non-existing file but on a valid mountpoint.
+ Path trashRoot = fsView.getTrashRoot(mountDataNonExistingFilePath);
+ assertEquals(mountDataRootTrashPath.toUri().getPath(),
+ trashRoot.toUri().getPath());
+
+ // Verify trash root for invalid mounts.
+ Path invalidMountRootPath = new Path("/invalid_mount");
+ Path invalidMountFilePath = new Path(invalidMountRootPath, "debug.log");
+ try {
+ fsView.getTrashRoot(invalidMountRootPath);
+ fail("ViewFileSystem getTashRoot should fail for non-mountpoint paths.");
+ } catch (NotInMountpointException e) {
+ //expected exception
+ }
+ try {
+ fsView.getTrashRoot(invalidMountFilePath);
+ fail("ViewFileSystem getTashRoot should fail for non-mountpoint paths.");
+ } catch (NotInMountpointException e) {
+ //expected exception
+ }
+ try {
+ fsView.getTrashRoot(null);
+ fail("ViewFileSystem getTashRoot should fail for empty paths.");
+ } catch (NotInMountpointException e) {
+ //expected exception
+ }
+
+ // Move the file to trash
+ FileStatus fileStatus = fsTarget.getFileStatus(fsTargetFilePath);
+ Configuration newConf = new Configuration(conf);
+ newConf.setLong("fs.trash.interval", 1000);
+ Trash lTrash = new Trash(fsTarget, newConf);
+ boolean trashed = lTrash.moveToTrash(fsTargetFilePath);
+ Assert.assertTrue("File " + fileStatus + " move to " +
+ "trash failed.", trashed);
+
+ // Verify ViewFileSystem trash roots shows the ones from
+ // target mounted FileSystem.
+ Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a09e597/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index 9221317..df84b4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -18,25 +18,37 @@
package org.apache.hadoop.fs.viewfs;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+
+import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.EnumSet;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Test;
public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
@@ -58,6 +70,19 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
+
+ // Encryption Zone settings
+ FileSystemTestHelper fsHelper = new FileSystemTestHelper();
+ String testRoot = fsHelper.getTestRootDir();
+ CONF.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" +
+ new Path(new File(testRoot).getAbsoluteFile().toString(), "test" +
+ ".jks").toUri());
+ CONF.setBoolean(DFSConfigKeys
+ .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
+ 2);
+
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
@@ -137,4 +162,36 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
int getExpectedDelegationTokenCountWithCredentials() {
return 2;
}
+
+ @Test
+ public void testTrashRootsAfterEncryptionZoneDeletion() throws Exception {
+ final Path zone = new Path("/EZ");
+ fsTarget.mkdirs(zone);
+ final Path zone1 = new Path("/EZ/zone1");
+ fsTarget.mkdirs(zone1);
+
+ DFSTestUtil.createKey("test_key", cluster, CONF);
+ HdfsAdmin hdfsAdmin = new HdfsAdmin(cluster.getURI(0), CONF);
+ final EnumSet<CreateEncryptionZoneFlag> provisionTrash =
+ EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH);
+ hdfsAdmin.createEncryptionZone(zone1, "test_key", provisionTrash);
+
+ final Path encFile = new Path(zone1, "encFile");
+ DFSTestUtil.createFile(fsTarget, encFile, 10240, (short) 1, 0xFEED);
+
+ Configuration clientConf = new Configuration(CONF);
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ clientConf.set("fs.default.name", fsTarget.getUri().toString());
+ FsShell shell = new FsShell(clientConf);
+
+ //Verify file deletion within EZ
+ DFSTestUtil.verifyDelete(shell, fsTarget, encFile, true);
+ Assert.assertTrue("ViewFileSystem trash roots should include EZ file trash",
+ (fsView.getTrashRoots(true).size() == 1));
+
+ //Verify deletion of EZ
+ DFSTestUtil.verifyDelete(shell, fsTarget, zone, true);
+ Assert.assertTrue("ViewFileSystem trash roots should include EZ zone trash",
+ (fsView.getTrashRoots(true).size() == 2));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org