You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2017/07/30 06:06:49 UTC
[1/2] hadoop git commit: HADOOP-14455. ViewFileSystem#rename should
support be supported within same nameservice with different mountpoints.
Contributed by Brahma Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7e643130f -> 8bfb9971c
refs/heads/branch-2.8 a1682876c -> 90f3108b8
HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bfb9971
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bfb9971
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bfb9971
Branch: refs/heads/branch-2
Commit: 8bfb9971cad1a83da852de4d12f1de4197d25d21
Parents: 7e64313
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Sun Jul 30 14:01:41 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Sun Jul 30 14:01:41 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/viewfs/Constants.java | 2 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 79 +++++++++++-----
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 43 ++++-----
.../src/main/resources/core-default.xml | 9 ++
.../conf/TestCommonConfigurationFields.java | 1 +
.../hadoop/fs/contract/ContractTestUtils.java | 54 +++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 79 +++++++++++++---
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 94 ++++++++++++++++----
.../fs/viewfs/TestViewFileSystemHdfs.java | 28 ++++--
9 files changed, 310 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index ec8ab2b..9882a8e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -66,4 +66,6 @@ public interface Constants {
static public final FsPermission PERMISSION_555 =
new FsPermission((short) 0555);
+
+ String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 6b48e72..27d8c0b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -108,7 +108,8 @@ public class ViewFileSystem extends FileSystem {
Configuration config;
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
-
+ // Default to rename within same mountpoint
+ private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT;
/**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
@@ -190,6 +191,9 @@ public class ViewFileSystem extends FileSystem {
}
};
workingDir = this.getHomeDirectory();
+ renameStrategy = RenameStrategy.valueOf(
+ conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ RenameStrategy.SAME_MOUNTPOINT.toString()));
} catch (URISyntaxException e) {
throw new IOException("URISyntax exception: " + theUri);
}
@@ -477,27 +481,55 @@ public class ViewFileSystem extends FileSystem {
if (resDst.isInternalDir()) {
throw readOnlyMountTable("rename", dst);
}
- /**
- // Alternate 1: renames within same file system - valid but we disallow
- // Alternate 2: (as described in next para - valid but we have disallowed it
- //
- // Note we compare the URIs. the URIs include the link targets.
- // hence we allow renames across mount links as long as the mount links
- // point to the same target.
- if (!resSrc.targetFileSystem.getUri().equals(
- resDst.targetFileSystem.getUri())) {
- throw new IOException("Renames across Mount points not supported");
- }
- */
-
- //
- // Alternate 3 : renames ONLY within the the same mount links.
- //
- if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
- throw new IOException("Renames across Mount points not supported");
+
+ URI srcUri = resSrc.targetFileSystem.getUri();
+ URI dstUri = resDst.targetFileSystem.getUri();
+
+ verifyRenameStrategy(srcUri, dstUri,
+ resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
+
+ ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem;
+ ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem;
+ return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath),
+ dstFS.fullPath(resDst.remainingPath));
+ }
+
+ static void verifyRenameStrategy(URI srcUri, URI dstUri,
+ boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy)
+ throws IOException {
+ switch (renameStrategy) {
+ case SAME_FILESYSTEM_ACROSS_MOUNTPOINT:
+ if (srcUri.getAuthority() != null) {
+ if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri
+ .getAuthority().equals(dstUri.getAuthority()))) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+ }
+
+ break;
+ case SAME_TARGET_URI_ACROSS_MOUNTPOINT:
+ // Alternate 2: Rename across mountpoints with same target.
+ // i.e. Rename across alias mountpoints.
+ //
+ // Note we compare the URIs. the URIs include the link targets.
+ // hence we allow renames across mount links as long as the mount links
+ // point to the same target.
+ if (!srcUri.equals(dstUri)) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+
+ break;
+ case SAME_MOUNTPOINT:
+ //
+ // Alternate 3 : renames ONLY within the the same mount links.
+ //
+ if (!isSrcDestSame) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+ break;
+ default:
+ throw new IllegalArgumentException ("Unexpected rename strategy");
}
- return resSrc.targetFileSystem.rename(resSrc.remainingPath,
- resDst.remainingPath);
}
@Override
@@ -1080,4 +1112,9 @@ public class ViewFileSystem extends FileSystem {
throw new NotInMountpointException(f, "getQuotaUsage");
}
}
+
+ enum RenameStrategy {
+ SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT,
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 3a34a91..364485f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem {
final Configuration config;
InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
-
+ private ViewFileSystem.RenameStrategy renameStrategy =
+ ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT;
+
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@@ -237,6 +239,9 @@ public class ViewFs extends AbstractFileSystem {
// return MergeFs.createMergeFs(mergeFsURIList, config);
}
};
+ renameStrategy = ViewFileSystem.RenameStrategy.valueOf(
+ conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString()));
}
@Override
@@ -495,37 +500,23 @@ public class ViewFs extends AbstractFileSystem {
+ " is readOnly");
}
- InodeTree.ResolveResult<AbstractFileSystem> resDst =
+ InodeTree.ResolveResult<AbstractFileSystem> resDst =
fsState.resolve(getUriPath(dst), false);
if (resDst.isInternalDir()) {
throw new AccessControlException(
"Cannot Rename within internal dirs of mount table: dest=" + dst
+ " is readOnly");
}
-
- /**
- // Alternate 1: renames within same file system - valid but we disallow
- // Alternate 2: (as described in next para - valid but we have disallowed it
- //
- // Note we compare the URIs. the URIs include the link targets.
- // hence we allow renames across mount links as long as the mount links
- // point to the same target.
- if (!resSrc.targetFileSystem.getUri().equals(
- resDst.targetFileSystem.getUri())) {
- throw new IOException("Renames across Mount points not supported");
- }
- */
-
- //
- // Alternate 3 : renames ONLY within the the same mount links.
- //
-
- if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
- throw new IOException("Renames across Mount points not supported");
- }
-
- resSrc.targetFileSystem.renameInternal(resSrc.remainingPath,
- resDst.remainingPath, overwrite);
+ //Alternate 1: renames within same file system
+ URI srcUri = resSrc.targetFileSystem.getUri();
+ URI dstUri = resDst.targetFileSystem.getUri();
+ ViewFileSystem.verifyRenameStrategy(srcUri, dstUri,
+ resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
+
+ ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem;
+ ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem;
+ srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath),
+ dstFS.fullPath(resDst.remainingPath), overwrite);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f72afc5..104ba60 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -783,6 +783,15 @@
</property>
<property>
+ <name>fs.viewfs.rename.strategy</name>
+ <value>SAME_MOUNTPOINT</value>
+ <description>Allowed rename strategy to rename between multiple mountpoints.
+ Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
+ </description>
+</property>
+
+<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
<description>The FileSystem for Ftp: uris.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 97c6279..228ee89 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -94,6 +94,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("nfs3.server.port");
xmlPropsToSkipCompare.add("test.fs.s3.name");
xmlPropsToSkipCompare.add("test.fs.s3n.name");
+ xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy");
// S3/S3A properties are in a different subtree.
// - org.apache.hadoop.fs.s3.S3FileSystemConfigKeys
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index ebb8220..e4fd97c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.contract;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -691,6 +692,21 @@ public class ContractTestUtils extends Assert {
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
+ *
+ * @param fileContext filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileContext fileContext, Path filename)
+ throws IOException {
+ assertPathExists(fileContext, "Expected file", filename);
+ FileStatus status = fileContext.getFileStatus(filename);
+ assertIsFile(filename, status);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
* @param filename name of the file
* @param status file status
*/
@@ -739,6 +755,25 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry.
+ *
+ * @param fileContext fileContext to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileContext fileContext, String message,
+ Path path) throws IOException {
+ if (!fileContext.util().exists(path)) {
+ //failure, report it
+ throw new FileNotFoundException(
+ message + ": not found " + path + " in " + path.getParent());
+ }
+ }
+
+ /**
* Assert that a path does not exist.
*
* @param fileSystem filesystem to examine
@@ -759,6 +794,25 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Assert that a path does not exist.
+ *
+ * @param fileContext fileContext to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileContext fileContext,
+ String message, Path path) throws IOException {
+ try {
+ FileStatus status = fileContext.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
* @param fs filesystem
* @param dir directory to scan
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 8c4ddc3..5ebcacb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
@@ -51,6 +52,7 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -351,28 +353,83 @@ abstract public class ViewFileSystemBaseTest {
}
// rename across mount points that point to same target also fail
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts1() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
- fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
- /* - code if we had wanted this to suceed
- Assert.assertFalse(fSys.exists(new Path("/user/foo")));
- Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo")));
- Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar")));
- Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar")));
- */
+ try {
+ fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts2() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
- fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ try {
+ fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
-
+
+ // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points that point to same target URI
+ @Test
+ public void testRenameAcrossMounts3() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
+ .toString());
+ FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+ fileSystemTestHelper.createFile(fsView2, "/user/foo");
+ fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsView2, "src should not exist after rename",
+ new Path("/user/foo"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fsView2,
+ fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar"));
+ ContractTestUtils
+ .assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar"));
+ }
+
+ // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points where the mount link targets are different
+ // but are part of the same target FS
+ @Test
+ public void testRenameAcrossMounts4() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ .toString());
+ FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+ fileSystemTestHelper.createFile(fsView2, "/user/foo");
+ fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsView2, "src should not exist after rename",
+ new Path("/user/foo"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fsView2,
+ fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar"));
+ ContractTestUtils
+ .assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar"));
+ }
+
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 50237d1..73e43e1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.local.LocalConfigKeys;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -66,6 +67,7 @@ import org.apache.hadoop.fs.viewfs.ViewFs.MountPoint;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -345,33 +347,93 @@ abstract public class ViewFsBaseTest {
}
// rename across mount points that point to same target also fail
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts1() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
- fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
- /* - code if we had wanted this to succeed
- Assert.assertFalse(exists(fc, new Path("/user/foo")));
- Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo")));
- Assert.assertTrue(isFile(fc,
- FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar")));
- Assert.assertTrue(isFile(fclocal,
- new Path(targetTestRoot,"user/fooBarBar")));
- */
+ try {
+ fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts2() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
- fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ try {
+ fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
-
-
-
-
+
+ // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points that point to same target URI
+ @Test
+ public void testRenameAcrossMounts3() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
+ .toString());
+
+ FileContext fcView2 =
+ FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
+ String user1Path = "/user/foo";
+ fileContextTestHelper.createFile(fcView2, user1Path);
+ String user2Path = "/user2/fooBarBar";
+ Path user2Dst = new Path(user2Path);
+ fcView2.rename(new Path(user1Path), user2Dst);
+ ContractTestUtils
+ .assertPathDoesNotExist(fcView2, "src should not exist after rename",
+ new Path(user1Path));
+ ContractTestUtils
+ .assertPathDoesNotExist(fcTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fcView2,
+ fileContextTestHelper.getTestRootPath(fcView2, user2Path));
+ ContractTestUtils
+ .assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar"));
+ }
+
+ // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points if the mount link targets are different
+ // but are part of the same target FS
+ @Test
+ public void testRenameAcrossMounts4() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ .toString());
+ FileContext fcView2 =
+ FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
+ String userPath = "/user/foo";
+ fileContextTestHelper.createFile(fcView2, userPath);
+ String anotherMountPath = "/data/fooBar";
+ Path anotherDst = new Path(anotherMountPath);
+ fcView2.rename(new Path(userPath), anotherDst);
+
+ ContractTestUtils
+ .assertPathDoesNotExist(fcView2, "src should not exist after rename",
+ new Path(userPath));
+ ContractTestUtils
+ .assertPathDoesNotExist(fcTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fcView2,
+ fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath));
+ ContractTestUtils
+ .assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar"));
+ }
+
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index 9221317..7a31a37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -29,15 +29,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.*;
public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
@@ -137,4 +135,24 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
int getExpectedDelegationTokenCountWithCredentials() {
return 2;
}
+
+ //Rename should fail on across different fileSystems
+ @Test
+ public void testRenameAccorssFilesystem() throws IOException {
+ //data is mountpoint in nn1
+ Path mountDataRootPath = new Path("/data");
+ //mountOnNn2 is nn2 mountpoint
+ Path fsTargetFilePath = new Path("/mountOnNn2");
+ Path filePath = new Path(mountDataRootPath + "/ttest");
+ Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2");
+ fsView.create(filePath);
+ try {
+ fsView.rename(filePath, hdfFilepath);
+ ContractTestUtils.fail("Should thrown IOE on Renames across filesytems");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-14455. ViewFileSystem#rename should
support be supported within same nameservice with different mountpoints.
Contributed by Brahma Reddy Battula.
Posted by br...@apache.org.
HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula.
(cherry picked from commit 8bfb9971cad1a83da852de4d12f1de4197d25d21)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90f3108b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90f3108b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90f3108b
Branch: refs/heads/branch-2.8
Commit: 90f3108b8ee157199e5800e294742062d462c167
Parents: a168287
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Sun Jul 30 14:01:41 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Sun Jul 30 14:02:25 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/viewfs/Constants.java | 2 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 79 +++++++++++-----
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 43 ++++-----
.../src/main/resources/core-default.xml | 9 ++
.../conf/TestCommonConfigurationFields.java | 1 +
.../hadoop/fs/contract/ContractTestUtils.java | 54 +++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 79 +++++++++++++---
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 94 ++++++++++++++++----
.../fs/viewfs/TestViewFileSystemHdfs.java | 28 ++++--
9 files changed, 310 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index ec8ab2b..9882a8e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -66,4 +66,6 @@ public interface Constants {
static public final FsPermission PERMISSION_555 =
new FsPermission((short) 0555);
+
+ String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index c315dc5..813642f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -108,7 +108,8 @@ public class ViewFileSystem extends FileSystem {
Configuration config;
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
-
+ // Default to rename within same mountpoint
+ private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT;
/**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
@@ -190,6 +191,9 @@ public class ViewFileSystem extends FileSystem {
}
};
workingDir = this.getHomeDirectory();
+ renameStrategy = RenameStrategy.valueOf(
+ conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ RenameStrategy.SAME_MOUNTPOINT.toString()));
} catch (URISyntaxException e) {
throw new IOException("URISyntax exception: " + theUri);
}
@@ -477,27 +481,55 @@ public class ViewFileSystem extends FileSystem {
if (resDst.isInternalDir()) {
throw readOnlyMountTable("rename", dst);
}
- /**
- // Alternate 1: renames within same file system - valid but we disallow
- // Alternate 2: (as described in next para - valid but we have disallowed it
- //
- // Note we compare the URIs. the URIs include the link targets.
- // hence we allow renames across mount links as long as the mount links
- // point to the same target.
- if (!resSrc.targetFileSystem.getUri().equals(
- resDst.targetFileSystem.getUri())) {
- throw new IOException("Renames across Mount points not supported");
- }
- */
-
- //
- // Alternate 3 : renames ONLY within the the same mount links.
- //
- if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
- throw new IOException("Renames across Mount points not supported");
+
+ URI srcUri = resSrc.targetFileSystem.getUri();
+ URI dstUri = resDst.targetFileSystem.getUri();
+
+ verifyRenameStrategy(srcUri, dstUri,
+ resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
+
+ ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem;
+ ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem;
+ return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath),
+ dstFS.fullPath(resDst.remainingPath));
+ }
+
+ static void verifyRenameStrategy(URI srcUri, URI dstUri,
+ boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy)
+ throws IOException {
+ switch (renameStrategy) {
+ case SAME_FILESYSTEM_ACROSS_MOUNTPOINT:
+ if (srcUri.getAuthority() != null) {
+ if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri
+ .getAuthority().equals(dstUri.getAuthority()))) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+ }
+
+ break;
+ case SAME_TARGET_URI_ACROSS_MOUNTPOINT:
+ // Alternate 2: Rename across mountpoints with same target.
+ // i.e. Rename across alias mountpoints.
+ //
+ // Note we compare the URIs. the URIs include the link targets.
+ // hence we allow renames across mount links as long as the mount links
+ // point to the same target.
+ if (!srcUri.equals(dstUri)) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+
+ break;
+ case SAME_MOUNTPOINT:
+ //
+ // Alternate 3 : renames ONLY within the the same mount links.
+ //
+ if (!isSrcDestSame) {
+ throw new IOException("Renames across Mount points not supported");
+ }
+ break;
+ default:
+ throw new IllegalArgumentException ("Unexpected rename strategy");
}
- return resSrc.targetFileSystem.rename(resSrc.remainingPath,
- resDst.remainingPath);
}
@Override
@@ -1080,4 +1112,9 @@ public class ViewFileSystem extends FileSystem {
throw new NotInMountpointException(f, "getQuotaUsage");
}
}
+
+ enum RenameStrategy {
+ SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT,
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 3a34a91..364485f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem {
final Configuration config;
InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
-
+ private ViewFileSystem.RenameStrategy renameStrategy =
+ ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT;
+
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@@ -237,6 +239,9 @@ public class ViewFs extends AbstractFileSystem {
// return MergeFs.createMergeFs(mergeFsURIList, config);
}
};
+ renameStrategy = ViewFileSystem.RenameStrategy.valueOf(
+ conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString()));
}
@Override
@@ -495,37 +500,23 @@ public class ViewFs extends AbstractFileSystem {
+ " is readOnly");
}
- InodeTree.ResolveResult<AbstractFileSystem> resDst =
+ InodeTree.ResolveResult<AbstractFileSystem> resDst =
fsState.resolve(getUriPath(dst), false);
if (resDst.isInternalDir()) {
throw new AccessControlException(
"Cannot Rename within internal dirs of mount table: dest=" + dst
+ " is readOnly");
}
-
- /**
- // Alternate 1: renames within same file system - valid but we disallow
- // Alternate 2: (as described in next para - valid but we have disallowed it
- //
- // Note we compare the URIs. the URIs include the link targets.
- // hence we allow renames across mount links as long as the mount links
- // point to the same target.
- if (!resSrc.targetFileSystem.getUri().equals(
- resDst.targetFileSystem.getUri())) {
- throw new IOException("Renames across Mount points not supported");
- }
- */
-
- //
- // Alternate 3 : renames ONLY within the the same mount links.
- //
-
- if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
- throw new IOException("Renames across Mount points not supported");
- }
-
- resSrc.targetFileSystem.renameInternal(resSrc.remainingPath,
- resDst.remainingPath, overwrite);
+ //Alternate 1: renames within same file system
+ URI srcUri = resSrc.targetFileSystem.getUri();
+ URI dstUri = resDst.targetFileSystem.getUri();
+ ViewFileSystem.verifyRenameStrategy(srcUri, dstUri,
+ resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
+
+ ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem;
+ ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem;
+ srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath),
+ dstFS.fullPath(resDst.remainingPath), overwrite);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b83a328..75a39c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -654,6 +654,15 @@
</property>
<property>
+ <name>fs.viewfs.rename.strategy</name>
+ <value>SAME_MOUNTPOINT</value>
+ <description>Allowed rename strategy to rename between multiple mountpoints.
+ Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
+ </description>
+</property>
+
+<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
<description>The FileSystem for Ftp: uris.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index eee0e25..52f28e5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -92,6 +92,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("nfs3.server.port");
xmlPropsToSkipCompare.add("test.fs.s3.name");
xmlPropsToSkipCompare.add("test.fs.s3n.name");
+ xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy");
// S3/S3A properties are in a different subtree.
// - org.apache.hadoop.fs.s3.S3FileSystemConfigKeys
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index 90ae974..4378394 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.contract;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -691,6 +692,21 @@ public class ContractTestUtils extends Assert {
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
+ *
+ * @param fileContext filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileContext fileContext, Path filename)
+ throws IOException {
+ assertPathExists(fileContext, "Expected file", filename);
+ FileStatus status = fileContext.getFileStatus(filename);
+ assertIsFile(filename, status);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
* @param filename name of the file
* @param status file status
*/
@@ -739,6 +755,25 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry.
+ *
+ * @param fileContext fileContext to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileContext fileContext, String message,
+ Path path) throws IOException {
+ if (!fileContext.util().exists(path)) {
+ //failure, report it
+ throw new FileNotFoundException(
+ message + ": not found " + path + " in " + path.getParent());
+ }
+ }
+
+ /**
* Assert that a path does not exist.
*
* @param fileSystem filesystem to examine
@@ -759,6 +794,25 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Assert that a path does not exist.
+ *
+ * @param fileContext fileContext to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileContext fileContext,
+ String message, Path path) throws IOException {
+ try {
+ FileStatus status = fileContext.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
* @param fs filesystem
* @param dir directory to scan
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 8c4ddc3..5ebcacb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
@@ -51,6 +52,7 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -351,28 +353,83 @@ abstract public class ViewFileSystemBaseTest {
}
// rename across mount points that point to same target also fail
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts1() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
- fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
- /* - code if we had wanted this to suceed
- Assert.assertFalse(fSys.exists(new Path("/user/foo")));
- Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo")));
- Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar")));
- Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar")));
- */
+ try {
+ fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts2() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
- fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ try {
+ fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
-
+
+ // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points that point to same target URI
+ @Test
+ public void testRenameAcrossMounts3() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
+ .toString());
+ FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+ fileSystemTestHelper.createFile(fsView2, "/user/foo");
+ fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsView2, "src should not exist after rename",
+ new Path("/user/foo"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fsView2,
+ fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar"));
+ ContractTestUtils
+ .assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar"));
+ }
+
+ // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points where the mount link targets are different
+ // but are part of the same target FS
+ @Test
+ public void testRenameAcrossMounts4() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ .toString());
+ FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+ fileSystemTestHelper.createFile(fsView2, "/user/foo");
+ fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsView2, "src should not exist after rename",
+ new Path("/user/foo"));
+ ContractTestUtils
+ .assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fsView2,
+ fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar"));
+ ContractTestUtils
+ .assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar"));
+ }
+
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 50237d1..73e43e1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.local.LocalConfigKeys;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -66,6 +67,7 @@ import org.apache.hadoop.fs.viewfs.ViewFs.MountPoint;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -345,33 +347,93 @@ abstract public class ViewFsBaseTest {
}
// rename across mount points that point to same target also fail
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts1() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
- fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
- /* - code if we had wanted this to succeed
- Assert.assertFalse(exists(fc, new Path("/user/foo")));
- Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo")));
- Assert.assertTrue(isFile(fc,
- FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar")));
- Assert.assertTrue(isFile(fclocal,
- new Path(targetTestRoot,"user/fooBarBar")));
- */
+ try {
+ fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
- @Test(expected=IOException.class)
+ @Test
public void testRenameAcrossMounts2() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
- fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ try {
+ fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
+ ContractTestUtils.fail("IOException is not thrown on rename operation");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
}
-
-
-
-
+
+ // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points that point to same target URI
+ @Test
+ public void testRenameAcrossMounts3() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
+ .toString());
+
+ FileContext fcView2 =
+ FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
+ String user1Path = "/user/foo";
+ fileContextTestHelper.createFile(fcView2, user1Path);
+ String user2Path = "/user2/fooBarBar";
+ Path user2Dst = new Path(user2Path);
+ fcView2.rename(new Path(user1Path), user2Dst);
+ ContractTestUtils
+ .assertPathDoesNotExist(fcView2, "src should not exist after rename",
+ new Path(user1Path));
+ ContractTestUtils
+ .assertPathDoesNotExist(fcTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fcView2,
+ fileContextTestHelper.getTestRootPath(fcView2, user2Path));
+ ContractTestUtils
+ .assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar"));
+ }
+
+ // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
+ // to rename across mount points if the mount link targets are different
+ // but are part of the same target FS
+ @Test
+ public void testRenameAcrossMounts4() throws IOException {
+ Configuration conf2 = new Configuration(conf);
+ conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+ ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+ .toString());
+ FileContext fcView2 =
+ FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
+ String userPath = "/user/foo";
+ fileContextTestHelper.createFile(fcView2, userPath);
+ String anotherMountPath = "/data/fooBar";
+ Path anotherDst = new Path(anotherMountPath);
+ fcView2.rename(new Path(userPath), anotherDst);
+
+ ContractTestUtils
+ .assertPathDoesNotExist(fcView2, "src should not exist after rename",
+ new Path(userPath));
+ ContractTestUtils
+ .assertPathDoesNotExist(fcTarget, "src should not exist after rename",
+ new Path(targetTestRoot, "user/foo"));
+ ContractTestUtils.assertIsFile(fcView2,
+ fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath));
+ ContractTestUtils
+ .assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar"));
+ }
+
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90f3108b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index 9221317..7a31a37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -29,15 +29,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.*;
public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
@@ -137,4 +135,24 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
int getExpectedDelegationTokenCountWithCredentials() {
return 2;
}
+
+ //Rename should fail on across different fileSystems
+ @Test
+ public void testRenameAccorssFilesystem() throws IOException {
+ //data is mountpoint in nn1
+ Path mountDataRootPath = new Path("/data");
+ //mountOnNn2 is nn2 mountpoint
+ Path fsTargetFilePath = new Path("/mountOnNn2");
+ Path filePath = new Path(mountDataRootPath + "/ttest");
+ Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2");
+ fsView.create(filePath);
+ try {
+ fsView.rename(filePath, hdfFilepath);
+ ContractTestUtils.fail("Should thrown IOE on Renames across filesytems");
+ } catch (IOException e) {
+ GenericTestUtils
+ .assertExceptionContains("Renames across Mount points not supported",
+ e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org