You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/05/04 19:27:36 UTC
[08/50] [abbrv] hadoop git commit: HDFS-13508. RBF: Normalize paths
(automatically) when adding, updating,
removing or listing mount table entries. Contributed by Ekanth S.
HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries. Contributed by Ekanth S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48444060
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48444060
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48444060
Branch: refs/heads/HDFS-12943
Commit: 484440602c5b69fbd8106010603c61ae051056dd
Parents: f469628
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Apr 27 16:28:17 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Apr 27 16:28:17 2018 -0700
----------------------------------------------------------------------
.../hdfs/tools/federation/RouterAdmin.java | 16 +++
.../federation/router/TestRouterAdminCLI.java | 117 ++++++++++++++++++-
2 files changed, 130 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48444060/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 17707dc..b0a2062 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -322,6 +323,7 @@ public class RouterAdmin extends Configured implements Tool {
public boolean addMount(String mount, String[] nss, String dest,
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
throws IOException {
+ mount = normalizeFileSystemPath(mount);
// Get the existing entry
MountTableManager mountTable = client.getMountTableManager();
GetMountTableEntriesRequest getRequest =
@@ -473,6 +475,7 @@ public class RouterAdmin extends Configured implements Tool {
public boolean updateMount(String mount, String[] nss, String dest,
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
throws IOException {
+ mount = normalizeFileSystemPath(mount);
MountTableManager mountTable = client.getMountTableManager();
// Create a new entry
@@ -519,6 +522,7 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException If it cannot be removed.
*/
public boolean removeMount(String path) throws IOException {
+ path = normalizeFileSystemPath(path);
MountTableManager mountTable = client.getMountTableManager();
RemoveMountTableEntryRequest request =
RemoveMountTableEntryRequest.newInstance(path);
@@ -538,6 +542,7 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException If it cannot be listed.
*/
public void listMounts(String path) throws IOException {
+ path = normalizeFileSystemPath(path);
MountTableManager mountTable = client.getMountTableManager();
GetMountTableEntriesRequest request =
GetMountTableEntriesRequest.newInstance(path);
@@ -798,6 +803,17 @@ public class RouterAdmin extends Configured implements Tool {
}
/**
+ * Normalize a path for that filesystem.
+ *
+ * @param path Path to normalize.
+ * @return Normalized path.
+ */
+ private static String normalizeFileSystemPath(final String path) {
+ Path normalizedPath = new Path(path);
+ return normalizedPath.toString();
+ }
+
+ /**
* Inner class that stores ACL info of mount table.
*/
static class ACLEntity {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48444060/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 4e84c33..2537c19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -159,6 +159,45 @@ public class TestRouterAdminCLI {
}
@Test
+ public void testAddMountTableNotNormalized() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-addmounttable-notnormalized";
+ String srcWithSlash = src + "/";
+ String dest = "/addmounttable-notnormalized";
+ String[] argv = new String[] {"-add", srcWithSlash, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(src);
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+
+ List<RemoteLocation> destinations = mountTable.getDestinations();
+ assertEquals(1, destinations.size());
+
+ assertEquals(src, mountTable.getSourcePath());
+ assertEquals(nsId, destinations.get(0).getNameserviceId());
+ assertEquals(dest, destinations.get(0).getDest());
+ assertFalse(mountTable.isReadOnly());
+
+ // test mount table update behavior
+ dest = dest + "-new";
+ argv = new String[] {"-add", srcWithSlash, nsId, dest, "-readonly"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ assertEquals(2, mountTable.getDestinations().size());
+ assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId());
+ assertEquals(dest, mountTable.getDestinations().get(1).getDest());
+ assertTrue(mountTable.isReadOnly());
+ }
+
+ @Test
public void testAddOrderMountTable() throws Exception {
testAddOrderMountTable(DestinationOrder.HASH);
testAddOrderMountTable(DestinationOrder.LOCAL);
@@ -192,6 +231,7 @@ public class TestRouterAdminCLI {
public void testListMountTable() throws Exception {
String nsId = "ns0";
String src = "/test-lsmounttable";
+ String srcWithSlash = src + "/";
String dest = "/lsmounttable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@@ -203,6 +243,11 @@ public class TestRouterAdminCLI {
assertEquals(0, ToolRunner.run(admin, argv));
assertTrue(out.toString().contains(src));
+ // Test with not-normalized src input
+ argv = new String[] {"-ls", srcWithSlash};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ assertTrue(out.toString().contains(src));
+
out.reset();
GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
.newInstance("/");
@@ -256,6 +301,33 @@ public class TestRouterAdminCLI {
}
@Test
+ public void testRemoveMountTableNotNormalized() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-rmmounttable-notnormalized";
+ String srcWithSlash = src + "/";
+ String dest = "/rmmounttable-notnormalized";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(src);
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ // ensure mount table added successfully
+ MountTable mountTable = getResponse.getEntries().get(0);
+ assertEquals(src, mountTable.getSourcePath());
+
+ argv = new String[] {"-rm", srcWithSlash};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ assertEquals(0, getResponse.getEntries().size());
+ }
+
+ @Test
public void testMountTableDefaultACL() throws Exception {
String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"};
assertEquals(0, ToolRunner.run(admin, argv));
@@ -552,12 +624,12 @@ public class TestRouterAdminCLI {
}
@Test
- public void testUpdateNameserviceDestinationForExistingMountTable() throws
+ public void testUpdateDestinationForExistingMountTable() throws
Exception {
// Add a mount table firstly
String nsId = "ns0";
- String src = "/test-updateNameserviceDestinationForExistingMountTable";
- String dest = "/UpdateNameserviceDestinationForExistingMountTable";
+ String src = "/test-updateDestinationForExistingMountTable";
+ String dest = "/UpdateDestinationForExistingMountTable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@@ -590,6 +662,45 @@ public class TestRouterAdminCLI {
}
@Test
+ public void testUpdateDestinationForExistingMountTableNotNormalized() throws
+ Exception {
+ // Add a mount table firstly
+ String nsId = "ns0";
+ String src = "/test-updateDestinationForExistingMountTableNotNormalized";
+ String srcWithSlash = src + "/";
+ String dest = "/UpdateDestinationForExistingMountTableNotNormalized";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance(src);
+ GetMountTableEntriesResponse getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ // Ensure mount table added successfully
+ MountTable mountTable = getResponse.getEntries().get(0);
+ assertEquals(src, mountTable.getSourcePath());
+ assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
+ assertEquals(dest, mountTable.getDestinations().get(0).getDest());
+
+ // Update the destination
+ String newNsId = "ns1";
+ String newDest = "/newDestination";
+ argv = new String[] {"-update", srcWithSlash, newNsId, newDest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ // Ensure the destination updated successfully
+ mountTable = getResponse.getEntries().get(0);
+ assertEquals(src, mountTable.getSourcePath());
+ assertEquals(newNsId,
+ mountTable.getDestinations().get(0).getNameserviceId());
+ assertEquals(newDest, mountTable.getDestinations().get(0).getDest());
+ }
+
+ @Test
public void testUpdateReadonlyUserGroupPermissionMountable()
throws Exception {
// Add a mount table
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org