You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/05/02 19:22:00 UTC
[25/50] [abbrv] hadoop git commit: HDFS-13453. RBF:
getMountPointDates should fetch latest subdir time/date when parent dir is
not present but /parent/child dirs are present in mount table. Contributed by
Dibyendu Karmakar.
HDFS-13453. RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table. Contributed by Dibyendu Karmakar.
(cherry picked from commit 1134af9ad1daf683204df8f95a8f03d7baaa74d4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3c17359
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3c17359
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3c17359
Branch: refs/heads/YARN-8200
Commit: a3c17359513f775e5768ad9c57c33f3e2fdc6bdd
Parents: 2444d70
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Apr 19 14:56:36 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Apr 19 14:58:02 2018 -0700
----------------------------------------------------------------------
.../federation/router/RouterRpcServer.java | 55 ++++++++++++++++----
.../federation/router/TestRouterMountTable.java | 21 +++++++-
2 files changed, 65 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index d626699..8a0cf27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -2197,18 +2197,11 @@ public class RouterRpcServer extends AbstractService
private Map<String, Long> getMountPointDates(String path) {
Map<String, Long> ret = new TreeMap<>();
if (subclusterResolver instanceof MountTableResolver) {
- MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
- String srcPath;
try {
final List<String> children = subclusterResolver.getMountPoints(path);
for (String child : children) {
- if (path.equals(Path.SEPARATOR)) {
- srcPath = Path.SEPARATOR + child;
- } else {
- srcPath = path + Path.SEPARATOR + child;
- }
- MountTable entry = mountTable.getMountPoint(srcPath);
- ret.put(child, entry.getDateModified());
+ Long modTime = getModifiedTime(ret, path, child);
+ ret.put(child, modTime);
}
} catch (IOException e) {
LOG.error("Cannot get mount point", e);
@@ -2218,6 +2211,50 @@ public class RouterRpcServer extends AbstractService
}
/**
+ * Get modified time for child. If the child is present in mount table it
+ * will return the modified time. If the child is not present but subdirs of
+ * this child are present then it will return latest modified subdir's time
+ * as modified time of the requested child.
+ * @param ret contains children and modified times.
+ * @param mountTable.
+ * @param path Name of the path to start checking dates from.
+ * @param child child of the requested path.
+ * @return modified time.
+ */
+ private long getModifiedTime(Map<String, Long> ret, String path,
+ String child) {
+ MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+ String srcPath;
+ if (path.equals(Path.SEPARATOR)) {
+ srcPath = Path.SEPARATOR + child;
+ } else {
+ srcPath = path + Path.SEPARATOR + child;
+ }
+ Long modTime = 0L;
+ try {
+ // Get mount table entry for the srcPath
+ MountTable entry = mountTable.getMountPoint(srcPath);
+ // if srcPath is not in mount table but its subdirs are in mount
+ // table we will display latest modified subdir date/time.
+ if (entry == null) {
+ List<MountTable> entries = mountTable.getMounts(srcPath);
+ for (MountTable eachEntry : entries) {
+ // Get the latest date
+ if (ret.get(child) == null ||
+ ret.get(child) < eachEntry.getDateModified()) {
+ modTime = eachEntry.getDateModified();
+ }
+ }
+ } else {
+ modTime = entry.getDateModified();
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get mount point", e);
+ }
+ return modTime;
+ }
+
+ /**
* Create a new file status for a mount point.
*
* @param name Name of the mount point.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index b33b998..4d8ffe1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -167,6 +168,12 @@ public class TestRouterMountTable {
addEntry = MountTable.newInstance(
"/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir3/subdir1", Collections.singletonMap("ns0", "/testdir3"));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testA/testB/testC/testD", Collections.singletonMap("ns0", "/test"));
+ assertTrue(addMountTable(addEntry));
// Create test dir in NN
final FileSystem nnFs = nnContext.getFileSystem();
@@ -174,8 +181,18 @@ public class TestRouterMountTable {
Map<String, Long> pathModTime = new TreeMap<>();
for (String mount : mountTable.getMountPoints("/")) {
- pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
- .getDateModified());
+ if (mountTable.getMountPoint("/"+mount) != null) {
+ pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
+ .getDateModified());
+ } else {
+ List<MountTable> entries = mountTable.getMounts("/"+mount);
+ for (MountTable entry : entries) {
+ if (pathModTime.get(mount) == null ||
+ pathModTime.get(mount) < entry.getDateModified()) {
+ pathModTime.put(mount, entry.getDateModified());
+ }
+ }
+ }
}
FileStatus[] iterator = nnFs.listStatus(new Path("/"));
for (FileStatus file : iterator) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org