You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2020/01/09 17:39:14 UTC
[hadoop] branch branch-3.2 updated: YARN-9018. Add functionality to
AuxiliaryLocalPathHandler to return all locations to read for a given path.
Contributed by Kuhu Shukla (kshukla)
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 3ba0fd1 YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
3ba0fd1 is described below
commit 3ba0fd1e50192336d6a0125768739973972917cd
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Thu Jan 9 17:18:44 2020 +0000
YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
(cherry picked from commit 93233a7d6e4d6b8098622a1aa830355cc18d9589)
---
.../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java | 8 ++++++++
.../apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java | 8 ++++++++
.../hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java | 4 ++++
.../server/nodemanager/containermanager/ContainerManagerImpl.java | 5 +++++
4 files changed, 25 insertions(+)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 8017248..6988219 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -173,6 +173,14 @@ public class TestShuffleHandler {
throws IOException {
return new Path(ABS_LOG_DIR.getAbsolutePath());
}
+
+ @Override
+ public Iterable<Path> getAllLocalPathsForRead(String path)
+ throws IOException {
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
+ return paths;
+ }
}
private static class MockShuffleHandler2 extends
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
index 50feecf..452edce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
@@ -55,4 +55,12 @@ public interface AuxiliaryLocalPathHandler {
* @throws IOException if the path creations fails
*/
Path getLocalPathForWrite(String path, long size) throws IOException;
+
+ /**
+ * Get all paths from the local FS for reading for a given Auxiliary Service.
+ * @param path the requested path
+ * @return the complete path list to the file on a local disk as an Iterable
+ * @throws IOException if the file read encounters a problem
+ */
+ Iterable<Path> getAllLocalPathsForRead(String path) throws IOException;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index ac0b84d..9af1645 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -624,6 +624,10 @@ public class LocalDirsHandlerService extends AbstractService {
return getPathToRead(pathStr, getLocalDirsForRead());
}
+ public Iterable<Path> getAllLocalPathsForRead(String pathStr) throws IOException {
+ return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig());
+ }
+
public Path getLogPathForWrite(String pathStr, boolean checkWrite)
throws IOException {
return logDirsAllocator.getLocalPathForWrite(pathStr,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8afc52a..0feb3c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1644,6 +1644,11 @@ public class ContainerManagerImpl extends CompositeService implements
throws IOException {
return dirhandlerService.getLocalPathForWrite(path, size, false);
}
+
+ @Override
+ public Iterable<Path> getAllLocalPathsForRead(String path) throws IOException {
+ return dirhandlerService.getAllLocalPathsForRead(path);
+ }
}
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org