You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2020/01/09 17:19:58 UTC
[hadoop] branch trunk updated: YARN-9018. Add functionality to
AuxiliaryLocalPathHandler to return all locations to read for a given path.
Contributed by Kuhu Shukla (kshukla)
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 93233a7 YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
93233a7 is described below
commit 93233a7d6e4d6b8098622a1aa830355cc18d9589
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Thu Jan 9 17:18:44 2020 +0000
YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
---
.../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java | 8 ++++++++
.../apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java | 8 ++++++++
.../hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java | 4 ++++
.../server/nodemanager/containermanager/ContainerManagerImpl.java | 5 +++++
4 files changed, 25 insertions(+)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index cd9dab0..af3cb87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -173,6 +173,14 @@ public class TestShuffleHandler {
throws IOException {
return new Path(ABS_LOG_DIR.getAbsolutePath());
}
+
+ @Override
+ public Iterable<Path> getAllLocalPathsForRead(String path)
+ throws IOException {
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
+ return paths;
+ }
}
private static class MockShuffleHandler2 extends
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
index 50feecf..452edce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
@@ -55,4 +55,12 @@ public interface AuxiliaryLocalPathHandler {
* @throws IOException if the path creations fails
*/
Path getLocalPathForWrite(String path, long size) throws IOException;
+
+ /**
+ * Get all paths from the local FS for reading for a given Auxiliary Service.
+ * @param path the requested path
+ * @return the complete path list to the file on a local disk as an Iterable
+ * @throws IOException if the file read encounters a problem
+ */
+ Iterable<Path> getAllLocalPathsForRead(String path) throws IOException;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index fee8847..591d2df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -644,6 +644,10 @@ public class LocalDirsHandlerService extends AbstractService
return getPathToRead(pathStr, getLocalDirsForRead());
}
+ public Iterable<Path> getAllLocalPathsForRead(String pathStr) throws IOException {
+ return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig());
+ }
+
public Path getLogPathForWrite(String pathStr, boolean checkWrite)
throws IOException {
return logDirsAllocator.getLocalPathForWrite(pathStr,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index d88c98f..07d6417 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1650,6 +1650,11 @@ public class ContainerManagerImpl extends CompositeService implements
throws IOException {
return dirhandlerService.getLocalPathForWrite(path, size, false);
}
+
+ @Override
+ public Iterable<Path> getAllLocalPathsForRead(String path) throws IOException {
+ return dirhandlerService.getAllLocalPathsForRead(path);
+ }
}
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org