You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/05/04 16:37:39 UTC
hadoop git commit: YARN-8223. Improved yarn auxiliary service to load
jar file from HDFS. Contributed by Zian Chen
Repository: hadoop
Updated Branches:
refs/heads/trunk 6795f8072 -> 8cdb032af
YARN-8223. Improved yarn auxiliary service to load jar file from HDFS.
Contributed by Zian Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cdb032a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cdb032a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cdb032a
Branch: refs/heads/trunk
Commit: 8cdb032aff4237d8d3970057d82290e4e32c4040
Parents: 6795f80
Author: Eric Yang <ey...@apache.org>
Authored: Fri May 4 12:36:31 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri May 4 12:36:31 2018 -0400
----------------------------------------------------------------------
.../PluggableShuffleAndPluggableSort.md | 44 ++++++++++++++++++++
.../containermanager/AuxServices.java | 19 ++++++++-
2 files changed, 61 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cdb032a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
index 5ea0567..9e24103 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
@@ -67,6 +67,50 @@ The collector class configuration may specify a comma-separated list of collecto
|:---- |:---- |:---- |
| `yarn.nodemanager.aux-services` | `...,mapreduce_shuffle` | The auxiliary service name |
| `yarn.nodemanager.aux-services.mapreduce_shuffle.class` | `org.apache.hadoop.mapred.ShuffleHandler` | The auxiliary service class to use |
+| `yarn.nodemanager.aux-services.%s.classpath` | NONE | local directory which includes the related jar file as well as all the dependencies’ jar file. We could specify the single jar file or use /dep/* to load all jars under the dep directory. |
+| `yarn.nodemanager.aux-services.%s.remote-classpath` | NONE | The remote absolute or relative path to jar file |
+
+#### Example of loading jar file from HDFS:
+
+```xml
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,AuxServiceFromHDFS</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.remote-classpath</name>
+ <value>/aux/test/aux-service-hdfs.jar</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class</name>
+ <value>org.apache.auxtest.AuxServiceFromHDFS2</value>
+ </property>
+</configuration>
+```
+
+#### Example of loading jar file from local file system:
+
+```xml
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,AuxServiceFromHDFS</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.classpath</name>
+ <value>/aux/test/aux-service-hdfs.jar</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class</name>
+ <value>org.apache.auxtest.AuxServiceFromHDFS2</value>
+ </property>
+</configuration>
+```
**IMPORTANT:** If setting an auxiliary service in addition the default
`mapreduce_shuffle` service, then a new service key should be added to the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cdb032a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index c8b7a76..3fe3cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -230,15 +230,30 @@ public class AuxServices extends AbstractService
}
}
if (reDownload) {
+ LocalResourceType srcType = null;
+ String lowerDst = StringUtils.toLowerCase(src.toString());
+ if (lowerDst.endsWith(".jar")) {
+ srcType = LocalResourceType.FILE;
+ } else if (lowerDst.endsWith(".zip") ||
+ lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz")
+ || lowerDst.endsWith(".tar")) {
+ srcType = LocalResourceType.ARCHIVE;
+ } else {
+ throw new YarnRuntimeException(
+ "Can not unpack file from remote-file-path:" + src
+ + "for aux-service:" + ".\n");
+ }
LocalResource scRsrc = LocalResource.newInstance(
URL.fromURI(src.toUri()),
- LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE,
+ srcType, LocalResourceVisibility.PRIVATE,
scFileStatus.getLen(), scFileStatus.getModificationTime());
FSDownload download = new FSDownload(localLFS, null, conf,
downloadDest, scRsrc, null);
try {
Path downloaded = download.call();
- dest = new Path(downloaded + Path.SEPARATOR + "*");
+ // don't need to convert downloaded path into a dir
+ // since its already a jar path.
+ dest = downloaded;
} catch (Exception ex) {
throw new YarnRuntimeException(
"Exception happend while downloading files "
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org