You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/06/22 00:26:34 UTC
[36/51] [abbrv] hadoop git commit: YARN-4958. The file localization
process should allow for wildcards to reduce the application footprint in the
state store (Daniel Templeton via sjlee)
YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5107a967
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5107a967
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5107a967
Branch: refs/heads/YARN-2915
Commit: 5107a967fa2558deba11c33a326d4d2e5748f452
Parents: fc6b50c
Author: Sangjin Lee <sj...@twitter.com>
Authored: Mon Jun 20 09:56:53 2016 -0700
Committer: Sangjin Lee <sj...@twitter.com>
Committed: Mon Jun 20 09:56:53 2016 -0700
----------------------------------------------------------------------
.../yarn/server/nodemanager/ContainerExecutor.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5107a967/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 98171af..98d45f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
public abstract class ContainerExecutor implements Configurable {
-
+ private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700);
@@ -281,7 +281,18 @@ public abstract class ContainerExecutor implements Configurable {
if (resources != null) {
for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
- sb.symlink(entry.getKey(), new Path(linkName));
+ if (new Path(linkName).getName().equals(WILDCARD)) {
+ // If this is a wildcarded path, link to everything in the
+ // directory from the working directory
+ File directory = new File(entry.getKey().toString());
+
+ for (File wildLink : directory.listFiles()) {
+ sb.symlink(new Path(wildLink.toString()),
+ new Path(wildLink.getName()));
+ }
+ } else {
+ sb.symlink(entry.getKey(), new Path(linkName));
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org