You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/08/16 22:45:34 UTC
hadoop git commit: YARN-8667. Cleanup symlinks when container
restarted by NM. Contributed by Chandni Singh
Repository: hadoop
Updated Branches:
refs/heads/trunk 8512e1a91 -> d42806160
YARN-8667. Cleanup symlinks when container restarted by NM.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4280616
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4280616
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4280616
Branch: refs/heads/trunk
Commit: d42806160eb95594f08f38bb753cf0306a191a38
Parents: 8512e1a
Author: Eric Yang <ey...@apache.org>
Authored: Thu Aug 16 18:41:58 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Aug 16 18:41:58 2018 -0400
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 62 +++++++++++++++-----
.../launcher/ContainerLaunch.java | 7 +++
.../nodemanager/TestContainerExecutor.java | 31 +++++++++-
3 files changed, 85 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4280616/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 9b604ce..ba272e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -27,6 +27,7 @@ import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.LinkedHashSet;
import java.util.Map;
@@ -35,6 +36,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -417,20 +419,9 @@ public abstract class ContainerExecutor implements Configurable {
if (resources != null) {
sb.echo("Setting up job resources");
- for (Map.Entry<Path, List<String>> resourceEntry :
- resources.entrySet()) {
- for (String linkName : resourceEntry.getValue()) {
- if (new Path(linkName).getName().equals(WILDCARD)) {
- // If this is a wildcarded path, link to everything in the
- // directory from the working directory
- for (File wildLink : readDirAsUser(user, resourceEntry.getKey())) {
- sb.symlink(new Path(wildLink.toString()),
- new Path(wildLink.getName()));
- }
- } else {
- sb.symlink(resourceEntry.getKey(), new Path(linkName));
- }
- }
+ Map<Path, Path> symLinks = resolveSymLinks(resources, user);
+ for (Map.Entry<Path, Path> symLink : symLinks.entrySet()) {
+ sb.symlink(symLink.getKey(), symLink.getValue());
}
}
@@ -792,6 +783,28 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Perform any cleanup before the next launch of the container.
+ * @param container container
+ */
+ public void cleanupBeforeRelaunch(Container container)
+ throws IOException, InterruptedException {
+ if (container.getLocalizedResources() != null) {
+
+ Map<Path, Path> symLinks = resolveSymLinks(
+ container.getLocalizedResources(), container.getUser());
+
+ for (Map.Entry<Path, Path> symLink : symLinks.entrySet()) {
+ LOG.debug("{} deleting {}", container.getContainerId(),
+ symLink.getValue());
+ deleteAsUser(new DeletionAsUserContext.Builder()
+ .setUser(container.getUser())
+ .setSubDir(symLink.getValue())
+ .build());
+ }
+ }
+ }
+
+ /**
* Get the process-identifier for the container.
*
* @param containerID the container ID
@@ -870,4 +883,25 @@ public abstract class ContainerExecutor implements Configurable {
}
}
}
+
+ private Map<Path, Path> resolveSymLinks(Map<Path,
+ List<String>> resources, String user) {
+ Map<Path, Path> symLinks = new HashMap<>();
+ for (Map.Entry<Path, List<String>> resourceEntry :
+ resources.entrySet()) {
+ for (String linkName : resourceEntry.getValue()) {
+ if (new Path(linkName).getName().equals(WILDCARD)) {
+ // If this is a wildcarded path, link to everything in the
+ // directory from the working directory
+ for (File wildLink : readDirAsUser(user, resourceEntry.getKey())) {
+ symLinks.put(new Path(wildLink.toString()),
+ new Path(wildLink.getName()));
+ }
+ } else {
+ symLinks.put(resourceEntry.getKey(), new Path(linkName));
+ }
+ }
+ }
+ return symLinks;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4280616/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 2195232..2aca5f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1868,6 +1868,13 @@ public class ContainerLaunch implements Callable<Integer> {
deleteAsUser(new Path(containerWorkDir, CONTAINER_SCRIPT));
// delete TokensPath
deleteAsUser(new Path(containerWorkDir, FINAL_CONTAINER_TOKENS_FILE));
+
+ // delete symlinks because launch script will create symlinks again
+ try {
+ exec.cleanupBeforeRelaunch(container);
+ } catch (IOException | InterruptedException e) {
+ LOG.warn("{} exec failed to cleanup", container.getContainerId(), e);
+ }
}
private void deleteAsUser(Path path) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4280616/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
index 3bcdc87..2890bb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
@@ -18,9 +18,17 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -28,13 +36,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
-import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Assert;
import org.junit.Test;
import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
@SuppressWarnings("deprecation")
public class TestContainerExecutor {
@@ -168,4 +176,25 @@ public class TestContainerExecutor {
builder.setContainer(container).setUser("foo");
assertTrue(containerExecutor.reapContainer(builder.build()));
}
+
+ @Test
+ public void testCleanupBeforeLaunch() throws Exception {
+ Container container = mock(Container.class);
+ java.nio.file.Path linkName = Paths.get("target/linkName");
+ java.nio.file.Path target = Paths.get("target");
+ //deletes the link if it already exists because of previous test failures
+ FileUtils.deleteQuietly(linkName.toFile());
+ Files.createSymbolicLink(linkName.toAbsolutePath(),
+ target.toAbsolutePath());
+
+ Map<Path, List<String>> localResources = new HashMap<>();
+ localResources.put(new Path(target.toFile().getAbsolutePath()),
+ Lists.newArrayList(linkName.toFile().getAbsolutePath()));
+
+ when(container.getLocalizedResources())
+ .thenReturn(localResources);
+ when(container.getUser()).thenReturn(System.getProperty("user.name"));
+ containerExecutor.cleanupBeforeRelaunch(container);
+ Assert.assertTrue(!Files.exists(linkName));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org