You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/21 05:54:04 UTC
[1/2] git commit: Missing file for YARN-2701 (cherry picked from
commit 4fa1fb3193bf39fcb1bd7f8f8391a78f69c3c302)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 debb3686d -> 23b4af488
Missing file for YARN-2701
(cherry picked from commit 4fa1fb3193bf39fcb1bd7f8f8391a78f69c3c302)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23b4af48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23b4af48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23b4af48
Branch: refs/heads/branch-2.6
Commit: 23b4af488e3efd525baa38fce0c052bcc5e97700
Parents: c7db305
Author: Jian He <ji...@apache.org>
Authored: Mon Oct 20 19:57:45 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Oct 20 20:53:28 2014 -0700
----------------------------------------------------------------------
.../nodemanager/MockContainerLocalizer.java | 41 ++++++++++++++++++++
1 file changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b4af48/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainerLocalizer.java
new file mode 100644
index 0000000..0ff8bf6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainerLocalizer.java
@@ -0,0 +1,41 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+
+public class MockContainerLocalizer {
+ public static void buildMainArgs(List<String> command,
+ String user, String appId, String locId,
+ InetSocketAddress nmAddr, List<String> localDirs) {
+ command.add(MockContainerLocalizer.class.getName());
+ command.add(user);
+ command.add(appId);
+ command.add(locId);
+ command.add(nmAddr.getHostName());
+ command.add(Integer.toString(nmAddr.getPort()));
+ for(String dir : localDirs) {
+ command.add(dir);
+ }
+ }
+
+ public static void main(String[] argv) throws Throwable {
+ //DO Nothing
+ }
+}
[2/2] git commit: YARN-2701. Potential race condition in
startLocalizer when using LinuxContainerExecutor. Contributed by Xuan Gong
(cherry picked from commit 2839365f230165222f63129979ea82ada79ec56e)
Posted by ji...@apache.org.
YARN-2701. Potential race condition in startLocalizer when using LinuxContainerExecutor. Contributed by Xuan Gong
(cherry picked from commit 2839365f230165222f63129979ea82ada79ec56e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7db3052
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7db3052
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7db3052
Branch: refs/heads/branch-2.6
Commit: c7db30522cf6a48801968e184b49ac8870ba6cd2
Parents: debb368
Author: Jian He <ji...@apache.org>
Authored: Mon Oct 20 18:42:30 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Oct 20 20:53:28 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/LinuxContainerExecutor.java | 10 ++-
.../impl/container-executor.c | 68 +++++++------------
.../nodemanager/TestLinuxContainerExecutor.java | 70 +++++++++++++++++++-
4 files changed, 104 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7db3052/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2d2f2f3..fff076d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -632,6 +632,9 @@ Release 2.6.0 - UNRELEASED
YARN-2588. Standby RM fails to transitionToActive if previous
transitionToActive failed with ZK exception. (Rohith Sharmaks via jianhe)
+ YARN-2701. Potential race condition in startLocalizer when using
+ LinuxContainerExecutor. (Xuan Gong via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7db3052/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 0c06c04..d628b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import java.io.File;
@@ -218,7 +219,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
- ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
+ buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
if (LOG.isDebugEnabled()) {
@@ -239,6 +240,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
}
+ @VisibleForTesting
+ public void buildMainArgs(List<String> command, String user, String appId,
+ String locId, InetSocketAddress nmAddr, List<String> localDirs) {
+ ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
+ localDirs);
+ }
+
@Override
public int launchContainer(Container container,
Path nmPrivateCotainerScriptPath, Path nmPrivateTokensPath,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7db3052/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f582d85..7624bef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -451,59 +451,39 @@ char *get_tmp_directory(const char *work_dir) {
* with the desired permissions.
*/
int mkdirs(const char* path, mode_t perm) {
- struct stat sb;
- char * npath;
- char * p;
- if (stat(path, &sb) == 0) {
- if (S_ISDIR (sb.st_mode)) {
- return 0;
- } else {
- fprintf(LOGFILE, "Path %s is file not dir\n", path);
- return -1;
- }
- }
- npath = strdup(path);
- if (npath == NULL) {
- fprintf(LOGFILE, "Not enough memory to copy path string");
+ char *buffer = strdup(path);
+ char *token;
+ int cwd = open("/", O_RDONLY);
+ if (cwd == -1) {
+ fprintf(LOGFILE, "Can't open / in %s - %s\n", path, strerror(errno));
+ free(buffer);
return -1;
}
- /* Skip leading slashes. */
- p = npath;
- while (*p == '/') {
- p++;
- }
-
- while (NULL != (p = strchr(p, '/'))) {
- *p = '\0';
- if (stat(npath, &sb) != 0) {
- if (mkdir(npath, perm) != 0) {
- fprintf(LOGFILE, "Can't create directory %s in %s - %s\n", npath,
- path, strerror(errno));
- free(npath);
- return -1;
+ for(token = strtok(buffer, "/"); token != NULL; token = strtok(NULL, "/")) {
+ if (mkdirat(cwd, token, perm) != 0) {
+ if (errno != EEXIST) {
+ fprintf(LOGFILE, "Can't create directory %s in %s - %s\n",
+ token, path, strerror(errno));
+ close(cwd);
+ free(buffer);
+ return -1;
}
- } else if (!S_ISDIR (sb.st_mode)) {
- fprintf(LOGFILE, "Path %s is file not dir\n", npath);
- free(npath);
+ }
+ int new_dir = openat(cwd, token, O_RDONLY);
+ close(cwd);
+ cwd = new_dir;
+ if (cwd == -1) {
+ fprintf(LOGFILE, "Can't open %s in %s - %s\n", token, path,
+ strerror(errno));
+ free(buffer);
return -1;
}
- *p++ = '/'; /* restore slash */
- while (*p == '/')
- p++;
}
-
- /* Create the final directory component. */
- if (mkdir(npath, perm) != 0) {
- fprintf(LOGFILE, "Can't create directory %s - %s\n", npath,
- strerror(errno));
- free(npath);
- return -1;
- }
- free(npath);
+ free(buffer);
+ close(cwd);
return 0;
}
-
/**
* Function to prepare the container directories.
* It creates the container work and log directories.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7db3052/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index cea2e6b..67175d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -29,8 +31,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.junit.Assert;
@@ -52,6 +57,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
import org.junit.After;
import org.junit.Before;
@@ -108,10 +115,12 @@ public class TestLinuxContainerExecutor {
private LinuxContainerExecutor exec = null;
private String appSubmitter = null;
private LocalDirsHandlerService dirsHandler;
+ private Configuration conf;
+ private FileContext files;
@Before
public void setup() throws Exception {
- FileContext files = FileContext.getLocalFSFileContext();
+ files = FileContext.getLocalFSFileContext();
Path workSpacePath = new Path(workSpace.getAbsolutePath());
files.mkdir(workSpacePath, null, true);
FileUtil.chmod(workSpace.getAbsolutePath(), "777");
@@ -123,7 +132,11 @@ public class TestLinuxContainerExecutor {
new FsPermission("777"), false);
String exec_path = System.getProperty("container-executor.path");
if(exec_path != null && !exec_path.isEmpty()) {
- Configuration conf = new Configuration(false);
+ conf = new Configuration(false);
+ conf.setClass("fs.AbstractFileSystem.file.impl",
+ org.apache.hadoop.fs.local.LocalFs.class,
+ org.apache.hadoop.fs.AbstractFileSystem.class);
+ conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, "xuan");
LOG.info("Setting "+YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
+"="+exec_path);
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
@@ -212,6 +225,59 @@ public class TestLinuxContainerExecutor {
dirsHandler.getLogDirs());
}
+ @Test
+ public void testContainerLocalizer() throws Exception {
+ if (!shouldRun()) {
+ return;
+ }
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> logDirs = dirsHandler.getLogDirs();
+ for (String localDir : localDirs) {
+ Path userDir =
+ new Path(localDir, ContainerLocalizer.USERCACHE);
+ files.mkdir(userDir, new FsPermission("777"), false);
+ // $local/filecache
+ Path fileDir =
+ new Path(localDir, ContainerLocalizer.FILECACHE);
+ files.mkdir(fileDir, new FsPermission("777"), false);
+ }
+ String locId = "container_01_01";
+ Path nmPrivateContainerTokensPath =
+ dirsHandler.getLocalPathForWrite(
+ ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
+ + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
+ locId));
+ files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
+ Configuration config = new YarnConfiguration(conf);
+ InetSocketAddress nmAddr = config.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
+ String appId = "application_01_01";
+ exec = new LinuxContainerExecutor() {
+ @Override
+ public void buildMainArgs(List<String> command, String user, String appId,
+ String locId, InetSocketAddress nmAddr, List<String> localDirs) {
+ MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
+ localDirs);
+ }
+ };
+ exec.setConf(conf);
+
+ exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
+ appId, locId, localDirs, logDirs);
+
+ String locId2 = "container_01_02";
+ Path nmPrivateContainerTokensPath2 =
+ dirsHandler
+ .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
+ + Path.SEPARATOR
+ + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
+ files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
+ exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
+ appId, locId2, localDirs, logDirs);
+ }
@Test
public void testContainerLaunch() throws IOException {