You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [8/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java Tue Aug 19 23:49:39 2014
@@ -36,6 +36,7 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
+ protected long totalVCoresAllocatedContainers;
protected boolean vmemCheckEnabled;
protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
@@ -62,6 +63,8 @@ public class NodeInfo {
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
+ this.totalVCoresAllocatedContainers = resourceView
+ .getVCoresAllocatedForContainers();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@@ -124,6 +127,10 @@ public class NodeInfo {
return this.totalVmemAllocatedContainersMB;
}
+ public long getTotalVCoresAllocated() {
+ return this.totalVCoresAllocatedContainers;
+ }
+
public boolean isVmemCheckEnabled() {
return this.vmemCheckEnabled;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c Tue Aug 19 23:49:39 2014
@@ -33,6 +33,7 @@
#include <limits.h>
#include <sys/stat.h>
#include <sys/mount.h>
+#include <sys/wait.h>
static const int DEFAULT_MIN_USERID = 1000;
@@ -111,16 +112,16 @@ int check_executor_permissions(char *exe
return -1;
}
- // check others do not have read/write/execute permissions
- if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH)
- == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) {
+ // check others do not have write/execute permissions
+ if ((filestat.st_mode & S_IWOTH) == S_IWOTH ||
+ (filestat.st_mode & S_IXOTH) == S_IXOTH) {
fprintf(LOGFILE,
- "The container-executor binary should not have read or write or"
- " execute for others.\n");
+ "The container-executor binary should not have write or execute "
+ "for others.\n");
return -1;
}
- // Binary should be setuid/setgid executable
+ // Binary should be setuid executable
if ((filestat.st_mode & S_ISUID) == 0) {
fprintf(LOGFILE, "The container-executor binary should be set setuid.\n");
return -1;
@@ -245,6 +246,85 @@ static int write_pid_to_file_as_nm(const
}
/**
+ * Write the exit code of the container into the exit code file
+ * exit_code_file: Path to exit code file where exit code needs to be written
+ */
+static int write_exit_code_file(const char* exit_code_file, int exit_code) {
+ char *tmp_ecode_file = concatenate("%s.tmp", "exit_code_path", 1,
+ exit_code_file);
+ if (tmp_ecode_file == NULL) {
+ return -1;
+ }
+
+ // create with 700
+ int ecode_fd = open(tmp_ecode_file, O_WRONLY|O_CREAT|O_EXCL, S_IRWXU);
+ if (ecode_fd == -1) {
+ fprintf(LOGFILE, "Can't open file %s - %s\n", tmp_ecode_file,
+ strerror(errno));
+ free(tmp_ecode_file);
+ return -1;
+ }
+
+ char ecode_buf[21];
+ snprintf(ecode_buf, sizeof(ecode_buf), "%d", exit_code);
+ ssize_t written = write(ecode_fd, ecode_buf, strlen(ecode_buf));
+ close(ecode_fd);
+ if (written == -1) {
+ fprintf(LOGFILE, "Failed to write exit code to file %s - %s\n",
+ tmp_ecode_file, strerror(errno));
+ free(tmp_ecode_file);
+ return -1;
+ }
+
+ // rename temp file to actual exit code file
+ // use rename as atomic
+ if (rename(tmp_ecode_file, exit_code_file)) {
+ fprintf(LOGFILE, "Can't move exit code file from %s to %s - %s\n",
+ tmp_ecode_file, exit_code_file, strerror(errno));
+ unlink(tmp_ecode_file);
+ free(tmp_ecode_file);
+ return -1;
+ }
+
+ free(tmp_ecode_file);
+ return 0;
+}
+
+/**
+ * Wait for the container process to exit and write the exit code to
+ * the exit code file.
+ * Returns the exit code of the container process.
+ */
+static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) {
+ int child_status = -1;
+ int exit_code = -1;
+ int waitpid_result;
+
+ if (change_effective_user(nm_uid, nm_gid) != 0) {
+ return -1;
+ }
+ do {
+ waitpid_result = waitpid(pid, &child_status, 0);
+ } while (waitpid_result == -1 && errno == EINTR);
+ if (waitpid_result < 0) {
+ fprintf(LOGFILE, "Error waiting for container process %d - %s\n",
+ pid, strerror(errno));
+ return -1;
+ }
+ if (WIFEXITED(child_status)) {
+ exit_code = WEXITSTATUS(child_status);
+ } else if (WIFSIGNALED(child_status)) {
+ exit_code = 0x80 + WTERMSIG(child_status);
+ } else {
+ fprintf(LOGFILE, "Unable to determine exit status for pid %d\n", pid);
+ }
+ if (write_exit_code_file(exit_code_file, exit_code) < 0) {
+ return -1;
+ }
+ return exit_code;
+}
+
+/**
* Change the real and effective user and group to abandon the super user
* priviledges.
*/
@@ -337,6 +417,10 @@ char *get_container_work_directory(const
nm_root, user, app_id, container_id);
}
+char *get_exit_code_file(const char* pid_file) {
+ return concatenate("%s.exitcode", "exit_code_file", 1, pid_file);
+}
+
char *get_container_launcher_file(const char* work_dir) {
return concatenate("%s/%s", "container launcher", 2, work_dir, CONTAINER_SCRIPT);
}
@@ -879,6 +963,8 @@ int launch_container_as_user(const char
int exit_code = -1;
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
+ char *exit_code_file = NULL;
+
script_file_dest = get_container_launcher_file(work_dir);
if (script_file_dest == NULL) {
exit_code = OUT_OF_MEMORY;
@@ -889,6 +975,11 @@ int launch_container_as_user(const char
exit_code = OUT_OF_MEMORY;
goto cleanup;
}
+ exit_code_file = get_exit_code_file(pid_file);
+ if (NULL == exit_code_file) {
+ exit_code = OUT_OF_MEMORY;
+ goto cleanup;
+ }
// open launch script
int container_file_source = open_file_as_nm(script_name);
@@ -902,6 +993,13 @@ int launch_container_as_user(const char
goto cleanup;
}
+ pid_t child_pid = fork();
+ if (child_pid != 0) {
+ // parent
+ exit_code = wait_and_write_exit_code(child_pid, exit_code_file);
+ goto cleanup;
+ }
+
// setsid
pid_t pid = setsid();
if (pid == -1) {
@@ -986,6 +1084,7 @@ int launch_container_as_user(const char
exit_code = 0;
cleanup:
+ free(exit_code_file);
free(script_file_dest);
free(cred_file_dest);
return exit_code;
@@ -1082,6 +1181,7 @@ static int delete_path(const char *full_
FTS* tree = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
FTSENT* entry = NULL;
int ret = 0;
+ int ret_errno = 0;
if (tree == NULL) {
fprintf(LOGFILE,
@@ -1099,7 +1199,13 @@ static int delete_path(const char *full_
if (rmdir(entry->fts_accpath) != 0) {
fprintf(LOGFILE, "Couldn't delete directory %s - %s\n",
entry->fts_path, strerror(errno));
- exit_code = -1;
+ if (errno == EROFS) {
+ exit_code = -1;
+ }
+ // record the first errno
+ if (errno != ENOENT && ret_errno == 0) {
+ ret_errno = errno;
+ }
}
}
break;
@@ -1111,7 +1217,13 @@ static int delete_path(const char *full_
if (unlink(entry->fts_accpath) != 0) {
fprintf(LOGFILE, "Couldn't delete file %s - %s\n", entry->fts_path,
strerror(errno));
- exit_code = -1;
+ if (errno == EROFS) {
+ exit_code = -1;
+ }
+ // record the first errno
+ if (errno != ENOENT && ret_errno == 0) {
+ ret_errno = errno;
+ }
}
break;
@@ -1154,6 +1266,9 @@ static int delete_path(const char *full_
}
}
ret = fts_close(tree);
+ if (ret_errno != 0) {
+ exit_code = -1;
+ }
if (exit_code == 0 && ret != 0) {
fprintf(LOGFILE, "Error in fts_close while deleting %s\n", full_path);
exit_code = -1;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Aug 19 23:49:39 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -75,7 +76,7 @@ public class DummyContainerManager exten
protected ResourceLocalizationService createResourceLocalizationService(
ContainerExecutor exec, DeletionService deletionContext) {
return new ResourceLocalizationService(super.dispatcher, exec,
- deletionContext, super.dirsHandler) {
+ deletionContext, super.dirsHandler, new NMNullStateStoreService()) {
@Override
public void handle(LocalizationEvent event) {
switch (event.getType()) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java Tue Aug 19 23:49:39 2014
@@ -18,16 +18,37 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileReader;
import java.io.InputStream;
import java.io.IOException;
+import java.io.LineNumberReader;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Random;
-import org.junit.Assert;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
@@ -45,15 +66,13 @@ import org.apache.hadoop.util.Progressab
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream;
-import static org.apache.hadoop.fs.CreateFlag.*;
-
-
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Matchers;
-import static org.mockito.Mockito.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestDefaultContainerExecutor {
@@ -191,6 +210,92 @@ public class TestDefaultContainerExecuto
}
}
+ @Test
+ public void testContainerLaunchError()
+ throws IOException, InterruptedException {
+
+ Path localDir = new Path(BASE_TMP_PATH, "localDir");
+ List<String> localDirs = new ArrayList<String>();
+ localDirs.add(localDir.toString());
+ List<String> logDirs = new ArrayList<String>();
+ Path logDir = new Path(BASE_TMP_PATH, "logDir");
+ logDirs.add(logDir.toString());
+
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.toString());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString());
+
+ FileContext lfs = FileContext.getLocalFSFileContext(conf);
+ DefaultContainerExecutor mockExec = spy(new DefaultContainerExecutor(lfs));
+ mockExec.setConf(conf);
+ doAnswer(
+ new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ String diagnostics = (String) invocationOnMock.getArguments()[0];
+ assertTrue("Invalid Diagnostics message: " + diagnostics,
+ diagnostics.contains("No such file or directory"));
+ return null;
+ }
+ }
+ ).when(mockExec).logOutput(any(String.class));
+
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ Container container = mock(Container.class);
+ ContainerId cId = mock(ContainerId.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String, String>();
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ try {
+ doAnswer(new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ ContainerDiagnosticsUpdateEvent event =
+ (ContainerDiagnosticsUpdateEvent) invocationOnMock
+ .getArguments()[0];
+ assertTrue("Invalid Diagnostics message: "
+ + event.getDiagnosticsUpdate(),
+ event.getDiagnosticsUpdate().contains("No such file or directory")
+ );
+ return null;
+ }
+ }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
+
+ when(cId.toString()).thenReturn(containerId);
+ when(cId.getApplicationAttemptId()).thenReturn(
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 0));
+
+ when(context.getEnvironment()).thenReturn(env);
+
+ mockExec.createUserLocalDirs(localDirs, appSubmitter);
+ mockExec.createUserCacheDirs(localDirs, appSubmitter);
+ mockExec.createAppDirs(localDirs, appSubmitter, appId);
+ mockExec.createAppLogDirs(appId, logDirs);
+
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+ Path workDir = localDir;
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ mockExec.init();
+ mockExec.activateContainer(cId, pidFile);
+ int ret = mockExec
+ .launchContainer(container, scriptPath, tokensPath, appSubmitter,
+ appId, workDir, localDirs, localDirs);
+ Assert.assertNotSame(0, ret);
+ } finally {
+ mockExec.deleteAsUser(appSubmitter, localDir);
+ mockExec.deleteAsUser(appSubmitter, logDir);
+ }
+ }
+
// @Test
// public void testInit() throws IOException, InterruptedException {
// Configuration conf = new Configuration();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java Tue Aug 19 23:49:39 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
import org.junit.AfterClass;
import org.junit.Test;
import org.mockito.Mockito;
@@ -285,4 +286,58 @@ public class TestDeletionService {
del.stop();
}
}
+
+ @Test
+ public void testRecovery() throws Exception {
+ Random r = new Random();
+ long seed = r.nextLong();
+ r.setSeed(seed);
+ System.out.println("SEED: " + seed);
+ List<Path> baseDirs = buildDirs(r, base, 4);
+ createDirs(new Path("."), baseDirs);
+ List<Path> content = buildDirs(r, new Path("."), 10);
+ for (Path b : baseDirs) {
+ createDirs(b, content);
+ }
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 1);
+ NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ DeletionService del =
+ new DeletionService(new FakeDefaultContainerExecutor(), stateStore);
+ try {
+ del.init(conf);
+ del.start();
+ for (Path p : content) {
+ assertTrue(lfs.util().exists(new Path(baseDirs.get(0), p)));
+ del.delete((Long.parseLong(p.getName()) % 2) == 0 ? null : "dingo",
+ p, baseDirs.toArray(new Path[4]));
+ }
+
+ // restart the deletion service
+ del.stop();
+ del = new DeletionService(new FakeDefaultContainerExecutor(),
+ stateStore);
+ del.init(conf);
+ del.start();
+
+ // verify paths are still eventually deleted
+ int msecToWait = 10 * 1000;
+ for (Path p : baseDirs) {
+ for (Path q : content) {
+ Path fp = new Path(p, q);
+ while (msecToWait > 0 && lfs.util().exists(fp)) {
+ Thread.sleep(100);
+ msecToWait -= 100;
+ }
+ assertFalse(lfs.util().exists(fp));
+ }
+ }
+ } finally {
+ del.close();
+ stateStore.close();
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Aug 19 23:49:39 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -79,7 +80,8 @@ public class TestEventFlow {
YarnConfiguration conf = new YarnConfiguration();
Context context = new NMContext(new NMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInNM(), null, null) {
+ new NMTokenSecretManagerInNM(), null, null,
+ new NMNullStateStoreService()) {
@Override
public int getHttpPort() {
return 1234;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Tue Aug 19 23:49:39 2014
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
@@ -34,8 +38,6 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,9 +48,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.junit.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestLinuxContainerExecutorWithMocks {
@@ -216,7 +222,19 @@ public class TestLinuxContainerExecutorW
conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
- mockExec = new LinuxContainerExecutor();
+ mockExec = spy(new LinuxContainerExecutor());
+ doAnswer(
+ new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ String diagnostics = (String) invocationOnMock.getArguments()[0];
+ assertTrue("Invalid Diagnostics message: " + diagnostics,
+ diagnostics.contains("badcommand"));
+ return null;
+ }
+ }
+ ).when(mockExec).logOutput(any(String.class));
dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
mockExec.setConf(conf);
@@ -233,7 +251,22 @@ public class TestLinuxContainerExecutorW
when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
-
+ doAnswer(
+ new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ ContainerDiagnosticsUpdateEvent event =
+ (ContainerDiagnosticsUpdateEvent) invocationOnMock
+ .getArguments()[0];
+ assertTrue("Invalid Diagnostics message: " +
+ event.getDiagnosticsUpdate(),
+ event.getDiagnosticsUpdate().contains("badcommand"));
+ return null;
+ }
+ }
+ ).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
+
when(cId.toString()).thenReturn(containerId);
when(context.getEnvironment()).thenReturn(env);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,17 +30,19 @@ import java.util.concurrent.ConcurrentMa
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.junit.Assert;
-
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
@@ -46,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
@@ -57,6 +63,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -77,6 +84,9 @@ public class TestNodeManagerResync {
private CyclicBarrier syncBarrier;
private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false);
+ private final NodeManagerEvent resyncEvent =
+ new NodeManagerEvent(NodeManagerEventType.RESYNC);
+
@Before
public void setup() throws UnsupportedFileSystemException {
@@ -94,34 +104,56 @@ public class TestNodeManagerResync {
assertionFailedInThread.set(false);
}
- @SuppressWarnings("unchecked")
@Test
public void testKillContainersOnResync() throws IOException,
InterruptedException, YarnException {
- NodeManager nm = new TestNodeManager1();
+ TestNodeManager1 nm = new TestNodeManager1(false);
+
+ testContainerPreservationOnResyncImpl(nm, false);
+ }
+
+ @Test
+ public void testPreserveContainersOnResyncKeepingContainers() throws
+ IOException,
+ InterruptedException, YarnException {
+ TestNodeManager1 nm = new TestNodeManager1(true);
+
+ testContainerPreservationOnResyncImpl(nm, true);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm,
+ boolean isWorkPreservingRestartEnabled)
+ throws IOException, YarnException, InterruptedException {
YarnConfiguration conf = createNMConfig();
- nm.init(conf);
- nm.start();
- ContainerId cId = TestNodeManagerShutdown.createContainerId();
- TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
- processStartFile);
+ conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
+ isWorkPreservingRestartEnabled);
- Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount());
- nm.getNMDispatcher().getEventHandler().
- handle( new NodeManagerEvent(NodeManagerEventType.RESYNC));
try {
- syncBarrier.await();
- } catch (BrokenBarrierException e) {
+ nm.init(conf);
+ nm.start();
+ ContainerId cId = TestNodeManagerShutdown.createContainerId();
+ TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
+ processStartFile);
+
+ nm.setExistingContainerId(cId);
+ Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount());
+ nm.getNMDispatcher().getEventHandler().handle(resyncEvent);
+ try {
+ syncBarrier.await();
+ } catch (BrokenBarrierException e) {
+ }
+ Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount());
+ // Only containers should be killed on resync, apps should lie around.
+ // That way local resources for apps can be used beyond resync without
+ // relocalization
+ Assert.assertTrue(nm.getNMContext().getApplications()
+ .containsKey(cId.getApplicationAttemptId().getApplicationId()));
+ Assert.assertFalse(assertionFailedInThread.get());
+ }
+ finally {
+ nm.stop();
}
- Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount());
- // Only containers should be killed on resync, apps should lie around. That
- // way local resources for apps can be used beyond resync without
- // relocalization
- Assert.assertTrue(nm.getNMContext().getApplications()
- .containsKey(cId.getApplicationAttemptId().getApplicationId()));
- Assert.assertFalse(assertionFailedInThread.get());
-
- nm.stop();
}
// This test tests new container requests are blocked when NM starts from
@@ -149,7 +181,7 @@ public class TestNodeManagerResync {
Assert.assertFalse(assertionFailedInThread.get());
nm.stop();
}
-
+
@SuppressWarnings("unchecked")
@Test(timeout=10000)
public void testNMshutdownWhenResyncThrowException() throws IOException,
@@ -161,7 +193,7 @@ public class TestNodeManagerResync {
Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount());
nm.getNMDispatcher().getEventHandler()
.handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
-
+
synchronized (isNMShutdownCalled) {
while (isNMShutdownCalled.get() == false) {
try {
@@ -170,7 +202,7 @@ public class TestNodeManagerResync {
}
}
}
-
+
Assert.assertTrue("NM shutdown not called.",isNMShutdownCalled.get());
nm.stop();
}
@@ -185,6 +217,9 @@ public class TestNodeManagerResync {
TestNodeStatusUpdater.createContainerStatus(2, ContainerState.COMPLETE);
final Container container =
TestNodeStatusUpdater.getMockContainer(testCompleteContainer);
+ NMContainerStatus report =
+ createNMContainerStatus(2, ContainerState.COMPLETE);
+ when(container.getNMContainerStatus()).thenReturn(report);
NodeManager nm = new NodeManager() {
int registerCount = 0;
@@ -203,7 +238,7 @@ public class TestNodeManagerResync {
if (registerCount == 0) {
// first register, no containers info.
try {
- Assert.assertEquals(0, request.getContainerStatuses()
+ Assert.assertEquals(0, request.getNMContainerStatuses()
.size());
} catch (AssertionError error) {
error.printStackTrace();
@@ -214,8 +249,8 @@ public class TestNodeManagerResync {
testCompleteContainer.getContainerId(), container);
} else {
// second register contains the completed container info.
- List<ContainerStatus> statuses =
- request.getContainerStatuses();
+ List<NMContainerStatus> statuses =
+ request.getNMContainerStatuses();
try {
Assert.assertEquals(1, statuses.size());
Assert.assertEquals(testCompleteContainer.getContainerId(),
@@ -302,6 +337,16 @@ public class TestNodeManagerResync {
class TestNodeManager1 extends NodeManager {
private int registrationCount = 0;
+ private boolean containersShouldBePreserved;
+ private ContainerId existingCid;
+
+ public TestNodeManager1(boolean containersShouldBePreserved) {
+ this.containersShouldBePreserved = containersShouldBePreserved;
+ }
+
+ public void setExistingContainerId(ContainerId cId) {
+ existingCid = cId;
+ }
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
@@ -333,10 +378,23 @@ public class TestNodeManagerResync {
.containermanager.container.Container> containers =
getNMContext().getContainers();
try {
- // ensure that containers are empty before restart nodeStatusUpdater
- Assert.assertTrue(containers.isEmpty());
- super.rebootNodeStatusUpdaterAndRegisterWithRM();
- syncBarrier.await();
+ try {
+ if (containersShouldBePreserved) {
+ Assert.assertFalse(containers.isEmpty());
+ Assert.assertTrue(containers.containsKey(existingCid));
+ } else {
+ // ensure that containers are empty before restart nodeStatusUpdater
+ Assert.assertTrue(containers.isEmpty());
+ }
+ super.rebootNodeStatusUpdaterAndRegisterWithRM();
+ }
+ catch (AssertionError ae) {
+ ae.printStackTrace();
+ assertionFailedInThread.set(true);
+ }
+ finally {
+ syncBarrier.await();
+ }
} catch (InterruptedException e) {
} catch (BrokenBarrierException e) {
} catch (AssertionError ae) {
@@ -510,4 +568,17 @@ public class TestNodeManagerResync {
}
}
}}
+
+ public static NMContainerStatus createNMContainerStatus(int id,
+ ContainerState containerState) {
+ ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId applicationAttemptId =
+ ApplicationAttemptId.newInstance(applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(applicationAttemptId, id);
+ NMContainerStatus containerReport =
+ NMContainerStatus.newInstance(containerId, containerState,
+ Resource.newInstance(1024, 1), "recover container", 0,
+ Priority.newInstance(10), 0);
+ return containerReport;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Tue Aug 19 23:49:39 2014
@@ -109,6 +109,36 @@ public class TestNodeManagerShutdown {
}
@Test
+ public void testStateStoreRemovalOnDecommission() throws IOException {
+ final File recoveryDir = new File(basedir, "nm-recovery");
+ nm = new TestNodeManager();
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath());
+
+ // verify state store is not removed on normal shutdown
+ nm.init(conf);
+ nm.start();
+ Assert.assertTrue(recoveryDir.exists());
+ Assert.assertTrue(recoveryDir.isDirectory());
+ nm.stop();
+ nm = null;
+ Assert.assertTrue(recoveryDir.exists());
+ Assert.assertTrue(recoveryDir.isDirectory());
+
+ // verify state store is removed on decommissioned shutdown
+ nm = new TestNodeManager();
+ nm.init(conf);
+ nm.start();
+ Assert.assertTrue(recoveryDir.exists());
+ Assert.assertTrue(recoveryDir.isDirectory());
+ nm.getNMContext().setDecommissioned(true);
+ nm.stop();
+ nm = null;
+ Assert.assertFalse(recoveryDir.exists());
+ }
+
+ @Test
public void testKillContainersOnShutdown() throws IOException,
YarnException {
nm = new TestNodeManager();
@@ -157,7 +187,7 @@ public class TestNodeManagerShutdown {
public static void startContainer(NodeManager nm, ContainerId cId,
FileContext localFS, File scriptFileDir, File processStartFile)
- throws IOException, YarnException {
+ throws IOException, YarnException {
File scriptFile =
createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Aug 19 23:49:39 2014
@@ -82,6 +82,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -199,6 +201,7 @@ public class TestNodeStatusUpdater {
Dispatcher mockDispatcher = mock(Dispatcher.class);
EventHandler mockEventHandler = mock(EventHandler.class);
when(mockDispatcher.getEventHandler()).thenReturn(mockEventHandler);
+ NMStateStoreService stateStore = new NMNullStateStoreService();
nodeStatus.setResponseId(heartBeatID++);
Map<ApplicationId, List<ContainerStatus>> appToContainers =
getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
@@ -224,9 +227,8 @@ public class TestNodeStatusUpdater {
firstContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container =
- new ContainerImpl(conf, mockDispatcher, launchContext, null,
- mockMetrics, containerToken);
+ Container container = new ContainerImpl(conf, mockDispatcher,
+ stateStore, launchContext, null, mockMetrics, containerToken);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -255,9 +257,8 @@ public class TestNodeStatusUpdater {
secondContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container =
- new ContainerImpl(conf, mockDispatcher, launchContext, null,
- mockMetrics, containerToken);
+ Container container = new ContainerImpl(conf, mockDispatcher,
+ stateStore, launchContext, null, mockMetrics, containerToken);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@@ -782,7 +783,7 @@ public class TestNodeStatusUpdater {
ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
- nodeStatusUpdater.updateStoppedContainersInCache(cId);
+ nodeStatusUpdater.addCompletedContainer(cId);
Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId));
long time1 = System.currentTimeMillis();
@@ -930,6 +931,7 @@ public class TestNodeStatusUpdater {
Thread.sleep(500);
}
Assert.assertFalse(heartBeatID < 1);
+ Assert.assertTrue(nm.getNMContext().getDecommissioned());
// NM takes a while to reach the STOPPED state.
waitCount = 0;
@@ -1158,7 +1160,8 @@ public class TestNodeStatusUpdater {
@Override
protected NMContext createNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
- NMTokenSecretManagerInNM nmTokenSecretManager) {
+ NMTokenSecretManagerInNM nmTokenSecretManager,
+ NMStateStoreService store) {
return new MyNMContext(containerTokenSecretManager,
nmTokenSecretManager);
}
@@ -1267,7 +1270,8 @@ public class TestNodeStatusUpdater {
public MyNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager) {
- super(containerTokenSecretManager, nmTokenSecretManager, null, null);
+ super(containerTokenSecretManager, nmTokenSecretManager, null, null,
+ new NMNullStateStoreService());
}
@Override
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Aug 19 23:49:39 2014
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -103,7 +104,8 @@ public abstract class BaseContainerManag
protected static final int HTTP_PORT = 5412;
protected Configuration conf = new YarnConfiguration();
protected Context context = new NMContext(new NMContainerTokenSecretManager(
- conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) {
+ conf), new NMTokenSecretManagerInNM(), null,
+ new ApplicationACLsManager(conf), new NMNullStateStoreService()) {
public int getHttpPort() {
return HTTP_PORT;
};
@@ -231,7 +233,7 @@ public abstract class BaseContainerManag
protected DeletionService createDeletionService() {
return new DeletionService(exec) {
@Override
- public void delete(String user, Path subDir, Path[] baseDirs) {
+ public void delete(String user, Path subDir, Path... baseDirs) {
// Don't do any deletions.
LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir
+ ", baseDirs - " + baseDirs);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Tue Aug 19 23:49:39 2014
@@ -33,8 +33,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,6 +44,7 @@ import org.apache.hadoop.service.Service
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -54,10 +53,9 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.junit.Assert;
import org.junit.Test;
public class TestAuxServices {
@@ -192,8 +190,9 @@ public class TestAuxServices {
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1, 1);
ContainerTokenIdentifier cti = new ContainerTokenIdentifier(
ContainerId.newInstance(attemptId, 1), "", "",
- Resource.newInstance(1, 1), 0,0,0);
- Container container = new ContainerImpl(null, null, null, null, null, cti);
+ Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0);
+ Container container = new ContainerImpl(null, null, null, null, null,
+ null, cti);
ContainerId containerId = container.getContainerId();
Resource resource = container.getResource();
event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Aug 19 23:49:39 2014
@@ -31,8 +31,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -49,6 +47,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -57,6 +56,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.security.C
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
@@ -80,6 +79,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -348,8 +348,7 @@ public class TestContainerManager extend
GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
- int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
- ExitCode.TERMINATED.getExitCode();
+ int expectedExitCode = ContainerExitStatus.KILLED_BY_APPMASTER;
Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
// Assert that the process is not alive anymore
@@ -799,7 +798,8 @@ public class TestContainerManager extend
Resource r = BuilderUtils.newResource(1024, 1);
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
- System.currentTimeMillis() + 100000L, 123, rmIdentifier);
+ System.currentTimeMillis() + 100000L, 123, rmIdentifier,
+ Priority.newInstance(0), 0);
Token containerToken =
BuilderUtils
.newContainerToken(nodeId, containerTokenSecretManager
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Aug 19 23:49:39 2014
@@ -32,14 +32,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -66,6 +65,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
@@ -538,7 +538,8 @@ public class TestApplication {
long currentTime = System.currentTimeMillis();
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(container.getContainerId(), "", "",
- null, currentTime + 2000, masterKey.getKeyId(), currentTime);
+ null, currentTime + 2000, masterKey.getKeyId(), currentTime,
+ Priority.newInstance(0), 0);
containerTokenIdentifierMap
.put(identifier.getContainerID(), identifier);
context.getContainerTokenSecretManager().startContainerSuccessful(
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Tue Aug 19 23:49:39 2014
@@ -48,17 +48,17 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
@@ -88,7 +88,9 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
@@ -319,7 +321,7 @@ public class TestContainer {
assertEquals(ContainerState.NEW, wc.c.getContainerState());
wc.killContainer();
assertEquals(ContainerState.DONE, wc.c.getContainerState());
- assertEquals(ExitCode.TERMINATED.getExitCode(),
+ assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
wc.c.cloneAndGetContainerStatus().getExitStatus());
assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics()
.contains("KillRequest"));
@@ -339,7 +341,7 @@ public class TestContainer {
assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
wc.killContainer();
assertEquals(ContainerState.KILLING, wc.c.getContainerState());
- assertEquals(ExitCode.TERMINATED.getExitCode(),
+ assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
wc.c.cloneAndGetContainerStatus().getExitStatus());
assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics()
.contains("KillRequest"));
@@ -721,6 +723,8 @@ public class TestContainer {
Context context = mock(Context.class);
when(context.getApplications()).thenReturn(
new ConcurrentHashMap<ApplicationId, Application>());
+ NMNullStateStoreService stateStore = new NMNullStateStoreService();
+ when(context.getNMStateStore()).thenReturn(stateStore);
ContainerExecutor executor = mock(ContainerExecutor.class);
launcher =
new ContainersLauncher(context, dispatcher, executor, null, null);
@@ -749,7 +753,7 @@ public class TestContainer {
long currentTime = System.currentTimeMillis();
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource,
- currentTime + 10000L, 123, currentTime);
+ currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0);
Token token =
BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port),
"password".getBytes(), identifier);
@@ -776,7 +780,8 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
- c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier);
+ c = new ContainerImpl(conf, dispatcher, new NMNullStateStoreService(),
+ ctxt, null, metrics, identifier);
dispatcher.register(ContainerEventType.class,
new EventHandler<ContainerEvent>() {
@Override
@@ -898,12 +903,14 @@ public class TestContainer {
}
public void killContainer() {
- c.handle(new ContainerKillEvent(cId, "KillRequest"));
+ c.handle(new ContainerKillEvent(cId,
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
+ "KillRequest"));
drainDispatcherEvents();
}
public void containerKilledOnRequest() {
- int exitCode = ExitCode.FORCE_KILLED.getExitCode();
+ int exitCode = ContainerExitStatus.KILLED_BY_RESOURCEMANAGER;
String diagnosticMsg = "Container completed with exit code " + exitCode;
c.handle(new ContainerExitEvent(cId,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST, exitCode,
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Tue Aug 19 23:49:39 2014
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.junit.matchers.JUnitMatchers.containsString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -27,6 +30,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -36,8 +40,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -62,6 +65,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
@@ -70,12 +74,13 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Apps;
@@ -83,6 +88,8 @@ import org.apache.hadoop.yarn.util.Auxil
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -475,7 +482,7 @@ public class TestContainerLaunch extends
containerLaunchContext.setCommands(commands);
StartContainerRequest scRequest =
StartContainerRequest.newInstance(containerLaunchContext,
- createContainerToken(cId));
+ createContainerToken(cId, Priority.newInstance(0), 0));
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests =
@@ -599,8 +606,7 @@ public class TestContainerLaunch extends
GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
- int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
- ExitCode.TERMINATED.getExitCode();
+ int expectedExitCode = ContainerExitStatus.KILLED_BY_APPMASTER;
Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
// Assert that the process is not alive anymore
@@ -675,7 +681,9 @@ public class TestContainerLaunch extends
// set up the rest of the container
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Token containerToken = createContainerToken(cId);
+ Priority priority = Priority.newInstance(10);
+ long createTime = 1234;
+ Token containerToken = createContainerToken(cId, priority, createTime);
StartContainerRequest scRequest =
StartContainerRequest.newInstance(containerLaunchContext,
@@ -694,6 +702,11 @@ public class TestContainerLaunch extends
Assert.assertTrue("ProcessStartFile doesn't exist!",
processStartFile.exists());
+ NMContainerStatus nmContainerStatus =
+ containerManager.getContext().getContainers().get(cId)
+ .getNMContainerStatus();
+ Assert.assertEquals(priority, nmContainerStatus.getPriority());
+
// Now test the stop functionality.
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cId);
@@ -712,7 +725,7 @@ public class TestContainerLaunch extends
ContainerStatus containerStatus =
containerManager.getContainerStatuses(gcsRequest)
.getContainerStatuses().get(0);
- Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(),
+ Assert.assertEquals(ContainerExitStatus.KILLED_BY_APPMASTER,
containerStatus.getExitStatus());
// Now verify the contents of the file. Script generates a message when it
@@ -743,18 +756,18 @@ public class TestContainerLaunch extends
}
}
- @Test
+ @Test (timeout = 30000)
public void testDelayedKill() throws Exception {
internalKillTest(true);
}
- @Test
+ @Test (timeout = 30000)
public void testImmediateKill() throws Exception {
internalKillTest(false);
}
@SuppressWarnings("rawtypes")
- @Test
+ @Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
Container container = mock(Container.class);
when(container.getContainerId()).thenReturn(ContainerId.newInstance(
@@ -779,11 +792,13 @@ public class TestContainerLaunch extends
launch.call();
}
- protected Token createContainerToken(ContainerId cId) throws InvalidToken {
+ protected Token createContainerToken(ContainerId cId, Priority priority,
+ long createTime) throws InvalidToken {
Resource r = BuilderUtils.newResource(1024, 1);
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
- r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER);
+ r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER,
+ priority, createTime);
Token containerToken =
BuilderUtils.newContainerToken(
context.getNodeId(),
@@ -792,4 +807,166 @@ public class TestContainerLaunch extends
return containerToken;
}
+ /**
+ * Test that script exists with non-zero exit code when command fails.
+ * @throws IOException
+ */
+ @Test (timeout = 10000)
+ public void testShellScriptBuilderNonZeroExitCode() throws IOException {
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+ builder.command(Arrays.asList(new String[] {"unknownCommand"}));
+ File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError");
+ PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
+ builder.write(writer);
+ writer.close();
+ try {
+ FileUtil.setExecutable(shellFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ try {
+ shexc.execute();
+ fail("builder shell command was expected to throw");
+ }
+ catch(IOException e) {
+ // expected
+ System.out.println("Received an expected exception: " + e.getMessage());
+ }
+ }
+ finally {
+ FileUtil.fullyDelete(shellFile);
+ }
+ }
+
+ private static final String expectedMessage = "The command line has a length of";
+
+ @Test (timeout = 10000)
+ public void testWindowsShellScriptBuilderCommand() throws IOException {
+ String callCmd = "@call ";
+
+ // Test is only relevant on Windows
+ Assume.assumeTrue(Shell.WINDOWS);
+
+ // The tests are built on assuming 8191 max command line length
+ assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);
+
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ // Basic tests: less length, exact length, max+1 length
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat("A", 1024)));
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat(
+ "E", Shell.WINDOWS_MAX_SHELL_LENGHT - callCmd.length())));
+ try {
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat(
+ "X", Shell.WINDOWS_MAX_SHELL_LENGHT -callCmd.length() + 1)));
+ fail("longCommand was expected to throw");
+ } catch(IOException e) {
+ assertThat(e.getMessage(), containsString(expectedMessage));
+ }
+
+ // Composite tests, from parts: less, exact and +
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat("A", 1024),
+ org.apache.commons.lang.StringUtils.repeat("A", 1024),
+ org.apache.commons.lang.StringUtils.repeat("A", 1024)));
+
+ // buildr.command joins the command parts with an extra space
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat("E", 4095),
+ org.apache.commons.lang.StringUtils.repeat("E", 2047),
+ org.apache.commons.lang.StringUtils.repeat("E", 2047 - callCmd.length())));
+
+ try {
+ builder.command(Arrays.asList(
+ org.apache.commons.lang.StringUtils.repeat("X", 4095),
+ org.apache.commons.lang.StringUtils.repeat("X", 2047),
+ org.apache.commons.lang.StringUtils.repeat("X", 2048 - callCmd.length())));
+ fail("long commands was expected to throw");
+ } catch(IOException e) {
+ assertThat(e.getMessage(), containsString(expectedMessage));
+ }
+ }
+
+ @Test (timeout = 10000)
+ public void testWindowsShellScriptBuilderEnv() throws IOException {
+ // Test is only relevant on Windows
+ Assume.assumeTrue(Shell.WINDOWS);
+
+ // The tests are built on assuming 8191 max command line length
+ assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);
+
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ // test env
+ builder.env("somekey", org.apache.commons.lang.StringUtils.repeat("A", 1024));
+ builder.env("somekey", org.apache.commons.lang.StringUtils.repeat(
+ "A", Shell.WINDOWS_MAX_SHELL_LENGHT - ("@set somekey=").length()));
+ try {
+ builder.env("somekey", org.apache.commons.lang.StringUtils.repeat(
+ "A", Shell.WINDOWS_MAX_SHELL_LENGHT - ("@set somekey=").length()) + 1);
+ fail("long env was expected to throw");
+ } catch(IOException e) {
+ assertThat(e.getMessage(), containsString(expectedMessage));
+ }
+ }
+
+ @Test (timeout = 10000)
+ public void testWindowsShellScriptBuilderMkdir() throws IOException {
+ String mkDirCmd = "@if not exist \"\" mkdir \"\"";
+
+ // Test is only relevant on Windows
+ Assume.assumeTrue(Shell.WINDOWS);
+
+ // The tests are built on assuming 8191 max command line length
+ assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);
+
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ // test mkdir
+ builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)));
+ builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat(
+ "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - mkDirCmd.length())/2)));
+ try {
+ builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat(
+ "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - mkDirCmd.length())/2 +1)));
+ fail("long mkdir was expected to throw");
+ } catch(IOException e) {
+ assertThat(e.getMessage(), containsString(expectedMessage));
+ }
+ }
+
+ @Test (timeout = 10000)
+ public void testWindowsShellScriptBuilderLink() throws IOException {
+ // Test is only relevant on Windows
+ Assume.assumeTrue(Shell.WINDOWS);
+
+ String linkCmd = "@" +Shell.WINUTILS + " symlink \"\" \"\"";
+
+ // The tests are built on assuming 8191 max command line length
+ assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);
+
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ // test link
+ builder.link(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)),
+ new Path(org.apache.commons.lang.StringUtils.repeat("B", 1024)));
+ builder.link(
+ new Path(org.apache.commons.lang.StringUtils.repeat(
+ "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)),
+ new Path(org.apache.commons.lang.StringUtils.repeat(
+ "F", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)));
+ try {
+ builder.link(
+ new Path(org.apache.commons.lang.StringUtils.repeat(
+ "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2 + 1)),
+ new Path(org.apache.commons.lang.StringUtils.repeat(
+ "Y", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2) + 1));
+ fail("long link was expected to throw");
+ } catch(IOException e) {
+ assertThat(e.getMessage(), containsString(expectedMessage));
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java Tue Aug 19 23:49:39 2014
@@ -23,6 +23,7 @@ import org.junit.Assert;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheDirectoryManager.Directory;
import org.junit.Test;
public class TestLocalCacheDirectoryManager {
@@ -73,7 +74,7 @@ public class TestLocalCacheDirectoryMana
conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
Exception e = null;
ResourceLocalizationService service =
- new ResourceLocalizationService(null, null, null, null);
+ new ResourceLocalizationService(null, null, null, null, null);
try {
service.init(conf);
} catch (Exception e1) {
@@ -109,4 +110,49 @@ public class TestLocalCacheDirectoryMana
// first sub directory
Assert.assertEquals(firstSubDir, dir.getRelativePathForLocalization());
}
+
+ @Test
+ public void testDirectoryConversion() {
+ for (int i = 0; i < 10000; ++i) {
+ String path = Directory.getRelativePath(i);
+ Assert.assertEquals("Incorrect conversion for " + i, i,
+ Directory.getDirectoryNumber(path));
+ }
+ }
+
+ @Test
+ public void testIncrementFileCountForPath() {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY,
+ LocalCacheDirectoryManager.DIRECTORIES_PER_LEVEL + 2);
+ LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf);
+ final String rootPath = "";
+ mgr.incrementFileCountForPath(rootPath);
+ Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization());
+ Assert.assertFalse("root dir should be full",
+ rootPath.equals(mgr.getRelativePathForLocalization()));
+ // finish filling the other directory
+ mgr.getRelativePathForLocalization();
+ // free up space in the root dir
+ mgr.decrementFileCountForPath(rootPath);
+ mgr.decrementFileCountForPath(rootPath);
+ Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization());
+ Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization());
+ String otherDir = mgr.getRelativePathForLocalization();
+ Assert.assertFalse("root dir should be full", otherDir.equals(rootPath));
+
+ final String deepDir0 = "d/e/e/p/0";
+ final String deepDir1 = "d/e/e/p/1";
+ final String deepDir2 = "d/e/e/p/2";
+ final String deepDir3 = "d/e/e/p/3";
+ mgr.incrementFileCountForPath(deepDir0);
+ Assert.assertEquals(otherDir, mgr.getRelativePathForLocalization());
+ Assert.assertEquals(deepDir0, mgr.getRelativePathForLocalization());
+ Assert.assertEquals("total dir count incorrect after increment",
+ deepDir1, mgr.getRelativePathForLocalization());
+ mgr.incrementFileCountForPath(deepDir2);
+ mgr.incrementFileCountForPath(deepDir1);
+ mgr.incrementFileCountForPath(deepDir2);
+ Assert.assertEquals(deepDir3, mgr.getRelativePathForLocalization());
+ }
}