You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2013/04/01 18:47:34 UTC
svn commit: r1463203 [4/8] - in
/hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/bin/ hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org...
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java Mon Apr 1 16:47:16 2013
@@ -22,5 +22,9 @@ public interface ResourceView {
long getVmemAllocatedForContainers();
+ boolean isVmemCheckEnabled();
+
long getPmemAllocatedForContainers();
+
+ boolean isPmemCheckEnabled();
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java Mon Apr 1 16:47:16 2013
@@ -34,5 +34,6 @@ public enum ApplicationEventType {
// Source: Log Handler
APPLICATION_LOG_HANDLING_INITED,
- APPLICATION_LOG_HANDLING_FINISHED
+ APPLICATION_LOG_HANDLING_FINISHED,
+ APPLICATION_LOG_HANDLING_FAILED
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Mon Apr 1 16:47:16 2013
@@ -149,6 +149,9 @@ public class ApplicationImpl implements
.addTransition(ApplicationState.INITING, ApplicationState.INITING,
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED,
new AppLogInitDoneTransition())
+ .addTransition(ApplicationState.INITING, ApplicationState.INITING,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED,
+ new AppLogInitFailTransition())
.addTransition(ApplicationState.INITING, ApplicationState.RUNNING,
ApplicationEventType.APPLICATION_INITED,
new AppInitDoneTransition())
@@ -238,6 +241,26 @@ public class ApplicationImpl implements
}
/**
+ * Handles the APPLICATION_LOG_HANDLING_FAILED event that occurs after
+ * {@link LogAggregationService} has failed to initialize the log
+ * aggregation service
+ *
+ * In particular, this requests that the {@link ResourceLocalizationService}
+ * localize the application-scoped resources.
+ */
+ @SuppressWarnings("unchecked")
+ static class AppLogInitFailTransition implements
+ SingleArcTransition<ApplicationImpl, ApplicationEvent> {
+ @Override
+ public void transition(ApplicationImpl app, ApplicationEvent event) {
+ LOG.warn("Log Aggregation service failed to initialize, there will " +
+ "be no logs for this application");
+ app.dispatcher.getEventHandler().handle(
+ new ApplicationLocalizationEvent(
+ LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+ }
+ }
+ /**
* Handles INIT_CONTAINER events which request that we launch a new
* container. When we're still in the INITTING state, we simply
* queue these up. When we're in the RUNNING state, we pass along
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Mon Apr 1 16:47:16 2013
@@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.Creat
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.File;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -37,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@ -69,7 +71,8 @@ public class ContainerLaunch implements
private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
- public static final String CONTAINER_SCRIPT = "launch_container.sh";
+ public static final String CONTAINER_SCRIPT = Shell.WINDOWS ?
+ "launch_container.cmd" : "launch_container.sh";
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
@@ -130,7 +133,7 @@ public class ContainerLaunch implements
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath()));
+ containerLogDir.toString()));
}
launchContext.setCommands(newCmds);
@@ -141,7 +144,7 @@ public class ContainerLaunch implements
entry.setValue(
value.replace(
ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath())
+ containerLogDir.toString())
);
}
// /////////////////////////// End of variable expansion
@@ -411,28 +414,17 @@ public class ContainerLaunch implements
+ appIdStr;
}
- private static class ShellScriptBuilder {
-
- private final StringBuilder sb;
-
- public ShellScriptBuilder() {
- this(new StringBuilder("#!/bin/bash\n\n"));
- }
-
- protected ShellScriptBuilder(StringBuilder sb) {
- this.sb = sb;
- }
-
- public ShellScriptBuilder env(String key, String value) {
- line("export ", key, "=\"", value, "\"");
- return this;
- }
-
- public ShellScriptBuilder symlink(Path src, String dst) throws IOException {
- return symlink(src, new Path(dst));
- }
-
- public ShellScriptBuilder symlink(Path src, Path dst) throws IOException {
+ private static abstract class ShellScriptBuilder {
+
+ private static final String LINE_SEPARATOR =
+ System.getProperty("line.separator");
+ private final StringBuilder sb = new StringBuilder();
+
+ public abstract void command(List<String> command);
+
+ public abstract void env(String key, String value);
+
+ public final void symlink(Path src, Path dst) throws IOException {
if (!src.isAbsolute()) {
throw new IOException("Source must be absolute");
}
@@ -440,28 +432,89 @@ public class ContainerLaunch implements
throw new IOException("Destination must be relative");
}
if (dst.toUri().getPath().indexOf('/') != -1) {
- line("mkdir -p ", dst.getParent().toString());
+ mkdir(dst.getParent());
}
- line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
- return this;
+ link(src, dst);
}
-
- public void write(PrintStream out) throws IOException {
+
+ @Override
+ public String toString() {
+ return sb.toString();
+ }
+
+ public final void write(PrintStream out) throws IOException {
out.append(sb);
}
-
- public void line(String... command) {
+
+ protected final void line(String... command) {
for (String s : command) {
sb.append(s);
}
- sb.append("\n");
+ sb.append(LINE_SEPARATOR);
}
-
+
+ protected abstract void link(Path src, Path dst) throws IOException;
+
+ protected abstract void mkdir(Path path);
+ }
+
+ private static final class UnixShellScriptBuilder extends ShellScriptBuilder {
+
+ public UnixShellScriptBuilder(){
+ line("#!/bin/bash");
+ line();
+ }
+
@Override
- public String toString() {
- return sb.toString();
+ public void command(List<String> command) {
+ line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\"");
}
+ @Override
+ public void env(String key, String value) {
+ line("export ", key, "=\"", value, "\"");
+ }
+
+ @Override
+ protected void link(Path src, Path dst) throws IOException {
+ line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
+ }
+
+ @Override
+ protected void mkdir(Path path) {
+ line("mkdir -p ", path.toString());
+ }
+ }
+
+ private static final class WindowsShellScriptBuilder
+ extends ShellScriptBuilder {
+
+ public WindowsShellScriptBuilder() {
+ line("@setlocal");
+ line();
+ }
+
+ @Override
+ public void command(List<String> command) {
+ line("@call ", StringUtils.join(" ", command));
+ }
+
+ @Override
+ public void env(String key, String value) {
+ line("@set ", key, "=", value);
+ }
+
+ @Override
+ protected void link(Path src, Path dst) throws IOException {
+ line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
+ new File(dst.toString()).getPath(),
+ new File(src.toUri().getPath()).getPath()));
+ }
+
+ @Override
+ protected void mkdir(Path path) {
+ line("@if not exist ", path.toString(), " mkdir ", path.toString());
+ }
}
private static void putEnvIfNotNull(
@@ -479,7 +532,7 @@ public class ContainerLaunch implements
}
public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs) {
+ Path pwd, List<Path> appDirs) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -513,6 +566,16 @@ public class ContainerLaunch implements
environment.put("JVM_PID", "$$");
}
+ // TODO: Remove Windows check and use this approach on all platforms after
+ // additional testing. See YARN-358.
+ if (Shell.WINDOWS) {
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+ if (inputClassPath != null && !inputClassPath.isEmpty()) {
+ environment.put(Environment.CLASSPATH.name(),
+ FileUtil.createJarWithClassPath(inputClassPath, pwd));
+ }
+ }
+
/**
* Modifiable environment variables
*/
@@ -537,7 +600,8 @@ public class ContainerLaunch implements
Map<String,String> environment, Map<Path,List<String>> resources,
List<String> command)
throws IOException {
- ShellScriptBuilder sb = new ShellScriptBuilder();
+ ShellScriptBuilder sb = Shell.WINDOWS ? new WindowsShellScriptBuilder() :
+ new UnixShellScriptBuilder();
if (environment != null) {
for (Map.Entry<String,String> env : environment.entrySet()) {
sb.env(env.getKey().toString(), env.getValue().toString());
@@ -546,21 +610,13 @@ public class ContainerLaunch implements
if (resources != null) {
for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
- sb.symlink(entry.getKey(), linkName);
+ sb.symlink(entry.getKey(), new Path(linkName));
}
}
}
- ArrayList<String> cmd = new ArrayList<String>(2 * command.size() + 5);
- cmd.add("exec /bin/bash ");
- cmd.add("-c ");
- cmd.add("\"");
- for (String cs : command) {
- cmd.add(cs.toString());
- cmd.add(" ");
- }
- cmd.add("\"");
- sb.line(cmd.toArray(new String[cmd.size()]));
+ sb.command(command);
+
PrintStream pout = null;
try {
pout = new PrintStream(out);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Mon Apr 1 16:47:16 2013
@@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.Creat
import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
@@ -53,8 +54,10 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.Credentials;
@@ -175,9 +178,11 @@ public class ResourceLocalizationService
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
try {
- // TODO queue deletions here, rather than NM init?
FileContext lfs = getLocalFileContext(conf);
lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
+
+ cleanUpLocalDir(lfs,delService);
+
List<String> localDirs = dirsHandler.getLocalDirs();
for (String localDir : localDirs) {
// $local/usercache
@@ -659,25 +664,23 @@ public class ResourceLocalizationService
new ContainerResourceFailedEvent(
assoc.getContext().getContainerId(),
assoc.getResource().getRequest(), e.getCause()));
+ List<LocalizerResourceRequestEvent> reqs;
synchronized (attempts) {
LocalResourceRequest req = assoc.getResource().getRequest();
- List<LocalizerResourceRequestEvent> reqs = attempts.get(req);
+ reqs = attempts.get(req);
if (null == reqs) {
LOG.error("Missing pending list for " + req);
return;
}
- if (reqs.isEmpty()) {
- attempts.remove(req);
- }
- /*
- * Do not retry for now. Once failed is failed!
- * LocalizerResourceRequestEvent request = reqs.remove(0);
-
- pending.put(queue.submit(new FSDownload(
- lfs, null, conf, publicDirs,
- request.getResource().getRequest(), new Random())),
- request);
- */ }
+ attempts.remove(req);
+ }
+ // let the other containers know about the localization failure
+ for (LocalizerResourceRequestEvent reqEvent : reqs) {
+ dispatcher.getEventHandler().handle(
+ new ContainerResourceFailedEvent(
+ reqEvent.getContext().getContainerId(),
+ reqEvent.getResource().getRequest(), e.getCause()));
+ }
} catch (CancellationException e) {
// ignore; shutting down
}
@@ -928,4 +931,76 @@ public class ResourceLocalizationService
}
+ private void cleanUpLocalDir(FileContext lfs, DeletionService del) {
+ long currentTimeStamp = System.currentTimeMillis();
+ for (String localDir : dirsHandler.getLocalDirs()) {
+ renameLocalDir(lfs, localDir, ContainerLocalizer.USERCACHE,
+ currentTimeStamp);
+ renameLocalDir(lfs, localDir, ContainerLocalizer.FILECACHE,
+ currentTimeStamp);
+ renameLocalDir(lfs, localDir, ResourceLocalizationService.NM_PRIVATE_DIR,
+ currentTimeStamp);
+ try {
+ deleteLocalDir(lfs, del, localDir);
+ } catch (IOException e) {
+ // Do nothing, just give the warning
+ LOG.warn("Failed to delete localDir: " + localDir);
+ }
+ }
+ }
+
+ private void renameLocalDir(FileContext lfs, String localDir,
+ String localSubDir, long currentTimeStamp) {
+ try {
+ lfs.rename(new Path(localDir, localSubDir), new Path(
+ localDir, localSubDir + "_DEL_" + currentTimeStamp));
+ } catch (FileNotFoundException ex) {
+ // No need to handle this exception
+ // localSubDir may not be exist
+ } catch (Exception ex) {
+ // Do nothing, just give the warning
+ LOG.warn("Failed to rename the local file under " +
+ localDir + "/" + localSubDir);
+ }
+ }
+
+ private void deleteLocalDir(FileContext lfs, DeletionService del,
+ String localDir) throws IOException {
+ RemoteIterator<FileStatus> fileStatus = lfs.listStatus(new Path(localDir));
+ if (fileStatus != null) {
+ while (fileStatus.hasNext()) {
+ FileStatus status = fileStatus.next();
+ try {
+ if (status.getPath().getName().matches(".*" +
+ ContainerLocalizer.USERCACHE + "_DEL_.*")) {
+ cleanUpFilesFromSubDir(lfs, del, status.getPath());
+ } else if (status.getPath().getName()
+ .matches(".*" + NM_PRIVATE_DIR + "_DEL_.*")
+ ||
+ status.getPath().getName()
+ .matches(".*" + ContainerLocalizer.FILECACHE + "_DEL_.*")) {
+ del.delete(null, status.getPath(), new Path[] {});
+ }
+ } catch (IOException ex) {
+ // Do nothing, just give the warning
+ LOG.warn("Failed to delete this local Directory: " +
+ status.getPath().getName());
+ }
+ }
+ }
+ }
+
+ private void cleanUpFilesFromSubDir(FileContext lfs, DeletionService del,
+ Path dirPath) throws IOException {
+ RemoteIterator<FileStatus> fileStatus = lfs.listStatus(dirPath);
+ if (fileStatus != null) {
+ while (fileStatus.hasNext()) {
+ FileStatus status = fileStatus.next();
+ String owner = status.getOwner();
+ del.delete(owner, status.getPath(), new Path[] {});
+ }
+ }
+ del.delete(null, dirPath, new Path[] {});
+ }
+
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Mon Apr 1 16:47:16 2013
@@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
@@ -97,7 +96,7 @@ public class LogAggregationService exten
private final ConcurrentMap<ApplicationId, AppLogAggregator> appLogAggregators;
private final ExecutorService threadPool;
-
+
public LogAggregationService(Dispatcher dispatcher, Context context,
DeletionService deletionService, LocalDirsHandlerService dirsHandler) {
super(LogAggregationService.class.getName());
@@ -129,7 +128,6 @@ public class LogAggregationService exten
// NodeId is only available during start, the following cannot be moved
// anywhere else.
this.nodeId = this.context.getNodeId();
- verifyAndCreateRemoteLogDir(getConfig());
super.start();
}
@@ -164,7 +162,7 @@ public class LogAggregationService exten
}
}
- private void verifyAndCreateRemoteLogDir(Configuration conf) {
+ void verifyAndCreateRemoteLogDir(Configuration conf) {
// Checking the existance of the TLD
FileSystem remoteFS = null;
try {
@@ -177,7 +175,7 @@ public class LogAggregationService exten
remoteExists = remoteFS.exists(this.remoteRootLogDir);
} catch (IOException e) {
throw new YarnException("Failed to check for existence of remoteLogDir ["
- + this.remoteRootLogDir + "]");
+ + this.remoteRootLogDir + "]", e);
}
if (remoteExists) {
try {
@@ -191,8 +189,8 @@ public class LogAggregationService exten
}
} catch (IOException e) {
throw new YarnException(
- "Failed while attempting to check permissions for dir ["
- + this.remoteRootLogDir + "]");
+ "Failed to check permissions for dir ["
+ + this.remoteRootLogDir + "]", e);
}
} else {
LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
@@ -208,7 +206,6 @@ public class LogAggregationService exten
+ this.remoteRootLogDir + "]", e);
}
}
-
}
Path getRemoteNodeLogFileForApp(ApplicationId appId, String user) {
@@ -296,12 +293,14 @@ public class LogAggregationService exten
Map<ApplicationAccessType, String> appAcls) {
ApplicationEvent eventResponse;
try {
+ verifyAndCreateRemoteLogDir(getConfig());
initAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls);
eventResponse = new ApplicationEvent(appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED);
} catch (YarnException e) {
- eventResponse = new ApplicationFinishEvent(appId,
- "Application failed to init aggregation: " + e.getMessage());
+ LOG.warn("Application failed to init aggregation: " + e.getMessage());
+ eventResponse = new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED);
}
this.dispatcher.getEventHandler().handle(eventResponse);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Mon Apr 1 16:47:16 2013
@@ -63,14 +63,13 @@ public class ContainersMonitorImpl exten
private Configuration conf;
private Class<? extends ResourceCalculatorProcessTree> processTreeClass;
- private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
- private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
+ private long maxVmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT;
+ private long maxPmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT;
- /**
- * A value which if set for memory related configuration options, indicates
- * that the options are turned off.
- */
- public static final long DISABLED_MEMORY_LIMIT = -1L;
+ private boolean pmemCheckEnabled;
+ private boolean vmemCheckEnabled;
+
+ private static final long UNKNOWN_MEMORY_LIMIT = -1L;
public ContainersMonitorImpl(ContainerExecutor exec,
AsyncDispatcher dispatcher, Context context) {
@@ -104,63 +103,55 @@ public class ContainersMonitorImpl exten
LOG.info(" Using ResourceCalculatorProcessTree : "
+ this.processTreeClass);
- long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
- if (this.resourceCalculatorPlugin != null) {
- totalPhysicalMemoryOnNM =
- this.resourceCalculatorPlugin.getPhysicalMemorySize();
- if (totalPhysicalMemoryOnNM <= 0) {
- LOG.warn("NodeManager's totalPmem could not be calculated. "
- + "Setting it to " + DISABLED_MEMORY_LIMIT);
- totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
- }
- }
+ long configuredPMemForContainers = conf.getLong(
+ YarnConfiguration.NM_PMEM_MB,
+ YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
+ // Setting these irrespective of whether checks are enabled. Required in
+ // the UI.
// ///////// Physical memory configuration //////
- this.maxPmemAllottedForContainers =
- conf.getLong(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
- this.maxPmemAllottedForContainers =
- this.maxPmemAllottedForContainers * 1024 * 1024L; //Normalize to bytes
-
- if (totalPhysicalMemoryOnNM != DISABLED_MEMORY_LIMIT &&
- this.maxPmemAllottedForContainers >
- totalPhysicalMemoryOnNM * 0.80f) {
- LOG.warn("NodeManager configured with " +
- TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers, "", 1) +
- " physical memory allocated to containers, which is more than " +
- "80% of the total physical memory available (" +
- TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "", 1) +
- "). Thrashing might happen.");
- }
+ this.maxPmemAllottedForContainers = configuredPMemForContainers;
// ///////// Virtual memory configuration //////
- float vmemRatio = conf.getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
Preconditions.checkArgument(vmemRatio > 0.99f,
- YarnConfiguration.NM_VMEM_PMEM_RATIO +
- " should be at least 1.0");
+ YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0");
this.maxVmemAllottedForContainers =
- (long)(vmemRatio * maxPmemAllottedForContainers);
+ (long) (vmemRatio * configuredPMemForContainers);
- super.init(conf);
- }
-
- /**
- * Is the total physical memory check enabled?
- *
- * @return true if total physical memory check is enabled.
- */
- boolean isPhysicalMemoryCheckEnabled() {
- return !(this.maxPmemAllottedForContainers == DISABLED_MEMORY_LIMIT);
- }
+ pmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_PMEM_CHECK_ENABLED);
+ vmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED);
+ LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
+ LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
+
+ if (pmemCheckEnabled) {
+ // Logging if actual pmem cannot be determined.
+ long totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
+ if (this.resourceCalculatorPlugin != null) {
+ totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin
+ .getPhysicalMemorySize();
+ if (totalPhysicalMemoryOnNM <= 0) {
+ LOG.warn("NodeManager's totalPmem could not be calculated. "
+ + "Setting it to " + UNKNOWN_MEMORY_LIMIT);
+ totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
+ }
+ }
- /**
- * Is the total virtual memory check enabled?
- *
- * @return true if total virtual memory check is enabled.
- */
- boolean isVirtualMemoryCheckEnabled() {
- return !(this.maxVmemAllottedForContainers == DISABLED_MEMORY_LIMIT);
+ if (totalPhysicalMemoryOnNM != UNKNOWN_MEMORY_LIMIT &&
+ this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) {
+ LOG.warn("NodeManager configured with "
+ + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers,
+ "", 1)
+ + " physical memory allocated to containers, which is more than "
+ + "80% of the total physical memory available ("
+ + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "",
+ 1) + "). Thrashing might happen.");
+ }
+ }
+ super.init(conf);
}
private boolean isEnabled() {
@@ -174,7 +165,7 @@ public class ContainersMonitorImpl exten
+ this.getClass().getName() + " is disabled.");
return false;
}
- if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) {
+ if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
"needed. Not running the monitor-thread");
return false;
@@ -412,7 +403,7 @@ public class ContainersMonitorImpl exten
boolean isMemoryOverLimit = false;
String msg = "";
- if (isVirtualMemoryCheckEnabled()
+ if (isVmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
// Container (the root process) is still alive and overflowing
@@ -423,7 +414,7 @@ public class ContainersMonitorImpl exten
currentPmemUsage, pmemLimit,
pId, containerId, pTree);
isMemoryOverLimit = true;
- } else if (isPhysicalMemoryCheckEnabled()
+ } else if (isPmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentPmemUsage, curRssMemUsageOfAgedProcesses,
pmemLimit)) {
@@ -507,11 +498,31 @@ public class ContainersMonitorImpl exten
return this.maxVmemAllottedForContainers;
}
+ /**
+ * Is the total physical memory check enabled?
+ *
+ * @return true if total physical memory check is enabled.
+ */
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return this.pmemCheckEnabled;
+ }
+
@Override
public long getPmemAllocatedForContainers() {
return this.maxPmemAllottedForContainers;
}
+ /**
+ * Is the total virtual memory check enabled?
+ *
+ * @return true if total virtual memory check is enabled.
+ */
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return this.vmemCheckEnabled;
+ }
+
@Override
public void handle(ContainersMonitorEvent monitoringEvent) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java Mon Apr 1 16:47:16 2013
@@ -25,6 +25,8 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Helper functionality to read the pid from a file.
@@ -62,14 +64,28 @@ public class ProcessIdFileReader {
}
String temp = line.trim();
if (!temp.isEmpty()) {
- try {
- Long pid = Long.valueOf(temp);
- if (pid > 0) {
+ if (Shell.WINDOWS) {
+ // On Windows, pid is expected to be a container ID, so find first
+ // line that parses successfully as a container ID.
+ try {
+ ConverterUtils.toContainerId(temp);
processId = temp;
break;
+ } catch (Exception e) {
+ // do nothing
+ }
+ }
+ else {
+ // Otherwise, find first line containing a numeric pid.
+ try {
+ Long pid = Long.valueOf(temp);
+ if (pid > 0) {
+ processId = temp;
+ break;
+ }
+ } catch (Exception e) {
+ // do nothing
}
- } catch (Exception e) {
- // do nothing
}
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java Mon Apr 1 16:47:16 2013
@@ -18,16 +18,32 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+
public class NavBlock extends HtmlBlock implements YarnWebParams {
+ private Configuration conf;
+
+ @Inject
+ public NavBlock(Configuration conf) {
+ this.conf = conf;
+ }
+
@Override
protected void render(Block html) {
- html
+
+ String RMWebAppURL = YarnConfiguration.getRMWebAppURL(this.conf);
+ html
.div("#nav")
- .h3()._("NodeManager")._() // TODO: Problem if no header like this
+ .h3()._("ResourceManager")._()
+ .ul()
+ .li().a(RMWebAppURL, "RM Home")._()._()
+ .h3()._("NodeManager")._() // TODO: Problem if no header like this
.ul()
.li()
.a(url("node"), "Node Information")._()
@@ -37,7 +53,7 @@ public class NavBlock extends HtmlBlock
.li()
.a(url("allContainers"), "List of Containers")._()
._()
- .h3("Tools")
+ .h3("Tools")
.ul()
.li().a("/conf", "Configuration")._()
.li().a("/logs", "Local logs")._()
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Mon Apr 1 16:47:16 2013
@@ -42,7 +42,7 @@ public class NodePage extends NMView {
protected void commonPreHead(HTML<_> html) {
super.commonPreHead(html);
- set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:1}");
}
@Override
@@ -67,8 +67,12 @@ public class NodePage extends NMView {
info("NodeManager information")
._("Total Vmem allocated for Containers",
StringUtils.byteDesc(info.getTotalVmemAllocated() * BYTES_IN_MB))
+ ._("Vmem enforcement enabled",
+ info.isVmemCheckEnabled())
._("Total Pmem allocated for Container",
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
+ ._("Pmem enforcement enabled",
+ info.isVmemCheckEnabled())
._("NodeHealthyStatus",
info.getHealthStatus())
._("LastNodeHealthTime", new Date(
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java Mon Apr 1 16:47:16 2013
@@ -36,6 +36,8 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
+ protected boolean vmemCheckEnabled;
+ protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
protected boolean nodeHealthy;
protected String nodeManagerVersion;
@@ -56,8 +58,10 @@ public class NodeInfo {
this.nodeHostName = context.getNodeId().getHost();
this.totalVmemAllocatedContainersMB = resourceView
.getVmemAllocatedForContainers() / BYTES_IN_MB;
+ this.vmemCheckEnabled = resourceView.isVmemCheckEnabled();
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
+ this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@@ -120,8 +124,16 @@ public class NodeInfo {
return this.totalVmemAllocatedContainersMB;
}
+ public boolean isVmemCheckEnabled() {
+ return this.vmemCheckEnabled;
+ }
+
public long getTotalPmemAllocated() {
return this.totalPmemAllocatedContainersMB;
}
+ public boolean isPmemCheckEnabled() {
+ return this.pmemCheckEnabled;
+ }
+
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java Mon Apr 1 16:47:16 2013
@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
-import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
public class LocalRMInterface implements ResourceTracker {
@@ -34,9 +33,7 @@ public class LocalRMInterface implements
@Override
public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException {
- RegistrationResponse registrationResponse = recordFactory.newRecordInstance(RegistrationResponse.class);
RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
- response.setRegistrationResponse(registrationResponse);
return response;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java Mon Apr 1 16:47:16 2013
@@ -29,9 +29,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
-import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
/**
@@ -63,12 +61,8 @@ public class MockNodeStatusUpdater exten
@Override
public RegisterNodeManagerResponse registerNodeManager(
RegisterNodeManagerRequest request) throws YarnRemoteException {
- RegistrationResponse regResponse = recordFactory
- .newRecordInstance(RegistrationResponse.class);
-
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
- response.setRegistrationResponse(regResponse);
return response;
}
@@ -79,13 +73,9 @@ public class MockNodeStatusUpdater exten
LOG.info("Got heartbeat number " + heartBeatID);
nodeStatus.setResponseId(heartBeatID++);
- HeartbeatResponse response = recordFactory
- .newRecordInstance(HeartbeatResponse.class);
- response.setResponseId(heartBeatID);
-
NodeHeartbeatResponse nhResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setHeartbeatResponse(response);
+ nhResponse.setResponseId(heartBeatID);
return nhResponse;
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Mon Apr 1 16:47:16 2013
@@ -27,6 +27,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
@@ -131,8 +132,41 @@ public class TestLinuxContainerExecutorW
}
+ @Test (timeout = 5000)
+ public void testContainerLaunchWithPriority() throws IOException {
+
+ // set the scheduler priority to make sure still works with nice -n prio
+ File f = new File("./src/test/resources/mock-container-executor");
+ if (!f.canExecute()) {
+ f.setExecutable(true);
+ }
+ String executorPath = f.getAbsolutePath();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 2);
+
+ mockExec.setConf(conf);
+ List<String> command = new ArrayList<String>();
+ mockExec.addSchedPriorityCommand(command);
+ assertEquals("first should be nice", "nice", command.get(0));
+ assertEquals("second should be -n", "-n", command.get(1));
+ assertEquals("third should be the priority", Integer.toString(2),
+ command.get(2));
+
+ testContainerLaunch();
+ }
+
+ @Test (timeout = 5000)
+ public void testLaunchCommandWithoutPriority() throws IOException {
+ // make sure the command doesn't contain the nice -n since priority
+ // not specified
+ List<String> command = new ArrayList<String>();
+ mockExec.addSchedPriorityCommand(command);
+ assertEquals("addSchedPriority should be empty", 0, command.size());
+ }
+
- @Test
+ @Test (timeout = 5000)
public void testStartLocalizer() throws IOException {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Mon Apr 1 16:47:16 2013
@@ -56,10 +56,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
-import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -124,12 +122,9 @@ public class TestNodeStatusUpdater {
Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString());
Assert.assertEquals(5 * 1024, resource.getMemory());
registeredNodes.add(nodeId);
- RegistrationResponse regResponse = recordFactory
- .newRecordInstance(RegistrationResponse.class);
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
- response.setRegistrationResponse(regResponse);
return response;
}
@@ -218,13 +213,10 @@ public class TestNodeStatusUpdater {
this.context.getContainers();
Assert.assertEquals(2, activeContainers.size());
}
- HeartbeatResponse response = recordFactory
- .newRecordInstance(HeartbeatResponse.class);
- response.setResponseId(heartBeatID);
NodeHeartbeatResponse nhResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setHeartbeatResponse(response);
+ nhResponse.setResponseId(heartBeatID);
return nhResponse;
}
}
@@ -267,6 +259,36 @@ public class TestNodeStatusUpdater {
}
}
+ private class MyNodeStatusUpdater4 extends NodeStatusUpdaterImpl {
+ public ResourceTracker resourceTracker =
+ new MyResourceTracker(this.context);
+ private Context context;
+ private final long waitStartTime;
+ private final long rmStartIntervalMS;
+ private final boolean rmNeverStart;
+
+ public MyNodeStatusUpdater4(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
+ long rmStartIntervalMS, boolean rmNeverStart) {
+ super(context, dispatcher, healthChecker, metrics);
+ this.context = context;
+ this.waitStartTime = System.currentTimeMillis();
+ this.rmStartIntervalMS = rmStartIntervalMS;
+ this.rmNeverStart = rmNeverStart;
+ }
+
+ @Override
+ protected ResourceTracker getRMClient() {
+ if(System.currentTimeMillis() - waitStartTime <= rmStartIntervalMS
+ || rmNeverStart) {
+ throw new YarnException("Faking RM start failure as start " +
+ "delay timer has not expired.");
+ } else {
+ return resourceTracker;
+ }
+ }
+ }
+
private class MyNodeManager extends NodeManager {
private MyNodeStatusUpdater3 nodeStatusUpdater;
@@ -294,10 +316,7 @@ public class TestNodeStatusUpdater {
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
- RegistrationResponse regResponse = recordFactory
- .newRecordInstance(RegistrationResponse.class);
- regResponse.setNodeAction(registerNodeAction );
- response.setRegistrationResponse(regResponse);
+ response.setNodeAction(registerNodeAction );
return response;
}
@Override
@@ -305,14 +324,11 @@ public class TestNodeStatusUpdater {
throws YarnRemoteException {
NodeStatus nodeStatus = request.getNodeStatus();
nodeStatus.setResponseId(heartBeatID++);
- HeartbeatResponse response = recordFactory
- .newRecordInstance(HeartbeatResponse.class);
- response.setResponseId(heartBeatID);
- response.setNodeAction(heartBeatNodeAction);
NodeHeartbeatResponse nhResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setHeartbeatResponse(response);
+ nhResponse.setResponseId(heartBeatID);
+ nhResponse.setNodeAction(heartBeatNodeAction);
return nhResponse;
}
}
@@ -335,10 +351,7 @@ public class TestNodeStatusUpdater {
RegisterNodeManagerResponse response =
recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
- RegistrationResponse regResponse =
- recordFactory.newRecordInstance(RegistrationResponse.class);
- regResponse.setNodeAction(registerNodeAction);
- response.setRegistrationResponse(regResponse);
+ response.setNodeAction(registerNodeAction);
return response;
}
@@ -348,10 +361,10 @@ public class TestNodeStatusUpdater {
LOG.info("Got heartBeatId: [" + heartBeatID +"]");
NodeStatus nodeStatus = request.getNodeStatus();
nodeStatus.setResponseId(heartBeatID++);
- HeartbeatResponse response =
- recordFactory.newRecordInstance(HeartbeatResponse.class);
- response.setResponseId(heartBeatID);
- response.setNodeAction(heartBeatNodeAction);
+ NodeHeartbeatResponse nhResponse =
+ recordFactory.newRecordInstance(NodeHeartbeatResponse.class);
+ nhResponse.setResponseId(heartBeatID);
+ nhResponse.setNodeAction(heartBeatNodeAction);
if (nodeStatus.getKeepAliveApplications() != null
&& nodeStatus.getKeepAliveApplications().size() > 0) {
@@ -367,11 +380,8 @@ public class TestNodeStatusUpdater {
if (heartBeatID == 2) {
LOG.info("Sending FINISH_APP for application: [" + appId + "]");
this.context.getApplications().put(appId, mock(Application.class));
- response.addAllApplicationsToCleanup(Collections.singletonList(appId));
+ nhResponse.addAllApplicationsToCleanup(Collections.singletonList(appId));
}
- NodeHeartbeatResponse nhResponse =
- recordFactory.newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setHeartbeatResponse(response);
return nhResponse;
}
}
@@ -580,6 +590,73 @@ public class TestNodeStatusUpdater {
+ "Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed");
}
+ @Test (timeout = 15000)
+ public void testNMConnectionToRM() {
+ final long delta = 1500;
+ final long connectionWaitSecs = 5;
+ final long connectionRetryIntervalSecs = 1;
+ //Waiting for rmStartIntervalMS, RM will be started
+ final long rmStartIntervalMS = 2*1000;
+ YarnConfiguration conf = createNMConfig();
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS,
+ connectionWaitSecs);
+ conf.setLong(YarnConfiguration
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
+ connectionRetryIntervalSecs);
+
+ //Test NM try to connect to RM Several times, but finally fail
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4(
+ context, dispatcher, healthChecker, metrics,
+ rmStartIntervalMS, true);
+ return nodeStatusUpdater;
+ }
+ };
+ nm.init(conf);
+ long waitStartTime = System.currentTimeMillis();
+ try {
+ nm.start();
+ Assert.fail("NM should have failed to start due to RM connect failure");
+ } catch(Exception e) {
+ Assert.assertTrue("NM should have tried re-connecting to RM during " +
+ "period of at least " + connectionWaitSecs + " seconds, but " +
+ "stopped retrying within " + (connectionWaitSecs + delta/1000) +
+ " seconds", (System.currentTimeMillis() - waitStartTime
+ >= connectionWaitSecs*1000) && (System.currentTimeMillis()
+ - waitStartTime < (connectionWaitSecs*1000+delta)));
+ }
+
+ //Test NM connect to RM, fail at first several attempts,
+ //but finally success.
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4(
+ context, dispatcher, healthChecker, metrics, rmStartIntervalMS,
+ false);
+ return nodeStatusUpdater;
+ }
+ };
+
+ nm.init(conf);
+ waitStartTime = System.currentTimeMillis();
+ try {
+ nm.start();
+ } catch (Exception ex){
+ Assert.fail("NM should have started successfully " +
+ "after connecting to RM.");
+ }
+ Assert.assertTrue("NM should have connected to RM within " + delta/1000
+ +" seconds of RM starting up.",
+ (System.currentTimeMillis() - waitStartTime >= rmStartIntervalMS)
+ && (System.currentTimeMillis() - waitStartTime
+ < (rmStartIntervalMS+delta)));
+ }
+
/**
* Verifies that if for some reason NM fails to start ContainerManager RPC
* server, RM is oblivious to NM's presence. The behaviour is like this
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Mon Apr 1 16:47:16 2013
@@ -27,13 +27,16 @@ import static org.mockito.Matchers.argTh
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.isNull;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.ArrayList;
@@ -46,6 +49,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
import junit.framework.Assert;
@@ -89,6 +94,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerTracker;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
@@ -102,6 +108,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatcher;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestResourceLocalizationService {
@@ -512,6 +520,111 @@ public class TestResourceLocalizationSer
}
}
+ @Test(timeout=20000)
+ @SuppressWarnings("unchecked") // mocked generics
+ public void testFailedPublicResource() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ AbstractFileSystem spylfs =
+ spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
+ final FileContext lfs = FileContext.getFileContext(spylfs, conf);
+ doNothing().when(spylfs).mkdir(
+ isA(Path.class), isA(FsPermission.class), anyBoolean());
+ List<Path> localDirs = new ArrayList<Path>();
+ String[] sDirs = new String[4];
+ for (int i = 0; i < 4; ++i) {
+ localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
+ sDirs[i] = localDirs.get(i).toString();
+ }
+ conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
+ String logDir = lfs.makeQualified(new Path(basedir, "logdir " )).toString();
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
+
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, applicationBus);
+ EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
+ dispatcher.register(ContainerEventType.class, containerBus);
+
+ ContainerExecutor exec = mock(ContainerExecutor.class);
+ DeletionService delService = mock(DeletionService.class);
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+
+ dispatcher.init(conf);
+ dispatcher.start();
+
+ try {
+ ResourceLocalizationService rawService =
+ new ResourceLocalizationService(dispatcher, exec, delService,
+ dirsHandler);
+ ResourceLocalizationService spyService = spy(rawService);
+ doReturn(mockServer).when(spyService).createServer();
+ doReturn(lfs).when(spyService).getLocalFileContext(
+ isA(Configuration.class));
+
+ spyService.init(conf);
+ spyService.start();
+
+ final String user = "user0";
+ // init application
+ final Application app = mock(Application.class);
+ final ApplicationId appId =
+ BuilderUtils.newApplicationId(314159265358979L, 3);
+ when(app.getUser()).thenReturn(user);
+ when(app.getAppId()).thenReturn(appId);
+ spyService.handle(new ApplicationLocalizationEvent(
+ LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+ dispatcher.await();
+
+ // init container.
+ final Container c = getMockContainer(appId, 42);
+
+ // init resources
+ Random r = new Random();
+ long seed = r.nextLong();
+ System.out.println("SEED: " + seed);
+ r.setSeed(seed);
+
+ // cause chmod to fail after a delay
+ final CyclicBarrier barrier = new CyclicBarrier(2);
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) throws IOException {
+ try {
+ barrier.await();
+ } catch (InterruptedException e) {
+ } catch (BrokenBarrierException e) {
+ }
+ throw new IOException("forced failure");
+ }
+ }).when(spylfs)
+ .setPermission(isA(Path.class), isA(FsPermission.class));
+
+ // Queue up two localization requests for the same public resource
+ final LocalResource pubResource = getPublicMockedResource(r);
+ final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
+
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ new HashMap<LocalResourceVisibility,
+ Collection<LocalResourceRequest>>();
+ req.put(LocalResourceVisibility.PUBLIC,
+ Collections.singletonList(pubReq));
+
+ Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
+ pubRsrcs.add(pubReq);
+
+ spyService.handle(new ContainerLocalizationRequestEvent(c, req));
+ spyService.handle(new ContainerLocalizationRequestEvent(c, req));
+ dispatcher.await();
+
+ // allow the chmod to fail now that both requests have been queued
+ barrier.await();
+ verify(containerBus, timeout(5000).times(2))
+ .handle(isA(ContainerResourceFailedEvent.class));
+ } finally {
+ dispatcher.stop();
+ }
+ }
+
private static URL getPath(String path) {
URL url = BuilderUtils.newURL("file", null, 0, path);
return url;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Mon Apr 1 16:47:16 2013
@@ -22,11 +22,13 @@ import static org.mockito.Mockito.*;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.reflect.Method;
@@ -40,7 +42,9 @@ import java.util.Set;
import junit.framework.Assert;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataInputBuffer;
@@ -76,7 +80,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent;
@@ -84,6 +87,7 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
import org.mortbay.util.MultiException;
@@ -390,7 +394,76 @@ public class TestLogAggregationService e
@Test
@SuppressWarnings("unchecked")
- public void testLogAggregationInitFailsWithoutKillingNM() throws Exception {
+ public void testVerifyAndCreateRemoteDirsFailure()
+ throws Exception {
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ this.remoteRootLogDir.getAbsolutePath());
+
+ DrainDispatcher dispatcher = createDispatcher();
+ EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, appEventHandler);
+
+ LogAggregationService logAggregationService = spy(
+ new LogAggregationService(dispatcher, this.context, this.delSrvc,
+ super.dirsHandler));
+ logAggregationService.init(this.conf);
+
+ YarnException e = new YarnException("KABOOM!");
+ doThrow(e)
+ .when(logAggregationService).verifyAndCreateRemoteLogDir(
+ any(Configuration.class));
+
+ logAggregationService.start();
+
+ // Now try to start an application
+ ApplicationId appId = BuilderUtils.newApplicationId(
+ System.currentTimeMillis(), (int)Math.random());
+ logAggregationService.handle(new LogHandlerAppStartedEvent(appId,
+ this.user, null,
+ ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY,
+ this.acls));
+ dispatcher.await();
+
+ // Verify that it failed
+ ApplicationEvent[] expectedEvents = new ApplicationEvent[] {
+ new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
+ };
+ checkEvents(appEventHandler, expectedEvents, false,
+ "getType", "getApplicationID", "getDiagnostic");
+
+ Mockito.reset(logAggregationService);
+
+ // Now try to start another one
+ ApplicationId appId2 = BuilderUtils.newApplicationId(
+ System.currentTimeMillis(), (int)Math.random());
+ File appLogDir =
+ new File(localLogDir, ConverterUtils.toString(appId2));
+ appLogDir.mkdir();
+
+ logAggregationService.handle(new LogHandlerAppStartedEvent(appId2,
+ this.user, null,
+ ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY,
+ this.acls));
+ dispatcher.await();
+
+ // Verify that it worked
+ expectedEvents = new ApplicationEvent[] {
+ new ApplicationEvent(appId, // original failure
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED),
+ new ApplicationEvent(appId2, // success
+ ApplicationEventType.APPLICATION_LOG_HANDLING_INITED)
+ };
+ checkEvents(appEventHandler, expectedEvents, false,
+ "getType", "getApplicationID", "getDiagnostic");
+
+ logAggregationService.stop();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS,
localLogDir.getAbsolutePath());
@@ -421,8 +494,8 @@ public class TestLogAggregationService e
dispatcher.await();
ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
- new ApplicationFinishEvent(appId,
- "Application failed to init aggregation: KABOOM!")
+ new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
};
checkEvents(appEventHandler, expectedEvents, false,
"getType", "getApplicationID", "getDiagnostic");
@@ -471,8 +544,8 @@ public class TestLogAggregationService e
dispatcher.await();
ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
- new ApplicationFinishEvent(appId,
- "Application failed to init aggregation: "+e)
+ new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
};
checkEvents(appEventHandler, expectedEvents, false,
"getType", "getApplicationID", "getDiagnostic");
@@ -531,24 +604,26 @@ public class TestLogAggregationService e
while (true) {
try {
- DataOutputBuffer dob = new DataOutputBuffer();
- LogReader.readAContainerLogsForALogType(valueStream, dob);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+ LogReader.readAContainerLogsForALogType(valueStream, ps);
- DataInputBuffer dib = new DataInputBuffer();
- dib.reset(dob.getData(), dob.getLength());
+ String writtenLines[] = baos.toString().split(
+ System.getProperty("line.separator"));
- Assert.assertEquals("\nLogType:", dib.readUTF());
- String fileType = dib.readUTF();
+ Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
+ String fileType = writtenLines[0].substring(9);
- Assert.assertEquals("\nLogLength:", dib.readUTF());
- String fileLengthStr = dib.readUTF();
+ Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
+ String fileLengthStr = writtenLines[1].substring(11);
long fileLength = Long.parseLong(fileLengthStr);
- Assert.assertEquals("\nLog Contents:\n", dib.readUTF());
- byte[] buf = new byte[(int) fileLength]; // cast is okay in this
- // test.
- dib.read(buf, 0, (int) fileLength);
- perContainerMap.put(fileType, new String(buf));
+ Assert.assertEquals("Log Contents:",
+ writtenLines[2].substring(0, 13));
+
+ String logContents = StringUtils.join(
+ Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
+ perContainerMap.put(fileType, logContents);
LOG.info("LogType:" + fileType);
LOG.info("LogType:" + fileLength);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Mon Apr 1 16:47:16 2013
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
import java.io.BufferedReader;
import java.io.File;
@@ -52,8 +54,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -281,4 +286,54 @@ public class TestContainersMonitor exten
exec.signalContainer(user,
pid, Signal.NULL));
}
+
+ @Test(timeout = 20000)
+ public void testContainerMonitorMemFlags() {
+ ContainersMonitor cm = null;
+
+ long expPmem = 8192 * 1024 * 1024l;
+ long expVmem = (long) (expPmem * 2.1f);
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(false, false, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(false, cm.isPmemCheckEnabled());
+ assertEquals(false, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(true, false, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(true, cm.isPmemCheckEnabled());
+ assertEquals(false, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(true, true, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(true, cm.isPmemCheckEnabled());
+ assertEquals(true, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(false, true, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(false, cm.isPmemCheckEnabled());
+ assertEquals(true, cm.isVmemCheckEnabled());
+ }
+
+ private YarnConfiguration getConfForCM(boolean pMemEnabled,
+ boolean vMemEnabled, int nmPmem, float vMemToPMemRatio) {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, nmPmem);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, pMemEnabled);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, vMemEnabled);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, vMemToPMemRatio);
+ return conf;
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java Mon Apr 1 16:47:16 2013
@@ -26,13 +26,14 @@ import java.io.PrintWriter;
import junit.framework.Assert;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.junit.Test;
public class TestProcessIdFileReader {
- @Test
+ @Test (timeout = 30000)
public void testNullPath() {
String pid = null;
try {
@@ -44,22 +45,25 @@ public class TestProcessIdFileReader {
assert(pid == null);
}
- @Test
+ @Test (timeout = 30000)
public void testSimpleGet() throws IOException {
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File testFile = null;
+ String expectedProcessId = Shell.WINDOWS ?
+ "container_1353742680940_0002_01_000001" :
+ "56789";
try {
testFile = new File(rootDir, "temp.txt");
PrintWriter fileWriter = new PrintWriter(testFile);
- fileWriter.println("56789");
+ fileWriter.println(expectedProcessId);
fileWriter.close();
String processId = null;
processId = ProcessIdFileReader.getProcessId(
new Path(rootDir + Path.SEPARATOR + "temp.txt"));
- Assert.assertEquals("56789", processId);
+ Assert.assertEquals(expectedProcessId, processId);
} finally {
if (testFile != null
@@ -70,12 +74,15 @@ public class TestProcessIdFileReader {
}
- @Test
+ @Test (timeout = 30000)
public void testComplexGet() throws IOException {
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File testFile = null;
-
+ String processIdInFile = Shell.WINDOWS ?
+ " container_1353742680940_0002_01_000001 " :
+ " 23 ";
+ String expectedProcessId = processIdInFile.trim();
try {
testFile = new File(rootDir, "temp.txt");
PrintWriter fileWriter = new PrintWriter(testFile);
@@ -84,14 +91,14 @@ public class TestProcessIdFileReader {
fileWriter.println("abc");
fileWriter.println("-123");
fileWriter.println("-123 ");
- fileWriter.println(" 23 ");
+ fileWriter.println(processIdInFile);
fileWriter.println("6236");
fileWriter.close();
String processId = null;
processId = ProcessIdFileReader.getProcessId(
new Path(rootDir + Path.SEPARATOR + "temp.txt"));
- Assert.assertEquals("23", processId);
+ Assert.assertEquals(expectedProcessId, processId);
} finally {
if (testFile != null
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java Mon Apr 1 16:47:16 2013
@@ -41,11 +41,11 @@ import org.junit.Test;
public class TestContainerLogsPage {
- @Test
+ @Test(timeout=30000)
public void testContainerLogDirs() throws IOException {
- String absLogDir = new File("target",
- TestNMWebServer.class.getSimpleName() + "LogDir").getAbsolutePath();
- String logdirwithFile = "file://" + absLogDir;
+ File absLogDir = new File("target",
+ TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
+ String logdirwithFile = absLogDir.toURI().toString();
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();