You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2013/03/24 16:56:22 UTC
svn commit: r1460410 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoco...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java Sun Mar 24 15:56:18 2013
@@ -19,10 +19,10 @@
package org.apache.hadoop.yarn.logaggregation;
import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.PrintStream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -30,6 +30,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileContext;
@@ -57,10 +58,13 @@ public class LogDumper extends Configure
public int run(String[] args) throws Exception {
Options opts = new Options();
- opts.addOption(APPLICATION_ID_OPTION, true, "ApplicationId");
- opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId");
- opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress");
- opts.addOption(APP_OWNER_OPTION, true, "AppOwner");
+ opts.addOption(APPLICATION_ID_OPTION, true, "ApplicationId (required)");
+ opts.addOption(CONTAINER_ID_OPTION, true,
+ "ContainerId (must be specified if node address is specified)");
+ opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format "
+ + "nodename:port (must be specified if container id is specified)");
+ opts.addOption(APP_OWNER_OPTION, true,
+ "AppOwner (assumed to be current user if not specified)");
if (args.length < 1) {
HelpFormatter formatter = new HelpFormatter();
@@ -99,14 +103,12 @@ public class LogDumper extends Configure
ApplicationId appId =
ConverterUtils.toApplicationId(recordFactory, appIdStr);
- DataOutputStream out = new DataOutputStream(System.out);
-
if (appOwner == null || appOwner.isEmpty()) {
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
}
int resultCode = 0;
if (containerIdStr == null && nodeAddress == null) {
- resultCode = dumpAllContainersLogs(appId, appOwner, out);
+ resultCode = dumpAllContainersLogs(appId, appOwner, System.out);
} else if ((containerIdStr == null && nodeAddress != null)
|| (containerIdStr != null && nodeAddress == null)) {
System.out.println("ContainerId or NodeAddress cannot be null!");
@@ -125,7 +127,7 @@ public class LogDumper extends Configure
appOwner,
ConverterUtils.toNodeId(nodeAddress),
LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf())));
- resultCode = dumpAContainerLogs(containerIdStr, reader, out);
+ resultCode = dumpAContainerLogs(containerIdStr, reader, System.out);
}
return resultCode;
@@ -149,12 +151,11 @@ public class LogDumper extends Configure
"Log aggregation has not completed or is not enabled.");
return -1;
}
- DataOutputStream out = new DataOutputStream(System.out);
- return dumpAContainerLogs(containerId, reader, out);
+ return dumpAContainerLogs(containerId, reader, System.out);
}
private int dumpAContainerLogs(String containerIdStr,
- AggregatedLogFormat.LogReader reader, DataOutputStream out)
+ AggregatedLogFormat.LogReader reader, PrintStream out)
throws IOException {
DataInputStream valueStream;
LogKey key = new LogKey();
@@ -183,7 +184,7 @@ public class LogDumper extends Configure
}
private int dumpAllContainersLogs(ApplicationId appId, String appOwner,
- DataOutputStream out) throws IOException {
+ PrintStream out) throws IOException {
Path remoteRootLogDir =
new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
@@ -216,6 +217,9 @@ public class LogDumper extends Configure
valueStream = reader.next(key);
while (valueStream != null) {
+ String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
try {
LogReader.readAContainerLogsForALogType(valueStream, out);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Sun Mar 24 15:56:18 2013
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -404,4 +405,21 @@ public class BuilderUtils {
allocateRequest.addAllReleases(containersToBeReleased);
return allocateRequest;
}
+
+ public static AllocateResponse newAllocateResponse(int responseId,
+ List<ContainerStatus> completedContainers,
+ List<Container> allocatedContainers, List<NodeReport> updatedNodes,
+ Resource availResources, boolean reboot, int numClusterNodes) {
+ AllocateResponse response = recordFactory
+ .newRecordInstance(AllocateResponse.class);
+ response.setNumClusterNodes(numClusterNodes);
+ response.setResponseId(responseId);
+ response.setCompletedContainersStatuses(completedContainers);
+ response.setAllocatedContainers(allocatedContainers);
+ response.setUpdatedNodes(updatedNodes);
+ response.setAvailableResources(availResources);
+ response.setReboot(reboot);
+
+ return response;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sun Mar 24 15:56:18 2013
@@ -448,6 +448,20 @@
</property>
<property>
+ <description>Whether physical memory limits will be enforced for
+ containers.</description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>Whether virtual memory limits will be enforced for
+ containers.</description>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>Ratio between virtual memory to physical memory when
setting memory limits for containers. Container allocations are
expressed in terms of physical memory, and virtual memory usage
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java Sun Mar 24 15:56:18 2013
@@ -23,9 +23,9 @@ import junit.framework.Assert;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
-import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.impl.pb.AMResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl;
import org.junit.Test;
public class TestRecordFactory {
@@ -35,15 +35,17 @@ public class TestRecordFactory {
RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
try {
- AMResponse response = pbRecordFactory.newRecordInstance(AMResponse.class);
- Assert.assertEquals(AMResponsePBImpl.class, response.getClass());
+ AllocateResponse response =
+ pbRecordFactory.newRecordInstance(AllocateResponse.class);
+ Assert.assertEquals(AllocateResponsePBImpl.class, response.getClass());
} catch (YarnException e) {
e.printStackTrace();
Assert.fail("Failed to crete record");
}
try {
- AllocateRequest response = pbRecordFactory.newRecordInstance(AllocateRequest.class);
+ AllocateRequest response =
+ pbRecordFactory.newRecordInstance(AllocateRequest.class);
Assert.assertEquals(AllocateRequestPBImpl.class, response.getClass());
} catch (YarnException e) {
e.printStackTrace();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Sun Mar 24 15:56:18 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.util;
+import static org.junit.Assert.fail;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@@ -188,11 +190,20 @@ public class TestProcfsBasedProcessTree
// destroy the process and all its subprocesses
destroyProcessTree(pid);
- if (isSetsidAvailable()) { // whole processtree should be gone
- Assert.assertFalse("Proceesses in process group live",
- isAnyProcessInTreeAlive(p));
- } else {// process should be gone
- Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
+ boolean isAlive = true;
+ for (int tries = 100; tries > 0; tries--) {
+ if (isSetsidAvailable()) {// whole processtree
+ isAlive = isAnyProcessInTreeAlive(p);
+ } else {// process
+ isAlive = isAlive(pid);
+ }
+ if (!isAlive) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ if (isAlive) {
+ fail("ProcessTree shouldn't be alive");
}
LOG.info("Process-tree dump follows: \n" + processTreeDump);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java Sun Mar 24 15:56:18 2013
@@ -63,6 +63,10 @@ public class TestRackResolver {
return returnList;
}
+ @Override
+ public void reloadCachedMappings() {
+ // nothing to do here, since RawScriptBasedMapping has no cache.
+ }
}
@Test
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java Sun Mar 24 15:56:18 2013
@@ -22,5 +22,9 @@ public interface ResourceView {
long getVmemAllocatedForContainers();
+ boolean isVmemCheckEnabled();
+
long getPmemAllocatedForContainers();
+
+ boolean isPmemCheckEnabled();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Sun Mar 24 15:56:18 2013
@@ -570,8 +570,10 @@ public class ContainerLaunch implements
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
String inputClassPath = environment.get(Environment.CLASSPATH.name());
- environment.put(Environment.CLASSPATH.name(),
- FileUtil.createJarWithClassPath(inputClassPath, pwd));
+ if (inputClassPath != null && !inputClassPath.isEmpty()) {
+ environment.put(Environment.CLASSPATH.name(),
+ FileUtil.createJarWithClassPath(inputClassPath, pwd));
+ }
}
/**
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Sun Mar 24 15:56:18 2013
@@ -63,14 +63,13 @@ public class ContainersMonitorImpl exten
private Configuration conf;
private Class<? extends ResourceCalculatorProcessTree> processTreeClass;
- private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
- private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
+ private long maxVmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT;
+ private long maxPmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT;
- /**
- * A value which if set for memory related configuration options, indicates
- * that the options are turned off.
- */
- public static final long DISABLED_MEMORY_LIMIT = -1L;
+ private boolean pmemCheckEnabled;
+ private boolean vmemCheckEnabled;
+
+ private static final long UNKNOWN_MEMORY_LIMIT = -1L;
public ContainersMonitorImpl(ContainerExecutor exec,
AsyncDispatcher dispatcher, Context context) {
@@ -104,63 +103,55 @@ public class ContainersMonitorImpl exten
LOG.info(" Using ResourceCalculatorProcessTree : "
+ this.processTreeClass);
- long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
- if (this.resourceCalculatorPlugin != null) {
- totalPhysicalMemoryOnNM =
- this.resourceCalculatorPlugin.getPhysicalMemorySize();
- if (totalPhysicalMemoryOnNM <= 0) {
- LOG.warn("NodeManager's totalPmem could not be calculated. "
- + "Setting it to " + DISABLED_MEMORY_LIMIT);
- totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
- }
- }
+ long configuredPMemForContainers = conf.getLong(
+ YarnConfiguration.NM_PMEM_MB,
+ YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
+ // Setting these irrespective of whether checks are enabled. Required in
+ // the UI.
// ///////// Physical memory configuration //////
- this.maxPmemAllottedForContainers =
- conf.getLong(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
- this.maxPmemAllottedForContainers =
- this.maxPmemAllottedForContainers * 1024 * 1024L; //Normalize to bytes
-
- if (totalPhysicalMemoryOnNM != DISABLED_MEMORY_LIMIT &&
- this.maxPmemAllottedForContainers >
- totalPhysicalMemoryOnNM * 0.80f) {
- LOG.warn("NodeManager configured with " +
- TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers, "", 1) +
- " physical memory allocated to containers, which is more than " +
- "80% of the total physical memory available (" +
- TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "", 1) +
- "). Thrashing might happen.");
- }
+ this.maxPmemAllottedForContainers = configuredPMemForContainers;
// ///////// Virtual memory configuration //////
- float vmemRatio = conf.getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
Preconditions.checkArgument(vmemRatio > 0.99f,
- YarnConfiguration.NM_VMEM_PMEM_RATIO +
- " should be at least 1.0");
+ YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0");
this.maxVmemAllottedForContainers =
- (long)(vmemRatio * maxPmemAllottedForContainers);
+ (long) (vmemRatio * configuredPMemForContainers);
- super.init(conf);
- }
-
- /**
- * Is the total physical memory check enabled?
- *
- * @return true if total physical memory check is enabled.
- */
- boolean isPhysicalMemoryCheckEnabled() {
- return !(this.maxPmemAllottedForContainers == DISABLED_MEMORY_LIMIT);
- }
+ pmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_PMEM_CHECK_ENABLED);
+ vmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED);
+ LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
+ LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
+
+ if (pmemCheckEnabled) {
+ // Logging if actual pmem cannot be determined.
+ long totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
+ if (this.resourceCalculatorPlugin != null) {
+ totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin
+ .getPhysicalMemorySize();
+ if (totalPhysicalMemoryOnNM <= 0) {
+ LOG.warn("NodeManager's totalPmem could not be calculated. "
+ + "Setting it to " + UNKNOWN_MEMORY_LIMIT);
+ totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT;
+ }
+ }
- /**
- * Is the total virtual memory check enabled?
- *
- * @return true if total virtual memory check is enabled.
- */
- boolean isVirtualMemoryCheckEnabled() {
- return !(this.maxVmemAllottedForContainers == DISABLED_MEMORY_LIMIT);
+ if (totalPhysicalMemoryOnNM != UNKNOWN_MEMORY_LIMIT &&
+ this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) {
+ LOG.warn("NodeManager configured with "
+ + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers,
+ "", 1)
+ + " physical memory allocated to containers, which is more than "
+ + "80% of the total physical memory available ("
+ + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "",
+ 1) + "). Thrashing might happen.");
+ }
+ }
+ super.init(conf);
}
private boolean isEnabled() {
@@ -174,7 +165,7 @@ public class ContainersMonitorImpl exten
+ this.getClass().getName() + " is disabled.");
return false;
}
- if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) {
+ if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
"needed. Not running the monitor-thread");
return false;
@@ -412,7 +403,7 @@ public class ContainersMonitorImpl exten
boolean isMemoryOverLimit = false;
String msg = "";
- if (isVirtualMemoryCheckEnabled()
+ if (isVmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
// Container (the root process) is still alive and overflowing
@@ -423,7 +414,7 @@ public class ContainersMonitorImpl exten
currentPmemUsage, pmemLimit,
pId, containerId, pTree);
isMemoryOverLimit = true;
- } else if (isPhysicalMemoryCheckEnabled()
+ } else if (isPmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentPmemUsage, curRssMemUsageOfAgedProcesses,
pmemLimit)) {
@@ -507,11 +498,31 @@ public class ContainersMonitorImpl exten
return this.maxVmemAllottedForContainers;
}
+ /**
+ * Is the total physical memory check enabled?
+ *
+ * @return true if total physical memory check is enabled.
+ */
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return this.pmemCheckEnabled;
+ }
+
@Override
public long getPmemAllocatedForContainers() {
return this.maxPmemAllottedForContainers;
}
+ /**
+ * Is the total virtual memory check enabled?
+ *
+ * @return true if total virtual memory check is enabled.
+ */
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return this.vmemCheckEnabled;
+ }
+
@Override
public void handle(ContainersMonitorEvent monitoringEvent) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Sun Mar 24 15:56:18 2013
@@ -67,8 +67,12 @@ public class NodePage extends NMView {
info("NodeManager information")
._("Total Vmem allocated for Containers",
StringUtils.byteDesc(info.getTotalVmemAllocated() * BYTES_IN_MB))
+ ._("Vmem enforcement enabled",
+ info.isVmemCheckEnabled())
._("Total Pmem allocated for Container",
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
+ ._("Pmem enforcement enabled",
+ info.isVmemCheckEnabled())
._("NodeHealthyStatus",
info.getHealthStatus())
._("LastNodeHealthTime", new Date(
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java Sun Mar 24 15:56:18 2013
@@ -36,6 +36,8 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
+ protected boolean vmemCheckEnabled;
+ protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
protected boolean nodeHealthy;
protected String nodeManagerVersion;
@@ -56,8 +58,10 @@ public class NodeInfo {
this.nodeHostName = context.getNodeId().getHost();
this.totalVmemAllocatedContainersMB = resourceView
.getVmemAllocatedForContainers() / BYTES_IN_MB;
+ this.vmemCheckEnabled = resourceView.isVmemCheckEnabled();
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
+ this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@@ -120,8 +124,16 @@ public class NodeInfo {
return this.totalVmemAllocatedContainersMB;
}
+ public boolean isVmemCheckEnabled() {
+ return this.vmemCheckEnabled;
+ }
+
public long getTotalPmemAllocated() {
return this.totalPmemAllocatedContainersMB;
}
+ public boolean isPmemCheckEnabled() {
+ return this.pmemCheckEnabled;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Sun Mar 24 15:56:18 2013
@@ -22,11 +22,13 @@ import static org.mockito.Mockito.*;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.reflect.Method;
@@ -40,6 +42,7 @@ import java.util.Set;
import junit.framework.Assert;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -531,24 +534,26 @@ public class TestLogAggregationService e
while (true) {
try {
- DataOutputBuffer dob = new DataOutputBuffer();
- LogReader.readAContainerLogsForALogType(valueStream, dob);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+ LogReader.readAContainerLogsForALogType(valueStream, ps);
- DataInputBuffer dib = new DataInputBuffer();
- dib.reset(dob.getData(), dob.getLength());
+ String writtenLines[] = baos.toString().split(
+ System.getProperty("line.separator"));
- Assert.assertEquals("\nLogType:", dib.readUTF());
- String fileType = dib.readUTF();
+ Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
+ String fileType = writtenLines[0].substring(9);
- Assert.assertEquals("\nLogLength:", dib.readUTF());
- String fileLengthStr = dib.readUTF();
+ Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
+ String fileLengthStr = writtenLines[1].substring(11);
long fileLength = Long.parseLong(fileLengthStr);
- Assert.assertEquals("\nLog Contents:\n", dib.readUTF());
- byte[] buf = new byte[(int) fileLength]; // cast is okay in this
- // test.
- dib.read(buf, 0, (int) fileLength);
- perContainerMap.put(fileType, new String(buf));
+ Assert.assertEquals("Log Contents:",
+ writtenLines[2].substring(0, 13));
+
+ String logContents = StringUtils.join(
+ Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
+ perContainerMap.put(fileType, logContents);
LOG.info("LogType:" + fileType);
LOG.info("LogType:" + fileLength);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Sun Mar 24 15:56:18 2013
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
import java.io.BufferedReader;
import java.io.File;
@@ -52,8 +54,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -281,4 +286,54 @@ public class TestContainersMonitor exten
exec.signalContainer(user,
pid, Signal.NULL));
}
+
+ @Test(timeout = 20000)
+ public void testContainerMonitorMemFlags() {
+ ContainersMonitor cm = null;
+
+ long expPmem = 8192 * 1024 * 1024l;
+ long expVmem = (long) (expPmem * 2.1f);
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(false, false, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(false, cm.isPmemCheckEnabled());
+ assertEquals(false, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(true, false, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(true, cm.isPmemCheckEnabled());
+ assertEquals(false, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(true, true, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(true, cm.isPmemCheckEnabled());
+ assertEquals(true, cm.isVmemCheckEnabled());
+
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(false, true, 8192, 2.1f));
+ assertEquals(expPmem, cm.getPmemAllocatedForContainers());
+ assertEquals(expVmem, cm.getVmemAllocatedForContainers());
+ assertEquals(false, cm.isPmemCheckEnabled());
+ assertEquals(true, cm.isVmemCheckEnabled());
+ }
+
+ private YarnConfiguration getConfForCM(boolean pMemEnabled,
+ boolean vMemEnabled, int nmPmem, float vMemToPMemRatio) {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, nmPmem);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, pMemEnabled);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, vMemEnabled);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, vMemToPMemRatio);
+ return conf;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java Sun Mar 24 15:56:18 2013
@@ -41,11 +41,11 @@ import org.junit.Test;
public class TestContainerLogsPage {
- @Test
+ @Test(timeout=30000)
public void testContainerLogDirs() throws IOException {
- String absLogDir = new File("target",
- TestNMWebServer.class.getSimpleName() + "LogDir").getAbsolutePath();
- String logdirwithFile = "file://" + absLogDir;
+ File absLogDir = new File("target",
+ TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
+ String logdirwithFile = absLogDir.toURI().toString();
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Sun Mar 24 15:56:18 2013
@@ -86,6 +86,14 @@ public class TestNMWebServer {
public long getPmemAllocatedForContainers() {
return 0;
}
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return true;
+ }
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return true;
+ }
};
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
@@ -126,6 +134,14 @@ public class TestNMWebServer {
public long getPmemAllocatedForContainers() {
return 0;
}
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return true;
+ }
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return true;
+ }
};
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Sun Mar 24 15:56:18 2013
@@ -100,6 +100,14 @@ public class TestNMWebServices extends J
// 16G in bytes
return new Long("17179869184");
}
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return true;
+ }
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return true;
+ }
};
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
@@ -294,6 +302,8 @@ public class TestNMWebServices extends J
"totalVmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalPmemAllocatedContainersMB"),
+ WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
+ WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
WebServicesTestUtils.getXmlBoolean(element, "nodeHealthy"),
WebServicesTestUtils.getXmlString(element, "nodeHostName"),
@@ -310,10 +320,12 @@ public class TestNMWebServices extends J
public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("nodeInfo");
- assertEquals("incorrect number of elements", 13, info.length());
+ assertEquals("incorrect number of elements", 15, info.length());
verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
info.getLong("totalVmemAllocatedContainersMB"),
info.getLong("totalPmemAllocatedContainersMB"),
+ info.getBoolean("vmemCheckEnabled"),
+ info.getBoolean("pmemCheckEnabled"),
info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
info.getString("nodeHostName"), info.getString("hadoopVersionBuiltOn"),
info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"),
@@ -325,6 +337,7 @@ public class TestNMWebServices extends J
public void verifyNodeInfoGeneric(String id, String healthReport,
long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
+ boolean vmemCheckEnabled, boolean pmemCheckEnabled,
long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
String hadoopVersionBuiltOn, String hadoopBuildVersion,
String hadoopVersion, String resourceManagerVersionBuiltOn,
@@ -337,6 +350,8 @@ public class TestNMWebServices extends J
totalVmemAllocatedContainersMB);
assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
totalPmemAllocatedContainersMB);
+ assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
+ assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext
.getNodeHealthStatus().getLastHealthReportTime());
assertTrue("nodeHealthy isn't true", nodeHealthy);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Sun Mar 24 15:56:18 2013
@@ -106,6 +106,16 @@ public class TestNMWebServicesApps exten
// 16G in bytes
return new Long("17179869184");
}
+
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return true;
+ }
+
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return true;
+ }
};
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Sun Mar 24 15:56:18 2013
@@ -106,6 +106,16 @@ public class TestNMWebServicesContainers
// 16G in bytes
return new Long("17179869184");
}
+
+ @Override
+ public boolean isVmemCheckEnabled() {
+ return true;
+ }
+
+ @Override
+ public boolean isPmemCheckEnabled() {
+ return true;
+ }
};
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sun Mar 24 15:56:18 2013
@@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -78,10 +77,12 @@ public class ApplicationMasterService ex
private YarnScheduler rScheduler;
private InetSocketAddress bindAddress;
private Server server;
- private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
- private final ConcurrentMap<ApplicationAttemptId, AMResponse> responseMap =
- new ConcurrentHashMap<ApplicationAttemptId, AMResponse>();
- private final AMResponse reboot = recordFactory.newRecordInstance(AMResponse.class);
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+ private final ConcurrentMap<ApplicationAttemptId, AllocateResponse> responseMap =
+ new ConcurrentHashMap<ApplicationAttemptId, AllocateResponse>();
+ private final AllocateResponse reboot =
+ recordFactory.newRecordInstance(AllocateResponse.class);
private final RMContext rmContext;
public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) {
@@ -166,7 +167,7 @@ public class ApplicationMasterService ex
authorizeRequest(applicationAttemptId);
ApplicationId appID = applicationAttemptId.getApplicationId();
- AMResponse lastResponse = responseMap.get(applicationAttemptId);
+ AllocateResponse lastResponse = responseMap.get(applicationAttemptId);
if (lastResponse == null) {
String message = "Application doesn't exist in cache "
+ applicationAttemptId;
@@ -214,7 +215,7 @@ public class ApplicationMasterService ex
.getApplicationAttemptId();
authorizeRequest(applicationAttemptId);
- AMResponse lastResponse = responseMap.get(applicationAttemptId);
+ AllocateResponse lastResponse = responseMap.get(applicationAttemptId);
if (lastResponse == null) {
String message = "Application doesn't exist in cache "
+ applicationAttemptId;
@@ -248,25 +249,20 @@ public class ApplicationMasterService ex
this.amLivelinessMonitor.receivedPing(appAttemptId);
/* check if its in cache */
- AllocateResponse allocateResponse = recordFactory
- .newRecordInstance(AllocateResponse.class);
- AMResponse lastResponse = responseMap.get(appAttemptId);
+ AllocateResponse lastResponse = responseMap.get(appAttemptId);
if (lastResponse == null) {
LOG.error("AppAttemptId doesnt exist in cache " + appAttemptId);
- allocateResponse.setAMResponse(reboot);
- return allocateResponse;
+ return reboot;
}
if ((request.getResponseId() + 1) == lastResponse.getResponseId()) {
/* old heartbeat */
- allocateResponse.setAMResponse(lastResponse);
- return allocateResponse;
+ return lastResponse;
} else if (request.getResponseId() + 1 < lastResponse.getResponseId()) {
LOG.error("Invalid responseid from appAttemptId " + appAttemptId);
// Oh damn! Sending reboot isn't enough. RM state is corrupted. TODO:
// Reboot is not useful since after AM reboots, it will send register and
// get an exception. Might as well throw an exception here.
- allocateResponse.setAMResponse(reboot);
- return allocateResponse;
+ return reboot;
}
// Allow only one thread in AM to do heartbeat at a time.
@@ -288,7 +284,8 @@ public class ApplicationMasterService ex
appAttemptId.getApplicationId());
RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
- AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
+ AllocateResponse allocateResponse =
+ recordFactory.newRecordInstance(AllocateResponse.class);
// update the response with the deltas of node status changes
List<RMNode> updatedNodes = new ArrayList<RMNode>();
@@ -311,34 +308,34 @@ public class ApplicationMasterService ex
updatedNodeReports.add(report);
}
- response.setUpdatedNodes(updatedNodeReports);
+ allocateResponse.setUpdatedNodes(updatedNodeReports);
}
- response.setAllocatedContainers(allocation.getContainers());
- response.setCompletedContainersStatuses(appAttempt
+ allocateResponse.setAllocatedContainers(allocation.getContainers());
+ allocateResponse.setCompletedContainersStatuses(appAttempt
.pullJustFinishedContainers());
- response.setResponseId(lastResponse.getResponseId() + 1);
- response.setAvailableResources(allocation.getResourceLimit());
+ allocateResponse.setResponseId(lastResponse.getResponseId() + 1);
+ allocateResponse.setAvailableResources(allocation.getResourceLimit());
- AMResponse oldResponse = responseMap.put(appAttemptId, response);
+ AllocateResponse oldResponse =
+ responseMap.put(appAttemptId, allocateResponse);
if (oldResponse == null) {
// appAttempt got unregistered, remove it back out
responseMap.remove(appAttemptId);
String message = "App Attempt removed from the cache during allocate"
+ appAttemptId;
LOG.error(message);
- allocateResponse.setAMResponse(reboot);
- return allocateResponse;
+ return reboot;
}
- allocateResponse.setAMResponse(response);
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
return allocateResponse;
}
}
public void registerAppAttempt(ApplicationAttemptId attemptId) {
- AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
+ AllocateResponse response =
+ recordFactory.newRecordInstance(AllocateResponse.class);
response.setResponseId(0);
LOG.info("Registering " + attemptId);
responseMap.put(attemptId, response);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Sun Mar 24 15:56:18 2013
@@ -28,8 +28,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -96,14 +94,14 @@ public class MockAM {
requests.addAll(createReq(hosts, memory, priority, containers));
}
- public AMResponse schedule() throws Exception {
- AMResponse response = allocate(requests, releases);
+ public AllocateResponse schedule() throws Exception {
+ AllocateResponse response = allocate(requests, releases);
requests.clear();
releases.clear();
return response;
}
- public AMResponse allocate(
+ public AllocateResponse allocate(
String host, int memory, int numContainers,
List<ContainerId> releases) throws Exception {
List<ResourceRequest> reqs = createReq(new String[]{host}, memory, 1, numContainers);
@@ -143,13 +141,12 @@ public class MockAM {
return req;
}
- public AMResponse allocate(
+ public AllocateResponse allocate(
List<ResourceRequest> resourceRequest, List<ContainerId> releases)
throws Exception {
AllocateRequest req = BuilderUtils.newAllocateRequest(attemptId,
++responseId, 0F, resourceRequest, releases);
- AllocateResponse resp = amRMProtocol.allocate(req);
- return resp.getAMResponse();
+ return amRMProtocol.allocate(req);
}
public void unregisterAppAttempt() throws Exception {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Sun Mar 24 15:56:18 2013
@@ -26,7 +26,7 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -99,32 +99,32 @@ public class TestFifoScheduler {
// add request for containers
am1.addRequests(new String[] { "h1", "h2" }, GB, 1, 1);
- AMResponse am1Response = am1.schedule(); // send the request
+ AllocateResponse alloc1Response = am1.schedule(); // send the request
// add request for containers
am2.addRequests(new String[] { "h1", "h2" }, 3 * GB, 0, 1);
- AMResponse am2Response = am2.schedule(); // send the request
+ AllocateResponse alloc2Response = am2.schedule(); // send the request
// kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0
nm1.nodeHeartbeat(true);
- while (am1Response.getAllocatedContainers().size() < 1) {
+ while (alloc1Response.getAllocatedContainers().size() < 1) {
LOG.info("Waiting for containers to be created for app 1...");
Thread.sleep(1000);
- am1Response = am1.schedule();
+ alloc1Response = am1.schedule();
}
- while (am2Response.getAllocatedContainers().size() < 1) {
+ while (alloc2Response.getAllocatedContainers().size() < 1) {
LOG.info("Waiting for containers to be created for app 2...");
Thread.sleep(1000);
- am2Response = am2.schedule();
+ alloc2Response = am2.schedule();
}
// kick the scheduler, nothing given remaining 2 GB.
nm2.nodeHeartbeat(true);
- List<Container> allocated1 = am1Response.getAllocatedContainers();
+ List<Container> allocated1 = alloc1Response.getAllocatedContainers();
Assert.assertEquals(1, allocated1.size());
Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory());
Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());
- List<Container> allocated2 = am2Response.getAllocatedContainers();
+ List<Container> allocated2 = alloc2Response.getAllocatedContainers();
Assert.assertEquals(1, allocated2.size());
Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory());
Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Sun Mar 24 15:56:18 2013
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -213,9 +213,10 @@ public class TestRMRestart {
// verify old AM is not accepted
// change running AM to talk to new RM
am1.setAMRMProtocol(rm2.getApplicationMasterService());
- AMResponse amResponse = am1.allocate(new ArrayList<ResourceRequest>(),
+ AllocateResponse allocResponse = am1.allocate(
+ new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>());
- Assert.assertTrue(amResponse.getReboot());
+ Assert.assertTrue(allocResponse.getReboot());
// NM should be rebooted on heartbeat, even first heartbeat for nm2
HeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java Sun Mar 24 15:56:18 2013
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.re
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -109,7 +109,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns no updated node
AllocateRequest allocateRequest1 = BuilderUtils.newAllocateRequest(attempt1
.getAppAttemptId(), 0, 0F, null, null);
- AMResponse response1 = amService.allocate(allocateRequest1).getAMResponse();
+ AllocateResponse response1 = amService.allocate(allocateRequest1);
List<NodeReport> updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
@@ -118,7 +118,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns updated node
allocateRequest1 = BuilderUtils.newAllocateRequest(attempt1
.getAppAttemptId(), response1.getResponseId(), 0F, null, null);
- response1 = amService.allocate(allocateRequest1).getAMResponse();
+ response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
NodeReport nr = updatedNodes.iterator().next();
@@ -126,7 +126,7 @@ public class TestAMRMRPCNodeUpdates {
Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
// resending the allocate request returns the same result
- response1 = amService.allocate(allocateRequest1).getAMResponse();
+ response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -138,7 +138,7 @@ public class TestAMRMRPCNodeUpdates {
// subsequent allocate request returns delta
allocateRequest1 = BuilderUtils.newAllocateRequest(attempt1
.getAppAttemptId(), response1.getResponseId(), 0F, null, null);
- response1 = amService.allocate(allocateRequest1).getAMResponse();
+ response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -158,7 +158,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns no updated node
AllocateRequest allocateRequest2 = BuilderUtils.newAllocateRequest(attempt2
.getAppAttemptId(), 0, 0F, null, null);
- AMResponse response2 = amService.allocate(allocateRequest2).getAMResponse();
+ AllocateResponse response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
@@ -167,7 +167,7 @@ public class TestAMRMRPCNodeUpdates {
// both AM's should get delta updated nodes
allocateRequest1 = BuilderUtils.newAllocateRequest(attempt1
.getAppAttemptId(), response1.getResponseId(), 0F, null, null);
- response1 = amService.allocate(allocateRequest1).getAMResponse();
+ response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -176,7 +176,7 @@ public class TestAMRMRPCNodeUpdates {
allocateRequest2 = BuilderUtils.newAllocateRequest(attempt2
.getAppAttemptId(), response2.getResponseId(), 0F, null, null);
- response2 = amService.allocate(allocateRequest2).getAMResponse();
+ response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -186,7 +186,7 @@ public class TestAMRMRPCNodeUpdates {
// subsequent allocate calls should return no updated nodes
allocateRequest2 = BuilderUtils.newAllocateRequest(attempt2
.getAppAttemptId(), response2.getResponseId(), 0F, null, null);
- response2 = amService.allocate(allocateRequest2).getAMResponse();
+ response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Sun Mar 24 15:56:18 2013
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.re
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
@@ -81,22 +81,22 @@ public class TestAMRMRPCResponseId {
AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest(attempt
.getAppAttemptId(), 0, 0F, null, null);
- AMResponse response = amService.allocate(allocateRequest).getAMResponse();
+ AllocateResponse response = amService.allocate(allocateRequest);
Assert.assertEquals(1, response.getResponseId());
Assert.assertFalse(response.getReboot());
allocateRequest = BuilderUtils.newAllocateRequest(attempt
.getAppAttemptId(), response.getResponseId(), 0F, null, null);
- response = amService.allocate(allocateRequest).getAMResponse();
+ response = amService.allocate(allocateRequest);
Assert.assertEquals(2, response.getResponseId());
/* try resending */
- response = amService.allocate(allocateRequest).getAMResponse();
+ response = amService.allocate(allocateRequest);
Assert.assertEquals(2, response.getResponseId());
/** try sending old request again **/
allocateRequest = BuilderUtils.newAllocateRequest(attempt
.getAppAttemptId(), 0, 0F, null, null);
- response = amService.allocate(allocateRequest).getAMResponse();
+ response = amService.allocate(allocateRequest);
Assert.assertTrue(response.getReboot());
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java Sun Mar 24 15:56:18 2013
@@ -201,8 +201,7 @@ public class TestApplicationTokens {
AllocateRequest allocateRequest =
Records.newRecord(AllocateRequest.class);
allocateRequest.setApplicationAttemptId(applicationAttemptId);
- Assert.assertFalse(rmClient.allocate(allocateRequest).getAMResponse()
- .getReboot());
+ Assert.assertFalse(rmClient.allocate(allocateRequest).getReboot());
// Simulate a master-key-roll-over
ApplicationTokenSecretManager appTokenSecretManager =
@@ -218,8 +217,7 @@ public class TestApplicationTokens {
rmClient = createRMClient(rm, conf, rpc, currentUser);
allocateRequest = Records.newRecord(AllocateRequest.class);
allocateRequest.setApplicationAttemptId(applicationAttemptId);
- Assert.assertFalse(rmClient.allocate(allocateRequest).getAMResponse()
- .getReboot());
+ Assert.assertFalse(rmClient.allocate(allocateRequest).getReboot());
} finally {
rm.stop();
if (rmClient != null) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Sun Mar 24 15:56:18 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.security.Access
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -401,10 +402,15 @@ public class TestContainerManagerSecurit
UnsupportedFileSystemException, YarnRemoteException,
InterruptedException {
+ // Use ping to simulate sleep on Windows.
+ List<String> cmd = Shell.WINDOWS ?
+ Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") :
+ Arrays.asList("sleep", "100");
+
ContainerLaunchContext amContainer = BuilderUtils
.newContainerLaunchContext(null, "testUser", BuilderUtils
.newResource(1024, 1), Collections.<String, LocalResource>emptyMap(),
- new HashMap<String, String>(), Arrays.asList("sleep", "100"),
+ new HashMap<String, String>(), cmd,
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
@@ -487,7 +493,7 @@ public class TestContainerManagerSecurit
BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask,
new ArrayList<ContainerId>());
List<Container> allocatedContainers = scheduler.allocate(allocateRequest)
- .getAMResponse().getAllocatedContainers();
+ .getAllocatedContainers();
// Modify ask to request no more.
allocateRequest.clearAsks();
@@ -499,7 +505,7 @@ public class TestContainerManagerSecurit
Thread.sleep(1000);
allocateRequest.setResponseId(allocateRequest.getResponseId() + 1);
allocatedContainers = scheduler.allocate(allocateRequest)
- .getAMResponse().getAllocatedContainers();
+ .getAllocatedContainers();
}
Assert.assertNotNull("Container is not allocted!", allocatedContainers);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1460410&r1=1460409&r2=1460410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm Sun Mar 24 15:56:18 2013
@@ -493,7 +493,7 @@ Hadoop MapReduce Next Generation - Writi
+---+
* The AllocateResponse sent back from the ResourceManager provides the
- following information via the AMResponse object:
+ following information:
* Reboot flag: For scenarios when the ApplicationMaster may get out of sync
with the ResourceManager.
@@ -511,7 +511,9 @@ Hadoop MapReduce Next Generation - Writi
allocated container, it will receive an update from the ResourceManager
when the container completes. The ApplicationMaster can look into the
status of the completed container and take appropriate actions such as
- re-trying a particular sub-task in case of a failure.
+ re-trying a particular sub-task in case of a failure.
+
+ * Number of cluster nodes: The number of hosts available on the cluster.
[]
@@ -525,13 +527,11 @@ Hadoop MapReduce Next Generation - Writi
containers.
+---+
- // Get AMResponse from AllocateResponse
- AMResponse amResp = allocateResponse.getAMResponse();
// Retrieve list of allocated containers from the response
// and on each allocated container, lets assume we are launching
// the same job.
- List<Container> allocatedContainers = amResp.getAllocatedContainers();
+ List<Container> allocatedContainers = allocateResponse.getAllocatedContainers();
for (Container allocatedContainer : allocatedContainers) {
LOG.info("Launching shell command on a new container."
+ ", containerId=" + allocatedContainer.getId()
@@ -553,7 +553,7 @@ Hadoop MapReduce Next Generation - Writi
}
// Check what the current available resources in the cluster are
- Resource availableResources = amResp.getAvailableResources();
+ Resource availableResources = allocateResponse.getAvailableResources();
// Based on this information, an ApplicationMaster can make appropriate
// decisions
@@ -561,7 +561,7 @@ Hadoop MapReduce Next Generation - Writi
// Let's assume we are keeping a count of total completed containers,
// containers that failed and ones that completed successfully.
List<ContainerStatus> completedContainers =
- amResp.getCompletedContainersStatuses();
+ allocateResponse.getCompletedContainersStatuses();
for (ContainerStatus containerStatus : completedContainers) {
LOG.info("Got container status for containerID= "
+ containerStatus.getContainerId()
@@ -611,7 +611,7 @@ Hadoop MapReduce Next Generation - Writi
+---+
- //Assuming an allocated Container obtained from AMResponse
+ //Assuming an allocated Container obtained from AllocateResponse
Container container;
// Connect to ContainerManager on the allocated container
String cmIpPortStr = container.getNodeId().getHost() + ":"