You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2014/08/20 03:34:41 UTC
svn commit: r1619019 - in /hadoop/common/branches/YARN-1051: ./
hadoop-assemblies/src/main/resources/assemblies/ hadoop-project-dist/
hadoop-project/ hadoop-project/src/site/
hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/
hadoop-to...
Author: curino
Date: Wed Aug 20 01:34:29 2014
New Revision: 1619019
URL: http://svn.apache.org/r1619019
Log:
Merge trunk
Added:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/
- copied from r1619017, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
- copied unchanged from r1619017, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/
- copied from r1619017, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
- copied unchanged from r1619017, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
Modified:
hadoop/common/branches/YARN-1051/ (props changed)
hadoop/common/branches/YARN-1051/BUILDING.txt
hadoop/common/branches/YARN-1051/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
hadoop/common/branches/YARN-1051/hadoop-project-dist/pom.xml
hadoop/common/branches/YARN-1051/hadoop-project/pom.xml
hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-openstack/pom.xml
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
Propchange: hadoop/common/branches/YARN-1051/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2:r1616481
Merged /hadoop/common/trunk:r1613508-1619017
Modified: hadoop/common/branches/YARN-1051/BUILDING.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/BUILDING.txt?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/BUILDING.txt (original)
+++ hadoop/common/branches/YARN-1051/BUILDING.txt Wed Aug 20 01:34:29 2014
@@ -189,6 +189,7 @@ Requirements:
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
+* CMake 2.6 or newer
* Windows SDK or Visual Studio 2010 Professional
* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
* zlib headers (if building native code bindings for zlib)
Modified: hadoop/common/branches/YARN-1051/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml Wed Aug 20 01:34:29 2014
@@ -29,6 +29,7 @@
<exclude>*-config.cmd</exclude>
<exclude>start-*.cmd</exclude>
<exclude>stop-*.cmd</exclude>
+ <exclude>hadoop-layout.sh.example</exclude>
</excludes>
<fileMode>0755</fileMode>
</fileSet>
@@ -42,6 +43,8 @@
<includes>
<include>*-config.sh</include>
<include>*-config.cmd</include>
+ <include>*-functions.sh</include>
+ <include>hadoop-layout.sh.example</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
@@ -57,6 +60,10 @@
<exclude>hadoop.cmd</exclude>
<exclude>hdfs.cmd</exclude>
<exclude>hadoop-config.cmd</exclude>
+ <exclude>hadoop-functions.sh</exclude>
+ <exclude>hadoop-layout.sh.example</exclude>
+ <exclude>hdfs-config.cmd</exclude>
+ <exclude>hdfs-config.sh</exclude>
</excludes>
<fileMode>0755</fileMode>
</fileSet>
Modified: hadoop/common/branches/YARN-1051/hadoop-project-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-project-dist/pom.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-project-dist/pom.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-project-dist/pom.xml Wed Aug 20 01:34:29 2014
@@ -100,6 +100,8 @@
<artifactId>findbugs-maven-plugin</artifactId>
<configuration>
<excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ <fork>true</fork>
+ <maxHeap>2048</maxHeap>
</configuration>
</plugin>
<plugin>
Modified: hadoop/common/branches/YARN-1051/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-project/pom.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-project/pom.xml Wed Aug 20 01:34:29 2014
@@ -104,6 +104,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-nfs</artifactId>
<version>${project.version}</version>
</dependency>
@@ -789,7 +795,13 @@
<groupId>com.microsoft.windowsazure.storage</groupId>
<artifactId>microsoft-windowsazure-storage-sdk</artifactId>
<version>0.6.0</version>
- </dependency>
+ </dependency>
+
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.9.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
Modified: hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml Wed Aug 20 01:34:29 2014
@@ -69,6 +69,7 @@
<menu name="HDFS" inherit="top">
<item name="HDFS User Guide" href="hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html"/>
+ <item name="HDFS Commands Reference" href="hadoop-project-dist/hadoop-hdfs/HDFSCommands.html"/>
<item name="High Availability With QJM" href="hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html"/>
<item name="High Availability With NFS" href="hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html"/>
<item name="Federation" href="hadoop-project-dist/hadoop-hdfs/Federation.html"/>
@@ -94,6 +95,7 @@
<menu name="MapReduce" inherit="top">
<item name="MapReduce Tutorial" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html"/>
+ <item name="MapReduce Commands Reference" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html"/>
<item name="Compatibilty between Hadoop 1.x and Hadoop 2.x" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html"/>
<item name="Encrypted Shuffle" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html"/>
<item name="Pluggable Shuffle/Sort" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/PluggableShuffleAndPluggableSort.html"/>
@@ -140,6 +142,7 @@
<item name="Common CHANGES.txt" href="hadoop-project-dist/hadoop-common/CHANGES.txt"/>
<item name="HDFS CHANGES.txt" href="hadoop-project-dist/hadoop-hdfs/CHANGES.txt"/>
<item name="MapReduce CHANGES.txt" href="hadoop-project-dist/hadoop-mapreduce/CHANGES.txt"/>
+ <item name="YARN CHANGES.txt" href="hadoop-project-dist/hadoop-yarn/CHANGES.txt"/>
<item name="Metrics" href="hadoop-project-dist/hadoop-common/Metrics.html"/>
</menu>
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java Wed Aug 20 01:34:29 2014
@@ -338,7 +338,7 @@ public class HadoopArchives implements T
* directories to
* @param paths the source paths provided by the user. They
* are glob free and have full path (not relative paths)
- * @param parentPath the parent path that you wnat the archives
+ * @param parentPath the parent path that you want the archives
* to be relative to. example - /home/user/dir1 can be archived with
* parent as /home or /home/user.
* @throws IOException
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java Wed Aug 20 01:34:29 2014
@@ -140,10 +140,10 @@ public class TestGridmixRecord {
final int chk = WritableComparator.compareBytes(
out1.getData(), 0, out1.getLength(),
out2.getData(), 0, out2.getLength());
- assertEquals(chk, x.compareTo(y));
- assertEquals(chk, cmp.compare(
+ assertEquals(Integer.signum(chk), Integer.signum(x.compareTo(y)));
+ assertEquals(Integer.signum(chk), Integer.signum(cmp.compare(
out1.getData(), 0, out1.getLength(),
- out2.getData(), 0, out2.getLength()));
+ out2.getData(), 0, out2.getLength())));
// write second copy, compare eq
final int s1 = out1.getLength();
x.write(out1);
@@ -153,8 +153,8 @@ public class TestGridmixRecord {
y.write(out2);
assertEquals(0, cmp.compare(out2.getData(), 0, s2,
out2.getData(), s2, out2.getLength() - s2));
- assertEquals(chk, cmp.compare(out1.getData(), 0, s1,
- out2.getData(), s2, out2.getLength() - s2));
+ assertEquals(Integer.signum(chk), Integer.signum(cmp.compare(out1.getData(), 0, s1,
+ out2.getData(), s2, out2.getLength() - s2)));
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-openstack/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-openstack/pom.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-openstack/pom.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-openstack/pom.xml Wed Aug 20 01:34:29 2014
@@ -146,6 +146,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>test</scope>
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java Wed Aug 20 01:34:29 2014
@@ -63,6 +63,8 @@ import org.apache.hadoop.yarn.security.A
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Logger;
@@ -133,8 +135,7 @@ public abstract class AMSimulator extend
* register with RM
*/
@Override
- public void firstStep()
- throws YarnException, IOException, InterruptedException {
+ public void firstStep() throws Exception {
simulateStartTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
@@ -149,8 +150,7 @@ public abstract class AMSimulator extend
}
@Override
- public void middleStep()
- throws InterruptedException, YarnException, IOException {
+ public void middleStep() throws Exception {
// process responses in the queue
processResponseQueue();
@@ -162,7 +162,7 @@ public abstract class AMSimulator extend
}
@Override
- public void lastStep() {
+ public void lastStep() throws Exception {
LOG.info(MessageFormat.format("Application {0} is shutting down.", appId));
// unregister tracking
if (isTracked) {
@@ -173,26 +173,19 @@ public abstract class AMSimulator extend
.newRecordInstance(FinishApplicationMasterRequest.class);
finishAMRequest.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
- try {
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token =
- rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
- ugi.addTokenIdentifier(token.decodeIdentifier());
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- rm.getApplicationMasterService()
- .finishApplicationMaster(finishAMRequest);
- return null;
- }
- });
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appId)
+ .getRMAppAttempt(appAttemptId).getAMRMToken();
+ ugi.addTokenIdentifier(token.decodeIdentifier());
+ ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ rm.getApplicationMasterService()
+ .finishApplicationMaster(finishAMRequest);
+ return null;
+ }
+ });
simulateFinishTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
@@ -230,11 +223,9 @@ public abstract class AMSimulator extend
return createAllocateRequest(ask, new ArrayList<ContainerId>());
}
- protected abstract void processResponseQueue()
- throws InterruptedException, YarnException, IOException;
+ protected abstract void processResponseQueue() throws Exception;
- protected abstract void sendContainerRequest()
- throws YarnException, IOException, InterruptedException;
+ protected abstract void sendContainerRequest() throws Exception;
protected abstract void checkStop();
@@ -280,11 +271,18 @@ public abstract class AMSimulator extend
// waiting until application ACCEPTED
RMApp app = rm.getRMContext().getRMApps().get(appId);
while(app.getState() != RMAppState.ACCEPTED) {
- Thread.sleep(50);
+ Thread.sleep(10);
}
- appAttemptId = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt().getAppAttemptId();
+ // Waiting until application attempt reach LAUNCHED
+ // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
+ this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
+ .getCurrentAppAttempt().getAppAttemptId();
+ RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
+ .getCurrentAppAttempt();
+ while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
+ Thread.sleep(10);
+ }
}
private void registerAM()
@@ -297,10 +295,9 @@ public abstract class AMSimulator extend
amRegisterRequest.setTrackingUrl("localhost:1000");
UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token =
- rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appId)
+ .getRMAppAttempt(appAttemptId).getAMRMToken();
ugi.addTokenIdentifier(token.decodeIdentifier());
ugi.doAs(
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java Wed Aug 20 01:34:29 2014
@@ -145,8 +145,7 @@ public class MRAMSimulator extends AMSim
}
@Override
- public void firstStep()
- throws YarnException, IOException, InterruptedException {
+ public void firstStep() throws Exception {
super.firstStep();
requestAMContainer();
@@ -390,7 +389,7 @@ public class MRAMSimulator extends AMSim
}
@Override
- public void lastStep() {
+ public void lastStep() throws Exception {
super.lastStep();
// clear data structures
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java Wed Aug 20 01:34:29 2014
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -107,12 +108,12 @@ public class NMSimulator extends TaskRun
}
@Override
- public void firstStep() throws YarnException, IOException {
+ public void firstStep() {
// do nothing
}
@Override
- public void middleStep() {
+ public void middleStep() throws Exception {
// we check the lifetime for each running containers
ContainerSimulator cs = null;
synchronized(completedContainerList) {
@@ -136,37 +137,31 @@ public class NMSimulator extends TaskRun
ns.setResponseId(RESPONSE_ID ++);
ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
beatRequest.setNodeStatus(ns);
- try {
- NodeHeartbeatResponse beatResponse =
- rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
- if (! beatResponse.getContainersToCleanup().isEmpty()) {
- // remove from queue
- synchronized(releasedContainerList) {
- for (ContainerId containerId : beatResponse.getContainersToCleanup()){
- if (amContainerList.contains(containerId)) {
- // AM container (not killed?, only release)
- synchronized(amContainerList) {
- amContainerList.remove(containerId);
- }
- LOG.debug(MessageFormat.format("NodeManager {0} releases " +
- "an AM ({1}).", node.getNodeID(), containerId));
- } else {
- cs = runningContainers.remove(containerId);
- containerQueue.remove(cs);
- releasedContainerList.add(containerId);
- LOG.debug(MessageFormat.format("NodeManager {0} releases a " +
- "container ({1}).", node.getNodeID(), containerId));
+ NodeHeartbeatResponse beatResponse =
+ rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
+ if (! beatResponse.getContainersToCleanup().isEmpty()) {
+ // remove from queue
+ synchronized(releasedContainerList) {
+ for (ContainerId containerId : beatResponse.getContainersToCleanup()){
+ if (amContainerList.contains(containerId)) {
+ // AM container (not killed?, only release)
+ synchronized(amContainerList) {
+ amContainerList.remove(containerId);
}
+ LOG.debug(MessageFormat.format("NodeManager {0} releases " +
+ "an AM ({1}).", node.getNodeID(), containerId));
+ } else {
+ cs = runningContainers.remove(containerId);
+ containerQueue.remove(cs);
+ releasedContainerList.add(containerId);
+ LOG.debug(MessageFormat.format("NodeManager {0} releases a " +
+ "container ({1}).", node.getNodeID(), containerId));
}
}
}
- if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
- lastStep();
- }
- } catch (YarnException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
+ }
+ if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
+ lastStep();
}
}
@@ -262,4 +257,19 @@ public class NMSimulator extends TaskRun
completedContainerList.add(containerId);
}
}
+
+ @VisibleForTesting
+ Map<ContainerId, ContainerSimulator> getRunningContainers() {
+ return runningContainers;
+ }
+
+ @VisibleForTesting
+ List<ContainerId> getAMContainers() {
+ return amContainerList;
+ }
+
+ @VisibleForTesting
+ List<ContainerId> getCompletedContainers() {
+ return completedContainerList;
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java Wed Aug 20 01:34:29 2014
@@ -22,10 +22,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
- .AppSchedulable;
+ .FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
- .FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import com.codahale.metrics.Gauge;
import org.apache.hadoop.yarn.sls.SLSRunner;
@@ -66,8 +65,7 @@ public class FairSchedulerMetrics extend
public void trackApp(ApplicationAttemptId appAttemptId, String oldAppId) {
super.trackApp(appAttemptId, oldAppId);
FairScheduler fair = (FairScheduler) scheduler;
- final AppSchedulable app = fair.getSchedulerApp(appAttemptId)
- .getAppSchedulable();
+ final FSAppAttempt app = fair.getSchedulerApp(appAttemptId);
metrics.register("variable.app." + oldAppId + ".demand.memory",
new Gauge<Integer>() {
@Override
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Wed Aug 20 01:34:29 2014
@@ -36,8 +36,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -61,12 +61,14 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
@@ -101,7 +103,6 @@ final public class ResourceSchedulerWrap
private static final String EOL = System.getProperty("line.separator");
private static final int SAMPLING_SIZE = 60;
private ScheduledExecutorService pool;
- private RMContext rmContext;
// counters for scheduler allocate/handle operations
private Counter schedulerAllocateCounter;
private Counter schedulerHandleCounter;
@@ -576,7 +577,7 @@ final public class ResourceSchedulerWrap
new Gauge<Integer>() {
@Override
public Integer getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
+ if (scheduler == null || scheduler.getRootQueueMetrics() == null) {
return 0;
} else {
return scheduler.getRootQueueMetrics().getAppsRunning();
@@ -723,17 +724,18 @@ final public class ResourceSchedulerWrap
public void addAMRuntime(ApplicationId appId,
long traceStartTimeMS, long traceEndTimeMS,
long simulateStartTimeMS, long simulateEndTimeMS) {
-
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
+ if (metricsON) {
+ try {
+ // write job runtime information
+ StringBuilder sb = new StringBuilder();
+ sb.append(appId).append(",").append(traceStartTimeMS).append(",")
+ .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
+ .append(",").append(simulateEndTimeMS);
+ jobRuntimeLogBW.write(sb.toString() + EOL);
+ jobRuntimeLogBW.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
}
@@ -919,4 +921,23 @@ final public class ResourceSchedulerWrap
public Resource getClusterResource() {
return null;
}
+
+ @Override
+ public synchronized List<Container> getTransferredContainers(
+ ApplicationAttemptId currentAttempt) {
+ return new ArrayList<Container>();
+ }
+
+ @Override
+ public Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>>
+ getSchedulerApplications() {
+ return new HashMap<ApplicationId,
+ SchedulerApplication<SchedulerApplicationAttempt>>();
+ }
+
+ @Override
+ protected void completedContainer(RMContainer rmContainer,
+ ContainerStatus containerStatus, RMContainerEventType event) {
+ // do nothing
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java Wed Aug 20 01:34:29 2014
@@ -729,16 +729,18 @@ public class SLSCapacityScheduler extend
long traceStartTimeMS, long traceEndTimeMS,
long simulateStartTimeMS, long simulateEndTimeMS) {
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
+ if (metricsON) {
+ try {
+ // write job runtime information
+ StringBuilder sb = new StringBuilder();
+ sb.append(appId).append(",").append(traceStartTimeMS).append(",")
+ .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
+ .append(",").append(simulateEndTimeMS);
+ jobRuntimeLogBW.write(sb.toString() + EOL);
+ jobRuntimeLogBW.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java Wed Aug 20 01:34:29 2014
@@ -99,12 +99,10 @@ public class TaskRunner {
} else {
lastStep();
}
- } catch (YarnException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
+ } catch (Exception e) {
e.printStackTrace();
+ Thread.getDefaultUncaughtExceptionHandler()
+ .uncaughtException(Thread.currentThread(), e);
}
}
@@ -124,13 +122,11 @@ public class TaskRunner {
}
- public abstract void firstStep()
- throws YarnException, IOException, InterruptedException;
+ public abstract void firstStep() throws Exception;
- public abstract void middleStep()
- throws YarnException, InterruptedException, IOException;
+ public abstract void middleStep() throws Exception;
- public abstract void lastStep() throws YarnException;
+ public abstract void lastStep() throws Exception;
public void setEndTime(long et) {
endTime = et;
Modified: hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java Wed Aug 20 01:34:29 2014
@@ -18,10 +18,13 @@
package org.apache.hadoop.yarn.sls;
-import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
import org.junit.Test;
import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.UUID;
public class TestSLSRunner {
@@ -30,6 +33,15 @@ public class TestSLSRunner {
@SuppressWarnings("all")
public void testSimulatorRunning() throws Exception {
File tempDir = new File("target", UUID.randomUUID().toString());
+ final List<Throwable> exceptionList =
+ Collections.synchronizedList(new ArrayList<Throwable>());
+
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ exceptionList.add(e);
+ }
+ });
// start the simulator
File slsOutputDir = new File(tempDir.getAbsolutePath() + "/slsoutput/");
@@ -38,8 +50,20 @@ public class TestSLSRunner {
"-output", slsOutputDir.getAbsolutePath()};
SLSRunner.main(args);
- // wait for 45 seconds before stop
- Thread.sleep(45 * 1000);
+ // wait for 20 seconds before stop
+ int count = 20;
+ while (count >= 0) {
+ Thread.sleep(1000);
+
+ if (! exceptionList.isEmpty()) {
+ SLSRunner.getRunner().stop();
+ Assert.fail("TestSLSRunner catched exception from child thread " +
+ "(TaskRunner.Task): " + exceptionList.get(0).getMessage());
+ break;
+ }
+ count--;
+ }
+
SLSRunner.getRunner().stop();
}