You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:07 UTC
svn commit: r1619012 [3/3] - in /hadoop/common/branches/HADOOP-10388: ./
hadoop-assemblies/src/main/resources/assemblies/ hadoop-client/
hadoop-dist/ hadoop-maven-plugins/
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/
hadoop...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java Tue Aug 19 23:49:39 2014
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
import org.apache.hadoop.fs.swift.exceptions.SwiftException;
import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
@@ -109,7 +110,7 @@ class SwiftNativeOutputStream extends Ou
*/
private synchronized void verifyOpen() throws SwiftException {
if (closed) {
- throw new SwiftException("Output stream is closed");
+ throw new SwiftConnectionClosedException();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java Tue Aug 19 23:49:39 2014
@@ -21,9 +21,9 @@ package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
@@ -245,7 +245,7 @@ public class TestSwiftFileSystemBasicOps
writeTextFile(fs, path, "parent", true);
try {
fs.mkdirs(child);
- } catch (SwiftNotDirectoryException expected) {
+ } catch (ParentNotDirectoryException expected) {
LOG.debug("Expected Exception", expected);
}
} finally {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java Tue Aug 19 23:49:39 2014
@@ -23,8 +23,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
@@ -47,6 +47,14 @@ public class TestSwiftFileSystemContract
private static final Log LOG =
LogFactory.getLog(TestSwiftFileSystemContract.class);
+ /**
+ * Override this if the filesystem is not case sensitive
+ * @return true if the case detection/preservation tests should run
+ */
+ protected boolean filesystemIsCaseSensitive() {
+ return false;
+ }
+
@Override
protected void setUp() throws Exception {
final URI uri = getFilesystemURI();
@@ -89,9 +97,8 @@ public class TestSwiftFileSystemContract
try {
fs.mkdirs(testSubDir);
fail("Should throw IOException.");
- } catch (SwiftNotDirectoryException e) {
+ } catch (ParentNotDirectoryException e) {
// expected
- assertEquals(filepath,e.getPath());
}
//now verify that the subdir path does not exist
SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir);
@@ -100,7 +107,7 @@ public class TestSwiftFileSystemContract
try {
fs.mkdirs(testDeepSubDir);
fail("Should throw IOException.");
- } catch (SwiftNotDirectoryException e) {
+ } catch (ParentNotDirectoryException e) {
// expected
}
SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir after mkdir",
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java Tue Aug 19 23:49:39 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
@@ -220,7 +221,11 @@ public class TestSwiftFileSystemRename e
fs.mkdirs(testdir);
Path parent = testdir.getParent();
//the outcome here is ambiguous, so is not checked
- fs.rename(testdir, parent);
+ try {
+ fs.rename(testdir, parent);
+ } catch (SwiftOperationFailedException e) {
+ // allowed
+ }
assertExists("Source directory has been deleted ", testdir);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java Tue Aug 19 23:49:39 2014
@@ -111,7 +111,7 @@ public class TestV2LsOperations extends
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListFilesSubDir() throws Throwable {
createTestSubdirs();
- Path dir = path("/test");
+ Path dir = path("/test/subdir");
Path child = new Path(dir, "text.txt");
SwiftTestUtils.writeTextFile(fs, child, "text", false);
assertListFilesFinds(fs, dir, child, false);
@@ -120,7 +120,7 @@ public class TestV2LsOperations extends
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListFilesRecursive() throws Throwable {
createTestSubdirs();
- Path dir = path("/test");
+ Path dir = path("/test/recursive");
Path child = new Path(dir, "hadoop/a/a.txt");
SwiftTestUtils.writeTextFile(fs, child, "text", false);
assertListFilesFinds(fs, dir, child, true);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java Tue Aug 19 23:49:39 2014
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobPriorityChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobStatusChangedEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobQueueChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent;
import org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished;
import org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinishedEvent;
@@ -144,6 +145,8 @@ public class JobBuilder {
processJobInitedEvent((JobInitedEvent) event);
} else if (event instanceof JobPriorityChangeEvent) {
processJobPriorityChangeEvent((JobPriorityChangeEvent) event);
+ } else if (event instanceof JobQueueChangeEvent) {
+ processJobQueueChangeEvent((JobQueueChangeEvent) event);
} else if (event instanceof JobStatusChangedEvent) {
processJobStatusChangedEvent((JobStatusChangedEvent) event);
} else if (event instanceof JobSubmittedEvent) {
@@ -611,6 +614,14 @@ public class JobBuilder {
}
}
+ private void processJobQueueChangeEvent(JobQueueChangeEvent event) {
+ // set the queue name if existing
+ String queue = event.getJobQueueName();
+ if (queue != null) {
+ result.setQueue(queue);
+ }
+ }
+
private void processJobStatusChangedEvent(JobStatusChangedEvent event) {
result.setOutcome(Pre21JobHistoryConstants.Values
.valueOf(event.getStatus()));
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java Tue Aug 19 23:49:39 2014
@@ -21,6 +21,8 @@ import org.apache.commons.cli.CommandLin
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.ObjectWriter;
@@ -42,6 +44,8 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+@Private
+@Unstable
public class RumenToSLSConverter {
private static final String EOL = System.getProperty("line.separator");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java Tue Aug 19 23:49:39 2014
@@ -32,6 +32,8 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Arrays;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.rumen.JobTraceReader;
import org.apache.hadoop.tools.rumen.LoggedJob;
@@ -66,6 +68,8 @@ import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
+@Private
+@Unstable
public class SLSRunner {
// RM, Runner
private ResourceManager rm;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java Tue Aug 19 23:49:39 2014
@@ -29,6 +29,8 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -61,6 +63,8 @@ import org.apache.hadoop.yarn.security.A
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Logger;
@@ -70,6 +74,8 @@ import org.apache.hadoop.yarn.sls.SLSRun
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+@Private
+@Unstable
public abstract class AMSimulator extends TaskRunner.Task {
// resource manager
protected ResourceManager rm;
@@ -129,8 +135,7 @@ public abstract class AMSimulator extend
* register with RM
*/
@Override
- public void firstStep()
- throws YarnException, IOException, InterruptedException {
+ public void firstStep() throws Exception {
simulateStartTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
@@ -145,8 +150,7 @@ public abstract class AMSimulator extend
}
@Override
- public void middleStep()
- throws InterruptedException, YarnException, IOException {
+ public void middleStep() throws Exception {
// process responses in the queue
processResponseQueue();
@@ -158,7 +162,7 @@ public abstract class AMSimulator extend
}
@Override
- public void lastStep() {
+ public void lastStep() throws Exception {
LOG.info(MessageFormat.format("Application {0} is shutting down.", appId));
// unregister tracking
if (isTracked) {
@@ -169,26 +173,19 @@ public abstract class AMSimulator extend
.newRecordInstance(FinishApplicationMasterRequest.class);
finishAMRequest.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
- try {
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token =
- rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
- ugi.addTokenIdentifier(token.decodeIdentifier());
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- rm.getApplicationMasterService()
- .finishApplicationMaster(finishAMRequest);
- return null;
- }
- });
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appId)
+ .getRMAppAttempt(appAttemptId).getAMRMToken();
+ ugi.addTokenIdentifier(token.decodeIdentifier());
+ ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ rm.getApplicationMasterService()
+ .finishApplicationMaster(finishAMRequest);
+ return null;
+ }
+ });
simulateFinishTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
@@ -226,11 +223,9 @@ public abstract class AMSimulator extend
return createAllocateRequest(ask, new ArrayList<ContainerId>());
}
- protected abstract void processResponseQueue()
- throws InterruptedException, YarnException, IOException;
+ protected abstract void processResponseQueue() throws Exception;
- protected abstract void sendContainerRequest()
- throws YarnException, IOException, InterruptedException;
+ protected abstract void sendContainerRequest() throws Exception;
protected abstract void checkStop();
@@ -276,11 +271,18 @@ public abstract class AMSimulator extend
// waiting until application ACCEPTED
RMApp app = rm.getRMContext().getRMApps().get(appId);
while(app.getState() != RMAppState.ACCEPTED) {
- Thread.sleep(50);
+ Thread.sleep(10);
}
- appAttemptId = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt().getAppAttemptId();
+ // Waiting until application attempt reach LAUNCHED
+ // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
+ this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
+ .getCurrentAppAttempt().getAppAttemptId();
+ RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
+ .getCurrentAppAttempt();
+ while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
+ Thread.sleep(10);
+ }
}
private void registerAM()
@@ -293,10 +295,9 @@ public abstract class AMSimulator extend
amRegisterRequest.setTrackingUrl("localhost:1000");
UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token =
- rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appId)
+ .getRMAppAttempt(appAttemptId).getAMRMToken();
ugi.addTokenIdentifier(token.decodeIdentifier());
ugi.doAs(
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java Tue Aug 19 23:49:39 2014
@@ -27,6 +27,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -45,6 +47,8 @@ import org.apache.hadoop.yarn.sls.schedu
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.log4j.Logger;
+@Private
+@Unstable
public class MRAMSimulator extends AMSimulator {
/*
Vocabulary Used:
@@ -141,8 +145,7 @@ public class MRAMSimulator extends AMSim
}
@Override
- public void firstStep()
- throws YarnException, IOException, InterruptedException {
+ public void firstStep() throws Exception {
super.firstStep();
requestAMContainer();
@@ -386,7 +389,7 @@ public class MRAMSimulator extends AMSim
}
@Override
- public void lastStep() {
+ public void lastStep() throws Exception {
super.lastStep();
// clear data structures
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.sls.conf;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
public class SLSConfiguration {
// sls
public static final String PREFIX = "yarn.sls.";
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java Tue Aug 19 23:49:39 2014
@@ -27,6 +27,9 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -54,6 +57,8 @@ import org.apache.hadoop.yarn.sls.schedu
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+@Private
+@Unstable
public class NMSimulator extends TaskRunner.Task {
// node resource
private RMNode node;
@@ -103,12 +108,12 @@ public class NMSimulator extends TaskRun
}
@Override
- public void firstStep() throws YarnException, IOException {
+ public void firstStep() {
// do nothing
}
@Override
- public void middleStep() {
+ public void middleStep() throws Exception {
// we check the lifetime for each running containers
ContainerSimulator cs = null;
synchronized(completedContainerList) {
@@ -132,37 +137,31 @@ public class NMSimulator extends TaskRun
ns.setResponseId(RESPONSE_ID ++);
ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
beatRequest.setNodeStatus(ns);
- try {
- NodeHeartbeatResponse beatResponse =
- rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
- if (! beatResponse.getContainersToCleanup().isEmpty()) {
- // remove from queue
- synchronized(releasedContainerList) {
- for (ContainerId containerId : beatResponse.getContainersToCleanup()){
- if (amContainerList.contains(containerId)) {
- // AM container (not killed?, only release)
- synchronized(amContainerList) {
- amContainerList.remove(containerId);
- }
- LOG.debug(MessageFormat.format("NodeManager {0} releases " +
- "an AM ({1}).", node.getNodeID(), containerId));
- } else {
- cs = runningContainers.remove(containerId);
- containerQueue.remove(cs);
- releasedContainerList.add(containerId);
- LOG.debug(MessageFormat.format("NodeManager {0} releases a " +
- "container ({1}).", node.getNodeID(), containerId));
+ NodeHeartbeatResponse beatResponse =
+ rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
+ if (! beatResponse.getContainersToCleanup().isEmpty()) {
+ // remove from queue
+ synchronized(releasedContainerList) {
+ for (ContainerId containerId : beatResponse.getContainersToCleanup()){
+ if (amContainerList.contains(containerId)) {
+ // AM container (not killed?, only release)
+ synchronized(amContainerList) {
+ amContainerList.remove(containerId);
}
+ LOG.debug(MessageFormat.format("NodeManager {0} releases " +
+ "an AM ({1}).", node.getNodeID(), containerId));
+ } else {
+ cs = runningContainers.remove(containerId);
+ containerQueue.remove(cs);
+ releasedContainerList.add(containerId);
+ LOG.debug(MessageFormat.format("NodeManager {0} releases a " +
+ "container ({1}).", node.getNodeID(), containerId));
}
}
}
- if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
- lastStep();
- }
- } catch (YarnException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
+ }
+ if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
+ lastStep();
}
}
@@ -258,4 +257,19 @@ public class NMSimulator extends TaskRun
completedContainerList.add(containerId);
}
}
+
+ @VisibleForTesting
+ Map<ContainerId, ContainerSimulator> getRunningContainers() {
+ return runningContainers;
+ }
+
+ @VisibleForTesting
+ List<ContainerId> getAMContainers() {
+ return amContainerList;
+ }
+
+ @VisibleForTesting
+ List<ContainerId> getCompletedContainers() {
+ return completedContainerList;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java Tue Aug 19 23:49:39 2014
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.nodem
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -36,6 +38,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
+@Private
+@Unstable
public class NodeInfo {
private static int NODE_ID = 0;
@@ -43,6 +47,8 @@ public class NodeInfo {
return NodeId.newInstance(host, port);
}
+ @Private
+ @Unstable
private static class FakeRMNodeImpl implements RMNode {
private NodeId nodeId;
private String hostName;
@@ -164,7 +170,7 @@ public class NodeInfo {
perNode = resourceOption;
}
}
-
+
public static RMNode newNodeInfo(String rackName, String hostName,
final ResourceOption resourceOption, int port) {
final NodeId nodeId = newNodeID(hostName, port);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
public class CapacitySchedulerMetrics extends SchedulerMetrics {
public CapacitySchedulerMetrics() {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java Tue Aug 19 23:49:39 2014
@@ -21,9 +21,13 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
+@Private
+@Unstable
public class ContainerSimulator implements Delayed {
// id
private ContainerId id;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java Tue Aug 19 23:49:39 2014
@@ -18,16 +18,19 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
- .AppSchedulable;
+ .FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
- .FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import com.codahale.metrics.Gauge;
import org.apache.hadoop.yarn.sls.SLSRunner;
+@Private
+@Unstable
public class FairSchedulerMetrics extends SchedulerMetrics {
private int totalMemoryMB = Integer.MAX_VALUE;
@@ -62,8 +65,7 @@ public class FairSchedulerMetrics extend
public void trackApp(ApplicationAttemptId appAttemptId, String oldAppId) {
super.trackApp(appAttemptId, oldAppId);
FairScheduler fair = (FairScheduler) scheduler;
- final AppSchedulable app = fair.getSchedulerApp(appAttemptId)
- .getAppSchedulable();
+ final FSAppAttempt app = fair.getSchedulerApp(appAttemptId);
metrics.register("variable.app." + oldAppId + ".demand.memory",
new Gauge<Integer>() {
@Override
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java Tue Aug 19 23:49:39 2014
@@ -18,12 +18,16 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo
.FifoScheduler;
import com.codahale.metrics.Gauge;
+@Private
+@Unstable
public class FifoSchedulerMetrics extends SchedulerMetrics {
public FifoSchedulerMetrics() {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java Tue Aug 19 23:49:39 2014
@@ -18,9 +18,13 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
.NodeUpdateSchedulerEvent;
+@Private
+@Unstable
public class NodeUpdateSchedulerEventWrapper extends NodeUpdateSchedulerEvent {
public NodeUpdateSchedulerEventWrapper(NodeUpdateSchedulerEvent event) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.server.res
import java.util.Collections;
import java.util.List;
+@Private
+@Unstable
public class RMNodeWrapper implements RMNode {
private RMNode node;
private List<UpdatedContainerInfo> updates;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Tue Aug 19 23:49:39 2014
@@ -36,6 +36,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,11 +61,16 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -87,8 +95,11 @@ import com.codahale.metrics.MetricRegist
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
-public class ResourceSchedulerWrapper implements
- SchedulerWrapper,ResourceScheduler,Configurable {
+@Private
+@Unstable
+final public class ResourceSchedulerWrapper
+ extends AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>
+ implements SchedulerWrapper, ResourceScheduler, Configurable {
private static final String EOL = System.getProperty("line.separator");
private static final int SAMPLING_SIZE = 60;
private ScheduledExecutorService pool;
@@ -144,6 +155,7 @@ public class ResourceSchedulerWrapper im
public final Logger LOG = Logger.getLogger(ResourceSchedulerWrapper.class);
public ResourceSchedulerWrapper() {
+ super(ResourceSchedulerWrapper.class.getName());
samplerLock = new ReentrantLock();
queueLock = new ReentrantLock();
}
@@ -565,7 +577,7 @@ public class ResourceSchedulerWrapper im
new Gauge<Integer>() {
@Override
public Integer getValue() {
- if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
+ if (scheduler == null || scheduler.getRootQueueMetrics() == null) {
return 0;
} else {
return scheduler.getRootQueueMetrics().getAppsRunning();
@@ -712,17 +724,18 @@ public class ResourceSchedulerWrapper im
public void addAMRuntime(ApplicationId appId,
long traceStartTimeMS, long traceEndTimeMS,
long simulateStartTimeMS, long simulateEndTimeMS) {
-
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
+ if (metricsON) {
+ try {
+ // write job runtime information
+ StringBuilder sb = new StringBuilder();
+ sb.append(appId).append(",").append(traceStartTimeMS).append(",")
+ .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
+ .append(",").append(simulateEndTimeMS);
+ jobRuntimeLogBW.write(sb.toString() + EOL);
+ jobRuntimeLogBW.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
}
@@ -792,10 +805,39 @@ public class ResourceSchedulerWrapper im
return conf;
}
+ @SuppressWarnings("unchecked")
@Override
- public void reinitialize(Configuration entries, RMContext rmContext)
- throws IOException {
- scheduler.reinitialize(entries, rmContext);
+ public void serviceInit(Configuration conf) throws Exception {
+ ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
+ scheduler).init(conf);
+ super.serviceInit(conf);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void serviceStart() throws Exception {
+ ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
+ scheduler).start();
+ super.serviceStart();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void serviceStop() throws Exception {
+ ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
+ scheduler).stop();
+ super.serviceStop();
+ }
+
+ @Override
+ public void setRMContext(RMContext rmContext) {
+ scheduler.setRMContext(rmContext);
+ }
+
+ @Override
+ public void reinitialize(Configuration conf, RMContext rmContext)
+ throws IOException {
+ scheduler.reinitialize(conf, rmContext);
}
@Override
@@ -872,4 +914,30 @@ public class ResourceSchedulerWrapper im
throws YarnException {
return scheduler.moveApplication(appId, newQueue);
}
+
+ @Override
+ @LimitedPrivate("yarn")
+ @Unstable
+ public Resource getClusterResource() {
+ return null;
+ }
+
+ @Override
+ public synchronized List<Container> getTransferredContainers(
+ ApplicationAttemptId currentAttempt) {
+ return new ArrayList<Container>();
+ }
+
+ @Override
+ public Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>>
+ getSchedulerApplications() {
+ return new HashMap<ApplicationId,
+ SchedulerApplication<SchedulerApplicationAttempt>>();
+ }
+
+ @Override
+ protected void completedContainer(RMContainer rmContainer,
+ ContainerStatus containerStatus, RMContainerEventType event) {
+ // do nothing
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java Tue Aug 19 23:49:39 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
@@ -100,6 +102,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+@Private
+@Unstable
public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable {
private static final String EOL = System.getProperty("line.separator");
@@ -725,16 +729,18 @@ public class SLSCapacityScheduler extend
long traceStartTimeMS, long traceEndTimeMS,
long simulateStartTimeMS, long simulateEndTimeMS) {
- try {
- // write job runtime information
- StringBuilder sb = new StringBuilder();
- sb.append(appId).append(",").append(traceStartTimeMS).append(",")
- .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
- .append(",").append(simulateEndTimeMS);
- jobRuntimeLogBW.write(sb.toString() + EOL);
- jobRuntimeLogBW.flush();
- } catch (IOException e) {
- e.printStackTrace();
+ if (metricsON) {
+ try {
+ // write job runtime information
+ StringBuilder sb = new StringBuilder();
+ sb.append(appId).append(",").append(traceStartTimeMS).append(",")
+ .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
+ .append(",").append(simulateEndTimeMS);
+ jobRuntimeLogBW.write(sb.toString() + EOL);
+ jobRuntimeLogBW.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java Tue Aug 19 23:49:39 2014
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.HashSet;
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.ResourceScheduler;
@@ -30,6 +32,8 @@ import org.apache.hadoop.yarn.server.res
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
+@Private
+@Unstable
public abstract class SchedulerMetrics {
protected ResourceScheduler scheduler;
protected Set<String> trackedQueues;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java Tue Aug 19 23:49:39 2014
@@ -19,11 +19,15 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import com.codahale.metrics.MetricRegistry;
+@Private
+@Unstable
public interface SchedulerWrapper {
public MetricRegistry getMetrics();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java Tue Aug 19 23:49:39 2014
@@ -25,9 +25,15 @@ import java.util.concurrent.Delayed;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.exceptions.YarnException;
+@Private
+@Unstable
public class TaskRunner {
+ @Private
+ @Unstable
public abstract static class Task implements Runnable, Delayed {
private long start;
private long end;
@@ -93,12 +99,10 @@ public class TaskRunner {
} else {
lastStep();
}
- } catch (YarnException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
+ } catch (Exception e) {
e.printStackTrace();
+ Thread.getDefaultUncaughtExceptionHandler()
+ .uncaughtException(Thread.currentThread(), e);
}
}
@@ -118,13 +122,11 @@ public class TaskRunner {
}
- public abstract void firstStep()
- throws YarnException, IOException, InterruptedException;
+ public abstract void firstStep() throws Exception;
- public abstract void middleStep()
- throws YarnException, InterruptedException, IOException;
+ public abstract void middleStep() throws Exception;
- public abstract void lastStep() throws YarnException;
+ public abstract void lastStep() throws Exception;
public void setEndTime(long et) {
endTime = et;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java Tue Aug 19 23:49:39 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.sls.utils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.rumen.JobTraceReader;
@@ -36,6 +38,8 @@ import java.util.Map;
import java.util.List;
import java.util.Iterator;
+@Private
+@Unstable
public class SLSUtils {
public static String[] getRackHostName(String hostname) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java Tue Aug 19 23:49:39 2014
@@ -30,6 +30,8 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
.SchedulerEventType;
import org.mortbay.jetty.Handler;
@@ -49,6 +51,8 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry;
import org.mortbay.jetty.handler.ResourceHandler;
+@Private
+@Unstable
public class SLSWebApp extends HttpServlet {
private static final long serialVersionUID = 1905162041950251407L;
private transient Server server;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,8 +18,6 @@
Yarn Scheduler Load Simulator (SLS)
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* Overview
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java Tue Aug 19 23:49:39 2014
@@ -18,10 +18,13 @@
package org.apache.hadoop.yarn.sls;
-import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
import org.junit.Test;
import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.UUID;
public class TestSLSRunner {
@@ -30,6 +33,15 @@ public class TestSLSRunner {
@SuppressWarnings("all")
public void testSimulatorRunning() throws Exception {
File tempDir = new File("target", UUID.randomUUID().toString());
+ final List<Throwable> exceptionList =
+ Collections.synchronizedList(new ArrayList<Throwable>());
+
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ exceptionList.add(e);
+ }
+ });
// start the simulator
File slsOutputDir = new File(tempDir.getAbsolutePath() + "/slsoutput/");
@@ -38,8 +50,20 @@ public class TestSLSRunner {
"-output", slsOutputDir.getAbsolutePath()};
SLSRunner.main(args);
- // wait for 45 seconds before stop
- Thread.sleep(45 * 1000);
+ // wait for 20 seconds before stop
+ int count = 20;
+ while (count >= 0) {
+ Thread.sleep(1000);
+
+ if (! exceptionList.isEmpty()) {
+ SLSRunner.getRunner().stop();
+ Assert.fail("TestSLSRunner catched exception from child thread " +
+ "(TaskRunner.Task): " + exceptionList.get(0).getMessage());
+ break;
+ }
+ count--;
+ }
+
SLSRunner.getRunner().stop();
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java Tue Aug 19 23:49:39 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.sls.utils;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.Test;
public class TestSLSUtils {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java Tue Aug 19 23:49:39 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.sls.web;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.junit.Test;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/TestKeyOnlyTextOutputReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/TestKeyOnlyTextOutputReader.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/TestKeyOnlyTextOutputReader.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/TestKeyOnlyTextOutputReader.java Tue Aug 19 23:49:39 2014
@@ -23,7 +23,7 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-tools-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-tools-dist/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-tools-dist/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-tools-dist/pom.xml Tue Aug 19 23:49:39 2014
@@ -85,6 +85,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
<scope>compile</scope>
</dependency>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/pom.xml Tue Aug 19 23:49:39 2014
@@ -42,6 +42,7 @@
<module>hadoop-pipes</module>
<module>hadoop-openstack</module>
<module>hadoop-sls</module>
+ <module>hadoop-azure</module>
</modules>
<build>