You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/12/20 02:01:22 UTC
svn commit: r1552467 - in /hadoop/common/branches/HDFS-4685: ./
hadoop-project/ hadoop-tools/hadoop-extras/
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/
ha...
Author: cnauroth
Date: Fri Dec 20 01:01:18 2013
New Revision: 1552467
URL: http://svn.apache.org/r1552467
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml
Modified:
hadoop/common/branches/HDFS-4685/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-extras/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-streaming/pom.xml
Propchange: hadoop/common/branches/HDFS-4685/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2:r1551915
Merged /hadoop/common/trunk:r1551332-1552465
Modified: hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml Fri Dec 20 01:01:18 2013
@@ -318,6 +318,11 @@
<version>11.0.2</version>
</dependency>
<dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.2.4</version>
+ </dependency>
+ <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
@@ -433,6 +438,12 @@
<artifactId>guice</artifactId>
<version>3.0</version>
</dependency>
+
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib</artifactId>
+ <version>2.2</version>
+ </dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-extras/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-extras/pom.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-extras/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-extras/pom.xml Fri Dec 20 01:01:18 2013
@@ -94,6 +94,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java Fri Dec 20 01:01:18 2013
@@ -40,12 +40,15 @@ import org.apache.hadoop.tools.rumen.Log
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
+import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -140,9 +143,9 @@ public class SLSRunner {
// start application masters
startAM();
// set queue & tracked apps information
- ((ResourceSchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler())
.setQueueSet(this.queueAppNumMap.keySet());
- ((ResourceSchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler())
.setTrackedAppSet(this.trackedApps);
// print out simulation info
printSimulationInfo();
@@ -151,13 +154,24 @@ public class SLSRunner {
// starting the runner once everything is ready to go,
runner.start();
}
-
+
private void startRM() throws IOException, ClassNotFoundException {
Configuration rmConf = new YarnConfiguration();
String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
- rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
- rmConf.set(YarnConfiguration.RM_SCHEDULER,
- ResourceSchedulerWrapper.class.getName());
+
+ // For CapacityScheduler we use a sub-classing instead of wrapping
+ // to allow scheduler-specific invocations from monitors to work
+ // this can be used for other schedulers as well if we care to
+ // exercise/track behaviors that are not common to the scheduler api
+ if(Class.forName(schedulerClass) == CapacityScheduler.class) {
+ rmConf.set(YarnConfiguration.RM_SCHEDULER,
+ SLSCapacityScheduler.class.getName());
+ } else {
+ rmConf.set(YarnConfiguration.RM_SCHEDULER,
+ ResourceSchedulerWrapper.class.getName());
+ rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
+ }
+
rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
rm = new ResourceManager();
rm.init(rmConf);
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java Fri Dec 20 01:01:18 2013
@@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.util.Recor
import org.apache.log4j.Logger;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
-import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
@@ -193,7 +193,7 @@ public abstract class AMSimulator extend
simulateFinishTimeMS = System.currentTimeMillis() -
SLSRunner.getRunner().getStartTimeMS();
// record job running information
- ((ResourceSchedulerWrapper)rm.getResourceScheduler())
+ ((SchedulerWrapper)rm.getResourceScheduler())
.addAMRuntime(appId,
traceStartTimeMS, traceFinishTimeMS,
simulateStartTimeMS, simulateFinishTimeMS);
@@ -314,13 +314,13 @@ public abstract class AMSimulator extend
private void trackApp() {
if (isTracked) {
- ((ResourceSchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler())
.addTrackedApp(appAttemptId, oldAppId);
}
}
public void untrackApp() {
if (isTracked) {
- ((ResourceSchedulerWrapper) rm.getResourceScheduler())
+ ((SchedulerWrapper) rm.getResourceScheduler())
.removeTrackedApp(appAttemptId, oldAppId);
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Fri Dec 20 01:01:18 2013
@@ -85,8 +85,8 @@ import com.codahale.metrics.MetricRegist
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
-public class ResourceSchedulerWrapper implements ResourceScheduler,
- Configurable {
+public class ResourceSchedulerWrapper implements
+ SchedulerWrapper,ResourceScheduler,Configurable {
private static final String EOL = System.getProperty("line.separator");
private static final int SAMPLING_SIZE = 60;
private ScheduledExecutorService pool;
@@ -150,9 +150,8 @@ public class ResourceSchedulerWrapper im
public void setConf(Configuration conf) {
this.conf = conf;
// set scheduler
- Class<? extends ResourceScheduler> klass =
- conf.getClass(SLSConfiguration.RM_SCHEDULER, null,
- ResourceScheduler.class);
+ Class<? extends ResourceScheduler> klass = conf.getClass(
+ SLSConfiguration.RM_SCHEDULER, null, ResourceScheduler.class);
scheduler = ReflectionUtils.newInstance(klass, conf);
// start metrics
@@ -861,4 +860,3 @@ public class ResourceSchedulerWrapper im
return scheduler.getAppsInQueue(queue);
}
}
-
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java Fri Dec 20 01:01:18 2013
@@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.sls.SLSRun
import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics;
import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
+
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
@@ -50,7 +52,7 @@ import org.mortbay.jetty.handler.Resourc
public class SLSWebApp extends HttpServlet {
private static final long serialVersionUID = 1905162041950251407L;
private transient Server server;
- private transient ResourceSchedulerWrapper wrapper;
+ private transient SchedulerWrapper wrapper;
private transient MetricRegistry metrics;
private transient SchedulerMetrics schedulerMetrics;
// metrics objects
@@ -90,7 +92,7 @@ public class SLSWebApp extends HttpServl
}
}
- public SLSWebApp(ResourceSchedulerWrapper wrapper, int metricsAddressPort) {
+ public SLSWebApp(SchedulerWrapper wrapper, int metricsAddressPort) {
this.wrapper = wrapper;
metrics = wrapper.getMetrics();
handleOperTimecostHistogramMap =
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml Fri Dec 20 01:01:18 2013
@@ -17,7 +17,18 @@
<configuration>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+ <!-- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> -->
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.monitor.policies</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-streaming/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-streaming/pom.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-streaming/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-streaming/pom.xml Fri Dec 20 01:01:18 2013
@@ -100,6 +100,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>