You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/07/20 15:18:00 UTC
hadoop git commit: YARN-6685. Add job count in to SLS JSON input
format. (Yufei Gu via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/trunk c21c26039 -> 0ba8cda13
YARN-6685. Add job count in to SLS JSON input format. (Yufei Gu via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ba8cda1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ba8cda1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ba8cda1
Branch: refs/heads/trunk
Commit: 0ba8cda13549cc4a3946c440016f9d2a9e78740d
Parents: c21c260
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Jul 20 08:15:46 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Jul 20 08:15:46 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 28 +++++++++++++++-----
.../src/site/markdown/SchedulerLoadSimulator.md | 3 ++-
2 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba8cda1/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index a534f03..477cc4a 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -395,18 +395,28 @@ public class SLSRunner extends Configured implements Tool {
String queue = jsonJob.get("job.queue.name").toString();
increaseQueueAppNum(queue);
- String oldAppId = (String)jsonJob.get("job.id");
- if (oldAppId == null) {
- oldAppId = Integer.toString(AM_ID);
- }
-
String amType = (String)jsonJob.get("am.type");
if (amType == null) {
amType = SLSUtils.DEFAULT_JOB_TYPE;
}
- runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
- getTaskContainers(jsonJob), null, getAMContainerResource(jsonJob));
+ int jobCount = 1;
+ if (jsonJob.containsKey("job.count")) {
+ jobCount = Integer.parseInt(jsonJob.get("job.count").toString());
+ }
+ jobCount = Math.max(jobCount, 1);
+
+ String oldAppId = (String)jsonJob.get("job.id");
+ // Job id is generated automatically if this job configuration allows
+ // multiple job instances
+ if(jobCount > 1) {
+ oldAppId = null;
+ }
+
+ for (int i = 0; i < jobCount; i++) {
+ runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
+ getTaskContainers(jsonJob), null, getAMContainerResource(jsonJob));
+ }
}
private List<ContainerSimulator> getTaskContainers(Map jsonJob)
@@ -732,6 +742,10 @@ public class SLSRunner extends Configured implements Tool {
SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS,
SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS_DEFAULT);
boolean isTracked = trackedApps.contains(oldJobId);
+
+ if (oldJobId == null) {
+ oldJobId = Integer.toString(AM_ID);
+ }
AM_ID++;
amSim.init(heartbeatInterval, containerList, rm, this, jobStartTimeMS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba8cda1/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
index 6e00e9a..d1848e8 100644
--- a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -336,8 +336,9 @@ Here we provide an example format of the sls json file, which contains 2 jobs. T
"job.start.ms" : 0, // job start time
"job.end.ms" : 95375, // job finish time, optional, the default value is 0
"job.queue.name" : "sls_queue_1", // the queue job will be submitted to
- "job.id" : "job_1", // the job id used to track the job, optional, the default value is an zero-based integer increasing with number of jobs
+ "job.id" : "job_1", // the job id used to track the job, optional. The default value, an zero-based integer increasing with number of jobs, is used if this is not specified or job.count > 1
"job.user" : "default", // user, optional, the default value is "default"
+ "job.count" : 1, // number of jobs, optional, the default value is 1
"job.tasks" : [ {
"count": 1, // number of tasks, optional, the default value is 1
"container.host" : "/default-rack/node1", // host the container asks for
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org