You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/01/29 05:10:19 UTC
[01/50] [abbrv] hadoop git commit: YARN-7782. Enable user re-mapping
for Docker containers in yarn-default.xml. Contributed by Eric Yang [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 c95d31fd7 -> 13d61cc42 (forced update)
YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39b999ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39b999ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39b999ab
Branch: refs/heads/YARN-1011
Commit: 39b999aba2c1a1fc1df17bdf952834ba516f3b2a
Parents: a72cdcc
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 23 14:03:53 2018 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Jan 23 14:03:53 2018 -0800
----------------------------------------------------------------------
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39b999ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a9938c3..45aa635 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1688,7 +1688,7 @@
<property>
<description>Property to enable docker user remapping</description>
<name>yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-7798. Refactor SLS
Reservation Creation. Contributed by Young Chen.
Posted by ha...@apache.org.
YARN-7798. Refactor SLS Reservation Creation. Contributed by Young Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16be42d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16be42d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16be42d3
Branch: refs/heads/YARN-1011
Commit: 16be42d3097c13b17d704e5b6dc8d66bd5ff6d9a
Parents: 0c139d5
Author: Yufei Gu <yu...@apache.org>
Authored: Thu Jan 25 13:06:01 2018 -0800
Committer: Yufei Gu <yu...@apache.org>
Committed: Thu Jan 25 13:06:01 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 75 +++++++++-----------
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 41 ++++++-----
.../yarn/sls/appmaster/MRAMSimulator.java | 74 ++++++++++++++-----
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 2 +-
.../yarn/sls/appmaster/TestAMSimulator.java | 7 +-
5 files changed, 119 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16be42d3/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index ad4310f..456602f 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.tools.rumen.TaskAttemptInfo;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -444,7 +443,7 @@ public class SLSRunner extends Configured implements Tool {
for (int i = 0; i < jobCount; i++) {
runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
- getTaskContainers(jsonJob), null, getAMContainerResource(jsonJob));
+ getTaskContainers(jsonJob), getAMContainerResource(jsonJob));
}
}
@@ -607,7 +606,7 @@ public class SLSRunner extends Configured implements Tool {
// Only supports the default job type currently
runNewAM(SLSUtils.DEFAULT_JOB_TYPE, user, jobQueue, oldJobId,
- jobStartTimeMS, jobFinishTimeMS, containerList, null,
+ jobStartTimeMS, jobFinishTimeMS, containerList,
getAMContainerResource(null));
}
@@ -628,10 +627,6 @@ public class SLSRunner extends Configured implements Tool {
localConf.set("fs.defaultFS", "file:///");
long baselineTimeMS = 0;
- // reservations use wall clock time, so need to have a reference for that
- UTCClock clock = new UTCClock();
- long now = clock.getTime();
-
try {
// if we use the nodeFile this could have been not initialized yet.
@@ -670,13 +665,12 @@ public class SLSRunner extends Configured implements Tool {
ArrayList<NodeId> keyAsArray = new ArrayList<NodeId>(nmMap.keySet());
Random rand = new Random(stjp.getSeed());
- Resource maxMapRes = Resource.newInstance(0, 0);
- long maxMapDur = 0;
// map tasks
for (int i = 0; i < job.getNumberMaps(); i++) {
TaskAttemptInfo tai = job.getTaskAttemptInfo(TaskType.MAP, i, 0);
- RMNode node = nmMap
- .get(keyAsArray.get(rand.nextInt(keyAsArray.size()))).getNode();
+ RMNode node =
+ nmMap.get(keyAsArray.get(rand.nextInt(keyAsArray.size())))
+ .getNode();
String hostname = "/" + node.getRackName() + "/" + node.getHostName();
long containerLifeTime = tai.getRuntime();
Resource containerResource =
@@ -684,55 +678,39 @@ public class SLSRunner extends Configured implements Tool {
(int) tai.getTaskInfo().getTaskVCores());
containerList.add(new ContainerSimulator(containerResource,
containerLifeTime, hostname, DEFAULT_MAPPER_PRIORITY, "map"));
- maxMapRes = Resources.componentwiseMax(maxMapRes, containerResource);
- maxMapDur =
- containerLifeTime > maxMapDur ? containerLifeTime : maxMapDur;
-
}
- Resource maxRedRes = Resource.newInstance(0, 0);
- long maxRedDur = 0;
// reduce tasks
for (int i = 0; i < job.getNumberReduces(); i++) {
TaskAttemptInfo tai = job.getTaskAttemptInfo(TaskType.REDUCE, i, 0);
- RMNode node = nmMap
- .get(keyAsArray.get(rand.nextInt(keyAsArray.size()))).getNode();
+ RMNode node =
+ nmMap.get(keyAsArray.get(rand.nextInt(keyAsArray.size())))
+ .getNode();
String hostname = "/" + node.getRackName() + "/" + node.getHostName();
long containerLifeTime = tai.getRuntime();
Resource containerResource =
Resource.newInstance((int) tai.getTaskInfo().getTaskMemory(),
(int) tai.getTaskInfo().getTaskVCores());
- containerList.add(new ContainerSimulator(containerResource,
- containerLifeTime, hostname, DEFAULT_REDUCER_PRIORITY, "reduce"));
- maxRedRes = Resources.componentwiseMax(maxRedRes, containerResource);
- maxRedDur =
- containerLifeTime > maxRedDur ? containerLifeTime : maxRedDur;
-
+ containerList.add(
+ new ContainerSimulator(containerResource, containerLifeTime,
+ hostname, DEFAULT_REDUCER_PRIORITY, "reduce"));
}
- // generating reservations for the jobs that require them
+ ReservationId reservationId = null;
- ReservationSubmissionRequest rr = null;
if (job.hasDeadline()) {
- ReservationId reservationId =
+ reservationId =
ReservationId.newInstance(this.rm.getStartTime(), AM_ID);
-
- rr = ReservationClientUtil.createMRReservation(reservationId,
- "reservation_" + AM_ID, maxMapRes, job.getNumberMaps(), maxMapDur,
- maxRedRes, job.getNumberReduces(), maxRedDur,
- now + jobStartTimeMS, now + job.getDeadline(),
- job.getQueueName());
-
}
runNewAM(SLSUtils.DEFAULT_JOB_TYPE, user, jobQueue, oldJobId,
- jobStartTimeMS, jobFinishTimeMS, containerList, rr,
- getAMContainerResource(null));
+ jobStartTimeMS, jobFinishTimeMS, containerList, reservationId,
+ job.getDeadline(), getAMContainerResource(null));
+
}
} finally {
stjp.close();
}
-
}
private Resource getAMContainerResource(Map jsonJob) {
@@ -772,7 +750,17 @@ public class SLSRunner extends Configured implements Tool {
private void runNewAM(String jobType, String user,
String jobQueue, String oldJobId, long jobStartTimeMS,
long jobFinishTimeMS, List<ContainerSimulator> containerList,
- ReservationSubmissionRequest rr, Resource amContainerResource) {
+ Resource amContainerResource) {
+ runNewAM(jobType, user, jobQueue, oldJobId, jobStartTimeMS,
+ jobFinishTimeMS, containerList, null, -1,
+ amContainerResource);
+ }
+
+ private void runNewAM(String jobType, String user,
+ String jobQueue, String oldJobId, long jobStartTimeMS,
+ long jobFinishTimeMS, List<ContainerSimulator> containerList,
+ ReservationId reservationId, long deadline,
+ Resource amContainerResource) {
AMSimulator amSim = (AMSimulator) ReflectionUtils.newInstance(
amClassMap.get(jobType), new Configuration());
@@ -787,10 +775,15 @@ public class SLSRunner extends Configured implements Tool {
oldJobId = Integer.toString(AM_ID);
}
AM_ID++;
-
amSim.init(heartbeatInterval, containerList, rm, this, jobStartTimeMS,
- jobFinishTimeMS, user, jobQueue, isTracked, oldJobId, rr,
+ jobFinishTimeMS, user, jobQueue, isTracked, oldJobId,
runner.getStartTimeMS(), amContainerResource);
+ if(reservationId != null) {
+ // if we have a ReservationId, delegate reservation creation to
+ // AMSim (reservation shape is impl specific)
+ UTCClock clock = new UTCClock();
+ amSim.initReservation(reservationId, deadline, clock.getTime());
+ }
runner.schedule(amSim);
maxRuntime = Math.max(maxRuntime, jobFinishTimeMS);
numTasks += containerList.size();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16be42d3/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index 72698ea..5727b5f 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -85,7 +85,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
protected final BlockingQueue<AllocateResponse> responseQueue;
private int responseId = 0;
// user name
- protected String user;
+ private String user;
// queue name
protected String queue;
// am type
@@ -105,7 +105,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
// waiting for AM container
volatile boolean isAMContainerRunning = false;
volatile Container amContainer;
-
+
private static final Logger LOG = LoggerFactory.getLogger(AMSimulator.class);
private Resource amContainerResource;
@@ -120,9 +120,8 @@ public abstract class AMSimulator extends TaskRunner.Task {
public void init(int heartbeatInterval,
List<ContainerSimulator> containerList, ResourceManager resourceManager,
SLSRunner slsRunnner, long startTime, long finishTime, String simUser,
- String simQueue, boolean tracked, String oldApp,
- ReservationSubmissionRequest rr, long baseTimeMS,
- Resource amContainerResource) {
+ String simQueue, boolean tracked, String oldApp, long baseTimeMS,
+ Resource amResource) {
super.init(startTime, startTime + 1000000L * heartbeatInterval,
heartbeatInterval);
this.user = simUser;
@@ -134,8 +133,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
this.baselineTimeMS = baseTimeMS;
this.traceStartTimeMS = startTime;
this.traceFinishTimeMS = finishTime;
- this.reservationRequest = rr;
- this.amContainerResource = amContainerResource;
+ this.amContainerResource = amResource;
}
/**
@@ -171,6 +169,10 @@ public abstract class AMSimulator extends TaskRunner.Task {
isAMContainerRunning = true;
}
+ protected void setReservationRequest(ReservationSubmissionRequest rr){
+ this.reservationRequest = rr;
+ }
+
private ReservationId submitReservationWhenSpecified()
throws IOException, InterruptedException {
if (reservationRequest != null) {
@@ -256,7 +258,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
simulateStartTimeMS, simulateFinishTimeMS);
}
}
-
+
protected ResourceRequest createResourceRequest(
Resource resource, String host, int priority, int numContainers) {
ResourceRequest request = recordFactory
@@ -269,7 +271,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
request.setPriority(prio);
return request;
}
-
+
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask,
List<ContainerId> toRelease) {
AllocateRequest allocateRequest =
@@ -279,36 +281,39 @@ public abstract class AMSimulator extends TaskRunner.Task {
allocateRequest.setReleaseList(toRelease);
return allocateRequest;
}
-
+
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask) {
return createAllocateRequest(ask, new ArrayList<ContainerId>());
}
protected abstract void processResponseQueue() throws Exception;
-
+
protected abstract void sendContainerRequest() throws Exception;
-
+
+ public abstract void initReservation(
+ ReservationId reservationId, long deadline, long now);
+
protected abstract void checkStop();
-
+
private void submitApp(ReservationId reservationId)
throws YarnException, InterruptedException, IOException {
// ask for new application
GetNewApplicationRequest newAppRequest =
Records.newRecord(GetNewApplicationRequest.class);
- GetNewApplicationResponse newAppResponse =
+ GetNewApplicationResponse newAppResponse =
rm.getClientRMService().getNewApplication(newAppRequest);
appId = newAppResponse.getApplicationId();
-
+
// submit the application
final SubmitApplicationRequest subAppRequest =
Records.newRecord(SubmitApplicationRequest.class);
- ApplicationSubmissionContext appSubContext =
+ ApplicationSubmissionContext appSubContext =
Records.newRecord(ApplicationSubmissionContext.class);
appSubContext.setApplicationId(appId);
appSubContext.setMaxAppAttempts(1);
appSubContext.setQueue(queue);
appSubContext.setPriority(Priority.newInstance(0));
- ContainerLaunchContext conLauContext =
+ ContainerLaunchContext conLauContext =
Records.newRecord(ContainerLaunchContext.class);
conLauContext.setApplicationACLs(new HashMap<>());
conLauContext.setCommands(new ArrayList<>());
@@ -379,7 +384,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
}
}
}
-
+
protected List<ResourceRequest> packageRequests(
List<ContainerSimulator> csList, int priority) {
// create requests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16be42d3/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
index 21bf054..18a155c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.sls.appmaster;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -42,8 +44,10 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.ReservationClientUtil;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,51 +55,51 @@ import org.slf4j.LoggerFactory;
@Unstable
public class MRAMSimulator extends AMSimulator {
/*
- Vocabulary Used:
+ Vocabulary Used:
pending -> requests which are NOT yet sent to RM
scheduled -> requests which are sent to RM but not yet assigned
assigned -> requests which are assigned to a container
completed -> request corresponding to which container has completed
-
+
Maps are scheduled as soon as their requests are received. Reduces are
scheduled when all maps have finished (not support slow-start currently).
*/
-
+
private static final int PRIORITY_REDUCE = 10;
private static final int PRIORITY_MAP = 20;
// pending maps
private LinkedList<ContainerSimulator> pendingMaps =
new LinkedList<>();
-
+
// pending failed maps
private LinkedList<ContainerSimulator> pendingFailedMaps =
new LinkedList<ContainerSimulator>();
-
+
// scheduled maps
private LinkedList<ContainerSimulator> scheduledMaps =
new LinkedList<ContainerSimulator>();
-
+
// assigned maps
private Map<ContainerId, ContainerSimulator> assignedMaps =
new HashMap<ContainerId, ContainerSimulator>();
-
+
// reduces which are not yet scheduled
private LinkedList<ContainerSimulator> pendingReduces =
new LinkedList<ContainerSimulator>();
-
+
// pending failed reduces
private LinkedList<ContainerSimulator> pendingFailedReduces =
new LinkedList<ContainerSimulator>();
-
+
// scheduled reduces
private LinkedList<ContainerSimulator> scheduledReduces =
new LinkedList<ContainerSimulator>();
-
+
// assigned reduces
private Map<ContainerId, ContainerSimulator> assignedReduces =
new HashMap<ContainerId, ContainerSimulator>();
-
+
// all maps & reduces
private LinkedList<ContainerSimulator> allMaps =
new LinkedList<ContainerSimulator>();
@@ -117,14 +121,14 @@ public class MRAMSimulator extends AMSimulator {
@SuppressWarnings("checkstyle:parameternumber")
public void init(int heartbeatInterval,
List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
- long traceStartTime, long traceFinishTime, String user, String queue,
- boolean isTracked, String oldAppId, ReservationSubmissionRequest rr,
- long baselineStartTimeMS, Resource amContainerResource) {
+ long traceStartTime, long traceFinishTime, String user, String queue,
+ boolean isTracked, String oldAppId, long baselineStartTimeMS,
+ Resource amContainerResource) {
super.init(heartbeatInterval, containerList, rm, se,
traceStartTime, traceFinishTime, user, queue, isTracked, oldAppId,
- rr, baselineStartTimeMS, amContainerResource);
+ baselineStartTimeMS, amContainerResource);
amtype = "mapreduce";
-
+
// get map/reduce tasks
for (ContainerSimulator cs : containerList) {
if (cs.getType().equals("map")) {
@@ -202,7 +206,7 @@ public class MRAMSimulator extends AMSimulator {
}
}
}
-
+
// check finished
if (isAMContainerRunning &&
(mapFinished >= mapTotal) &&
@@ -234,7 +238,7 @@ public class MRAMSimulator extends AMSimulator {
}
}
}
-
+
/**
* restart running because of the am container killed
*/
@@ -322,7 +326,7 @@ public class MRAMSimulator extends AMSimulator {
if (ask == null) {
ask = new ArrayList<>();
}
-
+
final AllocateRequest request = createAllocateRequest(ask);
if (totalContainers == 0) {
request.setProgress(1.0f);
@@ -349,6 +353,38 @@ public class MRAMSimulator extends AMSimulator {
}
@Override
+ public void initReservation(ReservationId reservationId, long deadline,
+ long now) {
+
+ Resource mapRes = getMaxResource(allMaps);
+ long mapDur = getMaxDuration(allMaps);
+ Resource redRes = getMaxResource(allReduces);
+ long redDur = getMaxDuration(allReduces);
+
+ ReservationSubmissionRequest rr = ReservationClientUtil.
+ createMRReservation(reservationId,
+ "reservation_" + reservationId.getId(), mapRes, allMaps.size(),
+ mapDur, redRes, allReduces.size(), redDur, now + traceStartTimeMS,
+ now + deadline, queue);
+
+ setReservationRequest(rr);
+ }
+
+ // Helper to compute the component-wise maximum resource used by any container
+ private Resource getMaxResource(Collection<ContainerSimulator> containers) {
+ return containers.parallelStream()
+ .map(ContainerSimulator::getResource)
+ .reduce(Resource.newInstance(0, 0), Resources::componentwiseMax);
+ }
+
+ // Helper to compute the maximum resource used by any map container
+ private long getMaxDuration(Collection<ContainerSimulator> containers) {
+ return containers.parallelStream()
+ .mapToLong(ContainerSimulator::getLifeTime)
+ .reduce(0L, Long::max);
+ }
+
+ @Override
protected void checkStop() {
if (isFinished) {
super.setEndTime(System.currentTimeMillis());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16be42d3/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
index 567f0d9..abb3b5e 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
@@ -78,7 +78,7 @@ public class TestSLSRunner extends BaseSLSRunnerTest {
exitInvariantFile = "src/test/resources/exit-invariants.txt";
}
- @Test(timeout = 60000)
+ @Test(timeout = 90000)
@SuppressWarnings("all")
public void testSimulatorRunning() throws Exception {
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16be42d3/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
index 02dc26e..a67845b 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.sls.appmaster;
import com.codahale.metrics.MetricRegistry;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -90,6 +91,10 @@ public class TestAMSimulator {
}
@Override
+ public void initReservation(ReservationId id, long deadline, long now){
+ }
+
+ @Override
protected void checkStop() {
}
}
@@ -134,7 +139,7 @@ public class TestAMSimulator {
String queue = "default";
List<ContainerSimulator> containers = new ArrayList<>();
app.init(1000, containers, rm, null, 0, 1000000L, "user1", queue, true,
- appId, null, 0, SLSConfiguration.getAMContainerResource(conf));
+ appId, 0, SLSConfiguration.getAMContainerResource(conf));
app.firstStep();
verifySchedulerMetrics(appId);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: Revert "YARN-7537 [Atsv2] load
hbase configuration from filesystem rather than URL. Contributed by Rohith
Sharma"
Posted by ha...@apache.org.
Revert "YARN-7537 [Atsv2] load hbase configuration from filesystem rather than URL. Contributed by Rohith Sharma"
This reverts commit ec8f47e7fadbe62c0c39390d0a46cefd50e98492.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3e33aeb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3e33aeb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3e33aeb
Branch: refs/heads/YARN-1011
Commit: f3e33aeb80dca88d452ebe70028f95f0efede4dd
Parents: e7642a3
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Jan 24 15:04:15 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Jan 24 15:04:15 2018 +0530
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 2 +-
.../common/HBaseTimelineStorageUtils.java | 40 ++++++--------------
2 files changed, 13 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3e33aeb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 45aa635..1348d6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2463,7 +2463,7 @@
</property>
<property>
- <description> Optional FS path to an hbase-site.xml configuration file to be
+ <description> Optional URL to an hbase-site.xml configuration file to be
used to connect to the timeline-service hbase cluster. If empty or not
specified, then the HBase configuration will be loaded from the classpath.
When specified the values in the specified configuration file will override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3e33aeb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
index c25a0d2..c115b18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -18,14 +18,13 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -270,43 +269,28 @@ public final class HBaseTimelineStorageUtils {
* @return a configuration with the HBase configuration from the classpath,
* optionally overwritten by the timeline service configuration URL if
* specified.
- * @throws IOException if a timeline service HBase configuration path
- * is specified but unable to read it.
+ * @throws MalformedURLException if a timeline service HBase configuration URL
+ * is specified but is a malformed URL.
*/
public static Configuration getTimelineServiceHBaseConf(Configuration conf)
- throws IOException {
+ throws MalformedURLException {
if (conf == null) {
throw new NullPointerException();
}
Configuration hbaseConf;
- String timelineServiceHBaseConfFilePath =
+ String timelineServiceHBaseConfFileURL =
conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
- if (timelineServiceHBaseConfFilePath != null
- && timelineServiceHBaseConfFilePath.length() > 0) {
+ if (timelineServiceHBaseConfFileURL != null
+ && timelineServiceHBaseConfFileURL.length() > 0) {
LOG.info("Using hbase configuration at " +
- timelineServiceHBaseConfFilePath);
+ timelineServiceHBaseConfFileURL);
// create a clone so that we don't mess with out input one
hbaseConf = new Configuration(conf);
Configuration plainHBaseConf = new Configuration(false);
- FileSystem fs = null;
- FSDataInputStream in = null;
- try {
- Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath);
- fs = FileSystem.newInstance(conf);
- in = fs.open(hbaseConfigPath);
- plainHBaseConf.addResource(in);
- HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
- } catch (IOException e) {
- throw e;
- } finally {
- if (in != null) {
- in.close();
- }
- if (fs != null) {
- fs.close();
- }
- }
+ URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
+ plainHBaseConf.addResource(hbaseSiteXML);
+ HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
} else {
// default to what is on the classpath
hbaseConf = HBaseConfiguration.create(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-4511. Common scheduler
changes to support scheduler-specific oversubscription implementations.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 1836919..02c0cc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -563,7 +563,7 @@ public class TestNodeLabelContainerAllocation {
int numContainers) {
CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
SchedulerNode node = cs.getSchedulerNode(nodeId);
- Assert.assertEquals(numContainers, node.getNumContainers());
+ Assert.assertEquals(numContainers, node.getNumGuaranteedContainers());
}
/**
@@ -1065,7 +1065,7 @@ public class TestNodeLabelContainerAllocation {
for (int i = 0; i < 50; i++) {
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
- if (schedulerNode1.getNumContainers() == 0) {
+ if (schedulerNode1.getNumGuaranteedContainers() == 0) {
cycleWaited++;
}
}
@@ -1131,7 +1131,7 @@ public class TestNodeLabelContainerAllocation {
CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG
+ nodeIdStr + " ( Partition : [x]"));
Assert.assertEquals(0, cs.getSchedulerNode(nm1.getNodeId())
- .getNumContainers());
+ .getNumGuaranteedContainers());
rm1.close();
}
@@ -1215,7 +1215,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(10, schedulerNode1.getNumContainers());
+ Assert.assertEquals(10, schedulerNode1.getNumGuaranteedContainers());
// check non-exclusive containers of LeafQueue is correctly updated
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
@@ -1943,7 +1943,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(5, schedulerNode1.getNumContainers());
+ Assert.assertEquals(5, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2043,7 +2043,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x (non-exclusive)
- Assert.assertEquals(3, schedulerNode1.getNumContainers());
+ Assert.assertEquals(3, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2074,7 +2074,7 @@ public class TestNodeLabelContainerAllocation {
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
// app1 gets all resource in default partition
- Assert.assertEquals(2, schedulerNode2.getNumContainers());
+ Assert.assertEquals(2, schedulerNode2.getNumGuaranteedContainers());
// 3GB is used from label x quota. 2GB used from default label.
// So total 2.5 GB is remaining.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 95dbaea..6e65c14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -20,16 +20,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +58,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Test;
@@ -326,7 +338,9 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
for (FSSchedulerNode node : clusterNodeTracker.getAllNodes()) {
int i = ThreadLocalRandom.current().nextInt(-30, 30);
synchronized (scheduler) {
- node.deductUnallocatedResource(Resource.newInstance(i * 1024, i));
+ RMContainer container = createRMContainer(
+ node.getRMNode(), Resource.newInstance(i * 1024, 1));
+ node.allocateContainer(container);
}
}
}
@@ -340,6 +354,32 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
}
}
+ private static RMContainer createRMContainer(
+ RMNode node, Resource resource) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+ newInstance(ApplicationId.newInstance(0, 0), 0);
+ ContainerId cId =
+ ContainerId.newContainerId(appAttemptId, 0);
+ Container container = Container.newInstance(
+ cId, node.getNodeID(), node.getNodeAddress(), resource,
+ Priority.newInstance(0), null, ExecutionType.GUARANTEED);
+
+ Dispatcher dispatcher = new AsyncDispatcher();
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ when(rmContext.getSystemMetricsPublisher()).
+ thenReturn(new NoOpSystemMetricPublisher());
+ when(rmContext.getYarnConfiguration()).
+ thenReturn(new YarnConfiguration());
+ when(rmContext.getContainerAllocationExpirer()).
+ thenReturn(new ContainerAllocationExpirer(dispatcher));
+ when(rmContext.getRMApplicationHistoryWriter()).
+ thenReturn(new RMApplicationHistoryWriter());
+ return new RMContainerImpl(container, null,
+ container.getId().getApplicationAttemptId(),
+ node.getNodeID(), "test", rmContext);
+ }
+
@Test
public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 0e3d344..6726f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -87,7 +87,7 @@ public class TestFSSchedulerNode {
while (!Resources.isNone(schedulerNode.getUnallocatedResource())) {
createDefaultContainer();
schedulerNode.allocateContainer(containers.get(containers.size() - 1));
- schedulerNode.containerStarted(containers.get(containers.size() - 1).
+ schedulerNode.containerLaunched(containers.get(containers.size() - 1).
getContainerId());
}
}
@@ -183,9 +183,9 @@ public class TestFSSchedulerNode {
assertEquals("Nothing should have been allocated, yet",
Resources.none(), schedulerNode.getAllocatedResource());
schedulerNode.allocateContainer(containers.get(0));
- schedulerNode.containerStarted(containers.get(0).getContainerId());
+ schedulerNode.containerLaunched(containers.get(0).getContainerId());
schedulerNode.allocateContainer(containers.get(1));
- schedulerNode.containerStarted(containers.get(1).getContainerId());
+ schedulerNode.containerLaunched(containers.get(1).getContainerId());
schedulerNode.allocateContainer(containers.get(2));
assertEquals("Container should be allocated",
Resources.multiply(containers.get(0).getContainer().getResource(), 3.0),
@@ -225,7 +225,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -266,7 +266,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -312,7 +312,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -360,7 +360,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -399,7 +399,7 @@ public class TestFSSchedulerNode {
when(starvingApp.isStopped()).thenReturn(true);
allocateContainers(schedulerNode);
assertNotEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -437,7 +437,7 @@ public class TestFSSchedulerNode {
// Container partially reassigned
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- Resources.subtract(schedulerNode.getTotalResource(),
+ Resources.subtract(schedulerNode.getCapacity(),
Resource.newInstance(512, 0)),
schedulerNode.getAllocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 77b6d04..ef5de39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3293,12 +3293,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(node2UpdateEvent);
if (invalid) {
assertEquals(0, app.getLiveContainers().size());
- assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(0,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
} else {
assertEquals(1, app.getLiveContainers().size());
- assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(1,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
}
}
@@ -5024,7 +5028,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Check total resource of scheduler node is also changed to 0 GB 0 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 0);
// Check the available resource is 0/0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 3f97b59..63da0c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -340,7 +340,7 @@ public class TestFifoScheduler {
// SchedulerNode's total resource and available resource are changed.
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID())
- .getTotalResource().getMemorySize());
+ .getCapacity().getMemorySize());
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).
getUnallocatedResource().getMemorySize(), 1024);
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
@@ -1266,7 +1266,7 @@ public class TestFifoScheduler {
// Check total resource of scheduler node is also changed to 1 GB 1 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 1);
// Check the available resource is 0/0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index 3508ab4..ad3413f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,17 +23,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.log4j.Level;
@@ -84,13 +80,13 @@ public class TestContainerResourceUsage {
RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
- "Before app submittion, memory seconds should have been 0 but was "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() == 0);
+ "Before app submission, memory seconds should have been 0 but was "
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() == 0);
Assert.assertTrue(
"Before app submission, vcore seconds should have been 0 but was "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() == 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
@@ -105,7 +101,8 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
+ while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0
+ && cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
@@ -113,27 +110,35 @@ public class TestContainerResourceUsage {
rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
"While app is running, memory seconds should be >0 but is "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() > 0);
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() > 0);
Assert.assertTrue(
"While app is running, vcore seconds should be >0 but is "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() > 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() > 0);
MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ ru.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ ru.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ ru.getOpportunisticMemorySeconds(),
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ ru.getOpportunisticVcoreSeconds(),
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
}
- @Test (timeout = 120000)
+ @Test ()
public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
@@ -191,7 +196,7 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (app0.getRMAppMetrics().getMemorySeconds() <= 0
+ while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0
&& cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
@@ -216,19 +221,29 @@ public class TestContainerResourceUsage {
rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
// Check that the container metrics match those from the app usage report.
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
for (RMContainer c : rmContainers) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, metricsBefore.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, metricsBefore.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ metricsBefore.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ metricsBefore.getOpportunisticVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
@@ -238,10 +253,22 @@ public class TestContainerResourceUsage {
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
- Assert.assertEquals("Vcore seconds were not the same after RM Restart",
- metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
- Assert.assertEquals("Memory seconds were not the same after RM Restart",
- metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+ Assert.assertEquals(
+ "Guaranteed vcore seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedVcoreSeconds(),
+ metricsAfter.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(
+ "Guaranteed memory seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedMemorySeconds(),
+ metricsAfter.getGuaranteedMemorySeconds());
+ Assert.assertEquals(
+ "Opportunistic vcore seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticVcoreSeconds(),
+ metricsAfter.getOpportunisticVcoreSeconds());
+ Assert.assertEquals(
+ "Opportunistic memory seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticMemorySeconds(),
+ metricsAfter.getOpportunisticMemorySeconds());
rm0.stop();
rm0.close();
@@ -264,10 +291,8 @@ public class TestContainerResourceUsage {
MockRM rm = new MockRM(conf);
rm.start();
- RMApp app =
- rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
- null, "MAPREDUCE", false, keepRunningContainers);
+ RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false,
+ "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
MockNM nm =
new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
nm.registerNode();
@@ -275,18 +300,17 @@ public class TestContainerResourceUsage {
MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
int NUM_CONTAINERS = 1;
// allocate NUM_CONTAINERS containers
- am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
List<Container> containers =
- am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ am0.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
- containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers());
+ containers.addAll(am0.allocate(new ArrayList<>(),
+ new ArrayList<>()).getAllocatedContainers());
Thread.sleep(200);
}
@@ -294,26 +318,27 @@ public class TestContainerResourceUsage {
ContainerId containerId2 =
ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- containerId2.getContainerId(), ContainerState.RUNNING);
+ containerId2.getContainerId(), ContainerState.RUNNING);
rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
- Collection<RMContainer> rmContainers =
- rm.scheduler
- .getSchedulerAppInfo(am0.getApplicationAttemptId())
- .getLiveContainers();
+ Collection<RMContainer> rmContainers = rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId =
app.getCurrentAppAttempt().getMasterContainer().getId();
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- amContainerId.getContainerId(), ContainerState.COMPLETE);
+ amContainerId.getContainerId(), ContainerState.COMPLETE);
rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.drainEvents();
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers) {
@@ -321,8 +346,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
if (c.getContainerId().equals(amContainerId)) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
} else {
// The remaining container should be RUNNING.
Assert.assertTrue("After first attempt failed, remaining container "
@@ -336,8 +363,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
}
@@ -355,14 +384,12 @@ public class TestContainerResourceUsage {
am1.registerAppAttempt();
rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// allocate NUM_CONTAINERS containers
- am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
- containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ containers = am1.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
@@ -388,19 +415,26 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
- return;
}
private void waitforContainerCompletion(MockRM rm, MockNM nm,
@@ -419,16 +453,10 @@ public class TestContainerResourceUsage {
private AggregateAppResourceUsage calculateContainerResourceMetrics(
RMContainer rmContainer) {
- Resource resource = rmContainer.getContainer().getResource();
- long usedMillis =
- rmContainer.getFinishTime() - rmContainer.getCreationTime();
- long memorySeconds = resource.getMemorySize()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- long vcoreSeconds = resource.getVirtualCores()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- Map<String, Long> map = new HashMap<>();
- map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- return new AggregateAppResourceUsage(map);
+ ContainerResourceUsageReport resourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ return new AggregateAppResourceUsage(
+ resourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ resourceUsageReport.getOpportunisticResourceSecondsMap());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 72de27c..3c2db3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -193,7 +193,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
}
@Override
@@ -353,7 +353,7 @@ public abstract class MockAsm extends MockApps {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
- new HashMap<>());
+ new HashMap<>(), null, new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
index 830d01c..d561cab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
@@ -453,7 +453,7 @@ public class TestCombinedSystemMetricsPublisher {
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(rmApp.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap,
- preemptedMap));
+ preemptedMap, new HashMap<>()));
when(rmApp.getApplicationTags()).thenReturn(
Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 2287617..c9e2d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
- app.getRMAppMetrics().getMemorySeconds(),
+ app.getRMAppMetrics().getGuaranteedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getGuaranteedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
- app.getRMAppMetrics().getVcoreSeconds(),
+ app.getRMAppMetrics().getOpportunisticMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getOpportunisticVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
app.getRMAppMetrics().getPreemptedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
@@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher {
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
- .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
+ .thenReturn(new RMAppMetrics(null, 0, 0,
+ resourceMap, preemptedMap, new HashMap<>()));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 68bb325..7aa1a60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0);
}
@Test(timeout = 10000)
@@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
- resourceSecondsMap
+ guaranteedResourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
- new HashMap<>()));
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 453d805..98199a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -194,7 +194,8 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -292,7 +293,8 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -369,7 +371,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -393,7 +396,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 6a8f47d..4d2c2da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -543,7 +543,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<>(),
+ new HashMap<>(0)));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -555,7 +556,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, new HashMap<>(), new HashMap<>());
+ finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -792,20 +793,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), 0L);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- 0L);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
+ Map<String, Long> guaranteedResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> opportunisticResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> preemptedResoureSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
+ amExitStatus, 0, guaranteedResourceSecondsMap,
+ preemptedResoureSecondsMap, opportunisticResourceSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 920bd4f..dde74a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -243,7 +242,7 @@ public class TestRMAppTransitions {
ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
doReturn(null).when(resourceScheduler)
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ .getAppActiveResourceUsageReport(Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
@@ -1249,7 +1248,8 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
- Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
+ Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT,
+ report.getApplicationResourceUsageReport());
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
Assert.assertTrue("bad proxy url for app",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 0e4f308..d889158 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions {
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L);
when(resourceScheduler
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any()))
.thenReturn(appResUsgRpt);
spyRMContext = spy(rmContext);
Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
@@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions {
ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
- when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
- .thenReturn(appResUsgRpt);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppActiveResourceUsageReport(
+ any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
// start and finish the attempt
Container amContainer = allocateApplicationAttempt();
@@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions {
// expect usage stats to come from the scheduler report
ApplicationResourceUsageReport report =
applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(123456L, report.getMemorySeconds());
- Assert.assertEquals(55544L, report.getVcoreSeconds());
+ Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds());
+ Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds());
// finish app attempt and remove it from scheduler
- when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions {
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
report = applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(223456, report.getMemorySeconds());
- Assert.assertEquals(75544, report.getVcoreSeconds());
+ Assert.assertEquals(223456, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 849a2c3..7748b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -103,7 +103,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -2859,7 +2858,7 @@ public class TestCapacityScheduler {
Thread.sleep(1000);
AllocateResponse allocResponse = am1.schedule();
ApplicationResourceUsageReport report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
@@ -2872,7 +2871,7 @@ public class TestCapacityScheduler {
allocResponse = am1.schedule();
}
report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index b4ebd15..a3ae7bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate {
RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
ApplicationResourceUsageReport appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(1024, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(1, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
// request a container.
am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(2048, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(2, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
LeafQueue queue =
(LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
.getQueue("a");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index f88ac8b..de97ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -66,7 +66,7 @@ public class TestAppPage {
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index b125608..93efe23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Token;
@@ -259,7 +258,7 @@ public class TestRMWebApp {
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
- (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
+ null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index cea0088..87846c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
- new HashMap<>(), new HashMap<>());
+ new HashMap<>(), new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index a381ed4..754d8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1603,7 +1603,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 44 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 40bdbd8..6f1ee3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -339,12 +339,22 @@ public final class RouterWebServiceUtil {
am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB());
am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores());
+ am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() +
+ uam.getAllocatedOpportunisticMB());
+ am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() +
+ uam.getAllocatedOpportunisticVCores());
am.setReservedMB(am.getReservedMB() + uam.getReservedMB());
am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB());
am.setRunningContainers(
am.getRunningContainers() + uam.getRunningContainers());
- am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds());
- am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds());
+ am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() +
+ uam.getGuaranteedMemorySeconds());
+ am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() +
+ uam.getGuaranteedVcoreSeconds());
+ am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() +
+ uam.getOpportunisticVcoreSeconds());
+ am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() +
+ uam.getOpportunisticMemorySeconds());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804..1a5c42e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil {
Assert.assertEquals(value * 3, app.getReservedMB());
Assert.assertEquals(value * 3, app.getReservedVCores());
Assert.assertEquals(value * 3, app.getRunningContainers());
- Assert.assertEquals(value * 3, app.getMemorySeconds());
- Assert.assertEquals(value * 3, app.getVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds());
Assert.assertEquals(3, app.getResourceRequests().size());
}
@@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil {
am.getResourceRequests().add(new ResourceRequestInfo());
am.setAllocatedMB(value);
am.setAllocatedVCores(value);
+ am.setAllocatedOpportunisticVCores(value);
+ am.setAllocatedOpportunisticMB(value);
am.setReservedMB(value);
am.setReservedVCores(value);
am.setRunningContainers(value);
- am.setMemorySeconds(value);
- am.setVcoreSeconds(value);
+ am.setGuaranteedMemorySeconds(value);
+ am.setGuaranteedVcoreSeconds(value);
+ am.setOpportunisticMemorySeconds(value);
+ am.setOpportunisticVcoreSeconds(value);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-7796. Container-executor
fails with segfault on certain OS configurations. Contributed by Gergo Repas.
Posted by ha...@apache.org.
YARN-7796. Container-executor fails with segfault on certain OS configurations. Contributed by Gergo Repas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7642a3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7642a3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7642a3e
Branch: refs/heads/YARN-1011
Commit: e7642a3e6f540b4b56367babfbaf35ee6b3c7675
Parents: 95743c6
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Jan 23 21:07:05 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Tue Jan 23 21:07:05 2018 -0800
----------------------------------------------------------------------
.../impl/container-executor.c | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7642a3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index b0b8e76..5ce6a00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1008,13 +1008,20 @@ static int open_file_as_nm(const char* filename) {
static int copy_file(int input, const char* in_filename,
const char* out_filename, mode_t perm) {
const int buffer_size = 128*1024;
- char buffer[buffer_size];
+ char* buffer = malloc(buffer_size);
+ if (buffer == NULL) {
+ fprintf(LOGFILE, "Failed to allocate buffer while copying file: %s -> %s",
+ in_filename, out_filename);
+ fflush(LOGFILE);
+ return -1;
+ }
int out_fd = open(out_filename, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW, perm);
if (out_fd == -1) {
fprintf(LOGFILE, "Can't open %s for output - %s\n", out_filename,
strerror(errno));
fflush(LOGFILE);
+ free(buffer);
return -1;
}
@@ -1024,15 +1031,18 @@ static int copy_file(int input, const char* in_filename,
while (pos < len) {
ssize_t write_result = write(out_fd, buffer + pos, len - pos);
if (write_result <= 0) {
- fprintf(LOGFILE, "Error writing to %s - %s\n", out_filename,
- strerror(errno));
- close(out_fd);
- return -1;
+ fprintf(LOGFILE, "Error writing to %s - %s\n", out_filename,
+ strerror(errno));
+ close(out_fd);
+ free(buffer);
+ return -1;
}
pos += write_result;
}
len = read(input, buffer, buffer_size);
}
+ free(buffer);
+
if (len < 0) {
fprintf(LOGFILE, "Failed to read file %s - %s\n", in_filename,
strerror(errno));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-2185. Use pipes when
localizing archives. Contributed by Miklos Szegedi
Posted by ha...@apache.org.
YARN-2185. Use pipes when localizing archives. Contributed by Miklos Szegedi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b0f265d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b0f265d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b0f265d
Branch: refs/heads/YARN-1011
Commit: 1b0f265db1a5bfccf1d870912237ea9618bd9c34
Parents: f2fa736
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 26 13:25:20 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 26 13:25:20 2018 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 251 ++++++++++++++++++-
.../java/org/apache/hadoop/util/RunJar.java | 65 +++++
.../org/apache/hadoop/yarn/util/FSDownload.java | 215 ++++++++++------
.../apache/hadoop/yarn/util/TestFSDownload.java | 30 ++-
4 files changed, 462 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b0f265d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 4d971aa..bf9b146 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -20,27 +20,35 @@ package org.apache.hadoop.fs;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
+import java.nio.charset.Charset;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.jar.Attributes;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
@@ -75,6 +83,11 @@ public class FileUtil {
public static final int SYMLINK_NO_PRIVILEGE = 2;
/**
+ * Buffer size for copy the content of compressed file to new file.
+ */
+ private static final int BUFFER_SIZE = 8_192;
+
+ /**
* convert an array of FileStatus to an array of Path
*
* @param stats
@@ -526,6 +539,22 @@ public class FileUtil {
}
/**
+ * Convert a os-native filename to a path that works for the shell
+ * and avoids script injection attacks.
+ * @param file The filename to convert
+ * @return The unix pathname
+ * @throws IOException on windows, there can be problems with the subprocess
+ */
+ public static String makeSecureShellPath(File file) throws IOException {
+ if (Shell.WINDOWS) {
+ // Currently it is never called, but it might be helpful in the future.
+ throw new UnsupportedOperationException("Not implemented for Windows");
+ } else {
+ return makeShellPath(file, false).replace("'", "'\\''");
+ }
+ }
+
+ /**
* Convert a os-native filename to a path that works for the shell.
* @param file The filename to convert
* @param makeCanonicalPath
@@ -576,11 +605,48 @@ public class FileUtil {
}
/**
- * Given a File input it will unzip the file in a the unzip directory
+ * Given a stream input it will unzip the it in the unzip directory.
+ * passed as the second parameter
+ * @param inputStream The zip file as input
+ * @param toDir The unzip directory where to unzip the zip file.
+ * @throws IOException an exception occurred
+ */
+ public static void unZip(InputStream inputStream, File toDir)
+ throws IOException {
+ try (ZipInputStream zip = new ZipInputStream(inputStream)) {
+ int numOfFailedLastModifiedSet = 0;
+ for(ZipEntry entry = zip.getNextEntry();
+ entry != null;
+ entry = zip.getNextEntry()) {
+ if (!entry.isDirectory()) {
+ File file = new File(toDir, entry.getName());
+ File parent = file.getParentFile();
+ if (!parent.mkdirs() &&
+ !parent.isDirectory()) {
+ throw new IOException("Mkdirs failed to create " +
+ parent.getAbsolutePath());
+ }
+ try (OutputStream out = new FileOutputStream(file)) {
+ IOUtils.copyBytes(zip, out, BUFFER_SIZE);
+ }
+ if (!file.setLastModified(entry.getTime())) {
+ numOfFailedLastModifiedSet++;
+ }
+ }
+ }
+ if (numOfFailedLastModifiedSet > 0) {
+ LOG.warn("Could not set last modfied time for {} file(s)",
+ numOfFailedLastModifiedSet);
+ }
+ }
+ }
+
+ /**
+ * Given a File input it will unzip it in the unzip directory.
* passed as the second parameter
* @param inFile The zip file as input
* @param unzipDir The unzip directory where to unzip the zip file.
- * @throws IOException
+ * @throws IOException An I/O exception has occurred
*/
public static void unZip(File inFile, File unzipDir) throws IOException {
Enumeration<? extends ZipEntry> entries;
@@ -621,6 +687,138 @@ public class FileUtil {
}
/**
+ * Run a command and send the contents of an input stream to it.
+ * @param inputStream Input stream to forward to the shell command
+ * @param command shell command to run
+ * @throws IOException read or write failed
+ * @throws InterruptedException command interrupted
+ * @throws ExecutionException task submit failed
+ */
+ private static void runCommandOnStream(
+ InputStream inputStream, String command)
+ throws IOException, InterruptedException, ExecutionException {
+ ExecutorService executor = null;
+ ProcessBuilder builder = new ProcessBuilder();
+ builder.command(
+ Shell.WINDOWS ? "cmd" : "bash",
+ Shell.WINDOWS ? "/c" : "-c",
+ command);
+ Process process = builder.start();
+ int exitCode;
+ try {
+ // Consume stdout and stderr, to avoid blocking the command
+ executor = Executors.newFixedThreadPool(2);
+ Future output = executor.submit(() -> {
+ try {
+ // Read until the output stream receives an EOF and closed.
+ if (LOG.isDebugEnabled()) {
+ // Log directly to avoid out of memory errors
+ try (BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(process.getInputStream(),
+ Charset.forName("UTF-8")))) {
+ String line;
+ while((line = reader.readLine()) != null) {
+ LOG.debug(line);
+ }
+ }
+ } else {
+ org.apache.commons.io.IOUtils.copy(
+ process.getInputStream(),
+ new IOUtils.NullOutputStream());
+ }
+ } catch (IOException e) {
+ LOG.debug(e.getMessage());
+ }
+ });
+ Future error = executor.submit(() -> {
+ try {
+ // Read until the error stream receives an EOF and closed.
+ if (LOG.isDebugEnabled()) {
+ // Log directly to avoid out of memory errors
+ try (BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(process.getErrorStream(),
+ Charset.forName("UTF-8")))) {
+ String line;
+ while((line = reader.readLine()) != null) {
+ LOG.debug(line);
+ }
+ }
+ } else {
+ org.apache.commons.io.IOUtils.copy(
+ process.getErrorStream(),
+ new IOUtils.NullOutputStream());
+ }
+ } catch (IOException e) {
+ LOG.debug(e.getMessage());
+ }
+ });
+
+ // Pass the input stream to the command to process
+ try {
+ org.apache.commons.io.IOUtils.copy(
+ inputStream, process.getOutputStream());
+ } finally {
+ process.getOutputStream().close();
+ }
+
+ // Wait for both stdout and stderr futures to finish
+ error.get();
+ output.get();
+ } finally {
+ // Clean up the threads
+ if (executor != null) {
+ executor.shutdown();
+ }
+ // Wait to avoid leaking the child process
+ exitCode = process.waitFor();
+ }
+
+ if (exitCode != 0) {
+ throw new IOException(
+ String.format(
+ "Error executing command. %s " +
+ "Process exited with exit code %d.",
+ command, exitCode));
+ }
+ }
+
+ /**
+ * Given a Tar File as input it will untar the file in a the untar directory
+ * passed as the second parameter
+ *
+ * This utility will untar ".tar" files and ".tar.gz","tgz" files.
+ *
+ * @param inputStream The tar file as input.
+ * @param untarDir The untar directory where to untar the tar file.
+ * @param gzipped The input stream is gzipped
+ * TODO Use magic number and PusbackInputStream to identify
+ * @throws IOException an exception occurred
+ * @throws InterruptedException command interrupted
+ * @throws ExecutionException task submit failed
+ */
+ public static void unTar(InputStream inputStream, File untarDir,
+ boolean gzipped)
+ throws IOException, InterruptedException, ExecutionException {
+ if (!untarDir.mkdirs()) {
+ if (!untarDir.isDirectory()) {
+ throw new IOException("Mkdirs failed to create " + untarDir);
+ }
+ }
+
+ if(Shell.WINDOWS) {
+ // Tar is not native to Windows. Use simple Java based implementation for
+ // tests and simple tar archives
+ unTarUsingJava(inputStream, untarDir, gzipped);
+ } else {
+ // spawn tar utility to untar archive for full fledged unix behavior such
+ // as resolving symlinks in tar archives
+ unTarUsingTar(inputStream, untarDir, gzipped);
+ }
+ }
+
+ /**
* Given a Tar File as input it will untar the file in a the untar directory
* passed as the second parameter
*
@@ -650,23 +848,41 @@ public class FileUtil {
}
}
+ private static void unTarUsingTar(InputStream inputStream, File untarDir,
+ boolean gzipped)
+ throws IOException, InterruptedException, ExecutionException {
+ StringBuilder untarCommand = new StringBuilder();
+ if (gzipped) {
+ untarCommand.append("gzip -dc | (");
+ }
+ untarCommand.append("cd '");
+ untarCommand.append(FileUtil.makeSecureShellPath(untarDir));
+ untarCommand.append("' && ");
+ untarCommand.append("tar -x ");
+
+ if (gzipped) {
+ untarCommand.append(")");
+ }
+ runCommandOnStream(inputStream, untarCommand.toString());
+ }
+
private static void unTarUsingTar(File inFile, File untarDir,
boolean gzipped) throws IOException {
StringBuffer untarCommand = new StringBuffer();
if (gzipped) {
untarCommand.append(" gzip -dc '");
- untarCommand.append(FileUtil.makeShellPath(inFile));
+ untarCommand.append(FileUtil.makeSecureShellPath(inFile));
untarCommand.append("' | (");
}
untarCommand.append("cd '");
- untarCommand.append(FileUtil.makeShellPath(untarDir));
- untarCommand.append("' ; ");
+ untarCommand.append(FileUtil.makeSecureShellPath(untarDir));
+ untarCommand.append("' && ");
untarCommand.append("tar -xf ");
if (gzipped) {
untarCommand.append(" -)");
} else {
- untarCommand.append(FileUtil.makeShellPath(inFile));
+ untarCommand.append(FileUtil.makeSecureShellPath(inFile));
}
String[] shellCmd = { "bash", "-c", untarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
@@ -701,6 +917,29 @@ public class FileUtil {
}
}
+ private static void unTarUsingJava(InputStream inputStream, File untarDir,
+ boolean gzipped) throws IOException {
+ TarArchiveInputStream tis = null;
+ try {
+ if (gzipped) {
+ inputStream = new BufferedInputStream(new GZIPInputStream(
+ inputStream));
+ } else {
+ inputStream =
+ new BufferedInputStream(inputStream);
+ }
+
+ tis = new TarArchiveInputStream(inputStream);
+
+ for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
+ unpackEntries(tis, entry, untarDir);
+ entry = tis.getNextTarEntry();
+ }
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, tis, inputStream);
+ }
+ }
+
private static void unpackEntries(TarArchiveInputStream tis,
TarArchiveEntry entry, File outputDir) throws IOException {
if (entry.isDirectory()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b0f265d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 19b51ad..89b7d76 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -34,9 +34,11 @@ import java.util.Enumeration;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
import java.util.jar.Manifest;
import java.util.regex.Pattern;
+import org.apache.commons.io.input.TeeInputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileUtil;
@@ -98,6 +100,69 @@ public class RunJar {
* Unpack matching files from a jar. Entries inside the jar that do
* not match the given pattern will be skipped.
*
+ * @param inputStream the jar stream to unpack
+ * @param toDir the destination directory into which to unpack the jar
+ * @param unpackRegex the pattern to match jar entries against
+ *
+ * @throws IOException if an I/O error has occurred or toDir
+ * cannot be created and does not already exist
+ */
+ public static void unJar(InputStream inputStream, File toDir,
+ Pattern unpackRegex)
+ throws IOException {
+ try (JarInputStream jar = new JarInputStream(inputStream)) {
+ int numOfFailedLastModifiedSet = 0;
+ for (JarEntry entry = jar.getNextJarEntry();
+ entry != null;
+ entry = jar.getNextJarEntry()) {
+ if (!entry.isDirectory() &&
+ unpackRegex.matcher(entry.getName()).matches()) {
+ File file = new File(toDir, entry.getName());
+ ensureDirectory(file.getParentFile());
+ try (OutputStream out = new FileOutputStream(file)) {
+ IOUtils.copyBytes(jar, out, BUFFER_SIZE);
+ }
+ if (!file.setLastModified(entry.getTime())) {
+ numOfFailedLastModifiedSet++;
+ }
+ }
+ }
+ if (numOfFailedLastModifiedSet > 0) {
+ LOG.warn("Could not set last modfied time for {} file(s)",
+ numOfFailedLastModifiedSet);
+ }
+ }
+ }
+
+ /**
+ * Unpack matching files from a jar. Entries inside the jar that do
+ * not match the given pattern will be skipped. Keep also a copy
+ * of the entire jar in the same directory for backward compatibility.
+ * TODO remove this feature in a new release and do only unJar
+ *
+ * @param inputStream the jar stream to unpack
+ * @param toDir the destination directory into which to unpack the jar
+ * @param unpackRegex the pattern to match jar entries against
+ *
+ * @throws IOException if an I/O error has occurred or toDir
+ * cannot be created and does not already exist
+ */
+ @Deprecated
+ public static void unJarAndSave(InputStream inputStream, File toDir,
+ String name, Pattern unpackRegex)
+ throws IOException{
+ File file = new File(toDir, name);
+ ensureDirectory(toDir);
+ try (OutputStream jar = new FileOutputStream(file);
+ TeeInputStream teeInputStream = new TeeInputStream(inputStream, jar)) {
+ unJar(teeInputStream, toDir, unpackRegex);
+ }
+ }
+
+ /**
+ * Unpack matching files from a jar. Entries inside the jar that do
+ * not match the given pattern will be skipped.
+ *
* @param jarFile the .jar file to unpack
* @param toDir the destination directory into which to unpack the jar
* @param unpackRegex the pattern to match jar entries against
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b0f265d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 6e59574..1a60948 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.Callable;
@@ -29,6 +31,7 @@ import java.util.concurrent.Future;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -54,6 +57,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.Futures;
+import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* Download a single URL to the local disk.
@@ -247,9 +251,21 @@ public class FSDownload implements Callable<Path> {
}
}
- private Path copy(Path sCopy, Path dstdir) throws IOException {
+ /**
+ * Localize files.
+ * @param destination destination directory
+ * @throws IOException cannot read or write file
+ * @throws YarnException subcommand returned an error
+ */
+ private void verifyAndCopy(Path destination)
+ throws IOException, YarnException {
+ final Path sCopy;
+ try {
+ sCopy = resource.getResource().toPath();
+ } catch (URISyntaxException e) {
+ throw new IOException("Invalid resource", e);
+ }
FileSystem sourceFs = sCopy.getFileSystem(conf);
- Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName());
FileStatus sStat = sourceFs.getFileStatus(sCopy);
if (sStat.getModificationTime() != resource.getTimestamp()) {
throw new IOException("Resource " + sCopy +
@@ -264,82 +280,108 @@ public class FSDownload implements Callable<Path> {
}
}
- FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false,
- true, conf);
- return dCopy;
+ downloadAndUnpack(sCopy, destination);
}
- private long unpack(File localrsrc, File dst) throws IOException {
- switch (resource.getType()) {
- case ARCHIVE: {
- String lowerDst = StringUtils.toLowerCase(dst.getName());
- if (lowerDst.endsWith(".jar")) {
- RunJar.unJar(localrsrc, dst);
- } else if (lowerDst.endsWith(".zip")) {
- FileUtil.unZip(localrsrc, dst);
- } else if (lowerDst.endsWith(".tar.gz") ||
- lowerDst.endsWith(".tgz") ||
- lowerDst.endsWith(".tar")) {
- FileUtil.unTar(localrsrc, dst);
+ /**
+ * Copy source path to destination with localization rules.
+ * @param source source path to copy. Typically HDFS
+ * @param destination destination path. Typically local filesystem
+ * @exception YarnException Any error has occurred
+ */
+ private void downloadAndUnpack(Path source, Path destination)
+ throws YarnException {
+ try {
+ FileSystem sourceFileSystem = source.getFileSystem(conf);
+ FileSystem destinationFileSystem = destination.getFileSystem(conf);
+ if (sourceFileSystem.getFileStatus(source).isDirectory()) {
+ FileUtil.copy(
+ sourceFileSystem, source,
+ destinationFileSystem, destination, false,
+ true, conf);
} else {
- LOG.warn("Cannot unpack " + localrsrc);
- if (!localrsrc.renameTo(dst)) {
- throw new IOException("Unable to rename file: [" + localrsrc
- + "] to [" + dst + "]");
- }
+ unpack(source, destination, sourceFileSystem, destinationFileSystem);
}
+ } catch (Exception e) {
+ throw new YarnException("Download and unpack failed", e);
}
- break;
- case PATTERN: {
+ }
+
+ /**
+ * Do the localization action on the input stream.
+ * We use the deprecated method RunJar.unJarAndSave for compatibility reasons.
+ * We should use the more efficient RunJar.unJar in the future.
+ * @param source Source path
+ * @param destination Destination pth
+ * @param sourceFileSystem Source filesystem
+ * @param destinationFileSystem Destination filesystem
+ * @throws IOException Could not read or write stream
+ * @throws InterruptedException Operation interrupted by caller
+ * @throws ExecutionException Could not create thread pool execution
+ */
+ @SuppressWarnings("deprecation")
+ private void unpack(Path source, Path destination,
+ FileSystem sourceFileSystem,
+ FileSystem destinationFileSystem)
+ throws IOException, InterruptedException, ExecutionException {
+ try (InputStream inputStream = sourceFileSystem.open(source)) {
+ File dst = new File(destination.toUri());
String lowerDst = StringUtils.toLowerCase(dst.getName());
- if (lowerDst.endsWith(".jar")) {
- String p = resource.getPattern();
- RunJar.unJar(localrsrc, dst,
- p == null ? RunJar.MATCH_ANY : Pattern.compile(p));
- File newDst = new File(dst, dst.getName());
- if (!dst.exists() && !dst.mkdir()) {
- throw new IOException("Unable to create directory: [" + dst + "]");
+ switch (resource.getType()) {
+ case ARCHIVE:
+ if (lowerDst.endsWith(".jar")) {
+ RunJar.unJar(inputStream, dst, RunJar.MATCH_ANY);
+ } else if (lowerDst.endsWith(".zip")) {
+ FileUtil.unZip(inputStream, dst);
+ } else if (lowerDst.endsWith(".tar.gz") ||
+ lowerDst.endsWith(".tgz") ||
+ lowerDst.endsWith(".tar")) {
+ FileUtil.unTar(inputStream, dst, lowerDst.endsWith("gz"));
+ } else {
+ LOG.warn("Cannot unpack " + source);
+ try (OutputStream outputStream =
+ destinationFileSystem.create(destination, true)) {
+ IOUtils.copy(inputStream, outputStream);
+ }
}
- if (!localrsrc.renameTo(newDst)) {
- throw new IOException("Unable to rename file: [" + localrsrc
- + "] to [" + newDst + "]");
+ break;
+ case PATTERN:
+ if (lowerDst.endsWith(".jar")) {
+ String p = resource.getPattern();
+ if (!dst.exists() && !dst.mkdir()) {
+ throw new IOException("Unable to create directory: [" + dst + "]");
+ }
+ RunJar.unJarAndSave(inputStream, dst, source.getName(),
+ p == null ? RunJar.MATCH_ANY : Pattern.compile(p));
+ } else if (lowerDst.endsWith(".zip")) {
+ LOG.warn("Treating [" + source + "] as an archive even though it " +
+ "was specified as PATTERN");
+ FileUtil.unZip(inputStream, dst);
+ } else if (lowerDst.endsWith(".tar.gz") ||
+ lowerDst.endsWith(".tgz") ||
+ lowerDst.endsWith(".tar")) {
+ LOG.warn("Treating [" + source + "] as an archive even though it " +
+ "was specified as PATTERN");
+ FileUtil.unTar(inputStream, dst, lowerDst.endsWith("gz"));
+ } else {
+ LOG.warn("Cannot unpack " + source);
+ try (OutputStream outputStream =
+ destinationFileSystem.create(destination, true)) {
+ IOUtils.copy(inputStream, outputStream);
+ }
}
- } else if (lowerDst.endsWith(".zip")) {
- LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
- "was specified as PATTERN");
- FileUtil.unZip(localrsrc, dst);
- } else if (lowerDst.endsWith(".tar.gz") ||
- lowerDst.endsWith(".tgz") ||
- lowerDst.endsWith(".tar")) {
- LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
- "was specified as PATTERN");
- FileUtil.unTar(localrsrc, dst);
- } else {
- LOG.warn("Cannot unpack " + localrsrc);
- if (!localrsrc.renameTo(dst)) {
- throw new IOException("Unable to rename file: [" + localrsrc
- + "] to [" + dst + "]");
+ break;
+ case FILE:
+ default:
+ try (OutputStream outputStream =
+ destinationFileSystem.create(destination, true)) {
+ IOUtils.copy(inputStream, outputStream);
}
+ break;
}
+ // TODO Should calculate here before returning
+ //return FileUtil.getDU(destDir);
}
- break;
- case FILE:
- default:
- if (!localrsrc.renameTo(dst)) {
- throw new IOException("Unable to rename file: [" + localrsrc
- + "] to [" + dst + "]");
- }
- break;
- }
- if(localrsrc.isFile()){
- try {
- files.delete(new Path(localrsrc.toString()), false);
- } catch (IOException ignore) {
- }
- }
- return 0;
- // TODO Should calculate here before returning
- //return FileUtil.getDU(destDir);
}
@Override
@@ -352,27 +394,34 @@ public class FSDownload implements Callable<Path> {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Starting to download " + sCopy);
+ LOG.debug(String.format("Starting to download %s %s %s",
+ sCopy,
+ resource.getType(),
+ resource.getPattern()));
}
- createDir(destDirPath, cachePerms);
- final Path dst_work = new Path(destDirPath + "_tmp");
- createDir(dst_work, cachePerms);
- Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName()));
+ final Path destinationTmp = new Path(destDirPath + "_tmp");
+ createDir(destinationTmp, PRIVATE_DIR_PERMS);
+ Path dFinal =
+ files.makeQualified(new Path(destinationTmp, sCopy.getName()));
try {
- Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work))
- : userUgi.doAs(new PrivilegedExceptionAction<Path>() {
- public Path run() throws Exception {
- return files.makeQualified(copy(sCopy, dst_work));
- };
- });
- unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
+ if (userUgi == null) {
+ verifyAndCopy(dFinal);
+ } else {
+ userUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ verifyAndCopy(dFinal);
+ return null;
+ }
+ });
+ }
changePermissions(dFinal.getFileSystem(conf), dFinal);
- files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+ files.rename(destinationTmp, destDirPath, Rename.OVERWRITE);
if (LOG.isDebugEnabled()) {
- LOG.debug("File has been downloaded to " +
- new Path(destDirPath, sCopy.getName()));
+ LOG.debug(String.format("File has been downloaded to %s from %s",
+ new Path(destDirPath, sCopy.getName()), sCopy));
}
} catch (Exception e) {
try {
@@ -382,7 +431,7 @@ public class FSDownload implements Callable<Path> {
throw e;
} finally {
try {
- files.delete(dst_work, true);
+ files.delete(destinationTmp, true);
} catch (FileNotFoundException ignore) {
}
conf = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b0f265d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 877dd08..fa8c039 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -82,6 +82,9 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+/**
+ * Unit test for the FSDownload class.
+ */
public class TestFSDownload {
private static final Log LOG = LogFactory.getLog(TestFSDownload.class);
@@ -90,7 +93,8 @@ public class TestFSDownload {
private enum TEST_FILE_TYPE {
TAR, JAR, ZIP, TGZ
};
-
+ private Configuration conf = new Configuration();
+
@AfterClass
public static void deleteTestDir() throws IOException {
FileContext fs = FileContext.getLocalFSFileContext();
@@ -132,6 +136,18 @@ public class TestFSDownload {
FileOutputStream stream = new FileOutputStream(jarFile);
LOG.info("Create jar out stream ");
JarOutputStream out = new JarOutputStream(stream, new Manifest());
+ ZipEntry entry = new ZipEntry("classes/1.class");
+ out.putNextEntry(entry);
+ out.write(1);
+ out.write(2);
+ out.write(3);
+ out.closeEntry();
+ ZipEntry entry2 = new ZipEntry("classes/2.class");
+ out.putNextEntry(entry2);
+ out.write(1);
+ out.write(2);
+ out.write(3);
+ out.closeEntry();
LOG.info("Done writing jar stream ");
out.close();
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
@@ -256,7 +272,6 @@ public class TestFSDownload {
@Test (timeout=10000)
public void testDownloadBadPublic() throws IOException, URISyntaxException,
InterruptedException {
- Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
FileContext files = FileContext.getLocalFSFileContext(conf);
final Path basedir = files.makeQualified(new Path("target",
@@ -307,7 +322,6 @@ public class TestFSDownload {
@Test (timeout=60000)
public void testDownloadPublicWithStatCache() throws IOException,
URISyntaxException, InterruptedException, ExecutionException {
- final Configuration conf = new Configuration();
FileContext files = FileContext.getLocalFSFileContext(conf);
Path basedir = files.makeQualified(new Path("target",
TestFSDownload.class.getSimpleName()));
@@ -382,7 +396,6 @@ public class TestFSDownload {
@Test (timeout=10000)
public void testDownload() throws IOException, URISyntaxException,
InterruptedException {
- Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
FileContext files = FileContext.getLocalFSFileContext(conf);
final Path basedir = files.makeQualified(new Path("target",
@@ -438,7 +451,7 @@ public class TestFSDownload {
FileStatus status = files.getFileStatus(localized.getParent());
FsPermission perm = status.getPermission();
assertEquals("Cache directory permissions are incorrect",
- new FsPermission((short)0755), perm);
+ new FsPermission((short)0700), perm);
status = files.getFileStatus(localized);
perm = status.getPermission();
@@ -455,7 +468,6 @@ public class TestFSDownload {
private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
URISyntaxException, InterruptedException{
- Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
FileContext files = FileContext.getLocalFSFileContext(conf);
final Path basedir = files.makeQualified(new Path("target",
@@ -530,7 +542,7 @@ public class TestFSDownload {
}
}
- @Test (timeout=10000)
+ @Test (timeout=10000)
public void testDownloadArchive() throws IOException, URISyntaxException,
InterruptedException {
downloadWithFileType(TEST_FILE_TYPE.TAR);
@@ -542,7 +554,7 @@ public class TestFSDownload {
downloadWithFileType(TEST_FILE_TYPE.JAR);
}
- @Test (timeout=10000)
+ @Test (timeout=10000)
public void testDownloadArchiveZip() throws IOException, URISyntaxException,
InterruptedException {
downloadWithFileType(TEST_FILE_TYPE.ZIP);
@@ -603,7 +615,6 @@ public class TestFSDownload {
@Test (timeout=10000)
public void testDirDownload() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
FileContext files = FileContext.getLocalFSFileContext(conf);
final Path basedir = files.makeQualified(new Path("target",
TestFSDownload.class.getSimpleName()));
@@ -668,7 +679,6 @@ public class TestFSDownload {
@Test (timeout=10000)
public void testUniqueDestinationPath() throws Exception {
- Configuration conf = new Configuration();
FileContext files = FileContext.getLocalFSFileContext(conf);
final Path basedir = files.makeQualified(new Path("target",
TestFSDownload.class.getSimpleName()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-6750. Add a configuration to
cap how much a NM can be overallocated. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6750. Add a configuration to cap how much a NM can be overallocated. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2bb70c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2bb70c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2bb70c6
Branch: refs/heads/YARN-1011
Commit: a2bb70c6c899d55fe4cba3eecb335a9ff2ffea4b
Parents: f096a17
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Nov 22 09:17:56 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:23:37 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 10 ++
.../scheduler/AbstractYarnScheduler.java | 4 +
.../scheduler/SchedulerNode.java | 41 +++++--
.../scheduler/fair/FSSchedulerNode.java | 5 +
.../scheduler/fair/FairScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 107 +++++++++++++++++++
7 files changed, 167 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 15f6480..8d55737 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -384,6 +384,11 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /** The global max overallocation per node in terms of their capacity. */
+ public static final String PER_NODE_MAX_OVERALLOCATION_RATIO =
+ RM_PREFIX + "overallocation.per-node-max-ratio";
+ public static final float DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO = 4.0f;
+
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 7962dae..c184226 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3182,6 +3182,16 @@
<property>
<description>
+ The maximum amount of resources, specified as a ratio to node capacity,
+ that can be allocated to opportunistic containers on any given node in
+ the cluster.
+ </description>
+ <name>yarn.resourcemanager.overallocation.per-node-max-ratio</name>
+ <value>4.0</value>
+ </property>
+
+ <property>
+ <description>
Frequency for computing least loaded NMs.
</description>
<name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ca66055..a755a4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -118,6 +118,7 @@ public abstract class AbstractYarnScheduler
new ClusterNodeTracker<>();
protected Resource minimumAllocation;
+ protected float maxOverAllocationRatioPerNode;
protected volatile RMContext rmContext;
@@ -200,6 +201,9 @@ public abstract class AbstractYarnScheduler
nodeTracker.setConfiguredMaxAllocationWaitTime(
configuredMaximumAllocationWaitTime);
maxClusterLevelAppPriority = getMaxPriorityFromConf(conf);
+ maxOverAllocationRatioPerNode = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
createReleaseCache();
autoUpdateContainers =
conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index e942981..b10cac5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -90,10 +90,14 @@ public abstract class SchedulerNode {
protected Resource resourceAllocatedPendingLaunch =
Resource.newInstance(0, 0);
+ // The max amount of resources that can be allocated to opportunistic
+ // containers on the node, specified as a ratio to its capacity
+ private final float maxOverAllocationRatio;
+
private volatile Set<String> labels = null;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
- Set<String> labels) {
+ Set<String> labels, float maxOverAllocationRatio) {
this.rmNode = node;
this.unallocatedResource = Resources.clone(node.getTotalCapability());
this.capacity = Resources.clone(node.getTotalCapability());
@@ -103,10 +107,24 @@ public abstract class SchedulerNode {
nodeName = rmNode.getHostName();
}
this.labels = ImmutableSet.copyOf(labels);
+ this.maxOverAllocationRatio = maxOverAllocationRatio;
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
+ this(node, usePortForNodeName, labels,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverAllocationRatio) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ maxOverAllocationRatio);
}
public SchedulerNode(RMNode node, boolean usePortForNodeName) {
- this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
}
public RMNode getRMNode() {
@@ -604,9 +622,11 @@ public abstract class SchedulerNode {
/**
* Get the amount of resources that can be allocated to opportunistic
- * containers in the case of overallocation. It is calculated as
+ * containers in the case of overallocation, calculated as
* node capacity - (node utilization + resources of allocated-yet-not-started
- * containers).
+ * containers), subject to the maximum amount of resources that can be
+ * allocated to opportunistic containers on the node specified as a ratio to
+ * its capacity.
* @return the amount of resources that are available to be allocated to
* opportunistic containers
*/
@@ -639,11 +659,20 @@ public abstract class SchedulerNode {
Resource resourceAllowedForOpportunisticContainers =
Resources.createResource(allowedMemory, allowedCpu);
- // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
- // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ // cap the total amount of resources allocated to OPPORTUNISTIC containers
+ Resource maxOverallocation = getMaxOverallocationAllowed();
+ Resources.subtractFrom(maxOverallocation, allocatedResourceOpportunistic);
+ resourceAllowedForOpportunisticContainers = Resources.componentwiseMin(
+ maxOverallocation, resourceAllowedForOpportunisticContainers);
+
return resourceAllowedForOpportunisticContainers;
}
+ private Resource getMaxOverallocationAllowed() {
+ long maxMemory = (long) (capacity.getMemorySize() * maxOverAllocationRatio);
+ int maxVcore = (int) (capacity.getVirtualCores() * maxOverAllocationRatio);
+ return Resource.newInstance(maxMemory, maxVcore);
+ }
private static class ContainerInfo {
private final RMContainer container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 95490f5..a53dda4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -66,10 +66,15 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ @VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
}
+ public FSSchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverallocationRatio) {
+ super(node, usePortForNodeName, maxOverallocationRatio);
+ }
/**
* Total amount of reserved resources including reservations and preempted
* containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a8e348c..3929f59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -751,7 +751,7 @@ public class FairScheduler extends
try {
writeLock.lock();
FSSchedulerNode schedulerNode = new FSSchedulerNode(node,
- usePortForNodeName);
+ usePortForNodeName, maxOverAllocationRatioPerNode);
nodeTracker.addNode(schedulerNode);
triggerUpdate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2bb70c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 1f89ace..1870ea5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3137,6 +3137,113 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test that max overallocation per node is enforced by Fair Scheduler.
+ * @throws Exception
+ */
+ @Test
+ public void testMaxOverallocationPerNode() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ float maxOverallocationRatio = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO, 1.5f);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 1G of memory and 1 vcores and an overallocation
+ // threshold of 1.0f and 1.0f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(1f, 1f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 1), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is zero after the container runs
+ ContainerStatus containerStatus1 = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus1),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create a scheduling request that should get allocated an OPPORTUNISTIC
+ // container because the node utilization is zero
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // node utilization is still zero after the container runs
+ ContainerStatus containerStatus2 = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus2),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create another scheduling request that should not get any allocation
+ // because of the max overallocation on the node will be exceeded.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ maxOverallocationRatio);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d61cc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d61cc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d61cc4
Branch: refs/heads/YARN-1011
Commit: 13d61cc425e3cea5e2d190b3df091ca3e3ba4ae7
Parents: 0a13be8
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Jan 23 22:34:49 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 19:51:15 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/TypeConverter.java | 4 +-
.../hadoop/mapreduce/TestTypeConverter.java | 4 +-
.../hadoop/mapred/TestResourceMgrDelegate.java | 2 +-
.../records/ApplicationResourceUsageReport.java | 158 +++++++++++---
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 23 ++-
.../ApplicationResourceUsageReportPBImpl.java | 205 +++++++++++++++----
.../hadoop/yarn/util/resource/Resources.java | 20 ++
...pplicationHistoryManagerOnTimelineStore.java | 83 +++++---
.../TestApplicationHistoryClientService.java | 8 +-
...pplicationHistoryManagerOnTimelineStore.java | 18 +-
.../metrics/ApplicationMetricsConstants.java | 10 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 39 +++-
.../server/resourcemanager/RMAppManager.java | 18 +-
.../server/resourcemanager/RMServerUtils.java | 3 +-
.../metrics/TimelineServiceV1Publisher.java | 14 +-
.../metrics/TimelineServiceV2Publisher.java | 14 +-
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../records/ApplicationAttemptStateData.java | 144 +++++++++++--
.../pb/ApplicationAttemptStateDataPBImpl.java | 110 ++++++++--
.../server/resourcemanager/rmapp/RMAppImpl.java | 38 ++--
.../resourcemanager/rmapp/RMAppMetrics.java | 38 ++--
.../attempt/AggregateAppResourceUsage.java | 51 +++--
.../rmapp/attempt/RMAppAttemptImpl.java | 21 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 47 +++--
.../ContainerResourceUsageReport.java | 46 +++++
.../rmcontainer/RMContainer.java | 7 +-
.../rmcontainer/RMContainerImpl.java | 119 ++++++++---
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 63 +++---
.../scheduler/YarnScheduler.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 9 +-
.../webapp/FairSchedulerAppsBlock.java | 8 +
.../resourcemanager/webapp/RMAppBlock.java | 9 +-
.../resourcemanager/webapp/RMAppsBlock.java | 10 +
.../resourcemanager/webapp/dao/AppInfo.java | 78 +++++--
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/TestAppManager.java | 39 ++--
.../resourcemanager/TestApplicationACLs.java | 4 +-
.../resourcemanager/TestClientRMService.java | 45 ++--
.../TestContainerResourceUsage.java | 184 ++++++++++-------
.../applicationsmanager/MockAsm.java | 4 +-
.../TestCombinedSystemMetricsPublisher.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 23 ++-
.../TestSystemMetricsPublisherForV2.java | 12 +-
.../recovery/RMStateStoreTestBase.java | 12 +-
.../recovery/TestZKRMStateStore.java | 40 ++--
.../rmapp/TestRMAppTransitions.java | 6 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +--
.../capacity/TestCapacityScheduler.java | 5 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 20 +-
.../resourcemanager/webapp/TestAppPage.java | 2 +-
.../resourcemanager/webapp/TestRMWebApp.java | 3 +-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../router/webapp/RouterWebServiceUtil.java | 14 +-
.../router/webapp/TestRouterWebServiceUtil.java | 14 +-
61 files changed, 1427 insertions(+), 517 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index aea931a..23409fc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -527,8 +527,8 @@ public class TypeConverter {
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
(int)resourceUsageReport.getReservedResources().getMemorySize());
- jobStatus.setUsedMem(
- (int) resourceUsageReport.getUsedResources().getMemorySize());
+ jobStatus.setUsedMem((int) resourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
}
return jobStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
index 629a246..acd9c35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
@@ -94,7 +94,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
applicationReport.setApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
Assert.assertEquals(appStartTime, jobStatus.getStartTime());
@@ -133,7 +133,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt);
JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
Assert.assertNotNull("fromYarn returned null status", status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
index d6754f0..0506b27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
@@ -137,7 +137,7 @@ public class TestResourceMgrDelegate {
Records.newRecord(Resource.class));
Mockito.when(appResources.getReservedResources()).thenReturn(
Records.newRecord(Resource.class));
- Mockito.when(appResources.getUsedResources()).thenReturn(
+ Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn(
Records.newRecord(Resource.class));
Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
appResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index d2e33ff..4517db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,21 +36,27 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public static ApplicationResourceUsageReport newInstance(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap, float queueUsagePerc,
- float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap, float queueUsagePerc,
+ float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
+
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
@@ -87,16 +93,40 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNumReservedContainers(int num_reserved_containers);
/**
- * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports.
- * @return the used <code>Resource</code>
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsedResources();
+ /**
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourcesUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourcesUsed(Resource resources);
+
+ /**
+ * Get the opportunistic <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the opportunistic <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourcesUsed();
+
@Private
@Unstable
- public abstract void setUsedResources(Resource resources);
+ public abstract void setOpportunisticResourcesUsed(Resource resources);
/**
* Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports.
@@ -123,42 +153,84 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNeededResources(Resource needed_resources);
/**
- * Set the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @param memory_seconds the aggregated amount of memory seconds
+ * Set the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @param memorySeconds the aggregated amount of guaranteed memory seconds
*/
@Private
@Unstable
- public abstract void setMemorySeconds(long memory_seconds);
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
/**
- * Get the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @return the aggregated amount of memory seconds
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ /**
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
/**
- * Set the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @param vcore_seconds the aggregated number of vcore seconds
+ * Set the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @param vcoreSeconds the aggregated number of guaranteed vcore seconds
*/
@Private
@Unstable
- public abstract void setVcoreSeconds(long vcore_seconds);
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @return the aggregated number of vcore seconds
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ /**
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
/**
+ * Get the aggregated amount of opportunistic memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of opportunistic memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the aggregated number of opportunistic vcores that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return the aggregated number of opportunistic vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the percentage of resources of the queue that the app is using.
* @return the percentage of resources of the queue that the app is using.
*/
@@ -231,23 +303,35 @@ public abstract class ApplicationResourceUsageReport {
public abstract long getPreemptedVcoreSeconds();
/**
- * Get the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
- * @return map containing the resource name and aggregated resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Private
@Unstable
- public abstract void setResourceSecondsMap(
+ public abstract void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
@@ -272,4 +356,24 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @param opportunisticResourceSecondsMap map containing the resource name
+ * and aggregated opportunistic resource-seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 44e1b88..c9f0a69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -251,6 +251,8 @@ message ApplicationResourceUsageReportProto {
optional int64 preempted_vcore_seconds = 11;
repeated StringLongMapProto application_resource_usage_map = 12;
repeated StringLongMapProto application_preempted_resource_usage_map = 13;
+ optional ResourceProto used_opportunistic_resources = 14;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 15;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index daca296..2b0f96a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -990,15 +990,20 @@ public class ApplicationCLI extends YarnCLI {
private void printResourceUsage(PrintWriter appReportStr,
ApplicationResourceUsageReport usageReport) {
- appReportStr.print("\tAggregate Resource Allocation : ");
+ appReportStr.print("\tAggregate Guaranteed Resource Allocation : ");
if (usageReport != null) {
- appReportStr.println(
- getResourceSecondsString(usageReport.getResourceSecondsMap()));
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getGuaranteedResourceSecondsMap()));
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getOpportunisticResourceSecondsMap()));
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println(getResourceSecondsString(
usageReport.getPreemptedResourceSecondsMap()));
} else {
appReportStr.println("N/A");
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index d6c33f4..ba6eb3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -170,13 +170,11 @@ public class TopCLI extends YarnCLI {
appReport.getApplicationResourceUsageReport()
.getNumReservedContainers();
displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers));
- usedVirtualCores =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getVirtualCores();
+ usedVirtualCores = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getVirtualCores();
displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores));
- usedMemory =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getMemorySize() / 1024;
+ usedMemory = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getMemorySize() / 1024;
displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G");
reservedVirtualCores =
appReport.getApplicationResourceUsageReport().getReservedResources()
@@ -195,11 +193,11 @@ public class TopCLI extends YarnCLI {
progress = appReport.getProgress() * 100;
displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress));
// store in GBSeconds
- memorySeconds =
- appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024;
+ memorySeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedMemorySeconds() / 1024;
displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds));
- vcoreSeconds =
- appReport.getApplicationResourceUsageReport().getVcoreSeconds();
+ vcoreSeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedVcoreSeconds();
displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index e172acd..c2e3d9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -121,18 +121,27 @@ public class TestYarnCLI {
for (int i = 0; i < 2; ++i) {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
preemptedResoureSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 2222L);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport
- .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(2, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
@@ -168,7 +177,9 @@ public class TestYarnCLI {
pw.println("\tTracking-URL : N/A");
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
- pw.println("\tAggregate Resource Allocation : " +
+ pw.println("\tAggregate Guaranteed Resource Allocation : " +
+ (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tAggregate Opportunistic Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tAggregate Resource Preempted : " +
(i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 14ede5d..8ff1631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -40,12 +40,14 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto.Builder builder = null;
boolean viaProto = false;
- Resource usedResources;
- Resource reservedResources;
- Resource neededResources;
+ private Resource guaranteedResourceUsed;
+ private Resource reservedResources;
+ private Resource neededResources;
+ private Resource opportunisticResourceUsed;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationResourceUsageReportPBImpl() {
builder = ApplicationResourceUsageReportProto.newBuilder();
@@ -55,8 +57,9 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto proto) {
this.proto = proto;
viaProto = true;
- getResourceSecondsMap();
+ getGuaranteedResourceSecondsMap();
getPreemptedResourceSecondsMap();
+ getOpportunisticResourceSecondsMap();
}
public synchronized ApplicationResourceUsageReportProto getProto() {
@@ -87,8 +90,13 @@ extends ApplicationResourceUsageReport {
}
private void mergeLocalToBuilder() {
- if (this.usedResources != null) {
- builder.setUsedResources(convertToProtoFormat(this.usedResources));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setUsedResources(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setUsedOpportunisticResources(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.reservedResources != null) {
builder.setReservedResources(
@@ -99,19 +107,28 @@ extends ApplicationResourceUsageReport {
}
builder.clearApplicationResourceUsageMap();
builder.clearApplicationPreemptedResourceUsageMap();
+ builder.clearApplicationOpportunisticResourceUsageMap();
if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap
.isEmpty()) {
builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
- if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) {
+ if (guaranteedResourceSecondsMap != null &&
+ !guaranteedResourceSecondsMap.isEmpty()) {
builder.addAllApplicationResourceUsageMap(
- ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ guaranteedResourceSecondsMap));
+ }
+ if (opportunisticResourceSecondsMap != null &&
+ !opportunisticResourceSecondsMap.isEmpty()) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ opportunisticResourceSecondsMap));
}
- builder.setMemorySeconds(this.getMemorySeconds());
- builder.setVcoreSeconds(this.getVcoreSeconds());
+ builder.setMemorySeconds(this.getGuaranteedMemorySeconds());
+ builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds());
builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds());
builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds());
}
@@ -157,24 +174,53 @@ extends ApplicationResourceUsageReport {
}
@Override
+ @Deprecated
public synchronized Resource getUsedResources() {
+ return getGuaranteedResourcesUsed();
+ }
+
+ @Override
+ public synchronized Resource getGuaranteedResourcesUsed() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- if (this.usedResources != null) {
- return this.usedResources;
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
}
if (!p.hasUsedResources()) {
return null;
}
- this.usedResources = convertFromProtoFormat(p.getUsedResources());
- return this.usedResources;
+ this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources());
+ return this.guaranteedResourceUsed;
}
@Override
- public synchronized void setUsedResources(Resource resources) {
+ public synchronized void setGuaranteedResourcesUsed(Resource resources) {
maybeInitBuilder();
if (resources == null)
builder.clearUsedResources();
- this.usedResources = resources;
+ this.guaranteedResourceUsed = resources;
+ }
+
+ @Override
+ public synchronized Resource getOpportunisticResourcesUsed() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
+ }
+ if (!p.hasUsedOpportunisticResources()) {
+ return null;
+ }
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getUsedOpportunisticResources());
+ return this.opportunisticResourceUsed;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourcesUsed(Resource resources) {
+ maybeInitBuilder();
+ if (resources == null) {
+ builder.clearUsedOpportunisticResources();
+ }
+ this.opportunisticResourceUsed = resources;
}
@Override
@@ -220,14 +266,14 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setMemorySeconds(long memory_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds);
+ public synchronized void setGuaranteedMemorySeconds(long memorySeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
}
@Override
- public synchronized long getMemorySeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ public synchronized long getGuaranteedMemorySeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
@@ -235,20 +281,50 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setVcoreSeconds(long vcore_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.VCORES.getName(), vcore_seconds);
+ @Deprecated
+ public synchronized long getMemorySeconds() {
+ return getGuaranteedMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
}
@Override
+ public synchronized long getGuaranteedVcoreSeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ @Deprecated
public synchronized long getVcoreSeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ return getGuaranteedVcoreSeconds();
+ }
+
+ @Override
+ public synchronized long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public synchronized long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
-
+
@Override
public synchronized void setPreemptedMemorySeconds(
long preemptedMemorySeconds) {
@@ -314,41 +390,47 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setResourceSecondsMap(
+ public synchronized void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap == null) {
return;
}
if (!resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(0L);
+ this.setGuaranteedMemorySeconds(0L);
}
if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(0L);
+ this.setGuaranteedVcoreSeconds(0L);
}
}
@Override
+ @Deprecated
public synchronized Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ public synchronized Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils
+ this.guaranteedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
}
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(p.getVcoreSeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
}
- this.setMemorySeconds(p.getMemorySeconds());
- this.setVcoreSeconds(p.getVcoreSeconds());
- return this.resourceSecondsMap;
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
+ return this.guaranteedResourceSecondsMap;
}
@Override
@@ -389,4 +471,45 @@ extends ApplicationResourceUsageReport {
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
return this.preemptedResourceSecondsMap;
}
+
+ @Override
+ public synchronized Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap = ProtoUtils
+ .convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ if (opportunisticResourceSecondsMap == null) {
+ return;
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 1c08844..010a0de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import java.util.Map;
+
/**
* Resources is a computation class which provides a set of apis to do
* mathematical operations on Resource object.
@@ -556,4 +558,22 @@ public class Resources {
Resource resource, Resource factor) {
return calculator.normalizeDown(resource, factor);
}
+
+ /**
+ * Merge resource usage entries from a map to another map.
+ * @param mergeFrom the map to merge from
+ * @param mergeTo the map to merge to
+ */
+ public static void mergeResourceSecondsMap(Map<String, Long> mergeFrom,
+ Map<String, Long> mergeTo) {
+ for (Map.Entry<String, Long> entry : mergeFrom.entrySet()) {
+ Long value = mergeTo.get(entry.getKey());
+ if (value != null) {
+ value += entry.getValue();
+ } else {
+ value = entry.getValue();
+ }
+ mergeTo.put(entry.getKey(), value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 0b57717..85cc006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -330,29 +329,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
.toString();
}
- if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_METRICS);
- long memorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_METRICS);
- long preemptedMemorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
- long preemptedVcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- preemptedMemorySeconds);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
-
+ if (entityInfo.containsKey(
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) {
+ Map<String, Long> guaranteedResourceSecondsMap =
+ extractGuaranteedResourceSecondsMap(entityInfo);
+ Map<String, Long> preemptedResoureSecondsMap =
+ extractPreemptedResourceSecondsMap(entityInfo);
+ Map<String, Long> opportunisticResourceSecondsMap =
+ extractOpportunisticResourceSecondsMap(entityInfo);
appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(0, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
@@ -456,6 +445,54 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static Map<String, Long> extractGuaranteedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+
+ long guaranteedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS);
+ long guaranteedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds);
+ guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(),
+ guaranteedVcoreSeconds);
+
+ return guaranteedResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractOpportunisticResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+
+ long opportunisticVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS);
+ long opportunisticMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(),
+ opportunisticMemorySeconds);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds);
+
+ return opportunisticResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractPreemptedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
+
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
+ preemptedMemorySeconds);
+ preemptedResoureSecondsMap
+ .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
+
+ return preemptedResoureSecondsMap;
+ }
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7ef6eca..6cb21ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -152,9 +152,13 @@ public class TestApplicationHistoryClientService {
ApplicationReport appReport = response.getApplicationReport();
Assert.assertNotNull(appReport);
Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
- .getMemorySeconds());
+ .getGuaranteedMemorySeconds());
Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
- .getVcoreSeconds());
+ .getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticMemorySeconds());
+ Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticVcoreSeconds());
Assert.assertEquals("application_0_0001", appReport.getApplicationId()
.toString());
Assert.assertEquals("test app type",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ecaaf1e..35996aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -250,9 +250,13 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
- applicationResourceUsageReport.getMemorySeconds());
- Assert
- .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+ applicationResourceUsageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123,
+ applicationResourceUsageReport.getOpportunisticMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getOpportunisticVcoreSeconds());
long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
@@ -534,8 +538,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345);
if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 4cec409..b4123ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -70,12 +70,18 @@ public class ApplicationMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_APPLICATION_STATE";
- public static final String APP_CPU_METRICS =
+ public static final String APP_GUARANTEED_CPU_METRICS =
"YARN_APPLICATION_CPU";
- public static final String APP_MEM_METRICS =
+ public static final String APP_GUARANTEED_MEM_METRICS =
"YARN_APPLICATION_MEMORY";
+ public static final String APP_OPPORTUNISTIC_CPU_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_CPU";
+
+ public static final String APP_OPPORTUNISTIC_MEM_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_MEMORY";
+
public static final String APP_RESOURCE_PREEMPTED_CPU =
"YARN_APPLICATION_RESOURCE_PREEMPTED_CPU";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 6abc617..860ebae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -466,19 +466,24 @@ public class BuilderUtils {
}
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index ac2f8da..fb856d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Times;
@@ -61,6 +62,8 @@ public class AppInfo {
protected int priority;
private long allocatedCpuVcores;
private long allocatedMemoryMB;
+ private long allocatedOpportunisticCpuVcores;
+ private long allocatedOpportunisticMemoryMB;
private long reservedCpuVcores;
private long reservedMemoryMB;
protected boolean unmanagedApplication;
@@ -98,15 +101,25 @@ public class AppInfo {
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
- if (app.getApplicationResourceUsageReport().getUsedResources() != null) {
- allocatedCpuVcores = app.getApplicationResourceUsageReport()
- .getUsedResources().getVirtualCores();
- allocatedMemoryMB = app.getApplicationResourceUsageReport()
- .getUsedResources().getMemorySize();
- reservedCpuVcores = app.getApplicationResourceUsageReport()
- .getReservedResources().getVirtualCores();
- reservedMemoryMB = app.getApplicationResourceUsageReport()
- .getReservedResources().getMemorySize();
+ Resource guaranteedResourceUsed =
+ app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed();
+ if (guaranteedResourceUsed != null) {
+ allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores();
+ allocatedMemoryMB = guaranteedResourceUsed.getMemorySize();
+ }
+ Resource opportunisticResourceUsed = app
+ .getApplicationResourceUsageReport().getOpportunisticResourcesUsed();
+ if (opportunisticResourceUsed != null) {
+ allocatedOpportunisticCpuVcores =
+ opportunisticResourceUsed.getVirtualCores();
+ allocatedOpportunisticMemoryMB =
+ opportunisticResourceUsed.getMemorySize();
+ }
+ Resource resourceReserved =
+ app.getApplicationResourceUsageReport().getReservedResources();
+ if (resourceReserved != null) {
+ reservedCpuVcores = resourceReserved.getVirtualCores();
+ reservedMemoryMB = resourceReserved.getMemorySize();
}
}
progress = app.getProgress() * 100; // in percent
@@ -166,6 +179,14 @@ public class AppInfo {
return allocatedMemoryMB;
}
+ public long getAllocatedOpportunisticCpuVcores() {
+ return allocatedOpportunisticCpuVcores;
+ }
+
+ public long getAllocatedOpportunisticMemoryMB() {
+ return allocatedOpportunisticMemoryMB;
+ }
+
public long getReservedCpuVcores() {
return reservedCpuVcores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 1e05fec..cb7d426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -187,19 +187,27 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
- .add("memorySeconds", metrics.getMemorySeconds())
- .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds())
+ .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds())
+ .add("opportunisticMemorySeconds",
+ metrics.getOpportunisticMemorySeconds())
+ .add("opportunisticVcoreSeconds",
+ metrics.getOpportunisticVcoreSeconds())
.add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds())
.add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds())
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
.add("applicationType", app.getApplicationType())
- .add("resourceSeconds", StringHelper
- .getResourceSecondsString(metrics.getResourceSecondsMap()))
+ .add("guaranteedResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getGuaranteedResourceSecondsMap()))
.add("preemptedResourceSeconds", StringHelper
.getResourceSecondsString(
- metrics.getPreemptedResourceSecondsMap()));
+ metrics.getPreemptedResourceSecondsMap()))
+ .add("opportunisticResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getOpportunisticResourceSecondsMap()));
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4934243..05b37cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -482,7 +482,8 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>());
+ Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(),
+ Resources.createResource(-1, -1), new HashMap<>());
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 73bb301..efe5239 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -140,10 +140,16 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
latestApplicationAttemptId);
}
RMAppMetrics appMetrics = app.getRMAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS,
+ appMetrics.getGuaranteedVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS,
+ appMetrics.getGuaranteedMemorySeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ appMetrics.getOpportunisticVcoreSeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ appMetrics.getOpportunisticMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,
appMetrics.getPreemptedMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 7eaa6e7..84f3e9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -190,11 +190,17 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_CPU_METRICS, timestamp,
- appMetrics.getVcoreSeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp,
+ appMetrics.getGuaranteedVcoreSeconds()));
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_MEM_METRICS, timestamp,
- appMetrics.getMemorySeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp,
+ appMetrics.getGuaranteedMemorySeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ timestamp, appMetrics.getOpportunisticVcoreSeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ timestamp, appMetrics.getOpportunisticMemorySeconds()));
entityMetrics.add(getTimelineMetric(
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp,
appMetrics.getPreemptedMemorySeconds()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index f0ab324..de66802 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -853,8 +853,9 @@ public abstract class RMStateStore extends AbstractService {
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
- resUsage.getResourceUsageSecondsMap(),
- attempMetrics.getPreemptedResourceSecondsMap());
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ attempMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
getRMStateStoreEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 2de071a..b08d2eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -45,8 +46,9 @@ public abstract class ApplicationAttemptStateData {
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
- long finishTime, Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ long finishTime, Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -59,19 +61,22 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
attemptStateData.setFinishTime(finishTime);
- attemptStateData.setMemorySeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap,
+ attemptStateData.setGuaranteedMemorySeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
- attemptStateData.setVcoreSeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L));
+ attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L));
attemptStateData.setPreemptedMemorySeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setPreemptedVcoreSeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L));
- attemptStateData.setResourceSecondsMap(resourceSecondsMap);
+ attemptStateData.setGuaranteedResourceSecondsMap(
+ guaranteedResourceSecondsMap);
+ attemptStateData.setOpportunisticResourceSecondsMap(
+ opportunisticResourcesSecondsMap);
attemptStateData
.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return attemptStateData;
@@ -80,11 +85,13 @@ public abstract class ApplicationAttemptStateData {
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime,
- Map<String, Long> resourceSeondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
return newInstance(attemptId, masterContainer, attemptTokens, startTime,
null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
- resourceSeondsMap, preemptedResourceSecondsMap);
+ guaranteedResourceSecondsMap, preemptedResourceSecondsMap,
+ opportunisticResourcesSecondsMap);
}
@@ -180,30 +187,78 @@ public abstract class ApplicationAttemptStateData {
public abstract void setFinishTime(long finishTime);
/**
- * Get the <em>memory seconds</em> (in MB seconds) of the application.
- * @return <em>memory seconds</em> (in MB seconds) of the application
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setMemorySeconds(long memorySeconds);
/**
- * Get the <em>vcore seconds</em> of the application.
- * @return <em>vcore seconds</em> of the application
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setVcoreSeconds(long vcoreSeconds);
/**
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ @Private
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ @Private
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the <em>opportunistic vcore seconds</em> of the application.
+ * @return <em>opportunistic vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the <em>preempted memory seconds</em>
* (in MB seconds) of the application.
* @return <em>preempted memory seconds</em>
@@ -232,28 +287,54 @@ public abstract class ApplicationAttemptStateData {
public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of resources preempted that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
- * @return map containing the resource name and aggregated preempted
- * resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
+
+ /**
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * guaranteed resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
+
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
@@ -276,4 +357,27 @@ public abstract class ApplicationAttemptStateData {
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * opportunistic resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: MAPREDUCE-7020. Task timeout in
uber mode can crash AM. Contributed by Peter Bacsko
Posted by ha...@apache.org.
MAPREDUCE-7020. Task timeout in uber mode can crash AM. Contributed by Peter Bacsko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eef3d7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eef3d7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eef3d7f
Branch: refs/heads/YARN-1011
Commit: 6eef3d7f1a1e5e3f27fb3bf7596663640d786181
Parents: e990904
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 26 15:31:43 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 26 15:31:43 2018 -0600
----------------------------------------------------------------------
.../apache/hadoop/mapred/TaskAttemptListenerImpl.java | 8 +++++---
.../hadoop/mapred/TestTaskAttemptListenerImpl.java | 6 ++++--
.../src/main/java/org/apache/hadoop/mapred/Task.java | 14 +++++++++++---
3 files changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eef3d7f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index b155af22..668d8ed 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -369,14 +369,16 @@ public class TaskAttemptListenerImpl extends CompositeService
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID =
TypeConverter.toYarn(taskAttemptID);
+ AMFeedback feedback = new AMFeedback();
AtomicReference<TaskAttemptStatus> lastStatusRef =
attemptIdToStatus.get(yarnAttemptID);
if (lastStatusRef == null) {
- throw new IllegalStateException("Status update was called"
- + " with illegal TaskAttemptId: " + yarnAttemptID);
+ LOG.error("Status update was called with illegal TaskAttemptId: "
+ + yarnAttemptID);
+ feedback.setTaskFound(false);
+ return feedback;
}
- AMFeedback feedback = new AMFeedback();
feedback.setTaskFound(true);
// Propagating preemption to the task if TASK_PREEMPTION is enabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eef3d7f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
index 4ff6fb2..da7421b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
@@ -487,13 +487,15 @@ public class TestTaskAttemptListenerImpl {
assertEquals(Phase.REDUCE, status.phase);
}
- @Test(expected = IllegalStateException.class)
+ @Test
public void testStatusUpdateFromUnregisteredTask()
throws IOException, InterruptedException{
configureMocks();
startListener(false);
- listener.statusUpdate(attemptID, firstReduceStatus);
+ AMFeedback feedback = listener.statusUpdate(attemptID, firstReduceStatus);
+
+ assertFalse(feedback.getTaskFound());
}
private void configureMocks() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eef3d7f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 87c9e16..5b98b35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -855,6 +855,9 @@ abstract public class Task implements Writable, Configurable {
long taskProgressInterval = MRJobConfUtil.
getTaskProgressReportInterval(conf);
+ boolean uberized = conf.getBoolean("mapreduce.task.uberized",
+ false);
+
while (!taskDone.get()) {
synchronized (lock) {
done = false;
@@ -893,9 +896,14 @@ abstract public class Task implements Writable, Configurable {
// if Task Tracker is not aware of our task ID (probably because it died and
// came back up), kill ourselves
if (!taskFound) {
- LOG.warn("Parent died. Exiting "+taskId);
- resetDoneFlag();
- System.exit(66);
+ if (uberized) {
+ taskDone.set(true);
+ break;
+ } else {
+ LOG.warn("Parent died. Exiting "+taskId);
+ resetDoneFlag();
+ System.exit(66);
+ }
}
// Set a flag that says we should preempt this is read by
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: MapFile.fix creates a wrong index
file in case of block-compressed data file. Contributed by Grigori Rybkine
Posted by ha...@apache.org.
MapFile.fix creates a wrong index file in case of block-compressed data file. Contributed by Grigori Rybkine
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56872cff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56872cff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56872cff
Branch: refs/heads/YARN-1011
Commit: 56872cff92f543bf77206a1324968559dceb7bc2
Parents: 8b5b045
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Jan 26 09:06:48 2018 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Jan 26 09:18:30 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/io/MapFile.java | 35 ++++++++++--
.../java/org/apache/hadoop/io/TestMapFile.java | 59 +++++++++++++++++++-
2 files changed, 88 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56872cff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
index d56822f..51db0b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
@@ -811,15 +811,40 @@ public class MapFile {
(LongWritable.class));
}
try {
- long pos = 0L;
+ /** What's the position (in bytes) we wrote when we got the last index */
+ long lastIndexPos = -1;
+ /**
+ * What was size when we last wrote an index. Set to MIN_VALUE to ensure
+ * that we have an index at position zero - midKey will throw an exception
+ * if this is not the case
+ */
+ long lastIndexKeyCount = Long.MIN_VALUE;
+ long pos = dataReader.getPosition();
LongWritable position = new LongWritable();
+ long nextBlock = pos;
+ boolean blockCompressed = dataReader.isBlockCompressed();
while(dataReader.next(key, value)) {
- cnt++;
- if (cnt % indexInterval == 0) {
+ if (blockCompressed) {
+ long curPos = dataReader.getPosition();
+ if (curPos > nextBlock) {
+ pos = nextBlock; // current block position
+ nextBlock = curPos;
+ }
+ }
+ // Follow the same logic as in
+ // {@link MapFile.Writer#append(WritableComparable, Writable)}
+ if (cnt >= lastIndexKeyCount + indexInterval && pos > lastIndexPos) {
position.set(pos);
- if (!dryrun) indexWriter.append(key, position);
+ if (!dryrun) {
+ indexWriter.append(key, position);
+ }
+ lastIndexPos = pos;
+ lastIndexKeyCount = cnt;
+ }
+ if (!blockCompressed) {
+ pos = dataReader.getPosition(); // next record position
}
- pos = dataReader.getPosition();
+ cnt++;
}
} catch(Throwable t) {
// truncated data file. swallow it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56872cff/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
index ff8df7c..7ec4227 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
@@ -485,6 +485,63 @@ public class TestMapFile {
IOUtils.cleanup(null, writer);
}
}
+
+ /**
+ * test {@link MapFile#fix(FileSystem, Path, Class<? extends Writable>,
+ * Class<? extends Writable>, boolean, Configuration)}
+ * method in case of BLOCK compression
+ */
+ @Test
+ public void testFixBlockCompress() throws Exception {
+ final String indexLessMapFile = "testFixBlockCompress.mapfile";
+ final int compressBlocksize = 100;
+ final int indexInterval = 4;
+ final int noBlocks = 4;
+ final String value = "value-";
+ final int size = noBlocks * compressBlocksize / (4 + value.length());
+
+ conf.setInt("io.seqfile.compress.blocksize", compressBlocksize);
+ MapFile.Writer.setIndexInterval(conf, indexInterval);
+ FileSystem fs = FileSystem.getLocal(conf);
+ Path dir = new Path(TEST_DIR, indexLessMapFile);
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
+ try {
+ writer =
+ new MapFile.Writer(conf, dir,
+ MapFile.Writer.keyClass(IntWritable.class),
+ MapFile.Writer.valueClass(Text.class),
+ MapFile.Writer.compression(CompressionType.BLOCK));
+ for (int i = 0; i < size; i++) {
+ writer.append(new IntWritable(i), new Text(value + i));
+ }
+ writer.close();
+ Path index = new Path(dir, MapFile.INDEX_FILE_NAME);
+ fs.rename(index, index.suffix(".orig"));
+
+ assertEquals("No of valid MapFile entries wrong", size,
+ MapFile.fix(fs, dir, IntWritable.class, Text.class,
+ false, conf));
+ reader = new MapFile.Reader(dir, conf);
+ IntWritable key;
+ Text val = new Text();
+ int notFound = 0;
+ for (int i = 0; i < size; i++) {
+ key = new IntWritable(i);
+ if (null == reader.get(key, val)) {
+ notFound++;
+ }
+ }
+ assertEquals("With MapFile.fix-ed index, could not get entries # ",
+ 0, notFound);
+ } finally {
+ IOUtils.cleanupWithLogger(null, writer, reader);
+ if (fs.exists(dir)) {
+ fs.delete(dir, true);
+ }
+ }
+ }
+
/**
* test all available constructor for {@code MapFile.Writer}
*/
@@ -619,7 +676,7 @@ public class TestMapFile {
} catch (Exception ex) {
fail("testMainMethodMapFile error !!!");
} finally {
- IOUtils.cleanup(null, writer);
+ IOUtils.cleanupWithLogger(null, writer);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-13042. RBF: Heartbeat Router
State. Contributed by Inigo Goiri.
Posted by ha...@apache.org.
HDFS-13042. RBF: Heartbeat Router State. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7721fff7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7721fff7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7721fff7
Branch: refs/heads/YARN-1011
Commit: 7721fff74494eb7fbbbba7f8bb4b4692d880d301
Parents: eb2dd08
Author: Yiqun Lin <yq...@apache.org>
Authored: Thu Jan 25 15:51:26 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Thu Jan 25 15:51:26 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +
.../hdfs/server/federation/router/Router.java | 71 +++++++
.../router/RouterHeartbeatService.java | 155 +++++++++++++++
.../federation/router/RouterServiceState.java | 2 +-
.../federation/store/StateStoreService.java | 9 +
.../src/main/resources/hdfs-default.xml | 20 ++
.../store/TestStateStoreRouterState.java | 194 +++++++++++++++++++
7 files changed, 458 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f53badc..84215f3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1222,6 +1222,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_ROUTER_MONITOR_LOCAL_NAMENODE =
FEDERATION_ROUTER_PREFIX + "monitor.localnamenode.enable";
public static final boolean DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT = true;
+ public static final String DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS =
+ FEDERATION_ROUTER_PREFIX + "heartbeat-state.interval";
+ public static final long DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(5);
// HDFS Router NN client
public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
@@ -1282,6 +1286,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
FEDERATION_STORE_PREFIX + "membership.expiration";
public static final long FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT =
TimeUnit.MINUTES.toMillis(5);
+ public static final String FEDERATION_STORE_ROUTER_EXPIRATION_MS =
+ FEDERATION_STORE_PREFIX + "router.expiration";
+ public static final long FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(5);
// HDFS Router-based federation mount table entries
/** Maximum number of cache entries to have. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index ea8a1c0..1e72c93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -37,11 +37,13 @@ import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +114,18 @@ public class Router extends CompositeService {
/** Quota cache manager. */
private RouterQuotaManager quotaManager;
+ /** Manages the current state of the router. */
+ private RouterStore routerStateManager;
+ /** Heartbeat our run status to the router state manager. */
+ private RouterHeartbeatService routerHeartbeatService;
+
+ /** The start time of the namesystem. */
+ private final long startTime = Time.now();
+
+ /** State of the Router. */
+ private RouterServiceState state = RouterServiceState.UNINITIALIZED;
+
+
/////////////////////////////////////////////////////////
// Constructor
/////////////////////////////////////////////////////////
@@ -127,6 +141,7 @@ public class Router extends CompositeService {
@Override
protected void serviceInit(Configuration configuration) throws Exception {
this.conf = configuration;
+ updateRouterState(RouterServiceState.INITIALIZING);
if (conf.getBoolean(
DFSConfigKeys.DFS_ROUTER_STORE_ENABLE,
@@ -188,6 +203,10 @@ public class Router extends CompositeService {
if (this.namenodeHearbeatServices.isEmpty()) {
LOG.error("Heartbeat is enabled but there are no namenodes to monitor");
}
+
+ // Periodically update the router state
+ this.routerHeartbeatService = new RouterHeartbeatService(this);
+ addService(this.routerHeartbeatService);
}
// Router metrics system
@@ -219,6 +238,8 @@ public class Router extends CompositeService {
@Override
protected void serviceStart() throws Exception {
+ updateRouterState(RouterServiceState.RUNNING);
+
if (this.pauseMonitor != null) {
this.pauseMonitor.start();
JvmMetrics jvmMetrics = this.metrics.getJvmMetrics();
@@ -233,6 +254,9 @@ public class Router extends CompositeService {
@Override
protected void serviceStop() throws Exception {
+ // Update state
+ updateRouterState(RouterServiceState.SHUTDOWN);
+
// JVM pause monitor
if (this.pauseMonitor != null) {
this.pauseMonitor.stop();
@@ -454,6 +478,31 @@ public class Router extends CompositeService {
}
/////////////////////////////////////////////////////////
+ // Router State Management
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Update the router state and heartbeat to the state store.
+ *
+ * @param state The new router state.
+ */
+ public void updateRouterState(RouterServiceState newState) {
+ this.state = newState;
+ if (this.routerHeartbeatService != null) {
+ this.routerHeartbeatService.updateStateAsync();
+ }
+ }
+
+ /**
+ * Get the status of the router.
+ *
+ * @return Status of the router.
+ */
+ public RouterServiceState getRouterState() {
+ return this.state;
+ }
+
+ /////////////////////////////////////////////////////////
// Submodule getters
/////////////////////////////////////////////////////////
@@ -508,11 +557,33 @@ public class Router extends CompositeService {
return this.namenodeResolver;
}
+ /**
+ * Get the state store interface for the router heartbeats.
+ *
+ * @return FederationRouterStateStore state store API handle.
+ */
+ public RouterStore getRouterStateManager() {
+ if (this.routerStateManager == null && this.stateStore != null) {
+ this.routerStateManager = this.stateStore.getRegisteredRecordStore(
+ RouterStore.class);
+ }
+ return this.routerStateManager;
+ }
+
/////////////////////////////////////////////////////////
// Router info
/////////////////////////////////////////////////////////
/**
+ * Get the start date of the Router.
+ *
+ * @return Start date of the router.
+ */
+ public long getStartTime() {
+ return this.startTime;
+ }
+
+ /**
* Unique ID for the router, typically the hostname:port string for the
* router's RPC server. This ID may be null on router startup before the RPC
* server has bound to a port.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
new file mode 100644
index 0000000..86a6210
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.RecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically update the Router current state in the State Store.
+ */
+public class RouterHeartbeatService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterHeartbeatService.class);
+
+ /** Router we are hearbeating. */
+ private final Router router;
+
+ /**
+ * Create a new Router heartbeat service.
+ *
+ * @param router Router to heartbeat.
+ */
+ public RouterHeartbeatService(Router router) {
+ super(RouterHeartbeatService.class.getSimpleName());
+ this.router = router;
+ }
+
+ /**
+ * Trigger the update of the Router state asynchronously.
+ */
+ protected void updateStateAsync() {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ updateStateStore();
+ }
+ }, "Router Heartbeat Async");
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ /**
+ * Update the state of the Router in the State Store.
+ */
+ private synchronized void updateStateStore() {
+ String routerId = router.getRouterId();
+ if (routerId == null) {
+ LOG.error("Cannot heartbeat for router: unknown router id");
+ return;
+ }
+ RouterStore routerStore = router.getRouterStateManager();
+ if (routerStore != null) {
+ try {
+ RouterState record = RouterState.newInstance(
+ routerId, router.getStartTime(), router.getRouterState());
+ StateStoreVersion stateStoreVersion = StateStoreVersion.newInstance(
+ getStateStoreVersion(MembershipStore.class),
+ getStateStoreVersion(MountTableStore.class));
+ record.setStateStoreVersion(stateStoreVersion);
+ RouterHeartbeatRequest request =
+ RouterHeartbeatRequest.newInstance(record);
+ RouterHeartbeatResponse response = routerStore.routerHeartbeat(request);
+ if (!response.getStatus()) {
+ LOG.warn("Cannot heartbeat router {}", routerId);
+ } else {
+ LOG.debug("Router heartbeat for router {}", routerId);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot heartbeat router {}: {}", routerId, e.getMessage());
+ }
+ } else {
+ LOG.warn("Cannot heartbeat router {}: State Store unavailable", routerId);
+ }
+ }
+
+ /**
+ * Get the version of the data in the State Store.
+ *
+ * @param clazz Class in the State Store.
+ * @return Version of the data.
+ */
+ private <R extends BaseRecord, S extends RecordStore<R>>
+ long getStateStoreVersion(final Class<S> clazz) {
+ long version = -1;
+ try {
+ StateStoreService stateStore = router.getStateStore();
+ S recordStore = stateStore.getRegisteredRecordStore(clazz);
+ if (recordStore != null) {
+ if (recordStore instanceof CachedRecordStore) {
+ CachedRecordStore<R> cachedRecordStore =
+ (CachedRecordStore<R>) recordStore;
+ List<R> records = cachedRecordStore.getCachedRecords();
+ for (BaseRecord record : records) {
+ if (record.getDateModified() > version) {
+ version = record.getDateModified();
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get version for {}: {}", clazz, e.getMessage());
+ }
+ return version;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ long interval = conf.getTimeDuration(
+ DFSConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS,
+ DFSConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ this.setIntervalMs(interval);
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ updateStateStore();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
index 25a6466..3accbe9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hdfs.server.federation.router;
* States of the Router.
*/
public enum RouterServiceState {
- NONE,
+ UNINITIALIZED,
INITIALIZING,
SAFEMODE,
RUNNING,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index 0289ba6..aa730ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -38,8 +39,10 @@ import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.impl.MembershipStoreImpl;
import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.impl.RouterStoreImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.service.CompositeService;
@@ -148,6 +151,7 @@ public class StateStoreService extends CompositeService {
// Add supported record stores
addRecordStore(MembershipStoreImpl.class);
addRecordStore(MountTableStoreImpl.class);
+ addRecordStore(RouterStoreImpl.class);
// Check the connection to the State Store periodically
this.monitorService = new StateStoreConnectionMonitorService(this);
@@ -158,6 +162,11 @@ public class StateStoreService extends CompositeService {
DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT));
+ RouterState.setExpirationMs(conf.getTimeDuration(
+ DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
+ DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
// Cache update service
this.cacheUpdater = new StateStoreCacheUpdateService(this);
addService(this.cacheUpdater);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 7a23eb4..d24310e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5111,6 +5111,26 @@
</property>
<property>
+ <name>dfs.federation.router.heartbeat-state.interval</name>
+ <value>5s</value>
+ <description>
+ How often the Router should heartbeat its state into the State Store in
+ milliseconds. This setting supports multiple time unit suffixes as
+ described in dfs.federation.router.quota-cache.update.interval.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.router.expiration</name>
+ <value>5m</value>
+ <description>
+ Expiration time in milliseconds for a router state record. This setting
+ supports multiple time unit suffixes as described in
+ dfs.federation.router.quota-cache.update.interval.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.monitor.namenode</name>
<value></value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7721fff7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
new file mode 100644
index 0000000..ae15ef6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link StateStoreService} {@link RouterStore} functionality.
+ */
+public class TestStateStoreRouterState extends TestStateStoreBase {
+
+ private static RouterStore routerStore;
+
+ @BeforeClass
+ public static void create() {
+ // Reduce expirations to 5 seconds
+ getConf().setTimeDuration(
+ DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
+ 5, TimeUnit.SECONDS);
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+
+ if (routerStore == null) {
+ routerStore =
+ getStateStore().getRegisteredRecordStore(RouterStore.class);
+ }
+
+ // Clear router status registrations
+ assertTrue(clearRecords(getStateStore(), RouterState.class));
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertEquals(false, getStateStore().isDriverReady());
+
+ // Test all APIs that access the data store to ensure they throw the correct
+ // exception.
+ GetRouterRegistrationRequest getSingleRequest =
+ GetRouterRegistrationRequest.newInstance();
+ verifyException(routerStore, "getRouterRegistration",
+ StateStoreUnavailableException.class,
+ new Class[] {GetRouterRegistrationRequest.class},
+ new Object[] {getSingleRequest});
+
+ GetRouterRegistrationsRequest getRequest =
+ GetRouterRegistrationsRequest.newInstance();
+ routerStore.loadCache(true);
+ verifyException(routerStore, "getRouterRegistrations",
+ StateStoreUnavailableException.class,
+ new Class[] {GetRouterRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ RouterHeartbeatRequest hbRequest = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance("test", 0, RouterServiceState.UNINITIALIZED));
+ verifyException(routerStore, "routerHeartbeat",
+ StateStoreUnavailableException.class,
+ new Class[] {RouterHeartbeatRequest.class},
+ new Object[] {hbRequest});
+ }
+
+ //
+ // Router
+ //
+ @Test
+ public void testUpdateRouterStatus()
+ throws IllegalStateException, IOException {
+
+ long dateStarted = Time.now();
+ String address = "testaddress";
+
+ // Set
+ RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ address, dateStarted, RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+
+ // Verify
+ GetRouterRegistrationRequest getRequest =
+ GetRouterRegistrationRequest.newInstance(address);
+ RouterState record =
+ routerStore.getRouterRegistration(getRequest).getRouter();
+ assertNotNull(record);
+ assertEquals(RouterServiceState.RUNNING, record.getStatus());
+ assertEquals(address, record.getAddress());
+ assertEquals(FederationUtil.getCompileInfo(), record.getCompileInfo());
+ // Build version may vary a bit
+ assertTrue(record.getBuildVersion().length() > 0);
+ }
+
+ @Test
+ public void testRouterStateExpired()
+ throws IOException, InterruptedException {
+
+ long dateStarted = Time.now();
+ String address = "testaddress";
+
+ RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ address, dateStarted, RouterServiceState.RUNNING));
+ // Set
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+
+ // Verify
+ GetRouterRegistrationRequest getRequest =
+ GetRouterRegistrationRequest.newInstance(address);
+ RouterState record =
+ routerStore.getRouterRegistration(getRequest).getRouter();
+ assertNotNull(record);
+
+ // Wait past expiration (set to 5 sec in config)
+ Thread.sleep(6000);
+
+ // Verify expired
+ RouterState r = routerStore.getRouterRegistration(getRequest).getRouter();
+ assertEquals(RouterServiceState.EXPIRED, r.getStatus());
+
+ // Heartbeat again and this shouldn't be EXPIRED anymore
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+ r = routerStore.getRouterRegistration(getRequest).getRouter();
+ assertEquals(RouterServiceState.RUNNING, r.getStatus());
+ }
+
+ @Test
+ public void testGetAllRouterStates()
+ throws StateStoreUnavailableException, IOException {
+
+ // Set 2 entries
+ RouterHeartbeatRequest heartbeatRequest1 =
+ RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ "testaddress1", Time.now(), RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest1).getStatus());
+
+ RouterHeartbeatRequest heartbeatRequest2 =
+ RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ "testaddress2", Time.now(), RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest2).getStatus());
+
+ // Verify
+ routerStore.loadCache(true);
+ GetRouterRegistrationsRequest request =
+ GetRouterRegistrationsRequest.newInstance();
+ List<RouterState> entries =
+ routerStore.getRouterRegistrations(request).getRouters();
+ assertEquals(2, entries.size());
+ Collections.sort(entries);
+ assertEquals("testaddress1", entries.get(0).getAddress());
+ assertEquals("testaddress2", entries.get(1).getAddress());
+ assertEquals(RouterServiceState.RUNNING, entries.get(0).getStatus());
+ assertEquals(RouterServiceState.RUNNING, entries.get(1).getStatus());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-13054. Handling
PathIsNotEmptyDirectoryException in DFSClient delete call. Contributed by
Nanda kumar.
Posted by ha...@apache.org.
HDFS-13054. Handling PathIsNotEmptyDirectoryException in DFSClient delete call. Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e990904d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e990904d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e990904d
Branch: refs/heads/YARN-1011
Commit: e990904dd568a1d8f98efb55c1dd2d598ae4752b
Parents: a37e7f0
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Jan 26 11:42:27 2018 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Jan 26 13:09:13 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +++-
.../hadoop/hdfs/protocol/ClientProtocol.java | 3 +++
.../hadoop/hdfs/TestDistributedFileSystem.java | 17 +++++++++++++++++
3 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f0769c1..92bb99e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -82,6 +82,7 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
@@ -1620,7 +1621,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
FileNotFoundException.class,
SafeModeException.class,
UnresolvedPathException.class,
- SnapshotAccessControlException.class);
+ SnapshotAccessControlException.class,
+ PathIsNotEmptyDirectoryException.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index fbef037..0d77037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
@@ -625,6 +626,8 @@ public interface ClientProtocol {
* @throws org.apache.hadoop.fs.UnresolvedLinkException If <code>src</code>
* contains a symlink
* @throws SnapshotAccessControlException if path is in RO snapshot
+ * @throws PathIsNotEmptyDirectoryException if path is a non-empty directory
+ * and <code>recursive</code> is set to false
* @throws IOException If an I/O error occurred
*/
@AtMostOnce
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 823c747..072ee9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageStatistics.LongStatistic;
import org.apache.hadoop.fs.StorageType;
@@ -571,6 +572,22 @@ public class TestDistributedFileSystem {
in.close();
fs.close();
}
+
+ {
+ // Test PathIsNotEmptyDirectoryException while deleting non-empty dir
+ FileSystem fs = cluster.getFileSystem();
+ fs.mkdirs(new Path("/test/nonEmptyDir"));
+ fs.create(new Path("/tmp/nonEmptyDir/emptyFile")).close();
+ try {
+ fs.delete(new Path("/tmp/nonEmptyDir"), false);
+ Assert.fail("Expecting PathIsNotEmptyDirectoryException");
+ } catch (PathIsNotEmptyDirectoryException ex) {
+ // This is the proper exception to catch; move on.
+ }
+ Assert.assertTrue(fs.exists(new Path("/test/nonEmptyDir")));
+ fs.delete(new Path("/tmp/nonEmptyDir"), true);
+ }
+
}
finally {
if (cluster != null) {cluster.shutdown();}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: MAPREDUCE-7022. Fast fail rogue
jobs based on task scratch dir size. Contributed by Johan Gustavsson
Posted by ha...@apache.org.
MAPREDUCE-7022. Fast fail rogue jobs based on task scratch dir size. Contributed by Johan Gustavsson
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a37e7f0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a37e7f0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a37e7f0a
Branch: refs/heads/YARN-1011
Commit: a37e7f0ad8b68c7ed16c242bedf62f4cde48d6fd
Parents: 1b0f265
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 26 14:36:45 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 26 14:36:45 2018 -0600
----------------------------------------------------------------------
.../hadoop/mapred/LocalContainerLauncher.java | 2 +-
.../hadoop/mapred/TaskAttemptListenerImpl.java | 7 +-
.../org/apache/hadoop/mapred/YarnChild.java | 4 +-
.../v2/app/job/event/TaskAttemptFailEvent.java | 53 ++++++++++++
.../app/job/event/TaskTAttemptFailedEvent.java | 39 +++++++++
.../v2/app/job/impl/TaskAttemptImpl.java | 40 ++++++---
.../mapreduce/v2/app/job/impl/TaskImpl.java | 6 +-
.../hadoop/mapreduce/v2/app/TestFail.java | 7 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 7 +-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 5 +-
.../v2/app/job/impl/TestTaskAttempt.java | 9 +-
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 42 ++++-----
.../apache/hadoop/mapred/LocalJobRunner.java | 4 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 3 +-
.../java/org/apache/hadoop/mapred/Task.java | 87 ++++++++++++++++++-
.../hadoop/mapred/TaskUmbilicalProtocol.java | 12 ++-
.../apache/hadoop/mapreduce/MRJobConfig.java | 14 +++
.../src/main/resources/mapred-default.xml | 22 +++++
.../hadoop/mapred/TestTaskProgressReporter.java | 90 +++++++++++++++++++-
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 9 +-
.../apache/hadoop/mapred/TestMapProgress.java | 4 +-
.../apache/hadoop/mapred/TestTaskCommit.java | 2 +-
22 files changed, 397 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 6f9cc34..fed500a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -510,7 +510,7 @@ public class LocalContainerLauncher extends AbstractService implements
String cause =
(tCause == null) ? throwable.getMessage() : StringUtils
.stringifyException(tCause);
- umbilical.fatalError(classicAttemptID, cause);
+ umbilical.fatalError(classicAttemptID, cause, false);
}
throw new RuntimeException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 556c90c..b155af22 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
@@ -281,7 +282,7 @@ public class TaskAttemptListenerImpl extends CompositeService
}
@Override
- public void fatalError(TaskAttemptID taskAttemptID, String msg)
+ public void fatalError(TaskAttemptID taskAttemptID, String msg, boolean fastFail)
throws IOException {
// This happens only in Child and in the Task.
LOG.error("Task: " + taskAttemptID + " - exited : " + msg);
@@ -294,7 +295,7 @@ public class TaskAttemptListenerImpl extends CompositeService
preemptionPolicy.handleFailedContainer(attemptID);
context.getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID, fastFail));
}
@Override
@@ -312,7 +313,7 @@ public class TaskAttemptListenerImpl extends CompositeService
preemptionPolicy.handleFailedContainer(attemptID);
context.getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 7ae7a1e..bd40e54 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -206,7 +206,7 @@ class YarnChild {
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fatalError(taskid,
- StringUtils.stringifyException(exception));
+ StringUtils.stringifyException(exception), false);
}
}
} catch (Throwable throwable) {
@@ -218,7 +218,7 @@ class YarnChild {
String cause =
tCause == null ? throwable.getMessage() : StringUtils
.stringifyException(tCause);
- umbilical.fatalError(taskid, cause);
+ umbilical.fatalError(taskid, cause, false);
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptFailEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptFailEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptFailEvent.java
new file mode 100644
index 0000000..6ea1d15
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptFailEvent.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public class TaskAttemptFailEvent extends TaskAttemptEvent {
+ private boolean fastFail;
+
+ /**
+ * Create a new TaskAttemptFailEvent, with task fastFail disabled.
+ *
+ * @param id the id of the task attempt
+ */
+ public TaskAttemptFailEvent(TaskAttemptId id) {
+ this(id, false);
+ }
+
+ /**
+ * Create a new TaskAttemptFailEvent.
+ *
+ * @param id the id of the task attempt
+ * @param fastFail should the task fastFail or not.
+ */
+ public TaskAttemptFailEvent(TaskAttemptId id, boolean fastFail) {
+ super(id, TaskAttemptEventType.TA_FAILMSG);
+ this.fastFail = fastFail;
+ }
+
+ /**
+ * Check if task should fast fail or retry
+ * @return boolean value where true indicates the task should not retry
+ */
+ public boolean isFastFail() {
+ return fastFail;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptFailedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptFailedEvent.java
new file mode 100644
index 0000000..30392ac
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptFailedEvent.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public class TaskTAttemptFailedEvent extends TaskTAttemptEvent {
+
+ private boolean fastFail;
+
+ public TaskTAttemptFailedEvent(TaskAttemptId id) {
+ this(id, false);
+ }
+
+ public TaskTAttemptFailedEvent(TaskAttemptId id, boolean fastFail) {
+ super(id, TaskEventType.T_ATTEMPT_FAILED);
+ this.fastFail = fastFail;
+ }
+
+ public boolean isFastFail() {
+ return fastFail;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 431128b..6632f27 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunched
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptRecoverEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
@@ -101,6 +102,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptKilledEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
@@ -194,6 +196,7 @@ public abstract class TaskAttemptImpl implements
private Locality locality;
private Avataar avataar;
private boolean rescheduleNextAttempt = false;
+ private boolean failFast = false;
private static final CleanupContainerTransition
CLEANUP_CONTAINER_TRANSITION = new CleanupContainerTransition();
@@ -1412,6 +1415,14 @@ public abstract class TaskAttemptImpl implements
public void setAvataar(Avataar avataar) {
this.avataar = avataar;
}
+
+ public void setTaskFailFast(boolean failFast) {
+ this.failFast = failFast;
+ }
+
+ public boolean isTaskFailFast() {
+ return failFast;
+ }
@SuppressWarnings("unchecked")
public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo,
@@ -1921,9 +1932,12 @@ public abstract class TaskAttemptImpl implements
switch(finalState) {
case FAILED:
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
- taskAttempt.attemptId,
- TaskEventType.T_ATTEMPT_FAILED));
+ boolean fastFail = false;
+ if (event instanceof TaskAttemptFailEvent) {
+ fastFail = ((TaskAttemptFailEvent) event).isFastFail();
+ }
+ taskAttempt.eventHandler.handle(new TaskTAttemptFailedEvent(
+ taskAttempt.attemptId, fastFail));
break;
case KILLED:
taskAttempt.eventHandler.handle(new TaskTAttemptKilledEvent(
@@ -2041,13 +2055,16 @@ public abstract class TaskAttemptImpl implements
private static class FailedTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+
+
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
// set the finish time
taskAttempt.setFinishTime();
- notifyTaskAttemptFailed(taskAttempt);
+
+ notifyTaskAttemptFailed(taskAttempt, taskAttempt.isTaskFailFast());
}
}
@@ -2154,8 +2171,8 @@ public abstract class TaskAttemptImpl implements
LOG.debug("Not generating HistoryFinish event since start event not " +
"generated for taskAttempt: " + taskAttempt.getID());
}
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
- taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
+ taskAttempt.eventHandler.handle(new TaskTAttemptFailedEvent(
+ taskAttempt.attemptId));
}
}
@@ -2332,6 +2349,8 @@ public abstract class TaskAttemptImpl implements
if (event instanceof TaskAttemptKillEvent) {
taskAttempt.setRescheduleNextAttempt(
((TaskAttemptKillEvent)event).getRescheduleAttempt());
+ } else if (event instanceof TaskAttemptFailEvent) {
+ taskAttempt.setTaskFailFast(((TaskAttemptFailEvent)event).isFastFail());
}
}
}
@@ -2400,12 +2419,13 @@ public abstract class TaskAttemptImpl implements
// register it to finishing state
taskAttempt.appContext.getTaskAttemptFinishingMonitor().register(
taskAttempt.attemptId);
- notifyTaskAttemptFailed(taskAttempt);
+ notifyTaskAttemptFailed(taskAttempt, false);
}
}
@SuppressWarnings("unchecked")
- private static void notifyTaskAttemptFailed(TaskAttemptImpl taskAttempt) {
+ private static void notifyTaskAttemptFailed(TaskAttemptImpl taskAttempt,
+ boolean fastFail) {
if (taskAttempt.getLaunchTime() == 0) {
sendJHStartEventForAssignedFailTask(taskAttempt);
}
@@ -2419,8 +2439,8 @@ public abstract class TaskAttemptImpl implements
taskAttempt.eventHandler.handle(new JobHistoryEvent(
taskAttempt.attemptId.getTaskId().getJobId(), tauce));
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
- taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
+ taskAttempt.eventHandler.handle(new TaskTAttemptFailedEvent(
+ taskAttempt.attemptId, fastFail));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
index 086d4d5..ce3b3cc 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskRecoverEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptKilledEvent;
import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
@@ -1054,7 +1055,7 @@ public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
@Override
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
- TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
+ TaskTAttemptFailedEvent castEvent = (TaskTAttemptFailedEvent) event;
TaskAttemptId taskAttemptId = castEvent.getTaskAttemptID();
task.failedAttempts.add(taskAttemptId);
if (taskAttemptId.equals(task.commitAttempt)) {
@@ -1068,7 +1069,8 @@ public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
}
task.finishedAttempts.add(taskAttemptId);
- if (task.failedAttempts.size() < task.maxAttempts) {
+ if (!castEvent.isFastFail()
+ && task.failedAttempts.size() < task.maxAttempts) {
task.handleTaskAttemptCompletion(
taskAttemptId,
TaskAttemptCompletionEventStatus.FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
index 4d3f6f4..a2f0aba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
@@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import java.util.Iterator;
import java.util.Map;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -288,8 +289,7 @@ public class TestFail {
if (attemptID.getTaskId().getId() == 0) {//check if it is first task
// send the Fail event
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID,
- TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else {
getContext().getEventHandler().handle(
new TaskAttemptEvent(attemptID,
@@ -310,8 +310,7 @@ public class TestFail {
//check if it is first task's first attempt
// send the Fail event
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID,
- TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else {
getContext().getEventHandler().handle(
new TaskAttemptEvent(attemptID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
index 893c4a0..b2807c1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
@@ -38,6 +38,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -167,9 +169,8 @@ public class TestRecovery {
/////////// Play some games with the TaskAttempts of the first task //////
//send the fail signal to the 1st map task attempt
app.getContext().getEventHandler().handle(
- new TaskAttemptEvent(
- task1Attempt1.getID(),
- TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(
+ task1Attempt1.getID()));
app.waitForState(task1Attempt1, TaskAttemptState.FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
index 1827ce4..8592b20 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
@@ -81,7 +81,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.InitTransition;
import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
@@ -437,8 +437,7 @@ public class TestJobImpl {
TaskImpl task = (TaskImpl) t;
task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
for(TaskAttempt ta: task.getAttempts().values()) {
- task.handle(new TaskTAttemptEvent(ta.getID(),
- TaskEventType.T_ATTEMPT_FAILED));
+ task.handle(new TaskTAttemptFailedEvent(ta.getID()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index fe5d95d..43571a9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -39,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -499,7 +500,7 @@ public class TestTaskAttempt{
new TaskAttemptDiagnosticsUpdateEvent(attemptID,
"Test Diagnostic Event"));
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
}
protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
@@ -1357,8 +1358,7 @@ public class TestTaskAttempt{
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
- taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
- TaskAttemptEventType.TA_FAILMSG));
+ taImpl.handle(new TaskAttemptFailEvent(taImpl.getID()));
assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
TaskAttemptState.FAILED);
@@ -1484,8 +1484,7 @@ public class TestTaskAttempt{
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
- taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
- TaskAttemptEventType.TA_FAILMSG));
+ taImpl.handle(new TaskAttemptFailEvent(taImpl.getID()));
assertEquals("Task attempt is not in RUNNING state", taImpl.getState(),
TaskAttemptState.FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
index 62d4cc0..1225c43 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptKilledEvent;
import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
import org.apache.hadoop.security.Credentials;
@@ -345,8 +346,7 @@ public class TestTaskImpl {
}
private void failRunningTaskAttempt(TaskAttemptId attemptId) {
- mockTask.handle(new TaskTAttemptEvent(attemptId,
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(attemptId));
assertTaskRunningState();
}
@@ -612,11 +612,16 @@ public class TestTaskImpl {
// The task should now have succeeded
assertTaskSucceededState();
-
+
// Now complete the first task attempt, after the second has succeeded
- mockTask.handle(new TaskTAttemptEvent(taskAttempts.get(0).getAttemptId(),
- firstAttemptFinishEvent));
-
+ if (firstAttemptFinishEvent.equals(TaskEventType.T_ATTEMPT_FAILED)) {
+ mockTask.handle(new TaskTAttemptFailedEvent(taskAttempts
+ .get(0).getAttemptId()));
+ } else {
+ mockTask.handle(new TaskTAttemptEvent(taskAttempts.get(0).getAttemptId(),
+ firstAttemptFinishEvent));
+ }
+
// The task should still be in the succeeded state
assertTaskSucceededState();
@@ -668,8 +673,8 @@ public class TestTaskImpl {
assertEquals(2, taskAttempts.size());
// speculative attempt retroactively fails from fetch failures
- mockTask.handle(new TaskTAttemptEvent(taskAttempts.get(1).getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(
+ taskAttempts.get(1).getAttemptId()));
assertTaskScheduledState();
assertEquals(3, taskAttempts.size());
@@ -683,8 +688,8 @@ public class TestTaskImpl {
assertEquals(2, taskAttempts.size());
// speculative attempt retroactively fails from fetch failures
- mockTask.handle(new TaskTAttemptEvent(taskAttempts.get(1).getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(
+ taskAttempts.get(1).getAttemptId()));
assertTaskScheduledState();
assertEquals(3, taskAttempts.size());
@@ -698,8 +703,8 @@ public class TestTaskImpl {
assertEquals(2, taskAttempts.size());
// speculative attempt retroactively fails from fetch failures
- mockTask.handle(new TaskTAttemptEvent(taskAttempts.get(1).getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(
+ taskAttempts.get(1).getAttemptId()));
assertTaskScheduledState();
assertEquals(3, taskAttempts.size());
@@ -734,8 +739,8 @@ public class TestTaskImpl {
// have the first attempt fail, verify task failed due to no retries
MockTaskAttemptImpl taskAttempt = taskAttempts.get(0);
taskAttempt.setState(TaskAttemptState.FAILED);
- mockTask.handle(new TaskTAttemptEvent(taskAttempt.getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(
+ taskAttempt.getAttemptId()));
assertEquals(TaskState.FAILED, mockTask.getState());
// verify task can no longer be killed
@@ -757,8 +762,7 @@ public class TestTaskImpl {
TaskEventType.T_ATTEMPT_COMMIT_PENDING));
assertEquals(TaskState.FAILED, mockTask.getState());
taskAttempt.setState(TaskAttemptState.FAILED);
- mockTask.handle(new TaskTAttemptEvent(taskAttempt.getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(taskAttempt.getAttemptId()));
assertEquals(TaskState.FAILED, mockTask.getState());
taskAttempt = taskAttempts.get(2);
taskAttempt.setState(TaskAttemptState.SUCCEEDED);
@@ -808,8 +812,7 @@ public class TestTaskImpl {
// max attempts is 4
MockTaskAttemptImpl taskAttempt = taskAttempts.get(0);
taskAttempt.setState(TaskAttemptState.FAILED);
- mockTask.handle(new TaskTAttemptEvent(taskAttempt.getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(taskAttempt.getAttemptId()));
assertEquals(TaskState.RUNNING, mockTask.getState());
// verify a new attempt(#3) added because the speculative attempt(#2)
@@ -829,8 +832,7 @@ public class TestTaskImpl {
// hasn't reach the max attempts which is 4
MockTaskAttemptImpl taskAttempt1 = taskAttempts.get(1);
taskAttempt1.setState(TaskAttemptState.FAILED);
- mockTask.handle(new TaskTAttemptEvent(taskAttempt1.getAttemptId(),
- TaskEventType.T_ATTEMPT_FAILED));
+ mockTask.handle(new TaskTAttemptFailedEvent(taskAttempt1.getAttemptId()));
assertEquals(TaskState.RUNNING, mockTask.getState());
// verify there's no new attempt added because of the running attempt(#3)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index c9dff6a..5e7a250 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -729,9 +729,9 @@ public class LocalJobRunner implements ClientProtocol {
LOG.error("shuffleError: "+ message + "from task: " + taskId);
}
- public synchronized void fatalError(TaskAttemptID taskId, String msg)
+ public synchronized void fatalError(TaskAttemptID taskId, String msg, boolean fastFail)
throws IOException {
- LOG.error("Fatal: "+ msg + "from task: " + taskId);
+ LOG.error("Fatal: "+ msg + " from task: " + taskId + " fast fail: " + fastFail);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 27c8976..ab7cba5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -1568,7 +1568,8 @@ public class MapTask extends Task {
if (lspillException instanceof Error) {
final String logMsg = "Task " + getTaskID() + " failed : " +
StringUtils.stringifyException(lspillException);
- mapTask.reportFatalError(getTaskID(), lspillException, logMsg);
+ mapTask.reportFatalError(getTaskID(), lspillException, logMsg,
+ false);
}
throw new IOException("Spill failed", lspillException);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 730f4ee..87c9e16 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
@@ -354,7 +355,7 @@ abstract public class Task implements Writable, Configurable {
* Report a fatal error to the parent (task) tracker.
*/
protected void reportFatalError(TaskAttemptID id, Throwable throwable,
- String logMsg) {
+ String logMsg, boolean fastFail) {
LOG.error(logMsg);
if (ShutdownHookManager.get().isShutdownInProgress()) {
@@ -366,7 +367,7 @@ abstract public class Task implements Writable, Configurable {
? StringUtils.stringifyException(throwable)
: StringUtils.stringifyException(tCause);
try {
- umbilical.fatalError(id, cause);
+ umbilical.fatalError(id, cause, fastFail);
} catch (IOException ioe) {
LOG.error("Failed to contact the tasktracker", ioe);
System.exit(-1);
@@ -652,6 +653,8 @@ abstract public class Task implements Writable, Configurable {
private Thread pingThread = null;
private boolean done = true;
private Object lock = new Object();
+ private volatile String diskLimitCheckStatus = null;
+ private Thread diskLimitCheckThread = null;
/**
* flag that indicates whether progress update needs to be sent to parent.
@@ -749,6 +752,65 @@ abstract public class Task implements Writable, Configurable {
}
/**
+ * disk limit checker, runs in separate thread when activated.
+ */
+ public class DiskLimitCheck implements Runnable {
+ private LocalFileSystem localFS;
+ private long fsLimit;
+ private long checkInterval;
+ private String[] localDirs;
+ private boolean killOnLimitExceeded;
+
+ public DiskLimitCheck(JobConf conf) throws IOException {
+ this.localFS = FileSystem.getLocal(conf);
+ this.fsLimit = conf.getLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES,
+ MRJobConfig.DEFAULT_JOB_SINGLE_DISK_LIMIT_BYTES);
+ this.localDirs = conf.getLocalDirs();
+ this.checkInterval = conf.getLong(
+ MRJobConfig.JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS,
+ MRJobConfig.DEFAULT_JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS);
+ this.killOnLimitExceeded = conf.getBoolean(
+ MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED,
+ MRJobConfig.DEFAULT_JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED);
+ }
+
+ @Override
+ public void run() {
+ while (!taskDone.get()) {
+ try {
+ long localWritesSize = 0L;
+ String largestWorkDir = null;
+ for (String local : localDirs) {
+ long size = FileUtil.getDU(localFS.pathToFile(new Path(local)));
+ if (localWritesSize < size) {
+ localWritesSize = size;
+ largestWorkDir = local;
+ }
+ }
+ if (localWritesSize > fsLimit) {
+ String localStatus =
+ "too much data in local scratch dir="
+ + largestWorkDir
+ + ". current size is "
+ + localWritesSize
+ + " the limit is " + fsLimit;
+ if (killOnLimitExceeded) {
+ LOG.error(localStatus);
+ diskLimitCheckStatus = localStatus;
+ } else {
+ LOG.warn(localStatus);
+ }
+ break;
+ }
+ Thread.sleep(checkInterval);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ /**
* check the counters to see whether the task has exceeded any configured
* limits.
* @throws TaskLimitException
@@ -773,6 +835,9 @@ abstract public class Task implements Writable, Configurable {
" the limit is " + limit);
}
}
+ if (diskLimitCheckStatus != null) {
+ throw new TaskLimitException(diskLimitCheckStatus);
+ }
}
/**
@@ -851,7 +916,7 @@ abstract public class Task implements Writable, Configurable {
StringUtils.stringifyException(e);
LOG.error(errMsg);
try {
- umbilical.fatalError(taskId, errMsg);
+ umbilical.fatalError(taskId, errMsg, true);
} catch (IOException ioe) {
LOG.error("Failed to update failure diagnosis", ioe);
}
@@ -884,6 +949,22 @@ abstract public class Task implements Writable, Configurable {
pingThread.setDaemon(true);
pingThread.start();
}
+ startDiskLimitCheckerThreadIfNeeded();
+ }
+ public void startDiskLimitCheckerThreadIfNeeded() {
+ if (diskLimitCheckThread == null && conf.getLong(
+ MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES,
+ MRJobConfig.DEFAULT_JOB_SINGLE_DISK_LIMIT_BYTES) >= 0) {
+ try {
+ diskLimitCheckThread = new Thread(new DiskLimitCheck(conf),
+ "disk limit check thread");
+ diskLimitCheckThread.setDaemon(true);
+ diskLimitCheckThread.start();
+ } catch (IOException e) {
+ LOG.error("Issues starting disk monitor thread: "
+ + e.getMessage(), e);
+ }
+ }
}
public void stopCommunicationThread() throws InterruptedException {
if (pingThread != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
index c3678d6..041ab39 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
@@ -68,9 +68,10 @@ public interface TaskUmbilicalProtocol extends VersionedProtocol {
* Version 18 Added numRequiredSlots to TaskStatus for MAPREDUCE-516
* Version 19 Added fatalError for child to communicate fatal errors to TT
* Version 20 Added methods to manage checkpoints
+ * Version 21 Added fastFail parameter to fatalError
* */
- public static final long versionID = 20L;
+ public static final long versionID = 21L;
/**
* Called when a child task process starts, to get its task.
@@ -140,8 +141,13 @@ public interface TaskUmbilicalProtocol extends VersionedProtocol {
/** Report that the task encounted a local filesystem error.*/
void fsError(TaskAttemptID taskId, String message) throws IOException;
- /** Report that the task encounted a fatal error.*/
- void fatalError(TaskAttemptID taskId, String message) throws IOException;
+ /**
+ * Report that the task encounted a fatal error.
+ * @param taskId task's id
+ * @param message fail message
+ * @param fastFail flag to enable fast fail for task
+ */
+ void fatalError(TaskAttemptID taskId, String message, boolean fastFail) throws IOException;
/** Called by a reduce task to get the map output locations for finished maps.
* Returns an update centered around the map-task-completion-events.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 6acf1bc..ca18bfe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -52,6 +52,20 @@ public interface MRJobConfig {
public static final String TASK_CLEANUP_NEEDED = "mapreduce.job.committer.task.cleanup.needed";
+ public static final String JOB_SINGLE_DISK_LIMIT_BYTES =
+ "mapreduce.job.local-fs.single-disk-limit.bytes";
+ // negative values disable the limit
+ public static final long DEFAULT_JOB_SINGLE_DISK_LIMIT_BYTES = -1;
+
+ public static final String JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED =
+ "mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed";
+ // setting to false only logs the kill
+ public static final boolean DEFAULT_JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED = true;
+
+ public static final String JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS =
+ "mapreduce.job.local-fs.single-disk-limit.check.interval-ms";
+ public static final long DEFAULT_JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS = 5000;
+
public static final String TASK_LOCAL_WRITE_LIMIT_BYTES =
"mapreduce.task.local-fs.write-limit.bytes";
// negative values disable the limit
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 62f3dfa..72f509c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -63,6 +63,28 @@
</property>
<property>
+ <name>mapreduce.job.local-fs.single-disk-limit.bytes</name>
+ <value>-1</value>
+ <description>Enable an in task monitor thread to watch for single disk
+ consumption by jobs. By setting this to x nr of bytes, the task will fast
+ fail in case it is reached. This is a per disk configuration.</description>
+</property>
+
+<property>
+ <name>mapreduce.job.local-fs.single-disk-limit.check.interval-ms</name>
+ <value>5000</value>
+ <description>Interval of disk limit check to run in ms.</description>
+</property>
+
+<property>
+ <name>mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed</name>
+ <value>true</value>
+ <description>If mapreduce.job.local-fs.single-disk-limit.bytes is triggered
+ should the task be killed or logged. If false the intent to kill the task
+ is only logged in the container logs.</description>
+</property>
+
+<property>
<name>mapreduce.job.maps</name>
<value>2</value>
<description>The default number of map tasks per job.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
index 18442d6..e5ff64e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
@@ -18,15 +18,19 @@
package org.apache.hadoop.mapred;
+import java.io.File;
import java.io.IOException;
import java.util.Random;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.mapred.SortedRanges.Range;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.util.ExitUtil;
@@ -43,6 +47,11 @@ public class TestTaskProgressReporter {
private FakeUmbilical fakeUmbilical = new FakeUmbilical();
+ private static final String TEST_DIR =
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")) + "/" +
+ TestTaskProgressReporter.class.getName();
+
private static class DummyTask extends Task {
@Override
public void run(JobConf job, TaskUmbilicalProtocol umbilical)
@@ -53,6 +62,11 @@ public class TestTaskProgressReporter {
public boolean isMapTask() {
return true;
}
+
+ @Override
+ public boolean isCommitRequired() {
+ return false;
+ }
}
private static class FakeUmbilical implements TaskUmbilicalProtocol {
@@ -118,7 +132,7 @@ public class TestTaskProgressReporter {
}
@Override
- public void fatalError(TaskAttemptID taskId, String message)
+ public void fatalError(TaskAttemptID taskId, String message, boolean fastFail)
throws IOException {
}
@@ -163,6 +177,78 @@ public class TestTaskProgressReporter {
}
}
+ @Test(timeout=60000)
+ public void testScratchDirSize() throws Exception {
+ String tmpPath = TEST_DIR + "/testBytesWrittenLimit-tmpFile-"
+ + new Random(System.currentTimeMillis()).nextInt();
+ File data = new File(tmpPath + "/out");
+ File testDir = new File(tmpPath);
+ testDir.mkdirs();
+ testDir.deleteOnExit();
+ JobConf conf = new JobConf();
+ conf.setStrings(MRConfig.LOCAL_DIR, "file://" + tmpPath);
+ conf.setLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES, 1024L);
+ conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED,
+ true);
+ getBaseConfAndWriteToFile(-1, data);
+ testScratchDirLimit(false, conf);
+ data.delete();
+ getBaseConfAndWriteToFile(100, data);
+ testScratchDirLimit(false, conf);
+ data.delete();
+ getBaseConfAndWriteToFile(1536, data);
+ testScratchDirLimit(true, conf);
+ conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED,
+ false);
+ testScratchDirLimit(false, conf);
+ conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED,
+ true);
+ conf.setLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES, -1L);
+ testScratchDirLimit(false, conf);
+ data.delete();
+ FileUtil.fullyDelete(testDir);
+ }
+
+ private void getBaseConfAndWriteToFile(int size, File data)
+ throws IOException {
+ if (size > 0) {
+ byte[] b = new byte[size];
+ for (int i = 0; i < size; i++) {
+ b[i] = 1;
+ }
+ FileUtils.writeByteArrayToFile(data, b);
+ }
+ }
+
+ public void testScratchDirLimit(boolean fastFail, JobConf conf)
+ throws Exception {
+ ExitUtil.disableSystemExit();
+ threadExited = false;
+ Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread th, Throwable ex) {
+ if (ex instanceof ExitUtil.ExitException) {
+ threadExited = true;
+ th.interrupt();
+ }
+ }
+ };
+ Task task = new DummyTask();
+ task.setConf(conf);
+ DummyTaskReporter reporter = new DummyTaskReporter(task);
+ reporter.startDiskLimitCheckerThreadIfNeeded();
+ Thread t = new Thread(reporter);
+ t.setUncaughtExceptionHandler(h);
+ reporter.setProgressFlag();
+ t.start();
+ while (!reporter.taskLimitIsChecked) {
+ Thread.yield();
+ }
+ task.done(fakeUmbilical, reporter);
+ reporter.resetDoneFlag();
+ t.join(1000L);
+ Assert.assertEquals(fastFail, threadExited);
+ }
+
@Test (timeout=10000)
public void testTaskProgress() throws Exception {
JobConf job = new JobConf();
@@ -214,7 +300,7 @@ public class TestTaskProgressReporter {
conf.getLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL, 0);
conf.setLong(MRJobConfig.TASK_LOCAL_WRITE_LIMIT_BYTES, limit);
LocalFileSystem localFS = FileSystem.getLocal(conf);
- Path tmpPath = new Path("/tmp/testBytesWrittenLimit-tmpFile-"
+ Path tmpPath = new Path(TEST_DIR + "/testBytesWrittenLimit-tmpFile-"
+ new Random(System.currentTimeMillis()).nextInt());
FSDataOutputStream out = localFS.create(tmpPath, true);
out.write(new byte[LOCAL_BYTES_WRITTEN]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
index 83e35fe..7b70f98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -712,7 +713,7 @@ public class TestJobHistoryParsing {
protected void attemptLaunched(TaskAttemptId attemptID) {
if (attemptID.getTaskId().getId() == 0 && attemptID.getId() == 0) {
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else {
getContext().getEventHandler().handle(
new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_DONE));
@@ -732,7 +733,7 @@ public class TestJobHistoryParsing {
protected void attemptLaunched(TaskAttemptId attemptID) {
if (attemptID.getTaskId().getId() == 0) {
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else {
getContext().getEventHandler().handle(
new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_DONE));
@@ -760,10 +761,10 @@ public class TestJobHistoryParsing {
new TaskEvent(attemptID.getTaskId(), TaskEventType.T_KILL));
} else if (taskType == TaskType.MAP && taskId == 1) {
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else if (taskType == TaskType.REDUCE && taskId == 0) {
getContext().getEventHandler().handle(
- new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ new TaskAttemptFailEvent(attemptID));
} else if (taskType == TaskType.REDUCE && taskId == 1) {
getContext().getEventHandler().handle(
new TaskEvent(attemptID.getTaskId(), TaskEventType.T_KILL));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
index f364c18..9b6ebda 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
@@ -91,8 +91,8 @@ public class TestMapProgress {
LOG.info("Task " + taskId + " reporting shuffle error: " + message);
}
- public void fatalError(TaskAttemptID taskId, String msg) throws IOException {
- LOG.info("Task " + taskId + " reporting fatal error: " + msg);
+ public void fatalError(TaskAttemptID taskId, String msg, boolean fastFail) throws IOException {
+ LOG.info("Task " + taskId + " reporting fatal error: " + msg + " fast fail: " + fastFail);
}
public JvmTask getTask(JvmContext context) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a37e7f0a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
index bed545e..a534cfa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
@@ -124,7 +124,7 @@ public class TestTaskCommit extends HadoopTestCase {
}
@Override
- public void fatalError(TaskAttemptID taskId, String message)
+ public void fatalError(TaskAttemptID taskId, String message, boolean fastFail)
throws IOException { }
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: MAPREDUCE-7041. MR should not try
to clean up at first job attempt. (Gergo Repas via Haibo Chen)
Posted by ha...@apache.org.
MAPREDUCE-7041. MR should not try to clean up at first job attempt. (Gergo Repas via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc108522
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc108522
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc108522
Branch: refs/heads/YARN-1011
Commit: cc10852252c2d69294eabc68bd032cc630a53b18
Parents: ff8378e
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Jan 25 16:11:01 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Jan 25 16:11:30 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc108522/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index e6a45cf..cb65b89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1400,7 +1400,7 @@ public class MRAppMaster extends CompositeService {
private void cleanUpPreviousJobOutput() {
// recovered application masters should not remove data from previous job
- if (!recovered()) {
+ if (!isFirstAttempt() && !recovered()) {
JobContext jobContext = getJobContextFromConf(getConfig());
try {
LOG.info("Starting to clean up previous job's temporary files");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-7540 and YARN-7605. Convert
yarn app cli to call yarn api services and implement doAs for Api Service
REST API. Contributed by Eric Yang and Jian He
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index d3ad53e..7364445 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -82,6 +82,7 @@ public class WebApps {
public Class<? extends HttpServlet> clazz;
public String name;
public String spec;
+ public Map<String, String> params;
}
final String name;
@@ -147,7 +148,19 @@ public class WebApps {
servlets.add(struct);
return this;
}
-
+
+ public Builder<T> withServlet(String name, String pathSpec,
+ Class<? extends HttpServlet> servlet,
+ Map<String, String> params) {
+ ServletStruct struct = new ServletStruct();
+ struct.clazz = servlet;
+ struct.name = name;
+ struct.spec = pathSpec;
+ struct.params = params;
+ servlets.add(struct);
+ return this;
+ }
+
public Builder<T> with(Configuration conf) {
this.conf = conf;
return this;
@@ -243,6 +256,11 @@ public class WebApps {
pathList.add("/" + wsName + "/*");
}
}
+ for (ServletStruct s : servlets) {
+ if (!pathList.contains(s.spec)) {
+ pathList.add(s.spec);
+ }
+ }
if (conf == null) {
conf = new Configuration();
}
@@ -315,7 +333,12 @@ public class WebApps {
HttpServer2 server = builder.build();
for(ServletStruct struct: servlets) {
- server.addServlet(struct.name, struct.spec, struct.clazz);
+ if (struct.params != null) {
+ server.addInternalServlet(struct.name, struct.spec,
+ struct.clazz, struct.params);
+ } else {
+ server.addServlet(struct.name, struct.spec, struct.clazz);
+ }
}
for(Map.Entry<String, Object> entry : attributes.entrySet()) {
server.setAttribute(entry.getKey(), entry.getValue());
@@ -394,22 +417,16 @@ public class WebApps {
}
public WebApp start(WebApp webapp) {
- return start(webapp, null, null);
+ return start(webapp, null);
}
- public WebApp start(WebApp webapp, WebAppContext ui2Context,
- Map<String, String> services) {
+ public WebApp start(WebApp webapp, WebAppContext ui2Context) {
WebApp webApp = build(webapp);
HttpServer2 httpServer = webApp.httpServer();
if (ui2Context != null) {
addFiltersForNewContext(ui2Context);
httpServer.addHandlerAtFront(ui2Context);
}
- if (services!=null) {
- String packageName = services.get("PackageName");
- String pathSpec = services.get("PathSpec");
- httpServer.addJerseyResourcePackage(packageName, pathSpec);
- }
try {
httpServer.start();
LOG.info("Web app " + name + " started at "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index 0006dfd..cfa2d65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -221,6 +221,7 @@ public interface RegistryConstants {
* No authentication; client is anonymous.
*/
String REGISTRY_CLIENT_AUTH_ANONYMOUS = "";
+ String REGISTRY_CLIENT_AUTH_SIMPLE = "simple";
/**
* Registry client authentication ID.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index ff6e8aa..521d8a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -99,7 +99,7 @@ public class RegistrySecurity extends AbstractService {
* Access policy options
*/
private enum AccessPolicy {
- anon, sasl, digest
+ anon, sasl, digest, simple
}
/**
@@ -214,6 +214,9 @@ public class RegistrySecurity extends AbstractService {
case REGISTRY_CLIENT_AUTH_ANONYMOUS:
access = AccessPolicy.anon;
break;
+ case REGISTRY_CLIENT_AUTH_SIMPLE:
+ access = AccessPolicy.simple;
+ break;
default:
throw new ServiceStateException(E_UNKNOWN_AUTHENTICATION_MECHANISM
+ "\"" + auth + "\"");
@@ -302,6 +305,7 @@ public class RegistrySecurity extends AbstractService {
break;
case anon:
+ case simple:
// nothing is needed; account is read only.
if (LOG.isDebugEnabled()) {
LOG.debug("Auth is anonymous");
@@ -758,6 +762,9 @@ public class RegistrySecurity extends AbstractService {
LOG.info(
"Enabling ZK sasl client: jaasClientEntry = " + jaasClientEntry
+ ", principal = " + principal + ", keytab = " + keytab);
+ default:
+ clearZKSaslClientProperties();
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 8641842..7bbf4aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.framework.AuthInfo;
@@ -1049,11 +1051,23 @@ public class ResourceManager extends CompositeService implements Recoverable {
RMWebAppUtil.setupSecurityAndFilters(conf,
getClientRMService().rmDTSecretManager);
+ Map<String, String> params = new HashMap<String, String>();
+ if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
+ false)) {
+ String apiPackages = "org.apache.hadoop.yarn.service.webapp;" +
+ "org.apache.hadoop.yarn.webapp";
+ params.put("com.sun.jersey.config.property.resourceConfigClass",
+ "com.sun.jersey.api.core.PackagesResourceConfig");
+ params.put("com.sun.jersey.config.property.packages", apiPackages);
+ }
+
Builder<ApplicationMasterService> builder =
WebApps
.$for("cluster", ApplicationMasterService.class, masterService,
"ws")
.with(conf)
+ .withServlet("API-Service", "/app/*",
+ ServletContainer.class, params)
.withHttpSpnegoPrincipalKey(
YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
@@ -1109,15 +1123,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
- false)) {
- serviceConfig = new HashMap<String, String>();
- String apiPackages = "org.apache.hadoop.yarn.service.webapp;" +
- "org.apache.hadoop.yarn.webapp";
- serviceConfig.put("PackageName", apiPackages);
- serviceConfig.put("PathSpec", "/app/*");
- }
- webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig);
+ webApp = builder.start(new RMWebApp(this), uiWebAppContext);
}
private String getWebAppsPath(String appName) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-4512 [YARN-1011]. Provide a
knob to turn on over-allocation. (kasha)
Posted by ha...@apache.org.
YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab91ebd2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab91ebd2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab91ebd2
Branch: refs/heads/YARN-1011
Commit: ab91ebd2cce89ff4e31433a0a179fdd18b4cf5ef
Parents: 443523f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Fri Jan 29 14:31:45 2016 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 17:43:50 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 13 ++-
.../src/main/resources/yarn-default.xml | 21 ++++
.../RegisterNodeManagerRequest.java | 14 ++-
.../pb/RegisterNodeManagerRequestPBImpl.java | 45 +++++++-
.../server/api/records/OverAllocationInfo.java | 45 ++++++++
.../server/api/records/ResourceThresholds.java | 45 ++++++++
.../impl/pb/OverAllocationInfoPBImpl.java | 106 +++++++++++++++++++
.../impl/pb/ResourceThresholdsPBImpl.java | 93 ++++++++++++++++
.../yarn_server_common_service_protos.proto | 10 ++
.../hadoop/yarn/server/nodemanager/Context.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 17 +++
.../nodemanager/NodeStatusUpdaterImpl.java | 7 +-
.../monitor/ContainersMonitorImpl.java | 34 ++++++
.../amrmproxy/BaseAMRMProxyTest.java | 11 ++
14 files changed, 455 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bbbfc52..d8f4d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1942,7 +1942,6 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
-
// Configurations for applicaiton life time monitor feature
public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =
RM_PREFIX + "application-timeouts.monitor.interval-ms";
@@ -1950,6 +1949,18 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
3000;
+ /** Overallocation (= allocation based on utilization) configs. */
+ public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD =
+ NM_PREFIX + "overallocation.allocation-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD
+ = 0f;
+ @Private
+ public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f;
+ public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
+ = 0f;
+
/**
* Interval of time the linux container executor should try cleaning up
* cgroups entry when cleaning up a container. This is required due to what
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0bb4fca..fc5430a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1668,6 +1668,27 @@
</property>
<property>
+ <description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node,
+ expressed as a float between 0 and 0.95. By default, over-allocation is
+ turned off (value = 0). When turned on, the node allows running
+ OPPORTUNISTIC containers when the aggregate utilization is under the
+ value specified here multiplied by the node's advertised capacity.
+ </description>
+ <name>yarn.nodemanager.overallocation.allocation-threshold</name>
+ <value>0f</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the utilization
+ beyond which OPPORTUNISTIC containers should start getting preempted.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold</name>
+ <value>1</value>
+ </property>
+
+ <property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
index fc30a80..da2987f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.util.Records;
public abstract class RegisterNodeManagerRequest {
@@ -42,14 +43,14 @@ public abstract class RegisterNodeManagerRequest {
List<NMContainerStatus> containerStatuses,
List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels) {
return newInstance(nodeId, httpPort, resource, nodeManagerVersionId,
- containerStatuses, runningApplications, nodeLabels, null);
+ containerStatuses, runningApplications, nodeLabels, null, null);
}
public static RegisterNodeManagerRequest newInstance(NodeId nodeId,
int httpPort, Resource resource, String nodeManagerVersionId,
List<NMContainerStatus> containerStatuses,
List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels,
- Resource physicalResource) {
+ Resource physicalResource, OverAllocationInfo overAllocationInfo) {
RegisterNodeManagerRequest request =
Records.newRecord(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
@@ -60,9 +61,10 @@ public abstract class RegisterNodeManagerRequest {
request.setRunningApplications(runningApplications);
request.setNodeLabels(nodeLabels);
request.setPhysicalResource(physicalResource);
+ request.setOverAllocationInfo(overAllocationInfo);
return request;
}
-
+
public abstract NodeId getNodeId();
public abstract int getHttpPort();
public abstract Resource getResource();
@@ -70,7 +72,11 @@ public abstract class RegisterNodeManagerRequest {
public abstract List<NMContainerStatus> getNMContainerStatuses();
public abstract Set<NodeLabel> getNodeLabels();
public abstract void setNodeLabels(Set<NodeLabel> nodeLabels);
-
+
+ public abstract OverAllocationInfo getOverAllocationInfo();
+ public abstract void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo);
+
/**
* We introduce this here because currently YARN RM doesn't persist nodes info
* for application running. When RM restart happened, we cannot determinate if
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
index eda06d0..079c8d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
@@ -41,11 +41,14 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto.Builder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
-
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.OverAllocationInfoPBImpl;
+
public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest {
RegisterNodeManagerRequestProto proto = RegisterNodeManagerRequestProto.getDefaultInstance();
RegisterNodeManagerRequestProto.Builder builder = null;
@@ -56,6 +59,7 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
private List<NMContainerStatus> containerStatuses = null;
private List<ApplicationId> runningApplications = null;
private Set<NodeLabel> labels = null;
+ private OverAllocationInfo overAllocationInfo = null;
/** Physical resources in the node. */
private Resource physicalResource = null;
@@ -100,6 +104,10 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
if (this.physicalResource != null) {
builder.setPhysicalResource(convertToProtoFormat(this.physicalResource));
}
+ if (this.overAllocationInfo != null) {
+ builder.setOverAllocationInfo(
+ convertToProtoFormat(this.overAllocationInfo));
+ }
}
private synchronized void addNMContainerStatusesToProto() {
@@ -341,7 +349,30 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
builder.clearNodeLabels();
this.labels = nodeLabels;
}
-
+
+ @Override
+ public synchronized OverAllocationInfo getOverAllocationInfo() {
+ RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.overAllocationInfo != null) {
+ return this.overAllocationInfo;
+ }
+ if (!p.hasOverAllocationInfo()) {
+ return null;
+ }
+ this.overAllocationInfo = convertFromProtoFormat(p.getOverAllocationInfo());
+ return this.overAllocationInfo;
+ }
+
+ @Override
+ public synchronized void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo) {
+ maybeInitBuilder();
+ if (this.overAllocationInfo == null) {
+ builder.clearOverAllocationInfo();
+ }
+ this.overAllocationInfo = overAllocationInfo;
+ }
+
private synchronized void initNodeLabels() {
if (this.labels != null) {
return;
@@ -400,4 +431,14 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
NMContainerStatus c) {
return ((NMContainerStatusPBImpl)c).getProto();
}
+
+ private static OverAllocationInfoProto convertToProtoFormat(
+ OverAllocationInfo overAllocationInfo) {
+ return ((OverAllocationInfoPBImpl)overAllocationInfo).getProto();
+ }
+
+ private static OverAllocationInfo convertFromProtoFormat(
+ OverAllocationInfoProto proto) {
+ return new OverAllocationInfoPBImpl(proto);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
new file mode 100644
index 0000000..77952bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb
+ .OverAllocationInfoPBImpl;
+
+/**
+ * Captures information on how aggressively the scheduler can over-allocate
+ * OPPORTUNISTIC containers on a node. This is node-specific, and is sent on
+ * the wire on each heartbeat.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class OverAllocationInfo {
+ public static OverAllocationInfo newInstance(
+ ResourceThresholds overAllocationThresholds) {
+ OverAllocationInfo info = new OverAllocationInfoPBImpl();
+ info.setOverAllocationThreshold(overAllocationThresholds);
+ return info;
+ }
+
+ public abstract ResourceThresholds getOverAllocationThresholds();
+
+ public abstract void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
new file mode 100644
index 0000000..d57706a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImpl;
+
+/**
+ * Captures resource thresholds to be used for allocation and preemption
+ * when over-allocation is turned on.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class ResourceThresholds {
+ public static ResourceThresholds newInstance(float threshold) {
+ ResourceThresholds thresholds = new ResourceThresholdsPBImpl();
+ thresholds.setMemoryThreshold(threshold);
+ thresholds.setCpuThreshold(threshold);
+ return thresholds;
+ }
+
+ public abstract float getMemoryThreshold();
+
+ public abstract float getCpuThreshold();
+
+ public abstract void setMemoryThreshold(float memoryThreshold);
+
+ public abstract void setCpuThreshold(float cpuThreshold);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
new file mode 100644
index 0000000..758f4fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class OverAllocationInfoPBImpl extends OverAllocationInfo {
+ private OverAllocationInfoProto proto =
+ OverAllocationInfoProto.getDefaultInstance();
+ private OverAllocationInfoProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ResourceThresholds overAllocationThresholds = null;
+
+ public OverAllocationInfoPBImpl() {
+ builder = OverAllocationInfoProto.newBuilder();
+ }
+
+ public OverAllocationInfoPBImpl(OverAllocationInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized OverAllocationInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ if (overAllocationThresholds != null) {
+ builder.setOverAllocationThresholds(
+ convertToProtoFormat(overAllocationThresholds));
+ }
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = OverAllocationInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized ResourceThresholds getOverAllocationThresholds() {
+ OverAllocationInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (overAllocationThresholds != null) {
+ return overAllocationThresholds;
+ }
+ if (!p.hasOverAllocationThresholds()) {
+ return null;
+ }
+ overAllocationThresholds =
+ convertFromProtoFormat(p.getOverAllocationThresholds());
+ return overAllocationThresholds;
+ }
+
+ @Override
+ public synchronized void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds) {
+ maybeInitBuilder();
+ if (this.overAllocationThresholds != null) {
+ builder.clearOverAllocationThresholds();
+ }
+ this.overAllocationThresholds = resourceThresholds;
+ }
+
+ private static ResourceThresholdsProto convertToProtoFormat(
+ ResourceThresholds overAllocationThresholds) {
+ return ((ResourceThresholdsPBImpl) overAllocationThresholds).getProto();
+ }
+
+ private static ResourceThresholds convertFromProtoFormat(
+ ResourceThresholdsProto overAllocationThresholdsProto) {
+ return new ResourceThresholdsPBImpl(overAllocationThresholdsProto);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
new file mode 100644
index 0000000..10fb284
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class ResourceThresholdsPBImpl extends ResourceThresholds{
+ private ResourceThresholdsProto proto =
+ ResourceThresholdsProto.getDefaultInstance();
+ private ResourceThresholdsProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public ResourceThresholdsPBImpl() {
+ builder = ResourceThresholdsProto.newBuilder();
+ }
+
+ public ResourceThresholdsPBImpl(ResourceThresholdsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized ResourceThresholdsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ /*
+ * Right now, we have only memory and cpu thresholds that are floats.
+ * This is a no-op until we add other non-static fields to the proto.
+ */
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceThresholdsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized float getMemoryThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemory();
+ }
+
+ @Override
+ public synchronized float getCpuThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getCpu();
+ }
+
+ @Override
+ public synchronized void setMemoryThreshold(float memoryThreshold) {
+ maybeInitBuilder();
+ builder.setMemory(memoryThreshold);
+ }
+
+ @Override
+ public synchronized void setCpuThreshold(float cpuThreshold) {
+ maybeInitBuilder();
+ builder.setCpu(cpuThreshold);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 8c4fc69..2512d93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -66,6 +66,7 @@ message RegisterNodeManagerRequestProto {
repeated ApplicationIdProto runningApplications = 7;
optional NodeLabelsProto nodeLabels = 8;
optional ResourceProto physicalResource = 9;
+ optional OverAllocationInfoProto overAllocationInfo = 10;
}
message RegisterNodeManagerResponseProto {
@@ -195,3 +196,12 @@ message SCMUploaderCanUploadRequestProto {
message SCMUploaderCanUploadResponseProto {
optional bool uploadable = 1;
}
+
+message OverAllocationInfoProto {
+ optional ResourceThresholdsProto over_allocation_thresholds = 1;
+}
+
+message ResourceThresholdsProto {
+ optional float memory = 1 [default = 0];
+ optional float cpu = 2 [default = 0];
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index d7e3b52..690b0bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -135,4 +136,8 @@ public interface Context {
* @return the NM {@code DeletionService}.
*/
DeletionService getDeletionService();
+
+ boolean isOverAllocationEnabled();
+
+ OverAllocationInfo getOverAllocationInfo();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 42b7b5f..1de274b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
@@ -621,6 +622,8 @@ public class NodeManager extends CompositeService
private ResourcePluginManager resourcePluginManager;
+ private OverAllocationInfo overAllocationInfo;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -768,6 +771,20 @@ public class NodeManager extends CompositeService
this.nodeStatusUpdater = nodeStatusUpdater;
}
+ @Override
+ public boolean isOverAllocationEnabled() {
+ return getOverAllocationInfo() != null;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return this.overAllocationInfo;
+ }
+
+ public void setOverAllocationInfo(OverAllocationInfo overAllocationInfo) {
+ this.overAllocationInfo = overAllocationInfo;
+ }
+
public boolean isDistributedSchedulingEnabled() {
return isDistSchedulingEnabled;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 3d3f573..61780df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -376,8 +376,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
RegisterNodeManagerRequest request =
RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
nodeManagerVersionId, containerReports, getRunningApplications(),
- nodeLabels, physicalResource);
-
+ nodeLabels, physicalResource, context.getOverAllocationInfo());
if (containerReports != null) {
LOG.info("Registering with RM using containers :" + containerReports);
}
@@ -494,8 +493,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
= getIncreasedContainers();
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
- createKeepAliveApplicationList(), nodeHealthStatus,
- containersUtilization, nodeUtilization, increasedContainers);
+ createKeepAliveApplicationList(), nodeHealthStatus,
+ containersUtilization, nodeUtilization, increasedContainers);
nodeStatus.setOpportunisticContainersStatus(
getOpportunisticContainersStatus());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 33986a0..a8e29bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -34,8 +34,11 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
@@ -102,6 +105,8 @@ public class ContainersMonitorImpl extends AbstractService implements
private ResourceUtilization containersUtilization;
+ private ResourceThresholds overAllocationPreemptionThresholds;
+
private volatile boolean stopped = false;
public ContainersMonitorImpl(ContainerExecutor exec,
@@ -176,6 +181,13 @@ public class ContainersMonitorImpl extends AbstractService implements
LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
+ initializeOverAllocation(conf);
+ if (context.isOverAllocationEnabled()) {
+ pmemCheckEnabled = true;
+ LOG.info("Force enabling physical memory checks because " +
+ "overallocation is enabled");
+ }
+
containersMonitorEnabled =
isContainerMonitorEnabled() && monitoringInterval > 0;
LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled);
@@ -227,6 +239,28 @@ public class ContainersMonitorImpl extends AbstractService implements
pId, processTreeClass, conf);
}
+ private void initializeOverAllocation(Configuration conf) {
+ float overAllocationTreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.min(overAllocationTreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.max(0, overAllocationTreshold);
+
+ if (overAllocationTreshold > 0f) {
+ ((NodeManager.NMContext) context).setOverAllocationInfo(
+ OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(overAllocationTreshold)));
+
+ float preemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+
+ this.overAllocationPreemptionThresholds =
+ ResourceThresholds.newInstance(preemptionThreshold);
+ }
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 9602142..affbc2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -764,6 +765,16 @@ public abstract class BaseAMRMProxyTest {
}
@Override
+ public boolean isOverAllocationEnabled() {
+ return false;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
+ @Override
public NodeResourceMonitor getNodeResourceMonitor() {
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node
resource utilization and allocate opportunistic containers if appropriate.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ef5de39..8756c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -55,14 +55,20 @@ import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -72,6 +78,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -93,6 +101,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -1056,15 +1065,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// verify metrics
QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
@@ -1099,7 +1108,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1109,7 +1118,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
// Now another node checks in with capacity
@@ -1123,7 +1132,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// The old reservation should still be there...
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
@@ -1133,7 +1142,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
- @Test (timeout = 5000)
+ @Test
public void testOffSwitchAppReservationThreshold() throws Exception {
conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f);
scheduler.init(conf);
@@ -1173,7 +1182,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(6144, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1205,7 +1214,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1266,7 +1275,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1311,7 +1320,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(10240, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1355,7 +1364,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Verify number of reservations have decremented
assertEquals(0,
@@ -1399,7 +1408,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1408,7 +1417,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1534,7 +1543,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1543,7 +1552,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1583,12 +1592,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure allocated memory of queue1 doesn't exceed its maximum
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//the reservation of queue1 should be reclaim
assertEquals(0, scheduler.getSchedulerApp(attId1).
getCurrentReservation().getMemorySize());
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
}
@Test
@@ -1628,7 +1637,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests below threshold
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1637,7 +1646,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 has no reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(0,
scheduler.getSchedulerApp(attId).getReservedContainers().size());
@@ -1648,7 +1657,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
.getVirtualCores());
@@ -1663,7 +1672,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(3, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// The old reservation should still be there...
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
@@ -2696,7 +2705,361 @@ public class TestFairScheduler extends FairSchedulerTestBase {
2, liveContainers.iterator().next().getContainer().
getPriority().getPriority());
}
-
+
+ /**
+ * Test that NO OPPORTUNISTIC containers can be allocated on a node that
+ * is fully allocated and with a very high utilization.
+ */
+ @Test
+ public void testAllocateNoOpportunisticContainersOnBusyNode()
+ throws IOException {
+ conf.setBoolean(
+ YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED, true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(2048, 2), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the node's full memory
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization shoots up after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(2000, 0, 0.8f));
+
+ // create another scheduling request
+ ApplicationAttemptId appAttempt2
+ = createSchedulingRequest(100, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue("Expecting no containers allocated",
+ allocatedContainers2.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that a reservation is made for the second resource request
+ Resource reserved = scheduler.getNode(node.getNodeID()).
+ getReservedContainer().getReservedResource();
+ assertTrue("Expect a reservation made for the second resource request",
+ reserved.equals(Resource.newInstance(100, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test that OPPORTUNISTIC containers can be allocated on a node with low
+ * utilization even though there is not enough unallocated resource on the
+ * node to accommodate the request.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnPartiallyOverAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node that is fully
+ * allocated but whose utilization is very low.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnFullyAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 = createSchedulingRequest(
+ 4096, "queue1", "user1", 4);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request now that there is no unallocated
+ // resources left on the node, the request should be served with an
+ // allocation of an opportunistic container
+ ApplicationAttemptId appAttempt2 = createSchedulingRequest(
+ 1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node with a low
+ * utilization even though there are GUARANTEED containers allocated.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersWithGuaranteedOnes()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3200, "queue1", "user1", 3);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3200, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(512, 0, 0.1f));
+
+ // create two other scheduling requests which in aggregate ask for more
+ // that what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(512, "queue2", "user1", 1);
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(512, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that no reservation is made given that the second request should
+ // be satisfied by a GUARANTEED container allocation, the third by an
+ // OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made.",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
@@ -3686,7 +4049,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.createAbnormalContainerStatus(container.getContainerId(),
SchedulerUtils.COMPLETED_APPLICATION),
RMContainerEventType.FINISHED);
- assertEquals(Resources.none(), app1.getResourceUsage());
+ assertEquals(Resources.none(), app1.getGuaranteedResourceUsage());
}
@Test
@@ -3786,7 +4149,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application1's AM should be finished",
0, app1.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app1.getResourceUsage());
+ Resources.none(), app1.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be running",
1, app3.getLiveContainers().size());
assertEquals("Application3's AM requests 1024 MB memory",
@@ -3806,7 +4169,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application4's AM should not be running",
0, app4.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app4.getResourceUsage());
+ Resources.none(), app4.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3822,7 +4185,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3835,7 +4198,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3851,11 +4214,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application2's AM should be finished",
0, app2.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app2.getResourceUsage());
+ Resources.none(), app2.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be finished",
0, app3.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app3.getResourceUsage());
+ Resources.none(), app3.getGuaranteedResourceUsage());
assertEquals("Application5's AM should be running",
1, app5.getLiveContainers().size());
assertEquals("Application5's AM requests 2048 MB memory",
@@ -3876,7 +4239,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should have 0 container",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
scheduler.update();
@@ -3900,7 +4263,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application6's AM should not be running",
0, app6.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app6.getResourceUsage());
+ Resources.none(), app6.getGuaranteedResourceUsage());
assertEquals("Application6's AM resource shouldn't be updated",
0, app6.getAMResource().getMemorySize());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -4617,17 +4980,25 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true);
FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true);
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2");
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
}
@Test (expected = YarnException.class)
@@ -4667,7 +5038,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(2048, 2),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
@@ -5091,7 +5463,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//container will be reserved at node1
RMContainer reservedContainer1 =
@@ -5111,7 +5483,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
app1, RMAppAttemptState.KILLED, false));
assertEquals(0, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// container will be allocated at node2
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -5259,10 +5631,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app1 = mock(FSAppAttempt.class);
Mockito.when(app1.getDemand()).thenReturn(maxResource);
- Mockito.when(app1.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app1.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
FSAppAttempt app2 = mock(FSAppAttempt.class);
Mockito.when(app2.getDemand()).thenReturn(maxResource);
- Mockito.when(app2.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app2.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
QueueManager queueManager = scheduler.getQueueManager();
FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
@@ -5318,7 +5692,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
child1.setMaxShare(new ConfigurableResource(resource));
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(resource);
- Mockito.when(app.getResourceUsage()).thenReturn(resource);
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(resource);
child1.addApp(app, true);
child1.updateDemand();
@@ -5354,7 +5728,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
+ " SteadyFairShare: <memory:0, vCores:0>,"
+ " MaxShare: <memory:4096, vCores:4>,"
+ " MinShare: <memory:0, vCores:0>,"
- + " ResourceUsage: <memory:4096, vCores:4>,"
+ + " Guaranteed ResourceUsage: <memory:4096, vCores:4>,"
+ " Demand: <memory:4096, vCores:4>,"
+ " MaxAMShare: 0.5,"
+ " Runnable: 0}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index b016c1b..6777b5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -243,11 +243,16 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public Resource getMinShare() {
return minShare;
}
@@ -278,7 +283,8 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node,
+ boolean opportunistic) {
throw new UnsupportedOperationException();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node
resource utilization and allocate opportunistic containers if appropriate.
Posted by ha...@apache.org.
YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdf3e661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdf3e661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdf3e661
Branch: refs/heads/YARN-1011
Commit: bdf3e661b7558020c284442f24ade677406b19e8
Parents: eba8436
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Nov 17 07:47:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:08:11 2018 -0800
----------------------------------------------------------------------
.../sls/scheduler/FairSchedulerMetrics.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 13 +
.../scheduler/SchedulerNode.java | 48 ++
.../scheduler/fair/FSAppAttempt.java | 166 ++++---
.../scheduler/fair/FSLeafQueue.java | 51 +-
.../scheduler/fair/FSParentQueue.java | 36 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 39 +-
.../scheduler/fair/FairScheduler.java | 97 ++--
.../fair/FairSchedulerConfiguration.java | 5 +
.../scheduler/fair/Schedulable.java | 17 +-
.../DominantResourceFairnessPolicy.java | 8 +-
.../fair/policies/FairSharePolicy.java | 4 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 60 ++-
.../TestWorkPreservingRMRestart.java | 2 +-
.../scheduler/fair/FakeSchedulable.java | 9 +-
.../scheduler/fair/TestAppRunnability.java | 9 +-
.../scheduler/fair/TestFSAppAttempt.java | 4 +-
.../scheduler/fair/TestFSLeafQueue.java | 4 +-
.../scheduler/fair/TestFSSchedulerNode.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 468 +++++++++++++++++--
.../scheduler/fair/TestSchedulingPolicy.java | 10 +-
23 files changed, 861 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index a5aee74..1f4e7c7 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -75,7 +75,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getMemorySize();
case USAGE:
- return schedulable.getResourceUsage().getMemorySize();
+ return schedulable.getGuaranteedResourceUsage().getMemorySize();
case MINSHARE:
return schedulable.getMinShare().getMemorySize();
case MAXSHARE:
@@ -96,7 +96,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getVirtualCores();
case USAGE:
- return schedulable.getResourceUsage().getVirtualCores();
+ return schedulable.getGuaranteedResourceUsage().getVirtualCores();
case MINSHARE:
return schedulable.getMinShare().getVirtualCores();
case MAXSHARE:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ed8404c..15f6480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -268,6 +268,11 @@ public class YarnConfiguration extends Configuration {
/** UserGroupMappingPlacementRule configuration string. */
public static final String USER_GROUP_PLACEMENT_RULE = "user-group";
+ public static final String RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED =
+ RM_PREFIX + "scheduler.oversubscription.enabled";
+ public static final boolean DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED
+ = false;
+
/** Enable Resource Manager webapp ui actions */
public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
RM_PREFIX + "webapp.ui-actions.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index da31694..7962dae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -389,6 +389,19 @@
</property>
<property>
+ <description>
+ If set to true, the scheduler will try to over-allocate resources on the
+ nodes that allow overallocation. To enable overallocatin on a node, set
+ {code}yarn.nodemanager.overallocation.memory-utilization-threshold{code}
+ and
+ {code}yarn.nodemanager.overallocation.cpu-utilization-threshold{code}
+ to a number in the range (0.0, 1.0)
+ </description>
+ <name>yarn.resourcemanager.scheduler.oversubscription.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified. </description>
<name>yarn.resourcemanager.recovery.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 349944e..e942981 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -59,6 +61,10 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
private Resource capacity;
+ // The resource available within the node's capacity that can be given out
+ // to run GUARANTEED containers, including reserved, preempted and any
+ // remaining free resources. Resources allocated to OPPORTUNISTIC containers
+ // are tracked in allocatedResourceOpportunistic
private Resource unallocatedResource = Resource.newInstance(0, 0);
private RMContainer reservedContainer;
@@ -596,6 +602,48 @@ public abstract class SchedulerNode {
return this.nodeUtilization;
}
+ /**
+ * Get the amount of resources that can be allocated to opportunistic
+ * containers in the case of overallocation. It is calculated as
+ * node capacity - (node utilization + resources of allocated-yet-not-started
+ * containers).
+ * @return the amount of resources that are available to be allocated to
+ * opportunistic containers
+ */
+ public synchronized Resource allowedResourceForOverAllocation() {
+ OverAllocationInfo overAllocationInfo = rmNode.getOverAllocationInfo();
+ if (overAllocationInfo == null) {
+ LOG.debug("Overallocation is disabled on node: " + rmNode.getHostName());
+ return Resources.none();
+ }
+
+ ResourceUtilization projectedNodeUtilization = ResourceUtilization.
+ newInstance(getNodeUtilization());
+ // account for resources allocated in this heartbeat
+ projectedNodeUtilization.addTo(
+ (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0,
+ (float) resourceAllocatedPendingLaunch.getVirtualCores() /
+ capacity.getVirtualCores());
+
+ ResourceThresholds thresholds =
+ overAllocationInfo.getOverAllocationThresholds();
+ Resource overAllocationThreshold = Resources.createResource(
+ (long) (capacity.getMemorySize() * thresholds.getMemoryThreshold()),
+ (int) (capacity.getVirtualCores() * thresholds.getCpuThreshold()));
+ long allowedMemory = Math.max(0, overAllocationThreshold.getMemorySize()
+ - projectedNodeUtilization.getPhysicalMemory());
+ int allowedCpu = Math.max(0, (int)
+ (overAllocationThreshold.getVirtualCores() -
+ projectedNodeUtilization.getCPU() * capacity.getVirtualCores()));
+
+ Resource resourceAllowedForOpportunisticContainers =
+ Resources.createResource(allowedMemory, allowedCpu);
+
+ // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
+ // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ return resourceAllowedForOpportunisticContainers;
+ }
+
private static class ContainerInfo {
private final RMContainer container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0305702..970ffcb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -169,7 +170,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedResource(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
@@ -178,30 +179,35 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
}
- private void unreserveInternal(
+ private boolean unreserveInternal(
SchedulerRequestKey schedulerKey, FSSchedulerNode node) {
try {
writeLock.lock();
Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(
schedulerKey);
- RMContainer reservedContainer = reservedContainers.remove(
- node.getNodeID());
- if (reservedContainers.isEmpty()) {
- this.reservedContainers.remove(schedulerKey);
- }
+ boolean unreserved = false;
+ if (reservedContainers != null) {
+ RMContainer reservedContainer = reservedContainers.remove(
+ node.getNodeID());
+ if (reservedContainers.isEmpty()) {
+ this.reservedContainers.remove(schedulerKey);
+ }
- // Reset the re-reservation count
- resetReReservations(schedulerKey);
+ // Reset the re-reservation count
+ resetReReservations(schedulerKey);
- Resource resource = reservedContainer.getContainer().getResource();
- this.attemptResourceUsage.decReserved(resource);
+ Resource resource = reservedContainer.getContainer().getResource();
+ this.attemptResourceUsage.decReserved(resource);
+ unreserved = true;
- LOG.info(
- "Application " + getApplicationId() + " unreserved " + " on node "
- + node + ", currently has " + reservedContainers.size()
- + " at priority " + schedulerKey.getPriority()
- + "; currentReservation " + this.attemptResourceUsage
- .getReserved());
+ LOG.info(
+ "Application " + getApplicationId() + " unreserved " + " on node "
+ + node + ", currently has " + reservedContainers.size()
+ + " at priority " + schedulerKey.getPriority()
+ + "; currentReservation " + this.attemptResourceUsage
+ .getReserved());
+ }
+ return unreserved;
} finally {
writeLock.unlock();
}
@@ -229,7 +235,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
SchedulingPolicy policy = fsQueue.getPolicy();
Resource queueFairShare = fsQueue.getFairShare();
- Resource queueUsage = fsQueue.getResourceUsage();
+ Resource queueUsage = fsQueue.getGuaranteedResourceUsage();
Resource clusterResource = this.scheduler.getClusterResource();
Resource clusterUsage = this.scheduler.getRootQueueMetrics()
.getAllocatedResources();
@@ -420,7 +426,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public RMContainer allocate(NodeType type, FSSchedulerNode node,
SchedulerRequestKey schedulerKey, PendingAsk pendingAsk,
- Container reservedContainer) {
+ Container reservedContainer, boolean opportunistic) {
RMContainer rmContainer;
Container container;
@@ -445,9 +451,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
container = reservedContainer;
+ ExecutionType executionType = opportunistic ?
+ ExecutionType.OPPORTUNISTIC : ExecutionType.GUARANTEED;
if (container == null) {
container = createContainer(node, pendingAsk.getPerAllocationResource(),
- schedulerKey);
+ schedulerKey, executionType);
}
// Create RMContainer
@@ -463,8 +471,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Update consumption and track allocations
ContainerRequest containerRequest = appSchedulingInfo.allocate(
type, node, schedulerKey, container);
- this.attemptResourceUsage.incUsed(container.getResource());
- getQueue().incUsedResource(container.getResource());
+ if (executionType.equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.incUsed(container.getResource());
+ getQueue().incUsedGuaranteedResource(container.getResource());
+ } else {
+ this.attemptOpportunisticResourceUsage.incUsed(container.getResource());
+ }
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl) rmContainer).setContainerRequest(containerRequest);
@@ -621,7 +633,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private Resource getUsageAfterPreemptingContainer(Resource containerResources,
Resource alreadyConsideringForPreemption) {
- Resource usageAfterPreemption = Resources.clone(getResourceUsage());
+ Resource usageAfterPreemption =
+ Resources.clone(getGuaranteedResourceUsage());
// Subtract resources of containers already queued for preemption
synchronized (preemptionVariablesLock) {
@@ -648,7 +661,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return Container
*/
private Container createContainer(FSSchedulerNode node, Resource capability,
- SchedulerRequestKey schedulerKey) {
+ SchedulerRequestKey schedulerKey, ExecutionType executionType) {
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(
@@ -658,7 +671,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability,
schedulerKey.getPriority(), null,
- schedulerKey.getAllocationRequestId());
+ executionType, schedulerKey.getAllocationRequestId());
}
@Override
@@ -670,7 +683,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super.recoverContainer(node, rmContainer);
if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
- getQueue().incUsedResource(rmContainer.getContainer().getResource());
+ getQueue().incUsedGuaranteedResource(
+ rmContainer.getContainer().getResource());
}
// If not running unmanaged, the first container we recover is always
@@ -708,7 +722,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (reservedContainer == null) {
reservedContainer =
createContainer(node, perAllocationResource,
- schedulerKey);
+ schedulerKey, ExecutionType.GUARANTEED);
getMetrics().reserveResource(node.getPartition(), getUser(),
reservedContainer.getResource());
RMContainer rmContainer =
@@ -765,11 +779,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public void unreserve(SchedulerRequestKey schedulerKey,
FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
- unreserveInternal(schedulerKey, node);
- node.unreserveResource(this);
- clearReservation(node);
- getMetrics().unreserveResource(node.getPartition(),
- getUser(), rmContainer.getContainer().getResource());
+ if (unreserveInternal(schedulerKey, node)) {
+ node.unreserveResource(this);
+ clearReservation(node);
+ getMetrics().unreserveResource(node.getPartition(),
+ getUser(), rmContainer.getContainer().getResource());
+ }
}
private void setReservation(SchedulerNode node) {
@@ -843,13 +858,15 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/
private Resource assignContainer(
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
- boolean reserved, SchedulerRequestKey schedulerKey) {
+ boolean reserved, boolean opportunistic,
+ SchedulerRequestKey schedulerKey) {
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
// How much does the node have?
- Resource available = node.getUnallocatedResource();
+ Resource available = opportunistic ?
+ node.allowedResourceForOverAllocation() : node.getUnallocatedResource();
Container reservedContainer = null;
if (reserved) {
@@ -861,33 +878,39 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Inform the application of the new container for this request
RMContainer allocatedContainer =
allocate(type, node, schedulerKey, pendingAsk,
- reservedContainer);
- if (allocatedContainer == null) {
- // Did the application need this resource?
- if (reserved) {
- unreserve(schedulerKey, node);
- }
- return Resources.none();
- }
+ reservedContainer, opportunistic);
- // If we had previously made a reservation, delete it
+ // delete the previous reservation, if any
if (reserved) {
unreserve(schedulerKey, node);
}
- // Inform the node
- node.allocateContainer(allocatedContainer);
+ if (allocatedContainer != null) {
+ if (opportunistic) {
+ // if an OPPORTUNISTIC container is allocated, we need to
+ // unreserve anything that we may have reserved in our
+ // previous attempt to assign GUARANTEED containers for this
+ // scheduling request.
+ unreserve(schedulerKey, node);
+ }
+
- // If not running unmanaged, the first container we allocate is always
- // the AM. Set the amResource for this app and update the leaf queue's AM
- // usage
- if (!isAmRunning() && !getUnmanagedAM()) {
- setAMResource(capability);
- getQueue().addAMResourceUsage(capability);
- setAmRunning(true);
- }
+ // Inform the node
+ node.allocateContainer(allocatedContainer);
+
+ // If not running unmanaged, the first container we allocate
+ // is always the AM. Set amResource for this app and update
+ // the leaf queue's AM usage
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ setAMResource(capability);
+ getQueue().addAMResourceUsage(capability);
+ setAmRunning(true);
+ }
- return capability;
+ return capability;
+ } else {
+ return Resources.none();
+ }
}
if (LOG.isDebugEnabled()) {
@@ -898,7 +921,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// The desired container won't fit here, so reserve
// Reserve only, if app does not wait for preempted resources on the node,
// otherwise we may end up with duplicate reservations
- if (isReservable(capability) &&
+ if (isReservable(capability) && !opportunistic &&
!node.isPreemptedForApp(this) &&
reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
type, schedulerKey)) {
@@ -944,7 +967,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@SuppressWarnings("deprecation")
- private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
+ private Resource assignContainer(FSSchedulerNode node, boolean opportunistic,
+ boolean reserved) {
if (LOG.isTraceEnabled()) {
LOG.trace("Node offered to app: " + getName() + " reserved: " + reserved);
}
@@ -1007,7 +1031,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, nodeLocalPendingAsk, NodeType.NODE_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
if (!appSchedulingInfo.canDelayTo(schedulerKey, node.getRackName())) {
@@ -1024,7 +1048,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, rackLocalPendingAsk, NodeType.RACK_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
PendingAsk offswitchAsk = getPendingAsk(schedulerKey,
@@ -1044,7 +1068,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, offswitchAsk, NodeType.OFF_SWITCH,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
}
@@ -1145,7 +1169,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getUnallocatedResource())) {
- assignContainer(node, true);
+ assignContainer(node, false, true);
}
return true;
}
@@ -1161,7 +1185,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource fairDemand = Resources.componentwiseMin(threshold, demand);
// Check if the queue is starved for fairshare
- boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand);
+ boolean starved =
+ isUsageBelowShare(getGuaranteedResourceUsage(), fairDemand);
if (!starved) {
lastTimeAtFairShare = now;
@@ -1173,8 +1198,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
fairshareStarvation = Resources.none();
} else {
// The app has been starved for longer than preemption-timeout.
- fairshareStarvation =
- Resources.subtractFromNonNegative(fairDemand, getResourceUsage());
+ fairshareStarvation = Resources.subtractFromNonNegative(fairDemand,
+ getGuaranteedResourceUsage());
}
return fairshareStarvation;
}
@@ -1193,7 +1218,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return true if the app is starved for fairshare, false otherwise
*/
boolean isStarvedForFairShare() {
- return isUsageBelowShare(getResourceUsage(), getFairShare());
+ return isUsageBelowShare(getGuaranteedResourceUsage(), getFairShare());
}
/**
@@ -1294,7 +1319,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* Get the current app's unsatisfied demand.
*/
Resource getPendingDemand() {
- return Resources.subtract(demand, getResourceUsage());
+ return Resources.subtract(demand, getGuaranteedResourceUsage());
}
@Override
@@ -1313,11 +1338,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return getCurrentConsumption();
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return attemptOpportunisticResourceUsage.getUsed();
+ }
+
+ @Override
public float getWeight() {
float weight = 1.0F;
@@ -1366,7 +1396,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
if (isOverAMShareLimit()) {
PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
@@ -1378,7 +1408,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
return Resources.none();
}
- return assignContainer(node, false);
+ return assignContainer(node, opportunistic, false);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 49d2166..2a90228 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -89,7 +89,7 @@ public class FSLeafQueue extends FSQueue {
} else {
nonRunnableApps.add(app);
}
- incUsedResource(app.getResourceUsage());
+ incUsedGuaranteedResource(app.getGuaranteedResourceUsage());
} finally {
writeLock.unlock();
}
@@ -124,7 +124,7 @@ public class FSLeafQueue extends FSQueue {
getMetrics().setAMResourceUsage(amResourceUsage);
}
- decUsedResource(app.getResourceUsage());
+ decUsedGuaranteedResource(app.getGuaranteedResourceUsage());
return runnable;
}
@@ -294,6 +294,42 @@ public class FSLeafQueue extends FSQueue {
return demand;
}
+ @Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
Resource getAmResourceUsage() {
return amResourceUsage;
}
@@ -327,14 +363,14 @@ public class FSLeafQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = none();
if (LOG.isDebugEnabled()) {
LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
getName() + " fairShare: " + getFairShare());
}
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
@@ -342,7 +378,7 @@ public class FSLeafQueue extends FSQueue {
if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) {
continue;
}
- assigned = sched.assignContainer(node);
+ assigned = sched.assignContainer(node, opportunistic);
if (!assigned.equals(none())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigned container in queue:" + getName() + " " +
@@ -540,7 +576,8 @@ public class FSLeafQueue extends FSQueue {
Resource desiredShare = Resources.min(policy.getResourceCalculator(),
scheduler.getClusterResource(), getMinShare(), getDemand());
- Resource starvation = Resources.subtract(desiredShare, getResourceUsage());
+ Resource starvation =
+ Resources.subtract(desiredShare, getGuaranteedResourceUsage());
boolean starved = !Resources.isNone(starvation);
long now = scheduler.getClock().getTime();
@@ -598,7 +635,7 @@ public class FSLeafQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", Runnable: " + getNumRunnableApps() +
", NumPendingApps: " + getNumPendingApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index a8e53fc..3e63e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -119,6 +119,34 @@ public class FSParentQueue extends FSQueue {
}
@Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(guaranteedResource, child.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(opportunisticResource,
+ child.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
@@ -177,11 +205,11 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = Resources.none();
// If this queue is over its limit, reject
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
@@ -204,7 +232,7 @@ public class FSParentQueue extends FSQueue {
readLock.lock();
try {
for (FSQueue child : childQueues) {
- assigned = child.assignContainer(node);
+ assigned = child.assignContainer(node, opportunistic);
if (!Resources.equals(assigned, Resources.none())) {
break;
}
@@ -288,7 +316,7 @@ public class FSParentQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", Guaranteed ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", MaxAMShare: " + maxAMShare +
", Runnable: " + getNumRunnableApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 4babfd5..f293eb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -57,7 +57,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
private Resource steadyFairShare = Resources.createResource(0, 0);
private Resource reservedResource = Resources.createResource(0, 0);
- private final Resource resourceUsage = Resource.newInstance(0, 0);
+ private final Resource guaranteedResourceUsage = Resource.newInstance(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final YarnAuthorizationProvider authorizer;
@@ -234,7 +234,8 @@ public abstract class FSQueue implements Queue, Schedulable {
if (getFairShare().getMemorySize() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
- queueInfo.setCurrentCapacity((float) getResourceUsage().getMemorySize() /
+ queueInfo.setCurrentCapacity(
+ (float) getGuaranteedResourceUsage().getMemorySize() /
getFairShare().getMemorySize());
}
@@ -418,14 +419,17 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @return true if check passes (can assign) or false otherwise
*/
- boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (node.getReservedContainer() != null) {
+ boolean assignContainerPreCheck(FSSchedulerNode node, boolean opportunistic) {
+ if (opportunistic) {
+ // always pre-approve OPPORTUNISTIC containers to be assigned on the node
+ return true;
+ } else if (node.getReservedContainer() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because it has reserved containers.");
}
return false;
- } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) {
+ } else if (!Resources.fitsIn(getGuaranteedResourceUsage(), getMaxShare())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because queue resource usage is larger than MaxShare: "
@@ -448,7 +452,8 @@ public abstract class FSQueue implements Queue, Schedulable {
@Override
public String toString() {
return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]",
- getName(), getDemand(), getResourceUsage(), fairShare, getWeight());
+ getName(), getDemand(), getGuaranteedResourceUsage(), fairShare,
+ getWeight());
}
@Override
@@ -480,8 +485,8 @@ public abstract class FSQueue implements Queue, Schedulable {
}
@Override
- public Resource getResourceUsage() {
- return resourceUsage;
+ public Resource getGuaranteedResourceUsage() {
+ return guaranteedResourceUsage;
}
/**
@@ -489,11 +494,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to increase
*/
- protected void incUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.addTo(resourceUsage, res);
+ protected void incUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.addTo(guaranteedResourceUsage, res);
if (parent != null) {
- parent.incUsedResource(res);
+ parent.incUsedGuaranteedResource(res);
}
}
}
@@ -503,11 +508,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to decrease
*/
- protected void decUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.subtractFrom(resourceUsage, res);
+ protected void decUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.subtractFrom(guaranteedResourceUsage, res);
if (parent != null) {
- parent.decUsedResource(res);
+ parent.decUsedGuaranteedResource(res);
}
}
}
@@ -520,7 +525,7 @@ public abstract class FSQueue implements Queue, Schedulable {
boolean fitsInMaxShare(Resource additionalResource) {
Resource usagePlusAddition =
- Resources.add(getResourceUsage(), additionalResource);
+ Resources.add(getGuaranteedResourceUsage(), additionalResource);
if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) {
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index e2a62ec..a8e348c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -165,7 +165,6 @@ public class FairScheduler extends
private float reservableNodesRatio; // percentage of available nodes
// an app can be reserved on
-
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
// Continuous Scheduling enabled or not
@Deprecated
@@ -188,6 +187,8 @@ public class FairScheduler extends
boolean maxAssignDynamic;
protected int maxAssign; // Max containers to assign per heartbeat
+ protected boolean oversubscriptionEnabled;
+
@VisibleForTesting
final MaxRunningAppsEnforcer maxRunningEnforcer;
@@ -1000,13 +1001,13 @@ public class FairScheduler extends
* resources for preempted containers.
* @param node Node to check
*/
- static void assignPreemptedContainers(FSSchedulerNode node) {
+ static void attemptToAssignPreemptedResources(FSSchedulerNode node) {
for (Entry<FSAppAttempt, Resource> entry :
node.getPreemptionList().entrySet()) {
FSAppAttempt app = entry.getKey();
Resource preemptionPending = Resources.clone(entry.getValue());
while (!app.isStopped() && !Resources.isNone(preemptionPending)) {
- Resource assigned = app.assignContainer(node);
+ Resource assigned = app.assignContainer(node, false);
if (Resources.isNone(assigned) ||
assigned.equals(FairScheduler.CONTAINER_RESERVED)) {
// Fail to assign, let's not try further
@@ -1038,44 +1039,82 @@ public class FairScheduler extends
// Assign new containers...
// 1. Ensure containers are assigned to the apps that preempted
// 2. Check for reserved applications
- // 3. Schedule if there are no reservations
+ // 3. Schedule GUARANTEED containers if there are no reservations
+ // 4. Schedule OPPORTUNISTIC containers if possible
// Apps may wait for preempted containers
// We have to satisfy these first to avoid cases, when we preempt
// a container for A from B and C gets the preempted containers,
// when C does not qualify for preemption itself.
- assignPreemptedContainers(node);
- FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
- boolean validReservation = false;
- if (reservedAppSchedulable != null) {
- validReservation = reservedAppSchedulable.assignReservedContainer(node);
- }
+ attemptToAssignPreemptedResources(node);
+
+ boolean validReservation = attemptToAssignReservedResources(node);
if (!validReservation) {
- // No reservation, schedule at queue which is farthest below fair share
- int assignedContainers = 0;
- Resource assignedResource = Resources.clone(Resources.none());
- Resource maxResourcesToAssign = Resources.multiply(
- node.getUnallocatedResource(), 0.5f);
- while (node.getReservedContainer() == null) {
- Resource assignment = queueMgr.getRootQueue().assignContainer(node);
- if (assignment.equals(Resources.none())) {
- break;
- }
+ // only attempt to assign GUARANTEED containers if there is no
+ // reservation on the node because
+ attemptToAssignResourcesAsGuaranteedContainers(node);
+ }
- assignedContainers++;
- Resources.addTo(assignedResource, assignment);
- if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
- assignedResource)) {
- break;
- }
- }
+ // attempt to assign OPPORTUNISTIC containers regardless of whether
+ // we have made a reservation or assigned a GUARANTEED container
+ if (oversubscriptionEnabled) {
+ attemptToAssignResourcesAsOpportunisticContainers(node);
}
+
updateRootQueueMetrics();
} finally {
writeLock.unlock();
}
}
+ /**
+ * Assign the reserved resource to the application that have reserved it.
+ */
+ private boolean attemptToAssignReservedResources(FSSchedulerNode node) {
+ boolean success = false;
+ FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
+ if (reservedAppSchedulable != null) {
+ success = reservedAppSchedulable.assignReservedContainer(node);
+ }
+ return success;
+ }
+
+ private void attemptToAssignResourcesAsGuaranteedContainers(
+ FSSchedulerNode node) {
+ // No reservation, schedule at queue which is farthest below fair share
+ int assignedContainers = 0;
+ Resource assignedResource = Resources.clone(Resources.none());
+ Resource maxResourcesToAssign = Resources.multiply(
+ node.getUnallocatedResource(), 0.5f);
+ while (node.getReservedContainer() == null) {
+ Resource assignment =
+ queueMgr.getRootQueue().assignContainer(node, false);
+ if (assignment.equals(Resources.none())) {
+ break;
+ }
+ assignedContainers++;
+ Resources.addTo(assignedResource, assignment);
+
+ if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
+ assignedResource)) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Try to assign OPPORTUNISTIC containers as long as there is resources
+ * to.
+ * @param node the node to assign OPPORTUNISTIC containers on
+ */
+ private void attemptToAssignResourcesAsOpportunisticContainers(
+ FSSchedulerNode node) {
+ while (!Resources.none().equals(
+ queueMgr.getRootQueue().assignContainer(node, true))) {
+ // nothing to do here
+ }
+ }
+
public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
return super.getApplicationAttempt(appAttemptId);
}
@@ -1316,6 +1355,7 @@ public class FairScheduler extends
sizeBasedWeight = this.conf.getSizeBasedWeight();
usePortForNodeName = this.conf.getUsePortForNodeName();
reservableNodesRatio = this.conf.getReservableNodes();
+ oversubscriptionEnabled = this.conf.isOversubscriptionEnabled();
updateInterval = this.conf.getUpdateInterval();
if (updateInterval < 0) {
@@ -1690,7 +1730,8 @@ public class FairScheduler extends
}
// maxShare
- if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption),
+ if (!Resources.fitsIn(
+ Resources.add(cur.getGuaranteedResourceUsage(), consumption),
cur.getMaxShare())) {
throw new YarnException("Moving app attempt " + appAttId + " to queue "
+ queueName + " would violate queue maxShare constraints on"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index b50e4bb..30bbb78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -344,6 +344,11 @@ public class FairSchedulerConfiguration extends Configuration {
DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS);
}
+ public boolean isOversubscriptionEnabled() {
+ return getBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED);
+ }
+
/**
* Delay in milliseconds for locality fallback node to rack.
* @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
index bd1ff7a..f018929 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
@@ -58,8 +58,17 @@ public interface Schedulable {
*/
Resource getDemand();
- /** Get the aggregate amount of resources consumed by the schedulable. */
- Resource getResourceUsage();
+ /**
+ * Get the aggregate amount of guaranteed resources consumed by the
+ * schedulable.
+ */
+ Resource getGuaranteedResourceUsage();
+
+ /**
+ * Get the aggregate amount of opportunistic resources consumed by the
+ * schedulable.
+ */
+ Resource getOpportunisticResourceUsage();
/** Minimum Resource share assigned to the schedulable. */
Resource getMinShare();
@@ -89,8 +98,10 @@ public interface Schedulable {
/**
* Assign a container on this node if possible, and return the amount of
* resources assigned.
+ * @param node the node to assign containers on
+ * @param opportunistic whether to assign OPPORTUNISTIC containers or not
*/
- Resource assignContainer(FSSchedulerNode node);
+ Resource assignContainer(FSSchedulerNode node, boolean opportunistic);
/** Get the fair share assigned to this Schedulable. */
Resource getFairShare();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 59635d9..f366891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -169,8 +169,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
extends DominantResourceFairnessComparator {
@Override
public int compare(Schedulable s1, Schedulable s2) {
- Resource usage1 = s1.getResourceUsage();
- Resource usage2 = s2.getResourceUsage();
+ Resource usage1 = s1.getGuaranteedResourceUsage();
+ Resource usage2 = s2.getGuaranteedResourceUsage();
Resource minShare1 = s1.getMinShare();
Resource minShare2 = s2.getMinShare();
Resource clusterCapacity = fsContext.getClusterResource();
@@ -370,9 +370,9 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public int compare(Schedulable s1, Schedulable s2) {
ResourceInformation[] resourceInfo1 =
- s1.getResourceUsage().getResources();
+ s1.getGuaranteedResourceUsage().getResources();
ResourceInformation[] resourceInfo2 =
- s2.getResourceUsage().getResources();
+ s2.getGuaranteedResourceUsage().getResources();
ResourceInformation[] minShareInfo1 = s1.getMinShare().getResources();
ResourceInformation[] minShareInfo2 = s2.getMinShare().getResources();
ResourceInformation[] clusterInfo =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 8179aa7..7ecbeea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -90,8 +90,8 @@ public class FairSharePolicy extends SchedulingPolicy {
int res = compareDemand(s1, s2);
// Pre-compute resource usages to avoid duplicate calculation
- Resource resourceUsage1 = s1.getResourceUsage();
- Resource resourceUsage2 = s2.getResourceUsage();
+ Resource resourceUsage1 = s1.getGuaranteedResourceUsage();
+ Resource resourceUsage2 = s2.getGuaranteedResourceUsage();
if (res == 0) {
res = compareMinShareUsage(s1, s2, resourceUsage1, resourceUsage2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 913513c..fe313a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -88,7 +88,7 @@ public class FairSchedulerQueueInfo {
amMaxResources = new ResourceInfo(Resource.newInstance(
queue.getMetrics().getMaxAMShareMB(),
queue.getMetrics().getMaxAMShareVCores()));
- usedResources = new ResourceInfo(queue.getResourceUsage());
+ usedResources = new ResourceInfo(queue.getGuaranteedResourceUsage());
demandResources = new ResourceInfo(queue.getDemand());
fractionMemUsed = (float)usedResources.getMemorySize() /
clusterResources.getMemorySize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index e81192e..6197a2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -103,7 +102,10 @@ public class MockNodes {
return rs;
}
- private static class MockRMNodeImpl implements RMNode {
+ /**
+ * A mock implementation of RMNode.
+ */
+ public static class MockRMNodeImpl implements RMNode {
private NodeId nodeId;
private String hostName;
private String nodeAddr;
@@ -118,12 +120,27 @@ public class MockNodes {
private ResourceUtilization containersUtilization;
private ResourceUtilization nodeUtilization;
private Resource physicalResource;
+ private OverAllocationInfo overAllocationInfo;
+ private List<UpdatedContainerInfo> containerUpdates =
+ Collections.EMPTY_LIST;
+
+ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
+ Resource perNode, String rackName, String healthReport,
+ long lastHealthReportTime, int cmdPort, String hostName,
+ NodeState state, Set<String> labels,
+ ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ this(nodeId, nodeAddr, httpAddress, perNode, rackName, healthReport,
+ lastHealthReportTime, cmdPort, hostName, state, labels,
+ containersUtilization, nodeUtilization, pPhysicalResource, null);
+ }
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
Set<String> labels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource,
+ OverAllocationInfo overAllocationInfo) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
@@ -138,6 +155,7 @@ public class MockNodes {
this.containersUtilization = containersUtilization;
this.nodeUtilization = nodeUtilization;
this.physicalResource = pPhysicalResource;
+ this.overAllocationInfo = overAllocationInfo;
}
@Override
@@ -226,7 +244,7 @@ public class MockNodes {
@Override
public List<UpdatedContainerInfo> pullContainerUpdates() {
- return new ArrayList<UpdatedContainerInfo>();
+ return containerUpdates;
}
@Override
@@ -264,7 +282,7 @@ public class MockNodes {
@Override
public OverAllocationInfo getOverAllocationInfo() {
- return null;
+ return this.overAllocationInfo;
}
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
@@ -289,6 +307,19 @@ public class MockNodes {
public Resource getPhysicalResource() {
return this.physicalResource;
}
+
+ public void updateResourceUtilization(ResourceUtilization utilization) {
+ this.nodeUtilization = utilization;
+ }
+
+ public void updateContainersAndNodeUtilization(
+ UpdatedContainerInfo updatedContainerInfo,
+ ResourceUtilization resourceUtilization) {
+ if (updatedContainerInfo != null) {
+ containerUpdates = Collections.singletonList(updatedContainerInfo);
+ }
+ this.nodeUtilization = resourceUtilization;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
@@ -312,6 +343,15 @@ public class MockNodes {
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Resource physicalResource) {
+ return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
+ labels, containersUtilization, nodeUtilization, physicalResource, null);
+ }
+
+ private static MockRMNodeImpl buildRMNode(int rack, final Resource perNode,
+ NodeState state, String httpAddr, int hostnum, String hostName, int port,
+ Set<String> labels, ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource physicalResource,
+ OverAllocationInfo overAllocationInfo) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
@@ -324,9 +364,9 @@ public class MockNodes {
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state, labels,
- containersUtilization, nodeUtilization, physicalResource);
+ containersUtilization, nodeUtilization,
+ physicalResource, overAllocationInfo);
}
-
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state) {
return buildRMNode(rack, perNode, state, "N/A");
@@ -355,4 +395,10 @@ public class MockNodes {
return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName, port);
}
+ public static MockRMNodeImpl newNodeInfo(int rack, final Resource perNode,
+ OverAllocationInfo overAllocationInfo) {
+ return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0",
+ NODE_ID++, null, 123, null, ResourceUtilization.newInstance(0, 0, 0.0f),
+ ResourceUtilization.newInstance(0, 0, 0.0f), null, overAllocationInfo);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 5b49303..075124f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -502,7 +502,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
FSParentQueue root = scheduler.getQueueManager().getRootQueue();
// ************ check cluster used Resources ********
assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
- assertEquals(usedResources,root.getResourceUsage());
+ assertEquals(usedResources, root.getGuaranteedResourceUsage());
// ************ check app headroom ****************
FSAppAttempt schedulerAttempt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 03332b2..6adc63c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
@@ -82,7 +82,7 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
return null;
}
@@ -112,11 +112,16 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public long getStartTime() {
return startTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index f581935..71db786 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -222,7 +222,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
scheduler.handle(nodeEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.update();
assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
@@ -231,8 +232,10 @@ public class TestAppRunnability extends FairSchedulerTestBase {
assertSame(targetQueue, app.getQueue());
assertFalse(oldQueue.isRunnableApp(app));
assertTrue(targetQueue.isRunnableApp(app));
- assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
- assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(0, 0),
+ oldQueue.getGuaranteedResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ targetQueue.getGuaranteedResourceUsage());
assertEquals(0, oldQueue.getNumRunnableApps());
assertEquals(1, targetQueue.getNumRunnableApps());
assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index 51ffd23..9bfbb91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -223,7 +223,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
- Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
+ Mockito.when(mockQueue.getGuaranteedResourceUsage()).thenReturn(queueUsage);
Mockito.when(mockScheduler.getClusterResource()).thenReturn
(clusterResource);
Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
@@ -305,7 +305,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
getApplicationId()));
FSAppAttempt app = scheduler.getSchedulerApp(id11);
assertNotNull(app);
- Resource queueUsage = app.getQueue().getResourceUsage();
+ Resource queueUsage = app.getQueue().getGuaranteedResourceUsage();
assertEquals(0, queueUsage.getMemorySize());
assertEquals(0, queueUsage.getVirtualCores());
SchedulerNode n1 = scheduler.getSchedulerNode(node1.getNodeID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 4a738ca..efe4ad1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -88,7 +88,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
- Mockito.when(app.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(Resources.none());
schedulable.addApp(app, true);
schedulable.addApp(app, true);
@@ -176,7 +176,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
@Override
public void run() {
for (int i=0; i < 500; i++) {
- schedulable.getResourceUsage();
+ schedulable.getGuaranteedResourceUsage();
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 6726f17..f79ba4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -98,7 +98,7 @@ public class TestFSSchedulerNode {
ApplicationAttemptId appAttemptId =
mock(ApplicationAttemptId.class);
when(starvingApp.getApplicationAttemptId()).thenReturn(appAttemptId);
- when(starvingApp.assignContainer(schedulerNode)).thenAnswer(
+ when(starvingApp.assignContainer(schedulerNode, false)).thenAnswer(
new Answer<Resource>() {
@Override
public Resource answer(InvocationOnMock invocationOnMock)
@@ -142,7 +142,7 @@ public class TestFSSchedulerNode {
}
private void allocateContainers(FSSchedulerNode schedulerNode) {
- FairScheduler.assignPreemptedContainers(schedulerNode);
+ FairScheduler.attemptToAssignPreemptedResources(schedulerNode);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-7737. prelaunch.err file not
found exception on container failure. Contributed by Keqiu Hu.
Posted by ha...@apache.org.
YARN-7737. prelaunch.err file not found exception on container failure. Contributed by Keqiu Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa8cf4d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa8cf4d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa8cf4d1
Branch: refs/heads/YARN-1011
Commit: fa8cf4d1b4896a602dc383d5e266768392a9790c
Parents: cff9edd
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Jan 24 12:54:34 2018 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Jan 24 12:54:54 2018 -0800
----------------------------------------------------------------------
.../nodemanager/containermanager/launcher/ContainerLaunch.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa8cf4d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 95ced22..7f43458 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -270,7 +270,7 @@ public class ContainerLaunch implements Callable<Integer> {
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
- new Path(containerLogDirs.get(0)), user);
+ containerLogDir, user);
}
// /////////// End of writing out container-script
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HDFS-12963. Error log level in
ShortCircuitRegistry#removeShm. Contributed by hu xiaodong.
Posted by ha...@apache.org.
HDFS-12963. Error log level in ShortCircuitRegistry#removeShm. Contributed by hu xiaodong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d95c1377
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d95c1377
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d95c1377
Branch: refs/heads/YARN-1011
Commit: d95c13774e1bd5b3cc61bf4da8bae4a93ed0040c
Parents: e307edc
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Jan 24 10:43:36 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Jan 24 10:43:36 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c1377/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
index edb64dd..ea9e72c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
@@ -114,7 +114,7 @@ public class ShortCircuitRegistry {
public synchronized void removeShm(ShortCircuitShm shm) {
if (LOG.isTraceEnabled()) {
- LOG.debug("removing shm " + shm);
+ LOG.trace("removing shm " + shm);
}
// Stop tracking the shmId.
RegisteredShm removedShm = segments.remove(shm.getShmId());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-7540 and YARN-7605. Convert
yarn app cli to call yarn api services and implement doAs for Api Service
REST API. Contributed by Eric Yang and Jian He
Posted by ha...@apache.org.
YARN-7540 and YARN-7605. Convert yarn app cli to call yarn api services and implement doAs for Api Service REST API. Contributed by Eric Yang and Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e307edcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e307edcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e307edcb
Branch: refs/heads/YARN-1011
Commit: e307edcb472207a39d1cbe4be6f7fcddc7b4fd6d
Parents: 39b999a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Jan 23 17:54:39 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Tue Jan 23 17:54:39 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/http/HttpServer2.java | 39 ++
.../hadoop-yarn-services-api/pom.xml | 9 +
.../yarn/service/client/ApiServiceClient.java | 450 +++++++++++++++++
.../yarn/service/client/package-info.java | 28 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 487 ++++++++++++++-----
.../yarn/service/webapp/package-info.java | 28 ++
.../hadoop/yarn/service/TestApiServer.java | 51 +-
.../service/client/TestApiServiceClient.java | 259 ++++++++++
.../src/test/resources/example-app.json | 15 +
.../src/test/resources/log4j.properties | 19 +
.../service/api/records/ReadinessCheck.java | 2 +
.../yarn/service/api/records/Resource.java | 1 -
.../yarn/service/api/records/ServiceState.java | 2 +-
.../yarn/service/client/ServiceClient.java | 4 +-
.../hadoop/yarn/service/utils/JsonSerDeser.java | 3 +
.../yarn/service/TestYarnNativeServices.java | 30 --
.../client/TestBuildExternalComponents.java | 2 +-
.../yarn/service/client/TestServiceCLI.java | 14 +-
.../hadoop/yarn/client/api/AppAdminClient.java | 6 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 3 +
.../org/apache/hadoop/yarn/webapp/WebApps.java | 37 +-
.../registry/client/api/RegistryConstants.java | 1 +
.../client/impl/zk/RegistrySecurity.java | 9 +-
.../server/resourcemanager/ResourceManager.java | 24 +-
24 files changed, 1313 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index fa447d8..65aadf3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -865,6 +865,45 @@ public final class HttpServer2 implements FilterContainer {
}
/**
+ * Add an internal servlet in the server, with initialization parameters.
+ * Note: This method is to be used for adding servlets that facilitate
+ * internal communication and not for user facing functionality. For
+ * servlets added using this method, filters (except internal Kerberos
+ * filters) are not enabled.
+ *
+ * @param name The name of the servlet (can be passed as null)
+ * @param pathSpec The path spec for the servlet
+ * @param clazz The servlet class
+ * @param params init parameters
+ */
+ public void addInternalServlet(String name, String pathSpec,
+ Class<? extends HttpServlet> clazz, Map<String, String> params) {
+ // Jetty doesn't like the same path spec mapping to different servlets, so
+ // if there's already a mapping for this pathSpec, remove it and assume that
+ // the newest one is the one we want
+ final ServletHolder sh = new ServletHolder(clazz);
+ sh.setName(name);
+ sh.setInitParameters(params);
+ final ServletMapping[] servletMappings =
+ webAppContext.getServletHandler().getServletMappings();
+ for (int i = 0; i < servletMappings.length; i++) {
+ if (servletMappings[i].containsPathSpec(pathSpec)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found existing " + servletMappings[i].getServletName() +
+ " servlet at path " + pathSpec + "; will replace mapping" +
+ " with " + sh.getName() + " servlet");
+ }
+ ServletMapping[] newServletMappings =
+ ArrayUtil.removeFromArray(servletMappings, servletMappings[i]);
+ webAppContext.getServletHandler()
+ .setServletMappings(newServletMappings);
+ break;
+ }
+ }
+ webAppContext.addServlet(sh, pathSpec);
+ }
+
+ /**
* Add the given handler to the front of the list of handlers.
*
* @param handler The handler to add
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index ddea2a1..bae62c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -65,6 +65,15 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
new file mode 100644
index 0000000..34e62b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.eclipse.jetty.util.UrlEncoded;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
+
+/**
+ * The rest API client for users to manage services on YARN.
+ */
+public class ApiServiceClient extends AppAdminClient {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApiServiceClient.class);
+ protected YarnClient yarnClient;
+
+ @Override protected void serviceInit(Configuration configuration)
+ throws Exception {
+ yarnClient = YarnClient.createYarnClient();
+ addService(yarnClient);
+ super.serviceInit(configuration);
+ }
+
+ /**
+ * Calculate Resource Manager address base on working REST API.
+ */
+ private String getRMWebAddress() {
+ Configuration conf = getConfig();
+ String scheme = "http://";
+ String path = "/app/v1/services/version";
+ String rmAddress = conf
+ .get("yarn.resourcemanager.webapp.address");
+ if (YarnConfiguration.useHttps(conf)) {
+ scheme = "https://";
+ rmAddress = conf
+ .get("yarn.resourcemanager.webapp.https.address");
+ }
+
+ List<String> rmServers = RMHAUtils
+ .getRMHAWebappAddresses(new YarnConfiguration(conf));
+ for (String host : rmServers) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(scheme);
+ sb.append(host);
+ sb.append(path);
+ Client client = Client.create();
+ WebResource webResource = client
+ .resource(sb.toString());
+ String test = webResource.get(String.class);
+ if (test.contains("hadoop_version")) {
+ rmAddress = host;
+ break;
+ }
+ }
+ return scheme+rmAddress;
+ }
+
+ /**
+ * Compute active resource manager API service location.
+ *
+ * @param appName - YARN service name
+ * @return URI to API Service
+ * @throws IOException
+ */
+ private String getApiUrl(String appName) throws IOException {
+ String url = getRMWebAddress();
+ StringBuilder api = new StringBuilder();
+ api.append(url);
+ api.append("/app/v1/services");
+ if (appName != null) {
+ api.append("/");
+ api.append(appName);
+ }
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ api.append("?user.name=" + UrlEncoded
+ .encodeString(System.getProperty("user.name")));
+ }
+ return api.toString();
+ }
+
+ private Builder getApiClient() throws IOException {
+ return getApiClient(null);
+ }
+
+ /**
+ * Setup API service web request.
+ *
+ * @param appName
+ * @return
+ * @throws IOException
+ */
+ private Builder getApiClient(String appName) throws IOException {
+ Client client = Client.create(getClientConfig());
+ Configuration conf = getConfig();
+ client.setChunkedEncodingSize(null);
+ Builder builder = client
+ .resource(getApiUrl(appName)).type(MediaType.APPLICATION_JSON);
+ if (conf.get("hadoop.security.authentication").equals("kerberos")) {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ builder.header("WWW-Authenticate", token);
+ }
+ return builder
+ .accept("application/json;charset=utf-8");
+ }
+
+ private ClientConfig getClientConfig() {
+ ClientConfig config = new DefaultClientConfig();
+ config.getProperties().put(
+ ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, 0);
+ config.getProperties().put(
+ ClientConfig.PROPERTY_BUFFER_RESPONSE_ENTITY_ON_EXCEPTION, true);
+ return config;
+ }
+
+ private int processResponse(ClientResponse response) {
+ response.bufferEntity();
+ String output;
+ if (response.getStatus() == 401) {
+ LOG.error("Authentication required");
+ return EXIT_EXCEPTION_THROWN;
+ }
+ try {
+ ServiceStatus ss = response.getEntity(ServiceStatus.class);
+ output = ss.getDiagnostics();
+ } catch (Throwable t) {
+ output = response.getEntity(String.class);
+ }
+ if (output==null) {
+ output = response.getEntity(String.class);
+ }
+ if (response.getStatus() <= 299) {
+ LOG.info(output);
+ return EXIT_SUCCESS;
+ } else {
+ LOG.error(output);
+ return EXIT_EXCEPTION_THROWN;
+ }
+ }
+
+ /**
+ * Utility method to load Service json from disk or from
+ * YARN examples.
+ *
+ * @param fileName - path to yarnfile
+ * @param serviceName - YARN Service Name
+ * @param lifetime - application lifetime
+ * @param queue - Queue to submit application
+ * @return
+ * @throws IOException
+ * @throws YarnException
+ */
+ public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
+ Long lifetime, String queue) throws IOException, YarnException {
+ File file = new File(fileName);
+ if (!file.exists() && fileName.equals(file.getName())) {
+ String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
+ String[] examplesDirs;
+ if (examplesDirStr == null) {
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ examplesDirs = new String[]{
+ yarnHome + "/share/hadoop/yarn/yarn-service-examples",
+ yarnHome + "/yarn-service-examples"
+ };
+ } else {
+ examplesDirs = StringUtils.split(examplesDirStr, ":");
+ }
+ for (String dir : examplesDirs) {
+ file = new File(MessageFormat.format("{0}/{1}/{2}.json",
+ dir, fileName, fileName));
+ if (file.exists()) {
+ break;
+ }
+ // Then look for secondary location.
+ file = new File(MessageFormat.format("{0}/{1}.json",
+ dir, fileName));
+ if (file.exists()) {
+ break;
+ }
+ }
+ }
+ if (!file.exists()) {
+ throw new YarnException("File or example could not be found: " +
+ fileName);
+ }
+ Path filePath = new Path(file.getAbsolutePath());
+ LOG.info("Loading service definition from local FS: " + filePath);
+ Service service = jsonSerDeser
+ .load(FileSystem.getLocal(getConfig()), filePath);
+ if (!StringUtils.isEmpty(serviceName)) {
+ service.setName(serviceName);
+ }
+ if (lifetime != null && lifetime > 0) {
+ service.setLifetime(lifetime);
+ }
+ if (!StringUtils.isEmpty(queue)) {
+ service.setQueue(queue);
+ }
+ return service;
+ }
+
+ /**
+ * Launch YARN service application.
+ *
+ * @param fileName - path to yarnfile
+ * @param appName - YARN Service Name
+ * @param lifetime - application lifetime
+ * @param queue - Queue to submit application
+ */
+ @Override
+ public int actionLaunch(String fileName, String appName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient()
+ .post(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to launch application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Stop YARN service application.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionStop(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.STOPPED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(appName)
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to stop application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Start YARN service application.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionStart(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.STARTED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(appName)
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to start application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Save Service configuration.
+ *
+ * @param fileName - path to Yarnfile
+ * @param appName - YARN Service Name
+ * @param lifetime - container life time
+ * @param queue - Queue to submit the application
+ */
+ @Override
+ public int actionSave(String fileName, String appName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
+ service.setState(ServiceState.STOPPED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient()
+ .post(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to save application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Decommission a YARN service.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionDestroy(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ ClientResponse response = getApiClient(appName)
+ .delete(ClientResponse.class);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to destroy application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Change number of containers associated with a service.
+ *
+ * @param appName - YARN Service Name
+ * @param componentCounts - list of components and desired container count
+ */
+ @Override
+ public int actionFlex(String appName, Map<String, String> componentCounts)
+ throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.FLEX);
+ for (Map.Entry<String, String> entry : componentCounts.entrySet()) {
+ Component component = new Component();
+ component.setName(entry.getKey());
+ Long numberOfContainers = Long.parseLong(entry.getValue());
+ component.setNumberOfContainers(numberOfContainers);
+ service.addComponent(component);
+ }
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(appName)
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to flex application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ @Override
+ public int enableFastLaunch(String destinationFolder) throws IOException, YarnException {
+ ServiceClient sc = new ServiceClient();
+ sc.init(getConfig());
+ sc.start();
+ int result = sc.enableFastLaunch(destinationFolder);
+ sc.close();
+ return result;
+ }
+
+ /**
+ * Retrieve Service Status through REST API.
+ *
+ * @param appIdOrName - YARN application ID or application name
+ * @return Status output
+ */
+ @Override
+ public String getStatusString(String appIdOrName) throws IOException,
+ YarnException {
+ String output = "";
+ String appName;
+ try {
+ ApplicationId appId = ApplicationId.fromString(appIdOrName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+ appName = appReport.getName();
+ } catch (IllegalArgumentException e) {
+ // not app Id format, may be app name
+ appName = appIdOrName;
+ ServiceApiUtil.validateNameFormat(appName, getConfig());
+ }
+ try {
+ ClientResponse response = getApiClient(appName).get(ClientResponse.class);
+ if (response.getStatus() != 200) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(appName);
+ sb.append(" Failed : HTTP error code : ");
+ sb.append(response.getStatus());
+ return sb.toString();
+ }
+ output = response.getEntity(String.class);
+ } catch (Exception e) {
+ LOG.error("Fail to check application status: ", e);
+ }
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
new file mode 100644
index 0000000..cf5ce11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.service.client contains classes
+ * for YARN Services Client API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 34ab8f0..16f8513 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -19,21 +19,23 @@ package org.apache.hadoop.yarn.service.webapp;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
@@ -42,15 +44,22 @@ import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_SUCCESS;
/**
* The rest API endpoints for users to manage services on YARN.
@@ -71,7 +80,8 @@ public class ApiServer {
private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();
- private static ServiceClient SERVICE_CLIENT;
+ private ServiceClient serviceClientUnitTest;
+ private boolean unitTest = false;
static {
init();
@@ -79,9 +89,6 @@ public class ApiServer {
// initialize all the common resources - order is important
private static void init() {
- SERVICE_CLIENT = new ServiceClient();
- SERVICE_CLIENT.init(YARN_CONFIG);
- SERVICE_CLIENT.start();
}
@GET
@@ -98,28 +105,62 @@ public class ApiServer {
@Path(SERVICE_ROOT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response createService(Service service) {
- LOG.info("POST: createService = {}", service);
+ public Response createService(@Context HttpServletRequest request,
+ Service service) {
ServiceStatus serviceStatus = new ServiceStatus();
try {
- ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
- LOG.info("Successfully created service " + service.getName()
- + " applicationId = " + applicationId);
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("POST: createService = {} user = {}", service, ugi);
+ if(service.getState()==ServiceState.STOPPED) {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionBuild(service);
+ sc.close();
+ return null;
+ }
+ });
+ serviceStatus.setDiagnostics("Service "+service.getName() +
+ " saved.");
+ } else {
+ ApplicationId applicationId = ugi
+ .doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ @Override
+ public ApplicationId run() throws IOException, YarnException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId applicationId = sc.actionCreate(service);
+ sc.close();
+ return applicationId;
+ }
+ });
+ serviceStatus.setDiagnostics("Application ID: " + applicationId);
+ }
serviceStatus.setState(ACCEPTED);
serviceStatus.setUri(
CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
.getName());
- return Response.status(Status.ACCEPTED).entity(serviceStatus).build();
- } catch (IllegalArgumentException e) {
+ return formatResponse(Status.ACCEPTED, serviceStatus);
+ } catch (AccessControlException e) {
serviceStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
- .build();
- } catch (Exception e) {
- String message = "Failed to create service " + service.getName();
+ return formatResponse(Status.FORBIDDEN, e.getCause().getMessage());
+ } catch (IllegalArgumentException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ String message = "Failed to create service " + service.getName()
+ + ": {}";
LOG.error(message, e);
- serviceStatus.setDiagnostics(message + ": " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(serviceStatus).build();
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ String message = "Failed to create service " + service.getName()
+ + ": {}";
+ LOG.error(message, e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
}
}
@@ -127,23 +168,42 @@ public class ApiServer {
@Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response getService(@PathParam(SERVICE_NAME) String appName) {
- LOG.info("GET: getService for appName = {}", appName);
+ public Response getService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName) {
ServiceStatus serviceStatus = new ServiceStatus();
try {
- Service app = SERVICE_CLIENT.getStatus(appName);
+ if (appName == null) {
+ throw new IllegalArgumentException("Service name can not be null.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("GET: getService for appName = {} user = {}", appName, ugi);
+ Service app = ugi.doAs(new PrivilegedExceptionAction<Service>() {
+ @Override
+ public Service run() throws IOException, YarnException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Service app = sc.getStatus(appName);
+ sc.close();
+ return app;
+ }
+ });
return Response.ok(app).build();
- } catch (IllegalArgumentException e) {
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (IllegalArgumentException |
+ FileNotFoundException e) {
serviceStatus.setDiagnostics(e.getMessage());
serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
return Response.status(Status.NOT_FOUND).entity(serviceStatus)
.build();
- } catch (Exception e) {
- LOG.error("Get service failed", e);
- serviceStatus
- .setDiagnostics("Failed to retrieve service: " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(serviceStatus).build();
+ } catch (IOException | InterruptedException e) {
+ LOG.error("Get service failed: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ LOG.error("Get service failed: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
}
}
@@ -151,62 +211,111 @@ public class ApiServer {
@Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response deleteService(@PathParam(SERVICE_NAME) String appName) {
- LOG.info("DELETE: deleteService for appName = {}", appName);
- return stopService(appName, true);
+ public Response deleteService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName) {
+ try {
+ if (appName == null) {
+ throw new IllegalArgumentException("Service name can not be null.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("DELETE: deleteService for appName = {} user = {}",
+ appName, ugi);
+ return stopService(appName, true, ugi);
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (IllegalArgumentException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ LOG.error("Fail to stop service: {}", e);
+ return formatResponse(Status.BAD_REQUEST,
+ e.getCause().getMessage());
+ } catch (YarnException | FileNotFoundException e) {
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ LOG.error("Fail to stop service: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ }
}
- private Response stopService(String appName, boolean destroy) {
- try {
- SERVICE_CLIENT.actionStop(appName, destroy);
- if (destroy) {
- SERVICE_CLIENT.actionDestroy(appName);
- LOG.info("Successfully deleted service {}", appName);
- } else {
- LOG.info("Successfully stopped service {}", appName);
+ private Response stopService(String appName, boolean destroy,
+ final UserGroupInformation ugi) throws IOException,
+ InterruptedException, YarnException, FileNotFoundException {
+ ugi.doAs(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws IOException, YarnException,
+ FileNotFoundException {
+ int result = 0;
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc.actionStop(appName, destroy);
+ if (destroy) {
+ result = sc.actionDestroy(appName);
+ LOG.info("Successfully deleted service {}", appName);
+ } else {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ sc.close();
+ return result;
}
- return Response.status(Status.OK).build();
- } catch (ApplicationNotFoundException e) {
- ServiceStatus serviceStatus = new ServiceStatus();
- serviceStatus.setDiagnostics(
- "Service " + appName + " is not found in YARN: " + e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
- .build();
- } catch (Exception e) {
- LOG.error("Fail to stop service:", e);
- ServiceStatus serviceStatus = new ServiceStatus();
- serviceStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(serviceStatus).build();
+ });
+ ServiceStatus serviceStatus = new ServiceStatus();
+ if (destroy) {
+ serviceStatus.setDiagnostics("Successfully destroyed service " +
+ appName);
+ } else {
+ serviceStatus.setDiagnostics("Successfully stopped service " +
+ appName);
}
+ return formatResponse(Status.OK, serviceStatus);
}
@PUT
@Path(COMPONENT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response updateComponent(@PathParam(SERVICE_NAME) String appName,
+ public Response updateComponent(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName,
@PathParam(COMPONENT_NAME) String componentName, Component component) {
- if (component.getNumberOfContainers() < 0) {
- return Response.status(Status.BAD_REQUEST).entity(
- "Service = " + appName + ", Component = " + component.getName()
- + ": Invalid number of containers specified " + component
- .getNumberOfContainers()).build();
- }
- ServiceStatus status = new ServiceStatus();
try {
- Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
- Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
+ UserGroupInformation ugi = getProxyUser(request);
+ if (component.getNumberOfContainers() < 0) {
+ String message =
+ "Service = " + appName + ", Component = " + component.getName()
+ + ": Invalid number of containers specified " + component
+ .getNumberOfContainers();
+ throw new YarnException(message);
+ }
+ Map<String, Long> original = ugi
+ .doAs(new PrivilegedExceptionAction<Map<String, Long>>() {
+ @Override
+ public Map<String, Long> run() throws YarnException, IOException {
+ ServiceClient sc = new ServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Map<String, Long> original = sc.flexByRestService(appName,
+ Collections.singletonMap(component.getName(),
+ component.getNumberOfContainers()));
+ sc.close();
+ return original;
+ }
+ });
+ ServiceStatus status = new ServiceStatus();
status.setDiagnostics(
"Updating component (" + componentName + ") size from " + original
.get(componentName) + " to " + component.getNumberOfContainers());
- return Response.ok().entity(status).build();
- } catch (YarnException | IOException e) {
- status.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
- .build();
+ return formatResponse(Status.OK, status);
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (YarnException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
}
}
@@ -214,75 +323,138 @@ public class ApiServer {
@Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response updateService(@PathParam(SERVICE_NAME) String appName,
+ public Response updateService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName,
Service updateServiceData) {
- LOG.info("PUT: updateService for app = {} with data = {}", appName,
- updateServiceData);
-
- // Ignore the app name provided in updateServiceData and always use appName
- // path param
- updateServiceData.setName(appName);
-
- // For STOP the app should be running. If already stopped then this
- // operation will be a no-op. For START it should be in stopped state.
- // If already running then this operation will be a no-op.
- if (updateServiceData.getState() != null
- && updateServiceData.getState() == ServiceState.STOPPED) {
- return stopService(appName, false);
- }
+ try {
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("PUT: updateService for app = {} with data = {} user = {}",
+ appName, updateServiceData, ugi);
+ // Ignore the app name provided in updateServiceData and always use
+ // appName path param
+ updateServiceData.setName(appName);
- // If a START is requested
- if (updateServiceData.getState() != null
- && updateServiceData.getState() == ServiceState.STARTED) {
- return startService(appName);
- }
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.FLEX) {
+ return flexService(updateServiceData, ugi);
+ }
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STOPPED) {
+ return stopService(appName, false, ugi);
+ }
+
+ // If a START is requested
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STARTED) {
+ return startService(appName, ugi);
+ }
- // If new lifetime value specified then update it
- if (updateServiceData.getLifetime() != null
- && updateServiceData.getLifetime() > 0) {
- return updateLifetime(appName, updateServiceData);
+ // If new lifetime value specified then update it
+ if (updateServiceData.getLifetime() != null
+ && updateServiceData.getLifetime() > 0) {
+ return updateLifetime(appName, updateServiceData, ugi);
+ }
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Status.BAD_REQUEST,
+ e.getCause().getMessage());
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (FileNotFoundException e) {
+ String message = "Application is not found app: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (YarnException e) {
+ String message = "Service is not found in hdfs: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ String message = "Error while performing operation for app: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
}
// If nothing happens consider it a no-op
return Response.status(Status.NO_CONTENT).build();
}
- private Response updateLifetime(String appName, Service updateAppData) {
- ServiceStatus status = new ServiceStatus();
- try {
- String newLifeTime =
- SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- status.setDiagnostics(
- "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
- + ", " + updateAppData.getLifetime()
- + " seconds remaining");
- return Response.ok(status).build();
- } catch (Exception e) {
- String message =
- "Failed to update service (" + appName + ")'s lifetime to "
- + updateAppData.getLifetime();
- LOG.error(message, e);
- status.setDiagnostics(message + ": " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
- .build();
+ private Response flexService(Service service, UserGroupInformation ugi)
+ throws IOException, InterruptedException {
+ String appName = service.getName();
+ Response response = Response.status(Status.BAD_REQUEST).build();
+ Map<String, String> componentCountStrings = new HashMap<String, String>();
+ for (Component c : service.getComponents()) {
+ componentCountStrings.put(c.getName(),
+ c.getNumberOfContainers().toString());
}
- }
+ Integer result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
- private Response startService(String appName) {
- ServiceStatus status = new ServiceStatus();
- try {
- SERVICE_CLIENT.actionStart(appName);
- LOG.info("Successfully started service " + appName);
- status.setDiagnostics("Service " + appName + " is successfully started.");
+ @Override
+ public Integer run() throws YarnException, IOException {
+ int result = 0;
+ ServiceClient sc = new ServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionFlex(appName, componentCountStrings);
+ sc.close();
+ return Integer.valueOf(result);
+ }
+ });
+ if (result == EXIT_SUCCESS) {
+ String message = "Service " + appName + " is successfully flexed.";
+ LOG.info(message);
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(message);
status.setState(ServiceState.ACCEPTED);
- return Response.ok(status).build();
- } catch (Exception e) {
- String message = "Failed to start service " + appName;
- status.setDiagnostics(message + ": " + e.getMessage());
- LOG.info(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(status).build();
+ response = formatResponse(Status.ACCEPTED, status);
}
+ return response;
+ }
+
+ private Response updateLifetime(String appName, Service updateAppData,
+ final UserGroupInformation ugi) throws IOException,
+ InterruptedException {
+ String newLifeTime = ugi.doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ String newLifeTime = sc.updateLifetime(appName,
+ updateAppData.getLifetime());
+ sc.close();
+ return newLifeTime;
+ }
+ });
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(
+ "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
+ + ", " + updateAppData.getLifetime() + " seconds remaining");
+ return formatResponse(Status.OK, status);
+ }
+
+ private Response startService(String appName,
+ final UserGroupInformation ugi) throws IOException,
+ InterruptedException {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionStart(appName);
+ sc.close();
+ return null;
+ }
+ });
+ LOG.info("Successfully started service " + appName);
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics("Service " + appName + " is successfully started.");
+ status.setState(ServiceState.ACCEPTED);
+ return formatResponse(Status.OK, status);
}
/**
@@ -290,10 +462,65 @@ public class ApiServer {
*
* @param mockServerClient - A mocked version of ServiceClient
*/
- public static void setServiceClient(ServiceClient mockServerClient) {
- SERVICE_CLIENT = mockServerClient;
- SERVICE_CLIENT.init(YARN_CONFIG);
- SERVICE_CLIENT.start();
+ public void setServiceClient(ServiceClient mockServerClient) {
+ serviceClientUnitTest = mockServerClient;
+ unitTest = true;
+ }
+
+ private ServiceClient getServiceClient() {
+ if (unitTest) {
+ return serviceClientUnitTest;
+ } else {
+ return new ServiceClient();
+ }
+ }
+
+ /**
+ * Configure impersonation callback.
+ *
+ * @param request - web request
+ * @return - configured UGI class for proxy callback
+ * @throws IOException - if user is not login.
+ */
+ private UserGroupInformation getProxyUser(HttpServletRequest request)
+ throws AccessControlException {
+ UserGroupInformation proxyUser;
+ UserGroupInformation ugi;
+ String remoteUser = request.getRemoteUser();
+ try {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ proxyUser = UserGroupInformation.getLoginUser();
+ ugi = UserGroupInformation.createProxyUser(remoteUser, proxyUser);
+ } else {
+ ugi = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return ugi;
+ } catch (IOException e) {
+ throw new AccessControlException(e.getCause());
+ }
+ }
+
+ /**
+ * Format HTTP response.
+ *
+ * @param status - HTTP Code
+ * @param message - Diagnostic message
+ * @return - HTTP response
+ */
+ private Response formatResponse(Status status, String message) {
+ ServiceStatus entity = new ServiceStatus();
+ entity.setDiagnostics(message);
+ return formatResponse(status, entity);
}
+ /**
+ * Format HTTP response.
+ *
+ * @param status - HTTP Code
+ * @param entity - ServiceStatus object
+ * @return - HTTP response
+ */
+ private Response formatResponse(Status status, ServiceStatus entity) {
+ return Response.status(status).entity(entity).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
new file mode 100644
index 0000000..1bdf05a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.service.webapp contains classes to be used
+ * for YARN Services API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
index 896b2f6..52057db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -26,12 +26,15 @@ import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.service.webapp.ApiServer;
+
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Path;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.List;
@@ -44,15 +47,19 @@ import static org.junit.Assert.*;
*/
public class TestApiServer {
private ApiServer apiServer;
+ private HttpServletRequest request;
@Before
public void setup() throws Exception {
+ request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteUser())
+ .thenReturn(System.getProperty("user.name"));
ServiceClient mockServerClient = new ServiceClientTest();
Configuration conf = new Configuration();
conf.set("yarn.api-service.service.client.class",
ServiceClientTest.class.getName());
- ApiServer.setServiceClient(mockServerClient);
- this.apiServer = new ApiServer(conf);
+ apiServer = new ApiServer(conf);
+ apiServer.setServiceClient(mockServerClient);
}
@Test
@@ -77,7 +84,7 @@ public class TestApiServer {
public void testBadCreateService() {
Service service = new Service();
// Test for invalid argument
- final Response actual = apiServer.createService(service);
+ final Response actual = apiServer.createService(request, service);
assertEquals("Create service is ", actual.getStatus(),
Response.status(Status.BAD_REQUEST).build().getStatus());
}
@@ -101,51 +108,51 @@ public class TestApiServer {
c.setResource(resource);
components.add(c);
service.setComponents(components);
- final Response actual = apiServer.createService(service);
+ final Response actual = apiServer.createService(request, service);
assertEquals("Create service is ", actual.getStatus(),
Response.status(Status.ACCEPTED).build().getStatus());
}
@Test
public void testBadGetService() {
- final Response actual = apiServer.getService("no-jenkins");
+ final Response actual = apiServer.getService(request, "no-jenkins");
assertEquals("Get service is ", actual.getStatus(),
Response.status(Status.NOT_FOUND).build().getStatus());
}
@Test
public void testBadGetService2() {
- final Response actual = apiServer.getService(null);
+ final Response actual = apiServer.getService(request, null);
assertEquals("Get service is ", actual.getStatus(),
- Response.status(Status.INTERNAL_SERVER_ERROR)
+ Response.status(Status.NOT_FOUND)
.build().getStatus());
}
@Test
public void testGoodGetService() {
- final Response actual = apiServer.getService("jenkins");
+ final Response actual = apiServer.getService(request, "jenkins");
assertEquals("Get service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
}
@Test
public void testBadDeleteService() {
- final Response actual = apiServer.deleteService("no-jenkins");
+ final Response actual = apiServer.deleteService(request, "no-jenkins");
assertEquals("Delete service is ", actual.getStatus(),
Response.status(Status.BAD_REQUEST).build().getStatus());
}
@Test
public void testBadDeleteService2() {
- final Response actual = apiServer.deleteService(null);
+ final Response actual = apiServer.deleteService(request, null);
assertEquals("Delete service is ", actual.getStatus(),
- Response.status(Status.INTERNAL_SERVER_ERROR)
+ Response.status(Status.BAD_REQUEST)
.build().getStatus());
}
@Test
public void testGoodDeleteService() {
- final Response actual = apiServer.deleteService("jenkins");
+ final Response actual = apiServer.deleteService(request, "jenkins");
assertEquals("Delete service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
}
@@ -170,7 +177,7 @@ public class TestApiServer {
c.setResource(resource);
components.add(c);
service.setComponents(components);
- final Response actual = apiServer.updateService("jenkins",
+ final Response actual = apiServer.updateService(request, "jenkins",
service);
assertEquals("update service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
@@ -197,7 +204,7 @@ public class TestApiServer {
components.add(c);
service.setComponents(components);
System.out.println("before stop");
- final Response actual = apiServer.updateService("no-jenkins",
+ final Response actual = apiServer.updateService(request, "no-jenkins",
service);
assertEquals("flex service is ", actual.getStatus(),
Response.status(Status.BAD_REQUEST).build().getStatus());
@@ -223,7 +230,7 @@ public class TestApiServer {
c.setResource(resource);
components.add(c);
service.setComponents(components);
- final Response actual = apiServer.updateService("jenkins",
+ final Response actual = apiServer.updateService(request, "jenkins",
service);
assertEquals("flex service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
@@ -249,10 +256,10 @@ public class TestApiServer {
c.setResource(resource);
components.add(c);
service.setComponents(components);
- final Response actual = apiServer.updateService("no-jenkins",
+ final Response actual = apiServer.updateService(request, "no-jenkins",
service);
assertEquals("start service is ", actual.getStatus(),
- Response.status(Status.INTERNAL_SERVER_ERROR).build()
+ Response.status(Status.BAD_REQUEST).build()
.getStatus());
}
@@ -276,7 +283,7 @@ public class TestApiServer {
c.setResource(resource);
components.add(c);
service.setComponents(components);
- final Response actual = apiServer.updateService("jenkins",
+ final Response actual = apiServer.updateService(request, "jenkins",
service);
assertEquals("start service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
@@ -303,7 +310,7 @@ public class TestApiServer {
components.add(c);
service.setComponents(components);
System.out.println("before stop");
- final Response actual = apiServer.updateService("no-jenkins",
+ final Response actual = apiServer.updateService(request, "no-jenkins",
service);
assertEquals("stop service is ", actual.getStatus(),
Response.status(Status.BAD_REQUEST).build().getStatus());
@@ -330,7 +337,7 @@ public class TestApiServer {
components.add(c);
service.setComponents(components);
System.out.println("before stop");
- final Response actual = apiServer.updateService("jenkins",
+ final Response actual = apiServer.updateService(request, "jenkins",
service);
assertEquals("stop service is ", actual.getStatus(),
Response.status(Status.OK).build().getStatus());
@@ -357,10 +364,10 @@ public class TestApiServer {
components.add(c);
service.setComponents(components);
System.out.println("before stop");
- final Response actual = apiServer.updateService("no-jenkins",
+ final Response actual = apiServer.updateService(request, "no-jenkins",
service);
assertEquals("update service is ", actual.getStatus(),
- Response.status(Status.INTERNAL_SERVER_ERROR)
+ Response.status(Status.BAD_REQUEST)
.build().getStatus());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
new file mode 100644
index 0000000..ffd9328
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
+
+/**
+ * Test case for CLI to API Service.
+ *
+ */
+public class TestApiServiceClient {
+ private static ApiServiceClient asc;
+ private static ApiServiceClient badAsc;
+ private static Server server;
+
+ /**
+ * A mocked version of API Service for testing purpose.
+ *
+ */
+ @SuppressWarnings("serial")
+ public static class TestServlet extends HttpServlet {
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ System.out.println("Get was called");
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doPut(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ server = new Server(8088);
+ ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10);
+ ServletContextHandler context = new ServletContextHandler();
+ context.setContextPath("/app");
+ server.setHandler(context);
+ context.addServlet(new ServletHolder(TestServlet.class), "/*");
+ ((ServerConnector)server.getConnectors()[0]).setHost("localhost");
+ server.start();
+
+ Configuration conf = new Configuration();
+ conf.set("yarn.resourcemanager.webapp.address",
+ "localhost:8088");
+ asc = new ApiServiceClient();
+ asc.serviceInit(conf);
+
+ Configuration conf2 = new Configuration();
+ conf2.set("yarn.resourcemanager.webapp.address",
+ "localhost:8089");
+ badAsc = new ApiServiceClient();
+ badAsc.serviceInit(conf2);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ server.stop();
+ }
+
+ @Test
+ public void testLaunch() {
+ String fileName = "target/test-classes/example-app.json";
+ String appName = "example-app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = asc.actionLaunch(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadLaunch() {
+ String fileName = "unknown_file";
+ String appName = "unknown_app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = badAsc.actionLaunch(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testStop() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionStop(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadStop() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionStop(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testStart() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionStart(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadStart() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionStart(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testSave() {
+ String fileName = "target/test-classes/example-app.json";
+ String appName = "example-app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = asc.actionSave(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadSave() {
+ String fileName = "unknown_file";
+ String appName = "unknown_app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = badAsc.actionSave(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFlex() {
+ String appName = "example-app";
+ HashMap<String, String> componentCounts = new HashMap<String, String>();
+ try {
+ int result = asc.actionFlex(appName, componentCounts);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadFlex() {
+ String appName = "unknown_app";
+ HashMap<String, String> componentCounts = new HashMap<String, String>();
+ try {
+ int result = badAsc.actionFlex(appName, componentCounts);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDestroy() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionDestroy(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadDestroy() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionDestroy(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
new file mode 100644
index 0000000..5dfbd64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
@@ -0,0 +1,15 @@
+{
+ "name": "example-app",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
new file mode 100644
index 0000000..81a3f6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 2bcf68b..af7c542 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -32,6 +32,7 @@ import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -49,6 +50,7 @@ import org.apache.hadoop.classification.InterfaceStability;
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index a7ce5be..c417ec0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlElement;
import java.util.Map;
import java.util.Objects;
-
/**
* Resource determines the amount of resources (vcores, memory, network, etc.)
* usable by a container. This field determines the resource to be applied for
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
index d2f5d06..902a0b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -29,5 +29,5 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "The current state of an service.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public enum ServiceState {
- ACCEPTED, STARTED, STABLE, STOPPED, FAILED;
+ ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index b8034b3..ee6e681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -987,7 +987,9 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
GetStatusResponseProto response =
amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
appSpec = jsonSerDeser.fromJson(response.getStatus());
-
+ if (lifetime != null) {
+ appSpec.setLifetime(lifetime.getRemainingTime());
+ }
return appSpec;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
index 7b22e3e..2c27ea7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -60,10 +60,13 @@ public class JsonSerDeser<T> {
* Create an instance bound to a specific type
* @param classType class type
*/
+ @SuppressWarnings("deprecation")
public JsonSerDeser(Class<T> classType) {
this.classType = classType;
this.mapper = new ObjectMapper();
mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ mapper.configure(SerializationConfig.Feature.WRITE_NULL_MAP_VALUES, false);
+ mapper.configure(SerializationConfig.Feature.WRITE_NULL_PROPERTIES, false);
}
public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 78670e2..5067ffc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -275,10 +275,6 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}
}
- private void checkRegistryAndCompDirDeleted() {
-
- }
-
private void checkEachCompInstancesInOrder(Component component) {
long expectedNumInstances = component.getNumberOfContainers();
Assert.assertEquals(expectedNumInstances, component.getContainers().size());
@@ -294,32 +290,6 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}
}
- private void waitForOneCompToBeReady(ServiceClient client,
- Service exampleApp, String readyComp)
- throws TimeoutException, InterruptedException {
- long numExpectedContainers =
- exampleApp.getComponent(readyComp).getNumberOfContainers();
- GenericTestUtils.waitFor(() -> {
- try {
- Service retrievedApp = client.getStatus(exampleApp.getName());
- Component retrievedComp = retrievedApp.getComponent(readyComp);
-
- if (retrievedComp.getContainers() != null
- && retrievedComp.getContainers().size() == numExpectedContainers) {
- LOG.info(readyComp + " found " + numExpectedContainers
- + " containers running");
- return true;
- } else {
- LOG.info(" Waiting for " + readyComp + "'s containers to be running");
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
/**
* Wait until all the containers for all components become ready state.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 1f4581e..6d5bb20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -60,7 +60,7 @@ public class TestBuildExternalComponents {
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
AppAdminClient client = AppAdminClient.createAppAdminClient(AppAdminClient
- .DEFAULT_TYPE, conf);
+ .UNIT_TEST_TYPE, conf);
client.actionSave(ExampleAppJson.resourceName(appDef), null, null,
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index df4b1df..a95818f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -61,15 +62,20 @@ public class TestServiceCLI {
}
private void buildApp(String serviceName, String appDef) throws Throwable {
- String[] args = {"app", "-D", basedirProp, "-save", serviceName,
- ExampleAppJson.resourceName(appDef)};
+ String[] args = {"app",
+ "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef),
+ "-appTypes", AppAdminClient.UNIT_TEST_TYPE};
runCLI(args);
}
private void buildApp(String serviceName, String appDef, String lifetime,
String queue) throws Throwable {
- String[] args = {"app", "-D", basedirProp, "-save", serviceName,
- ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
+ String[] args = {"app",
+ "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef),
+ "-appTypes", AppAdminClient.UNIT_TEST_TYPE,
+ "-updateLifetime", lifetime,
"-changeQueue", queue};
runCLI(args);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index 55be13b..a09663e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -39,6 +39,9 @@ public abstract class AppAdminClient extends CompositeService {
".application.admin.client.class.";
public static final String DEFAULT_TYPE = "yarn-service";
public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" +
+ ".service.client.ApiServiceClient";
+ public static final String UNIT_TEST_TYPE = "unit-test";
+ public static final String UNIT_TEST_CLASS_NAME = "org.apache.hadoop.yarn" +
".service.client.ServiceClient";
@Private
@@ -64,6 +67,9 @@ public abstract class AppAdminClient extends CompositeService {
if (!clientClassMap.containsKey(DEFAULT_TYPE)) {
clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME);
}
+ if (!clientClassMap.containsKey(UNIT_TEST_TYPE)) {
+ clientClassMap.put(UNIT_TEST_TYPE, UNIT_TEST_CLASS_NAME);
+ }
if (!clientClassMap.containsKey(appType)) {
throw new IllegalArgumentException("App admin client class name not " +
"specified for type " + appType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e307edcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 471b4d6..daca296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -322,6 +322,9 @@ public class ApplicationCLI extends YarnCLI {
System.err.println("Application with name '" + appIdOrName
+ "' doesn't exist in RM or Timeline Server.");
return -1;
+ } catch (Exception ie) {
+ System.err.println(ie.getMessage());
+ return -1;
}
}
} else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-7797. Docker host network can
not obtain IP address for RegistryDNS. Contributed by Eric Yang
Posted by ha...@apache.org.
YARN-7797. Docker host network can not obtain IP address for RegistryDNS. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2fa736f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2fa736f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2fa736f
Branch: refs/heads/YARN-1011
Commit: f2fa736f0ab139b5251d115fd75b833d1d7d1dcd
Parents: 56872cf
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Jan 26 09:32:23 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Jan 26 09:32:23 2018 -0800
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 28 ++++++++++++++++++++
1 file changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fa736f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2868dea..f3ce73d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.Contai
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -1013,6 +1015,32 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String ips = output.substring(0, index).trim();
String host = output.substring(index+1).trim();
+ if (ips.equals("")) {
+ String network;
+ try {
+ network = container.getLaunchContext().getEnvironment()
+ .get("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK");
+ if (network == null || network.isEmpty()) {
+ network = defaultNetwork;
+ }
+ } catch (NullPointerException e) {
+ network = defaultNetwork;
+ }
+ boolean useHostNetwork = network.equalsIgnoreCase("host");
+ if (useHostNetwork) {
+ // Report back node manager IP in the event where docker
+ // inspect reports no IP address. This is for bridging a gap for
+ // docker environment to run with host network.
+ InetAddress address;
+ try {
+ address = InetAddress.getLocalHost();
+ ips = address.getHostAddress();
+ } catch (UnknownHostException e) {
+ LOG.error("Can not determine IP for container:"
+ + containerId);
+ }
+ }
+ }
String[] ipAndHost = new String[2];
ipAndHost[0] = ips;
ipAndHost[1] = host;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: MAPREDUCE-6926. Allow MR jobs to
opt out of oversubscription. Contributed by Haibo Chen.
Posted by ha...@apache.org.
MAPREDUCE-6926. Allow MR jobs to opt out of oversubscription. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a13be84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a13be84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a13be84
Branch: refs/heads/YARN-1011
Commit: 0a13be84e774562a4aaf3a04000735b35044e3fa
Parents: ee49eb9
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jan 10 13:21:11 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:23:37 2018 -0800
----------------------------------------------------------------------
.../v2/app/rm/RMContainerRequestor.java | 48 ++---
.../v2/app/rm/TestRMContainerAllocator.java | 192 +++++++++++++++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../src/main/resources/mapred-default.xml | 8 +
4 files changed, 231 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a13be84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index bb3e1fa..d996690 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -111,6 +111,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
private final Set<String> blacklistRemovals = Collections
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private boolean optOutOfOversubscription;
public RMContainerRequestor(ClientService clientService, AppContext context) {
super(clientService, context);
@@ -136,9 +137,11 @@ public abstract class RMContainerRequestor extends RMCommunicator {
public ContainerRequest(ContainerRequestEvent event, Priority priority,
String nodeLabelExpression) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
- event.getRacks(), priority, nodeLabelExpression);
+ event.getRacks(), priority, System.currentTimeMillis(),
+ nodeLabelExpression);
}
+ @VisibleForTesting
public ContainerRequest(ContainerRequestEvent event, Priority priority,
long requestTimeMs) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
@@ -146,13 +149,6 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
public ContainerRequest(TaskAttemptId attemptID,
- Resource capability, String[] hosts, String[] racks,
- Priority priority, String nodeLabelExpression) {
- this(attemptID, capability, hosts, racks, priority,
- System.currentTimeMillis(), nodeLabelExpression);
- }
-
- public ContainerRequest(TaskAttemptId attemptID,
Resource capability, String[] hosts, String[] racks,
Priority priority, long requestTimeMs,String nodeLabelExpression) {
this.attemptID = attemptID;
@@ -186,6 +182,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
MRJobConfig.DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT);
LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
+ optOutOfOversubscription = conf.getBoolean(
+ MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ MRJobConfig.DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT);
+ LOG.info("optOutOfOversubscription is " + optOutOfOversubscription);
if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
+ blacklistDisablePercent
@@ -398,20 +398,20 @@ public abstract class RMContainerRequestor extends RMCommunicator {
for (String host : req.hosts) {
// Data-local
if (!isNodeBlacklisted(host)) {
- addResourceRequest(req.priority, host, req.capability,
+ addGuaranteedResourceRequest(req.priority, host, req.capability,
null);
}
}
// Nothing Rack-local for now
for (String rack : req.racks) {
- addResourceRequest(req.priority, rack, req.capability,
+ addGuaranteedResourceRequest(req.priority, rack, req.capability,
null);
}
// Off-switch
- addResourceRequest(req.priority, ResourceRequest.ANY, req.capability,
- req.nodeLabelExpression);
+ addGuaranteedResourceRequest(req.priority, ResourceRequest.ANY,
+ req.capability, req.nodeLabelExpression);
}
protected void decContainerReq(ContainerRequest req) {
@@ -430,18 +430,18 @@ public abstract class RMContainerRequestor extends RMCommunicator {
protected void addOpportunisticResourceRequest(Priority priority,
Resource capability) {
addResourceRequest(priority, ResourceRequest.ANY, capability, null,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, true);
}
- private void addResourceRequest(Priority priority, String resourceName,
- Resource capability, String nodeLabelExpression) {
+ private void addGuaranteedResourceRequest(Priority priority,
+ String resourceName, Resource capability, String nodeLabelExpression) {
addResourceRequest(priority, resourceName, capability, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, optOutOfOversubscription);
}
private void addResourceRequest(Priority priority, String resourceName,
Resource capability, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, boolean enforceExecutionType) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
if (remoteRequests == null) {
@@ -464,8 +464,8 @@ public abstract class RMContainerRequestor extends RMCommunicator {
remoteRequest.setCapability(capability);
remoteRequest.setNumContainers(0);
remoteRequest.setNodeLabelExpression(nodeLabelExpression);
- remoteRequest.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(executionType, true));
+ remoteRequest.setExecutionTypeRequest(ExecutionTypeRequest.
+ newInstance(executionType, enforceExecutionType));
reqMap.put(capability, remoteRequest);
}
remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
@@ -473,9 +473,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
- LOG.debug("addResourceRequest:" + " applicationId="
+ LOG.debug("addGuaranteedResourceRequest:" + " applicationId="
+ applicationId.getId() + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
+ + " resourceName=" + resourceName + " ExecutionType=" + executionType
+ + " enforceExecutionType=" + enforceExecutionType + " numContainers="
+ remoteRequest.getNumContainers() + " #asks=" + ask.size());
}
}
@@ -559,8 +560,9 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
}
String[] hosts = newHosts.toArray(new String[newHosts.size()]);
- ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability,
- hosts, orig.racks, orig.priority, orig.nodeLabelExpression);
+ ContainerRequest newReq = new ContainerRequest(orig.attemptID,
+ orig.capability, hosts, orig.racks, orig.priority,
+ System.currentTimeMillis(), orig.nodeLabelExpression);
return newReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a13be84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 85e4181..d95793d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -105,6 +105,8 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -844,6 +846,196 @@ public class TestRMContainerAllocator {
allocator.close();
}
+ /**
+ * Test A MapReduce job can be configured to opt out of oversubscription,
+ * that is, it always wait for guaranteed resources to execute its tasks.
+ * This is done by setting up a MapReduce job with 2 mappers and 1 reducers
+ * and capturing all ResourceRequests sent from the AM to RM, then checking
+ * if all ResourceRequests are guaranteed and their enforceExecutionType is
+ * true.
+ */
+ @Test
+ public void testMapReduceOptingOutOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(true);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or not enforced.");
+ }
+ }
+ }
+
+ /**
+ * Test a MapReduce job can be configured to opt in oversubscription (
+ * true by default). This is done by setting up a MapReduce job with 2
+ * mappers and 1 reducers and capturing all ResourceRequests sent from
+ * the AM to RM, then checking if all ResourceRequests are guaranteed
+ * but their enforceExecutionType is always set to false.
+ */
+ @Test
+ public void testMapReduceOptingInOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(false);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, false))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or it is enforced.");
+ }
+ }
+ }
+
+ /**
+ * Set up a mapreduce job with 2 mappers and 1 reducer and return
+ * all ResourceRequests sent from the AM to RM.
+ */
+ private List<ResourceRequest> captureResourceRequests(
+ boolean optOutOfOversubscription) throws Exception {
+ List<ResourceRequest> resourceRequests = new ArrayList<>();
+
+ final Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ optOutOfOversubscription);
+
+ // start the resource manager
+ final MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // submit an application
+ RMApp rmApp = rm.submitApp(1024);
+ rm.drainEvents();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ rm.drainEvents();
+
+ // start the MR AM and wait until it is in running state
+ MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
+ appAttemptId, 0), 2, 1, false,
+ this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+ mrApp.submit(conf);
+
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+ mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING);
+ amDispatcher.await();
+
+ // wait until all attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
+ .iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
+ }
+ amDispatcher.await();
+
+ // send map resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for both map tasks to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish both map tasks so that the reduce task can be scheduled
+ Iterator<Task> it = job.getTasks().values().iterator();
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // send the reduce resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for the reduce task to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish the reduce task
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ return resourceRequests;
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a13be84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index ca18bfe..48887c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1179,6 +1179,12 @@ public interface MRJobConfig {
public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
+ * Opt out of YARN oversubscription so that the job always waits for
+ * GUARANTEED resources available in the cluster.
+ */
+ String MR_OVERSUBSCRIPTION_OPT_OUT = "mapreduce.job.oversubscription-opt-out";
+ boolean DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT = false;
+ /**
* A comma-separated list of properties whose value will be redacted.
*/
String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a13be84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 72f509c..6d63a15 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2098,4 +2098,12 @@
</description>
</property>
+<property>
+ <description>
+ Opts out of YARN oversubscription so that the job always waits for GUARANTEED
+ resources available.
+ </description>
+ <name>mapreduce.job.oversubscription-opt-out</name>
+ <value>false</value>
+</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-7814. Remove automatic
mounting of the cgroups root directory into Docker containers. Contributed by
Shane Kumpf.
Posted by ha...@apache.org.
YARN-7814. Remove automatic mounting of the cgroups root directory into Docker containers. Contributed by Shane Kumpf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e586560
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e586560
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e586560
Branch: refs/heads/YARN-1011
Commit: 2e5865606b7701ee79d0d297238ab58a07a9f61f
Parents: cc10852
Author: Miklos Szegedi <sz...@apache.org>
Authored: Thu Jan 25 16:32:01 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Thu Jan 25 17:37:23 2018 -0800
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 7 -------
.../runtime/TestDockerContainerRuntime.java | 19 -------------------
2 files changed, 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e586560/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index f54323c..2868dea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -230,7 +230,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private PrivilegedOperationExecutor privilegedOperationExecutor;
private Set<String> allowedNetworks = new HashSet<>();
private String defaultNetwork;
- private String cgroupsRootDirectory;
private CGroupsHandler cGroupsHandler;
private AccessControlList privilegedContainersAcl;
private boolean enableUserReMapping;
@@ -290,7 +289,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOG.info("cGroupsHandler is null - cgroups not in use.");
} else {
this.cGroupsHandler = cGroupsHandler;
- this.cgroupsRootDirectory = cGroupsHandler.getCGroupMountPath();
}
}
@@ -796,11 +794,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
setHostname(runCommand, containerIdStr, hostname);
runCommand.setCapabilities(capabilities);
- if(cgroupsRootDirectory != null) {
- runCommand.addReadOnlyMountLocation(cgroupsRootDirectory,
- cgroupsRootDirectory, false);
- }
-
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
allDirs.add(containerWorkDir.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e586560/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index a825828..48a96e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
@@ -332,24 +331,6 @@ public class TestDockerContainerRuntime {
return expectedCapabilitiesString.toString();
}
- private String getExpectedCGroupsMountString() {
- CGroupsHandler cgroupsHandler = ResourceHandlerModule.getCGroupsHandler();
- if(cgroupsHandler == null) {
- return "";
- }
-
- String cgroupMountPath = cgroupsHandler.getCGroupMountPath();
- boolean cGroupsMountExists = new File(
- cgroupMountPath).exists();
-
- if(cGroupsMountExists) {
- return "-v " + cgroupMountPath
- + ":" + cgroupMountPath + ":ro ";
- } else {
- return "";
- }
- }
-
@Test
public void testDockerContainerLaunch()
throws ContainerExecutionException, PrivilegedOperationException,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-7728. Expose container
preemptions related information in Capacity Scheduler queue metrics.
Contributed by Eric Payne.
Posted by ha...@apache.org.
YARN-7728. Expose container preemptions related information in Capacity Scheduler queue metrics. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82cc6f69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82cc6f69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82cc6f69
Branch: refs/heads/YARN-1011
Commit: 82cc6f6968d2897ae006c5ef29eba74023f3483e
Parents: 7721fff
Author: Sunil G <su...@apache.org>
Authored: Thu Jan 25 14:32:54 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Jan 25 14:32:54 2018 +0530
----------------------------------------------------------------------
.../resourcemanager/scheduler/QueueMetrics.java | 18 ++++++++++++++++++
.../scheduler/capacity/CapacityScheduler.java | 19 +++++++++++++++++++
.../TestCapacitySchedulerSurgicalPreemption.java | 4 ++++
3 files changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cc6f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index eafe8ed..20a5a1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -68,6 +68,10 @@ public class QueueMetrics implements MetricsSource {
MutableCounterLong aggregateOffSwitchContainersAllocated;
@Metric("Aggregate # of preempted containers") MutableCounterLong
aggregateContainersPreempted;
+ @Metric("Aggregate # of preempted memory seconds") MutableCounterLong
+ aggregateMemoryMBSecondsPreempted;
+ @Metric("Aggregate # of preempted vcore seconds") MutableCounterLong
+ aggregateVcoreSecondsPreempted;
@Metric("# of active users") MutableGaugeInt activeUsers;
@Metric("# of active applications") MutableGaugeInt activeApplications;
@Metric("App Attempt First Container Allocation Delay")
@@ -534,6 +538,20 @@ public class QueueMetrics implements MetricsSource {
}
}
+ public void updatePreemptedMemoryMBSeconds(long mbSeconds) {
+ aggregateMemoryMBSecondsPreempted.incr(mbSeconds);
+ if (parent != null) {
+ parent.updatePreemptedMemoryMBSeconds(mbSeconds);
+ }
+ }
+
+ public void updatePreemptedVcoreSeconds(long vcoreSeconds) {
+ aggregateVcoreSecondsPreempted.incr(vcoreSeconds);
+ if (parent != null) {
+ parent.updatePreemptedVcoreSeconds(vcoreSeconds);
+ }
+ }
+
public void reserveResource(String partition, String user, Resource res) {
if(partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
reserveResource(user, res);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cc6f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index ba2f85a..99f4456 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -34,6 +34,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -46,6 +47,7 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -1821,6 +1823,23 @@ public class CapacityScheduler extends
LeafQueue queue = (LeafQueue) application.getQueue();
queue.completedContainer(getClusterResource(), application, node,
rmContainer, containerStatus, event, null, true);
+ if (ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) {
+ updateQueuePreemptionMetrics(queue, rmContainer);
+ }
+ }
+
+ private void updateQueuePreemptionMetrics(
+ CSQueue queue, RMContainer rmc) {
+ QueueMetrics qMetrics = queue.getMetrics();
+ long usedMillis = rmc.getFinishTime() - rmc.getCreationTime();
+ Resource containerResource = rmc.getAllocatedResource();
+ qMetrics.preemptContainer();
+ long mbSeconds = (containerResource.getMemorySize() * usedMillis)
+ / DateUtils.MILLIS_PER_SECOND;
+ long vcSeconds = (containerResource.getVirtualCores() * usedMillis)
+ / DateUtils.MILLIS_PER_SECOND;
+ qMetrics.updatePreemptedMemoryMBSeconds(mbSeconds);
+ qMetrics.updatePreemptedVcoreSeconds(vcSeconds);
}
@Lock(Lock.NoLock.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cc6f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index c20e091..9b183c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -171,6 +171,10 @@ public class TestCapacitySchedulerSurgicalPreemption
waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode2.getNodeID()),
am1.getApplicationAttemptId(), 16);
+ // Ensure preemption metrics were recored.
+ Assert.assertEquals("Number of preempted containers incorrectly recorded:",
+ 4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers());
+
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-7749. [UI2] GPU information
tab in left hand side disappears when we click other tabs below. Contributed
by Sunil G and Vasudevan Skm.
Posted by ha...@apache.org.
YARN-7749. [UI2] GPU information tab in left hand side disappears when we click other tabs below. Contributed by Sunil G and Vasudevan Skm.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20148634
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20148634
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20148634
Branch: refs/heads/YARN-1011
Commit: 2014863470446e945acb83bc235c9d4dd1074378
Parents: f3e33ae
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Jan 24 15:56:59 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Jan 24 15:56:59 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js | 1 +
.../hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js | 1 +
.../src/main/webapp/app/routes/yarn-node-containers.js | 1 +
3 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20148634/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
index 8b2ad80..05f0f0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
@@ -25,6 +25,7 @@ export default AbstractRoute.extend({
return Ember.RSVP.hash({
nodeApp: this.store.queryRecord('yarn-node-app',
{ nodeAddr : param.node_addr, appId: param.app_id }),
+ nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
nodeInfo: { id: param.node_id, addr: param.node_addr, appId: param.app_id }
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20148634/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js
index 6044076..3a98558 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js
@@ -25,6 +25,7 @@ export default AbstractRoute.extend({
// Get all apps running on a specific node. Node is contacted by using node_addr.
return Ember.RSVP.hash({
apps: this.store.query('yarn-node-app', { nodeAddr: param.node_addr }),
+ nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
nodeInfo: { id: param.node_id, addr: param.node_addr }
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20148634/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js
index 3c709f7..6968ce0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js
@@ -24,6 +24,7 @@ export default AbstractRoute.extend({
// Get all containers running on specific node.
return Ember.RSVP.hash({
containers: this.store.query('yarn-node-container', { nodeHttpAddr: param.node_addr }),
+ nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
nodeInfo: { id: param.node_id, addr: param.node_addr }
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-7817. Add Resource reference
to RM's NodeInfo object so REST API can get non memory/vcore resource usages.
(Sunil G via wangda)
Posted by ha...@apache.org.
YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0cfb0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0cfb0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0cfb0a3
Branch: refs/heads/YARN-1011
Commit: e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f
Parents: 2e58656
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Jan 26 15:43:27 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Jan 26 15:43:27 2018 +0800
----------------------------------------------------------------------
.../webapp/JAXBContextResolver.java | 2 +-
.../webapp/dao/ClusterMetricsInfo.java | 24 ++++++++++
.../resourcemanager/webapp/dao/NodeInfo.java | 20 ++++++++
.../webapp/dao/ResourceInfo.java | 18 ++++++--
.../webapp/dao/ResourceInformationsInfo.java | 48 ++++++++++++++++++++
.../main/webapp/app/models/cluster-metric.js | 4 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 4 +-
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
8 files changed, 114 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index 2f50a24..2e4204e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
- AppTimeoutInfo.class };
+ AppTimeoutInfo.class, ResourceInformationsInfo.class };
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3214cb9..84f70d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
private int activeNodes;
private int shutdownNodes;
+ // Total used resource of the cluster, including all partitions
+ private ResourceInfo totalUsedResourcesAcrossPartition;
+
+ // Total registered resources of the cluster, including all partitions
+ private ResourceInfo totalClusterResourcesAcrossPartition;
+
public ClusterMetricsInfo() {
} // JAXB needs this
@@ -92,9 +98,20 @@ public class ClusterMetricsInfo {
this.containersReserved = metrics.getReservedContainers();
if (rs instanceof CapacityScheduler) {
+ CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores =
availableVirtualCores + allocatedVirtualCores + containersReserved;
+ // TODO, add support of other schedulers to get total used resources
+ // across partition.
+ if (cs.getRootQueue() != null
+ && cs.getRootQueue().getQueueResourceUsage() != null
+ && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
+ totalUsedResourcesAcrossPartition = new ResourceInfo(
+ cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new ResourceInfo(
+ cs.getClusterResource());
+ }
} else {
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@@ -310,4 +327,11 @@ public class ClusterMetricsInfo {
this.shutdownNodes = shutdownNodes;
}
+ public ResourceInfo getTotalUsedResourcesAcrossPartition() {
+ return totalUsedResourcesAcrossPartition;
+ }
+
+ public ResourceInfo getTotalClusterResourcesAcrossPartition() {
+ return totalClusterResourcesAcrossPartition;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 2530c8e..3cec215 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -58,6 +58,8 @@ public class NodeInfo {
private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected ResourceUtilizationInfo resourceUtilization;
+ protected ResourceInfo usedResource;
+ protected ResourceInfo availableResource;
public NodeInfo() {
} // JAXB needs this
@@ -75,6 +77,8 @@ public class NodeInfo {
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores =
report.getAvailableResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getUsedResource());
+ this.availableResource = new ResourceInfo(report.getAvailableResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
@@ -183,6 +187,22 @@ public class NodeInfo {
return this.nodeLabels;
}
+ public ResourceInfo getUsedResource() {
+ return usedResource;
+ }
+
+ public void setUsedResource(ResourceInfo used) {
+ this.usedResource = used;
+ }
+
+ public ResourceInfo getAvailableResource() {
+ return availableResource;
+ }
+
+ public void setAvailableResource(ResourceInfo avail) {
+ this.availableResource = avail;
+ }
+
public ResourceUtilizationInfo getResourceUtilization() {
return this.resourceUtilization;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index e13980a..5bed936 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
+
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo {
@@ -34,6 +35,9 @@ public class ResourceInfo {
long memory;
@XmlElement
int vCores;
+ @XmlElement
+ ResourceInformationsInfo resourceInformations =
+ new ResourceInformationsInfo();
private Resource resources;
@@ -41,9 +45,13 @@ public class ResourceInfo {
}
public ResourceInfo(Resource res) {
- memory = res.getMemorySize();
- vCores = res.getVirtualCores();
- resources = Resources.clone(res);
+ // Make sure no NPE.
+ if (res != null) {
+ memory = res.getMemorySize();
+ vCores = res.getVirtualCores();
+ resources = Resources.clone(res);
+ resourceInformations.addAll(res.getAllResourcesListCopy());
+ }
}
public long getMemorySize() {
@@ -84,4 +92,8 @@ public class ResourceInfo {
public Resource getResource() {
return Resource.newInstance(resources);
}
+
+ public ResourceInformationsInfo getResourcesInformations() {
+ return resourceInformations;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
new file mode 100644
index 0000000..651d7df
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+@XmlRootElement(name = "resourceInformations")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceInformationsInfo {
+
+ @XmlElement(name = "resourceInformation")
+ protected ArrayList<ResourceInformation> resourceInformation =
+ new ArrayList<ResourceInformation>();
+
+ public ResourceInformationsInfo() {
+ } // JAXB needs this
+
+ public ArrayList<ResourceInformation> getApps() {
+ return resourceInformation;
+ }
+
+ public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
+ resourceInformation.addAll(resourcesInformationsInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index bbc6f08..764723a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -169,8 +169,8 @@ export default DS.Model.extend({
&& this.get("totalUsedResourcesAcrossPartition")) {
var usages = [];
- var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
- var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+ var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
+ var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
clusterResourceInformations.forEach(function(cluster) {
var perResourceTypeUsage = {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index 3b2385f..64fe7b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -99,7 +99,7 @@ export default DS.Model.extend({
const usedResource = this.get("usedResource");
const availableResource = this.get("availableResource");
- var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
+ var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
@@ -108,7 +108,7 @@ export default DS.Model.extend({
}
var available = 0;
- resourceInformations = availableResource ? availableResource.resourcesInformations : [];
+ resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
index a3a1d59..c27fd92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
@@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
nodeLabels: payload.nodeLabels,
- usedResource: payload.used,
- availableResource: payload.avail
+ usedResource: payload.usedResource,
+ availableResource: payload.availableResource
}
};
return fixedPayload;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-7102. NM heartbeat stuck when
responseId overflows MAX_INT. Contributed by Botong Huang
Posted by ha...@apache.org.
YARN-7102. NM heartbeat stuck when responseId overflows MAX_INT. Contributed by Botong Huang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff8378eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff8378eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff8378eb
Branch: refs/heads/YARN-1011
Commit: ff8378eb1b960c72d18a984c7e5d145b407ca11a
Parents: 16be42d
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 25 17:47:19 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 25 17:47:19 2018 -0600
----------------------------------------------------------------------
.../yarn/sls/nodemanager/NMSimulator.java | 4 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 11 +---
.../yarn/sls/scheduler/RMNodeWrapper.java | 12 +---
.../resourcemanager/ResourceTrackerService.java | 67 ++++++++++++++------
.../server/resourcemanager/rmnode/RMNode.java | 13 ++--
.../resourcemanager/rmnode/RMNodeImpl.java | 47 ++++----------
.../rmnode/RMNodeStatusEvent.java | 13 +---
.../yarn/server/resourcemanager/MockNM.java | 18 ++++--
.../yarn/server/resourcemanager/MockNodes.java | 9 +--
.../resourcemanager/TestRMNodeTransitions.java | 14 +---
.../TestResourceTrackerService.java | 35 ++++++++--
.../TestRMAppLogAggregationStatus.java | 10 +--
12 files changed, 125 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
index 6b19128..ba0fd56 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
@@ -71,7 +71,7 @@ public class NMSimulator extends TaskRunner.Task {
// resource manager
private ResourceManager rm;
// heart beat response id
- private int RESPONSE_ID = 1;
+ private int responseId = 0;
private final static Logger LOG = LoggerFactory.getLogger(NMSimulator.class);
public void init(String nodeIdStr, Resource nodeResource,
@@ -131,7 +131,7 @@ public class NMSimulator extends TaskRunner.Task {
ns.setContainersStatuses(generateContainerStatusList());
ns.setNodeId(node.getNodeID());
ns.setKeepAliveApplications(new ArrayList<ApplicationId>());
- ns.setResponseId(RESPONSE_ID ++);
+ ns.setResponseId(responseId++);
ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
beatRequest.setNodeStatus(ns);
NodeHeartbeatResponse beatResponse =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index e71ddff..1016ce1 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -144,8 +144,8 @@ public class NodeInfo {
return runningApplications;
}
- public void updateNodeHeartbeatResponseForCleanup(
- NodeHeartbeatResponse response) {
+ public void setAndUpdateNodeHeartbeatResponse(
+ NodeHeartbeatResponse response) {
}
public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
@@ -179,13 +179,6 @@ public class NodeInfo {
}
@Override
- public void updateNodeHeartbeatResponseForUpdatedContainers(
- NodeHeartbeatResponse response) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
public List<Container> pullNewlyIncreasedContainers() {
// TODO Auto-generated method stub
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 6b7ac3c..fdad826 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -127,9 +127,9 @@ public class RMNodeWrapper implements RMNode {
}
@Override
- public void updateNodeHeartbeatResponseForCleanup(
- NodeHeartbeatResponse nodeHeartbeatResponse) {
- node.updateNodeHeartbeatResponseForCleanup(nodeHeartbeatResponse);
+ public void setAndUpdateNodeHeartbeatResponse(
+ NodeHeartbeatResponse nodeHeartbeatResponse) {
+ node.setAndUpdateNodeHeartbeatResponse(nodeHeartbeatResponse);
}
@Override
@@ -167,12 +167,6 @@ public class RMNodeWrapper implements RMNode {
return RMNodeLabelsManager.EMPTY_STRING_SET;
}
- @Override
- public void updateNodeHeartbeatResponseForUpdatedContainers(
- NodeHeartbeatResponse response) {
- // TODO Auto-generated method stub
- }
-
@SuppressWarnings("unchecked")
@Override
public List<Container> pullNewlyIncreasedContainers() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index a42d053..9d95f63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -32,6 +32,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -81,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeReconnectEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
@@ -403,14 +405,37 @@ public class ResourceTrackerService extends AbstractService implements
} else {
LOG.info("Reconnect from the node at: " + host);
this.nmLivelinessMonitor.unregister(nodeId);
- // Reset heartbeat ID since node just restarted.
- oldNode.resetLastNodeHeartBeatResponse();
- this.rmContext
- .getDispatcher()
- .getEventHandler()
- .handle(
- new RMNodeReconnectEvent(nodeId, rmNode, request
- .getRunningApplications(), request.getNMContainerStatuses()));
+
+ if (CollectionUtils.isEmpty(request.getRunningApplications())
+ && rmNode.getState() != NodeState.DECOMMISSIONING
+ && rmNode.getHttpPort() != oldNode.getHttpPort()) {
+ // Reconnected node differs, so replace old node and start new node
+ switch (rmNode.getState()) {
+ case RUNNING:
+ ClusterMetrics.getMetrics().decrNumActiveNodes();
+ break;
+ case UNHEALTHY:
+ ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
+ break;
+ default:
+ LOG.debug("Unexpected Rmnode state");
+ }
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new NodeRemovedSchedulerEvent(rmNode));
+
+ this.rmContext.getRMNodes().put(nodeId, rmNode);
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeStartedEvent(nodeId, null, null));
+
+ } else {
+ // Reset heartbeat ID since node just restarted.
+ oldNode.resetLastNodeHeartBeatResponse();
+
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeReconnectEvent(nodeId, rmNode,
+ request.getRunningApplications(),
+ request.getNMContainerStatuses()));
+ }
}
// On every node manager register we will be clearing NMToken keys if
// present for any running application.
@@ -508,12 +533,13 @@ public class ResourceTrackerService extends AbstractService implements
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse();
- if (remoteNodeStatus.getResponseId() + 1 == lastNodeHeartbeatResponse
- .getResponseId()) {
+ if (getNextResponseId(
+ remoteNodeStatus.getResponseId()) == lastNodeHeartbeatResponse
+ .getResponseId()) {
LOG.info("Received duplicate heartbeat from node "
+ rmNode.getNodeAddress()+ " responseId=" + remoteNodeStatus.getResponseId());
return lastNodeHeartbeatResponse;
- } else if (remoteNodeStatus.getResponseId() + 1 < lastNodeHeartbeatResponse
+ } else if (remoteNodeStatus.getResponseId() != lastNodeHeartbeatResponse
.getResponseId()) {
String message =
"Too far behind rm response id:"
@@ -549,13 +575,11 @@ public class ResourceTrackerService extends AbstractService implements
}
// Heartbeat response
- NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
- .newNodeHeartbeatResponse(lastNodeHeartbeatResponse.
- getResponseId() + 1, NodeAction.NORMAL, null, null, null, null,
- nextHeartBeatInterval);
- rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
- rmNode.updateNodeHeartbeatResponseForUpdatedContainers(
- nodeHeartBeatResponse);
+ NodeHeartbeatResponse nodeHeartBeatResponse =
+ YarnServerBuilderUtils.newNodeHeartbeatResponse(
+ getNextResponseId(lastNodeHeartbeatResponse.getResponseId()),
+ NodeAction.NORMAL, null, null, null, null, nextHeartBeatInterval);
+ rmNode.setAndUpdateNodeHeartbeatResponse(nodeHeartBeatResponse);
populateKeys(request, nodeHeartBeatResponse);
@@ -573,7 +597,7 @@ public class ResourceTrackerService extends AbstractService implements
// 4. Send status to RMNode, saving the latest response.
RMNodeStatusEvent nodeStatusEvent =
- new RMNodeStatusEvent(nodeId, remoteNodeStatus, nodeHeartBeatResponse);
+ new RMNodeStatusEvent(nodeId, remoteNodeStatus);
if (request.getLogAggregationReportsForApps() != null
&& !request.getLogAggregationReportsForApps().isEmpty()) {
nodeStatusEvent.setLogAggregationReportsForApps(request
@@ -614,6 +638,11 @@ public class ResourceTrackerService extends AbstractService implements
return nodeHeartBeatResponse;
}
+ private int getNextResponseId(int responseId) {
+ // Loop between 0 and Integer.MAX_VALUE
+ return (responseId + 1) & Integer.MAX_VALUE;
+ }
+
private void setAppCollectorsMapToResponse(
List<ApplicationId> runningApps, NodeHeartbeatResponse response) {
Map<ApplicationId, AppCollectorData> liveAppCollectorsMap = new
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 328c040..a5615ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -141,10 +141,11 @@ public interface RMNode {
/**
* Update a {@link NodeHeartbeatResponse} with the list of containers and
- * applications to clean up for this node.
+ * applications to clean up for this node, and the containers to be updated.
+ *
* @param response the {@link NodeHeartbeatResponse} to update
*/
- void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response);
+ void setAndUpdateNodeHeartbeatResponse(NodeHeartbeatResponse response);
public NodeHeartbeatResponse getLastNodeHeartBeatResponse();
@@ -167,13 +168,7 @@ public interface RMNode {
* @return labels in this node
*/
public Set<String> getNodeLabels();
-
- /**
- * Update containers to be updated
- */
- void updateNodeHeartbeatResponseForUpdatedContainers(
- NodeHeartbeatResponse response);
-
+
public List<Container> pullNewlyIncreasedContainers();
OpportunisticContainersStatus getOpportunisticContainersStatus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 2b013a0..da54eb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -598,7 +598,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
};
@Override
- public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response) {
+ public void setAndUpdateNodeHeartbeatResponse(
+ NodeHeartbeatResponse response) {
this.writeLock.lock();
try {
@@ -613,38 +614,30 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.finishedApplications.clear();
this.containersToSignal.clear();
this.containersToBeRemovedFromNM.clear();
- } finally {
- this.writeLock.unlock();
- }
- };
-
- @VisibleForTesting
- public Collection<Container> getToBeUpdatedContainers() {
- return toBeUpdatedContainers.values();
- }
-
- @Override
- public void updateNodeHeartbeatResponseForUpdatedContainers(
- NodeHeartbeatResponse response) {
- this.writeLock.lock();
-
- try {
+
response.addAllContainersToUpdate(toBeUpdatedContainers.values());
toBeUpdatedContainers.clear();
// NOTE: This is required for backward compatibility.
response.addAllContainersToDecrease(toBeDecreasedContainers.values());
toBeDecreasedContainers.clear();
+
+ // Synchronously update the last response in rmNode with updated
+ // responseId
+ this.latestNodeHeartBeatResponse = response;
} finally {
this.writeLock.unlock();
}
+ };
+
+ @VisibleForTesting
+ public Collection<Container> getToBeUpdatedContainers() {
+ return toBeUpdatedContainers.values();
}
@Override
public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
-
this.readLock.lock();
-
try {
return this.latestNodeHeartBeatResponse;
} finally {
@@ -818,7 +811,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private static NodeHealthStatus updateRMNodeFromStatusEvents(
RMNodeImpl rmNode, RMNodeStatusEvent statusEvent) {
// Switch the last heartbeatresponse.
- rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();
NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
rmNode.setLastHealthReportTime(remoteNodeHealthStatus
@@ -912,21 +904,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeAddedSchedulerEvent(rmNode));
}
- } else {
- // Reconnected node differs, so replace old node and start new node
- switch (rmNode.getState()) {
- case RUNNING:
- ClusterMetrics.getMetrics().decrNumActiveNodes();
- break;
- case UNHEALTHY:
- ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
- break;
- default:
- LOG.debug("Unexpected Rmnode state");
- }
- rmNode.context.getRMNodes().put(newNode.getNodeID(), newNode);
- rmNode.context.getDispatcher().getEventHandler().handle(
- new RMNodeStartedEvent(newNode.getNodeID(), null, null));
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
index f9fe159..c79f270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
-import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -35,20 +34,16 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus;
public class RMNodeStatusEvent extends RMNodeEvent {
private final NodeStatus nodeStatus;
- private final NodeHeartbeatResponse latestResponse;
private List<LogAggregationReport> logAggregationReportsForApps;
- public RMNodeStatusEvent(NodeId nodeId, NodeStatus nodeStatus,
- NodeHeartbeatResponse latestResponse) {
- this(nodeId, nodeStatus, latestResponse, null);
+ public RMNodeStatusEvent(NodeId nodeId, NodeStatus nodeStatus) {
+ this(nodeId, nodeStatus, null);
}
public RMNodeStatusEvent(NodeId nodeId, NodeStatus nodeStatus,
- NodeHeartbeatResponse latestResponse,
List<LogAggregationReport> logAggregationReportsForApps) {
super(nodeId, RMNodeEventType.STATUS_UPDATE);
this.nodeStatus = nodeStatus;
- this.latestResponse = latestResponse;
this.logAggregationReportsForApps = logAggregationReportsForApps;
}
@@ -60,10 +55,6 @@ public class RMNodeStatusEvent extends RMNodeEvent {
return this.nodeStatus.getContainersStatuses();
}
- public NodeHeartbeatResponse getLatestResponse() {
- return this.latestResponse;
- }
-
public List<ApplicationId> getKeepAliveAppIds() {
return this.nodeStatus.getKeepAliveApplications();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 05b51e3..0a06e82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -131,7 +131,7 @@ public class MockNM {
container.getResource());
List<Container> increasedConts = Collections.singletonList(container);
nodeHeartbeat(Collections.singletonList(containerStatus), increasedConts,
- true, ++responseId);
+ true, responseId);
}
public void addRegisteringCollector(ApplicationId appId,
@@ -190,12 +190,13 @@ public class MockNM {
}
}
}
+ responseId = 0;
return registrationResponse;
}
public NodeHeartbeatResponse nodeHeartbeat(boolean isHealthy) throws Exception {
return nodeHeartbeat(Collections.<ContainerStatus>emptyList(),
- Collections.<Container>emptyList(), isHealthy, ++responseId);
+ Collections.<Container>emptyList(), isHealthy, responseId);
}
public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId,
@@ -208,12 +209,12 @@ public class MockNM {
containerStatusList.add(containerStatus);
Log.getLog().info("ContainerStatus: " + containerStatus);
return nodeHeartbeat(containerStatusList,
- Collections.<Container>emptyList(), true, ++responseId);
+ Collections.<Container>emptyList(), true, responseId);
}
public NodeHeartbeatResponse nodeHeartbeat(Map<ApplicationId,
List<ContainerStatus>> conts, boolean isHealthy) throws Exception {
- return nodeHeartbeat(conts, isHealthy, ++responseId);
+ return nodeHeartbeat(conts, isHealthy, responseId);
}
public NodeHeartbeatResponse nodeHeartbeat(Map<ApplicationId,
@@ -229,7 +230,7 @@ public class MockNM {
public NodeHeartbeatResponse nodeHeartbeat(
List<ContainerStatus> updatedStats, boolean isHealthy) throws Exception {
return nodeHeartbeat(updatedStats, Collections.<Container>emptyList(),
- isHealthy, ++responseId);
+ isHealthy, responseId);
}
public NodeHeartbeatResponse nodeHeartbeat(List<ContainerStatus> updatedStats,
@@ -265,7 +266,8 @@ public class MockNM {
NodeHeartbeatResponse heartbeatResponse =
resourceTracker.nodeHeartbeat(req);
-
+ responseId = heartbeatResponse.getResponseId();
+
MasterKey masterKeyFromRM = heartbeatResponse.getContainerTokenMasterKey();
if (masterKeyFromRM != null
&& masterKeyFromRM.getKeyId() != this.currentContainerTokenMasterKey
@@ -303,4 +305,8 @@ public class MockNM {
public String getVersion() {
return version;
}
+
+ public void setResponseId(int id) {
+ this.responseId = id;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 317c648..d6549b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -205,7 +205,8 @@ public class MockNodes {
}
@Override
- public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response) {
+ public void setAndUpdateNodeHeartbeatResponse(
+ NodeHeartbeatResponse response) {
}
@Override
@@ -246,12 +247,6 @@ public class MockNodes {
}
@Override
- public void updateNodeHeartbeatResponseForUpdatedContainers(
- NodeHeartbeatResponse response) {
-
- }
-
- @Override
public List<Container> pullNewlyIncreasedContainers() {
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 3657123..487d226 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -164,15 +164,12 @@ public class TestRMNodeTransitions {
private RMNodeStatusEvent getMockRMNodeStatusEvent(
List<ContainerStatus> containerStatus) {
- NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);
-
NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
Boolean yes = new Boolean(true);
doReturn(yes).when(healthStatus).getIsNodeHealthy();
RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
doReturn(healthStatus).when(event).getNodeHealthStatus();
- doReturn(response).when(event).getLatestResponse();
doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
if (containerStatus != null) {
doReturn(containerStatus).when(event).getContainers();
@@ -181,15 +178,12 @@ public class TestRMNodeTransitions {
}
private RMNodeStatusEvent getMockRMNodeStatusEventWithRunningApps() {
- NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);
-
NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
Boolean yes = new Boolean(true);
doReturn(yes).when(healthStatus).getIsNodeHealthy();
RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
doReturn(healthStatus).when(event).getNodeHealthStatus();
- doReturn(response).when(event).getLatestResponse();
doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
doReturn(getAppIdList()).when(event).getKeepAliveAppIds();
return event;
@@ -202,15 +196,12 @@ public class TestRMNodeTransitions {
}
private RMNodeStatusEvent getMockRMNodeStatusEventWithoutRunningApps() {
- NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);
-
NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
Boolean yes = new Boolean(true);
doReturn(yes).when(healthStatus).getIsNodeHealthy();
RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
doReturn(healthStatus).when(event).getNodeHealthStatus();
- doReturn(response).when(event).getLatestResponse();
doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
doReturn(null).when(event).getKeepAliveAppIds();
return event;
@@ -646,7 +637,7 @@ public class TestRMNodeTransitions {
Assert.assertEquals(1, node.getContainersToCleanUp().size());
Assert.assertEquals(1, node.getAppsToCleanup().size());
NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
- node.updateNodeHeartbeatResponseForCleanup(hbrsp);
+ node.setAndUpdateNodeHeartbeatResponse(hbrsp);
Assert.assertEquals(0, node.getContainersToCleanUp().size());
Assert.assertEquals(0, node.getAppsToCleanup().size());
Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
@@ -1108,7 +1099,8 @@ public class TestRMNodeTransitions {
NodeHeartbeatResponse hbrsp =
Records.newRecord(NodeHeartbeatResponse.class);
- node.updateNodeHeartbeatResponseForCleanup(hbrsp);
+ node.setAndUpdateNodeHeartbeatResponse(hbrsp);
+
Assert.assertEquals(1, hbrsp.getContainersToBeRemovedFromNM().size());
Assert.assertEquals(0, node.getCompletedContainers().size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index fc6326e..96e4451 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -801,7 +801,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Records.newRecord(NodeHeartbeatRequest.class);
heartbeatReq.setNodeLabels(null); // Node heartbeat label update
nodeStatusObject = getNodeStatusObject(nodeId);
- nodeStatusObject.setResponseId(responseId+2);
+ nodeStatusObject.setResponseId(responseId+1);
heartbeatReq.setNodeStatus(nodeStatusObject);
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse
.getNMTokenMasterKey());
@@ -1128,8 +1128,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
"", System.currentTimeMillis());
NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
statusList, null, nodeHealth, null, null, null);
- node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus,
- nodeHeartbeat1));
+ node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus));
Assert.assertEquals(1, node1.getRunningApps().size());
Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0));
@@ -1145,8 +1144,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
statusList.add(status2);
nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
statusList, null, nodeHealth, null, null, null);
- node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus,
- nodeHeartbeat2));
+ node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus));
Assert.assertEquals(1, node2.getRunningApps().size());
Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0));
@@ -2290,4 +2288,31 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
}
}
+
+ @Test
+ public void testResponseIdOverflow() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+
+ NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeat.getNodeAction());
+
+ // prepare the responseId that's about to overflow
+ RMNode node = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ node.getLastNodeHeartBeatResponse().setResponseId(Integer.MAX_VALUE);
+
+ nm1.setResponseId(Integer.MAX_VALUE);
+
+ // heartbeat twice and check responseId
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeat.getNodeAction());
+ Assert.assertEquals(0, nodeHeartbeat.getResponseId());
+
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeat.getNodeAction());
+ Assert.assertEquals(1, nodeHeartbeat.getResponseId());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff8378eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index 677990b..c2bc611 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -172,7 +172,7 @@ public class TestRMAppLogAggregationStatus {
NodeStatus nodeStatus1 = NodeStatus.newInstance(node1.getNodeID(), 0,
new ArrayList<ContainerStatus>(), null,
NodeHealthStatus.newInstance(true, null, 0), null, null, null);
- node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null,
+ node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1,
node1ReportForApp));
List<LogAggregationReport> node2ReportForApp =
@@ -186,7 +186,7 @@ public class TestRMAppLogAggregationStatus {
NodeStatus nodeStatus2 = NodeStatus.newInstance(node2.getNodeID(), 0,
new ArrayList<ContainerStatus>(), null,
NodeHealthStatus.newInstance(true, null, 0), null, null, null);
- node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null,
+ node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2,
node2ReportForApp));
// node1 and node2 has updated its log aggregation status
// verify that the log aggregation status for node1, node2
@@ -223,7 +223,7 @@ public class TestRMAppLogAggregationStatus {
LogAggregationReport.newInstance(appId,
LogAggregationStatus.RUNNING, messageForNode1_2);
node1ReportForApp2.add(report1_2);
- node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null,
+ node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1,
node1ReportForApp2));
// verify that the log aggregation status for node1
@@ -291,7 +291,7 @@ public class TestRMAppLogAggregationStatus {
LogAggregationStatus.SUCCEEDED, ""));
// For every logAggregationReport cached in memory, we can only save at most
// 10 diagnostic messages/failure messages
- node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null,
+ node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1,
node1ReportForApp3));
logAggregationStatus = rmApp.getLogAggregationReportsForApp();
@@ -335,7 +335,7 @@ public class TestRMAppLogAggregationStatus {
LogAggregationStatus.FAILED, "");
node2ReportForApp2.add(report2_2);
node2ReportForApp2.add(report2_3);
- node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null,
+ node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2,
node2ReportForApp2));
Assert.assertEquals(LogAggregationStatus.FAILED,
rmApp.getLogAggregationStatusForAppReport());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-7801. AmFilterInitializer
should addFilter after fill all parameters. Contributed by Wangda Tan
Posted by ha...@apache.org.
YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55c32776
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55c32776
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55c32776
Branch: refs/heads/YARN-1011
Commit: 55c32776b1f130c6b81ef7aff5fc1562180f4601
Parents: 9760761
Author: Jian He <ji...@apache.org>
Authored: Wed Jan 24 11:16:54 2018 -0800
Committer: Jian He <ji...@apache.org>
Committed: Wed Jan 24 11:17:12 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c32776/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c3ddc54..27742e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer {
}
sb.setLength(sb.length() - 1);
params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
- container.addFilter(FILTER_NAME, FILTER_CLASS, params);
// Handle RM HA urls
// Include yarn-site.xml in the classpath
@@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer {
params.put(RM_HA_URLS, StringUtils.join(",", urls));
}
}
+
+ container.addFilter(FILTER_NAME, FILTER_CLASS, params);
}
private Collection<String> getRmIds(Configuration conf) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-7806. Distributed Shell
should use timeline async api's. Contributed by Rohith Sharma K S
Posted by ha...@apache.org.
YARN-7806. Distributed Shell should use timeline async api's. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97607617
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97607617
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97607617
Branch: refs/heads/YARN-1011
Commit: 97607617ce01ee21af4fb9eb8df26fc6986a38a2
Parents: 2014863
Author: Sunil G <su...@apache.org>
Authored: Wed Jan 24 16:56:13 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Jan 24 16:56:13 2018 +0530
----------------------------------------------------------------------
.../yarn/applications/distributedshell/ApplicationMaster.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97607617/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index bd810c1..270ef1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1511,7 +1511,7 @@ public class ApplicationMaster {
appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
- timelineV2Client.putEntities(entity);
+ timelineV2Client.putEntitiesAsync(entity);
return null;
}
});
@@ -1545,7 +1545,7 @@ public class ApplicationMaster {
appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
- timelineV2Client.putEntities(entity);
+ timelineV2Client.putEntitiesAsync(entity);
return null;
}
});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: MAPREDUCE-7015. Possible race
condition in JHS if the job is not loaded. Contributed by Peter Bacsko
Posted by ha...@apache.org.
MAPREDUCE-7015. Possible race condition in JHS if the job is not loaded. Contributed by Peter Bacsko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cff9edd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cff9edd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cff9edd4
Branch: refs/heads/YARN-1011
Commit: cff9edd4b514bdcfe22cd49964e3707fb78ab876
Parents: 55c3277
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jan 24 14:44:07 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jan 24 14:44:07 2018 -0600
----------------------------------------------------------------------
.../mapreduce/v2/hs/CachedHistoryStorage.java | 8 +++++-
.../mapreduce/v2/hs/HistoryFileManager.java | 30 ++++++++++++++++----
.../hadoop/mapreduce/v2/hs/TestJobHistory.java | 26 +++++++++++++++++
3 files changed, 57 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cff9edd4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
index b001ae4..69f4831 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
@@ -173,9 +173,14 @@ public class CachedHistoryStorage extends AbstractService implements
HistoryFileInfo fileInfo;
fileInfo = hsManager.getFileInfo(jobId);
+
if (fileInfo == null) {
throw new HSFileRuntimeException("Unable to find job " + jobId);
- } else if (fileInfo.isDeleted()) {
+ }
+
+ fileInfo.waitUntilMoved();
+
+ if (fileInfo.isDeleted()) {
throw new HSFileRuntimeException("Cannot load deleted job " + jobId);
} else {
return fileInfo.loadJob();
@@ -211,6 +216,7 @@ public class CachedHistoryStorage extends AbstractService implements
for (HistoryFileInfo mi : hsManager.getAllFileInfo()) {
if (mi != null) {
JobId id = mi.getJobId();
+ mi.waitUntilMoved();
result.put(id, new PartialJob(mi.getJobIndexInfo(), id));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cff9edd4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index b418db7..a07ca26 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -452,6 +452,8 @@ public class HistoryFileManager extends AbstractService {
} catch (Throwable t) {
LOG.error("Error while trying to move a job to done", t);
this.state = HistoryInfoState.MOVE_FAILED;
+ } finally {
+ notifyAll();
}
}
@@ -485,12 +487,16 @@ public class HistoryFileManager extends AbstractService {
}
protected synchronized void delete() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("deleting " + historyFile + " and " + confFile);
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("deleting " + historyFile + " and " + confFile);
+ }
+ state = HistoryInfoState.DELETED;
+ doneDirFc.delete(doneDirFc.makeQualified(historyFile), false);
+ doneDirFc.delete(doneDirFc.makeQualified(confFile), false);
+ } finally {
+ notifyAll();
}
- state = HistoryInfoState.DELETED;
- doneDirFc.delete(doneDirFc.makeQualified(historyFile), false);
- doneDirFc.delete(doneDirFc.makeQualified(confFile), false);
}
public JobIndexInfo getJobIndexInfo() {
@@ -517,6 +523,17 @@ public class HistoryFileManager extends AbstractService {
jobIndexInfo.getNumMaps();
return (maxTasksForLoadedJob > 0) && (totalTasks > maxTasksForLoadedJob);
}
+
+ public synchronized void waitUntilMoved() {
+ while (isMovePending() && !didMoveFail()) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ LOG.warn("Waiting has been interrupted");
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
private SerialNumberIndex serialNumberIndex = null;
@@ -956,6 +973,7 @@ public class HistoryFileManager extends AbstractService {
if (LOG.isDebugEnabled()) {
LOG.debug("Scheduling move to done of " +found);
}
+
moveToDoneExecutor.execute(new Runnable() {
@Override
public void run() {
@@ -1193,5 +1211,5 @@ public class HistoryFileManager extends AbstractService {
@VisibleForTesting
void setMaxHistoryAge(long newValue){
maxHistoryAge=newValue;
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cff9edd4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
index 936c772..9f36477 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
@@ -446,6 +446,32 @@ public class TestJobHistory {
}
@Test
+ public void testCachedStorageWaitsForFileMove() throws IOException {
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+
+ Configuration conf = new Configuration();
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ Job job = mock(Job.class);
+ JobId jobId = mock(JobId.class);
+ when(job.getID()).thenReturn(jobId);
+ when(job.getTotalMaps()).thenReturn(10);
+ when(job.getTotalReduces()).thenReturn(2);
+ HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
+ when(historyManager.getFileInfo(eq(jobId))).thenReturn(fileInfo);
+ when(fileInfo.loadJob()).thenReturn(job);
+
+ storage.getFullJob(jobId);
+ verify(fileInfo).waitUntilMoved();
+ }
+
+ @Test
public void testRefreshLoadedJobCacheUnSupportedOperation() {
jobHistory = spy(new JobHistory());
HistoryStorage storage = new HistoryStorage() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-7777. Fix user name format in
YARN Registry DNS name. Contributed by Jian He
Posted by ha...@apache.org.
YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c559b27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c559b27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c559b27
Branch: refs/heads/YARN-1011
Commit: 0c559b27820d12ebe5c315c6e2d1685c6de6bd33
Parents: fa8cf4d
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Jan 24 13:50:47 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Wed Jan 24 13:50:47 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 11 +++-------
.../yarn/service/utils/ServiceApiUtil.java | 15 +++++++++----
.../client/binding/RegistryPathUtils.java | 20 ++++++++++++++++++
.../registry/client/binding/RegistryUtils.java | 5 ++++-
.../server/dns/BaseServiceRecordProcessor.java | 22 ++------------------
.../client/binding/TestRegistryPathUtils.java | 10 ++++++++-
.../markdown/yarn-service/Configurations.md | 2 +-
7 files changed, 50 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 6cf4e14..dfe9808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -392,14 +392,9 @@ public class ServiceScheduler extends CompositeService {
// ZK
globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
.getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
- String user = null;
- try {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- LOG.error("Failed to get user.", e);
- }
- globalTokens
- .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
+ String user = RegistryUtils.currentUser();
+ globalTokens.put(SERVICE_ZK_PATH,
+ ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
globalTokens.put(ServiceApiConstants.USER, user);
String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 7f85c95..a5716bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -61,6 +61,9 @@ public class ServiceApiUtil {
private static final PatternValidator namePattern
= new PatternValidator("[a-z][a-z0-9-]*");
+ private static final PatternValidator userNamePattern
+ = new PatternValidator("[a-z][a-z0-9-.]*");
+
@VisibleForTesting
public static void setJsonSerDeser(JsonSerDeser jsd) {
jsonSerDeser = jsd;
@@ -72,11 +75,15 @@ public class ServiceApiUtil {
IOException {
boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
RegistryConstants.DEFAULT_DNS_ENABLED);
- if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_USER_NAME_INVALID);
+ if (dnsEnabled) {
+ if (RegistryUtils.currentUser().length()
+ > RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_USER_NAME_INVALID);
+ }
+ userNamePattern.validate(RegistryUtils.currentUser());
}
+
if (StringUtils.isEmpty(service.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
index 5fa45f9..b8e9ba1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
@@ -24,11 +24,13 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor;
import org.apache.zookeeper.common.PathUtils;
import java.net.IDN;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -45,6 +47,9 @@ public class RegistryPathUtils {
private static final Pattern PATH_ENTRY_VALIDATION_PATTERN =
Pattern.compile(RegistryInternalConstants.VALID_PATH_ENTRY_PATTERN);
+ private static final Pattern USER_NAME =
+ Pattern.compile("/users/([a-z][a-z0-9-.]*)");
+
/**
* Validate ZK path with the path itself included in
* the exception text
@@ -215,4 +220,19 @@ public class RegistryPathUtils {
public static String encodeYarnID(String yarnId) {
return yarnId.replace("container", "ctr").replace("_", "-");
}
+
+ /**
+ * Return the username found in the ZK path.
+ *
+ * @param recPath the ZK recPath.
+ * @return the user name.
+ */
+ public static String getUsername(String recPath) {
+ String user = "anonymous";
+ Matcher matcher = USER_NAME.matcher(recPath);
+ if (matcher.find()) {
+ user = matcher.group(1);
+ }
+ return user;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
index 858b6b1..4ef7b8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
@@ -296,7 +296,10 @@ public class RegistryUtils {
*/
public static String currentUser() {
String shortUserName = currentUsernameUnencoded();
- return encodeForRegistry(shortUserName);
+ String encodedName = encodeForRegistry(shortUserName);
+ // DNS name doesn't allow "_", replace it with "-"
+ encodedName = RegistryUtils.convertUsername(encodedName);
+ return encodedName.replace("_", "-");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index fd5c74f..51ae99a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -36,8 +36,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
/**
* Provides common service record processing logic.
@@ -51,7 +49,6 @@ public abstract class BaseServiceRecordProcessor
private String path;
private String domain;
- private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
private static final String YARN_SERVICE_API_PREFIX =
"classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
@@ -76,21 +73,6 @@ public abstract class BaseServiceRecordProcessor
}
/**
- * Return the username found in the ZK path.
- *
- * @param recPath the ZK recPath.
- * @return the user name.
- */
- protected String getUsername(String recPath) {
- String user = "anonymous";
- Matcher matcher = USER_NAME.matcher(recPath);
- if (matcher.find()) {
- user = matcher.group(1);
- }
- return user;
- }
-
- /**
* Return the IPv6 mapped address for the provided IPv4 address. Utilized
* to create corresponding AAAA records.
*
@@ -300,7 +282,7 @@ public abstract class BaseServiceRecordProcessor
String service = RegistryPathUtils.lastPathEntry(
RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
String description = getRecord().description.toLowerCase();
- String user = getUsername(getPath());
+ String user = RegistryPathUtils.getUsername(getPath());
return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
description,
service,
@@ -352,7 +334,7 @@ public abstract class BaseServiceRecordProcessor
* @throws TextParseException
*/
protected Name getServiceName() throws TextParseException {
- String user = getUsername(getPath());
+ String user = RegistryPathUtils.getUsername(getPath());
String service =
String.format("%s.%s.%s",
RegistryPathUtils.lastPathEntry(getPath()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
index 9a24f1c..4346c9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
@@ -80,7 +80,15 @@ public class TestRegistryPathUtils extends Assert {
assertCreatedPathEquals("/alice", "/alice", "/");
}
-
+ @Test
+ public void testGetUserFromPath() throws Exception {
+ assertEquals("bob", RegistryPathUtils
+ .getUsername("/registry/users/bob/services/yarn-service/test1/"));
+ assertEquals("bob-dev", RegistryPathUtils
+ .getUsername("/registry/users/bob-dev/services/yarn-service/test1"));
+ assertEquals("bob.dev", RegistryPathUtils
+ .getUsername("/registry/users/bob.dev/services/yarn-service/test1"));
+ }
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index a6fd998..c2e6d26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -155,7 +155,7 @@ where `regionserver-0` is the actual component instance name assigned by the sys
| Name | Description |
| ------------ | ------------- |
| SERVICE_NAME | name of the service defined by the user
-| USER | user who submits the service |
+| USER | user who submits the service. Note that user name which has "\_" will be converted to use "-", to conform with DNS hostname RFC format which doesn't allow "\_", and all characters will be lowercased E.g. "Bob_dev" will be converted to "bob-dev" |
| DOMAIN | the domain name for the cluster |
| COMPONENT_NAME | the name for a given component |
| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-7742. [UI2] Duplicated
containers are rendered per attempt. (Vasudevan Skm / Sunil G via wangda)
Posted by ha...@apache.org.
YARN-7742. [UI2] Duplicated containers are rendered per attempt. (Vasudevan Skm / Sunil G via wangda)
Change-Id: Ib2bb3c64d7f09a7c3488f498d0079bcd2e6ac023
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b5b045b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b5b045b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b5b045b
Branch: refs/heads/YARN-1011
Commit: 8b5b045bd2b5aaf567c289fd34bab4bda216c835
Parents: 75618c0
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Jan 26 15:46:06 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Jan 26 15:46:06 2018 +0800
----------------------------------------------------------------------
.../src/main/webapp/app/components/timeline-view.js | 5 ++++-
.../app/serializers/yarn-timeline-container.js | 16 +++++++---------
.../src/main/webapp/app/utils/date-utils.js | 2 +-
3 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b5b045b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
index 420d793..e23d991 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
@@ -30,6 +30,7 @@ export default Ember.Component.extend({
clusterMetrics: undefined,
modelArr: [],
+ containerIdArr: [],
colors: d3.scale.category10().range(),
_selected: undefined,
gridColumns: [],
@@ -232,19 +233,21 @@ export default Ember.Component.extend({
// init tooltip
this.initTooltip();
this.modelArr = [];
+ this.containerIdArr = [];
// init model
if (this.get("rmModel")) {
this.get("rmModel").forEach(function(o) {
if(!this.modelArr.contains(o)) {
this.modelArr.push(o);
+ this.containerIdArr.push(o.id);
}
}.bind(this));
}
if (this.get("tsModel")) {
this.get("tsModel").forEach(function(o) {
- if(!this.modelArr.contains(o)) {
+ if(!this.containerIdArr.contains(o.id)) {
this.modelArr.push(o);
}
}.bind(this));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b5b045b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
index f2c60f6..471f910 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
@@ -23,12 +23,10 @@ import Converter from 'yarn-ui/utils/converter';
export default DS.JSONAPISerializer.extend({
internalNormalizeSingleResponse(store, primaryModelClass, payload) {
var payloadEvents = payload.events,
- createdEvent = payloadEvents.filterBy('id', 'YARN_RM_CONTAINER_CREATED')[0],
+ createdEvent = payloadEvents.filterBy('id', 'YARN_CONTAINER_CREATED')[0],
startedTime = createdEvent? createdEvent.timestamp : Date.now(),
- finishedEvent = payloadEvents.filterBy('id', 'YARN_RM_CONTAINER_FINISHED')[0],
- finishedTime = finishedEvent? finishedEvent.timestamp : Date.now(),
- containerExitStatus = finishedEvent? finishedEvent.info.YARN_CONTAINER_EXIT_STATUS : '',
- containerState = finishedEvent? finishedEvent.info.YARN_CONTAINER_STATE : '';
+ finishedEvent = payloadEvents.filterBy('id', 'YARN_CONTAINER_FINISHED')[0],
+ finishedTime = finishedEvent? finishedEvent.timestamp : Date.now()
var fixedPayload = {
id: payload.id,
@@ -38,11 +36,11 @@ export default DS.JSONAPISerializer.extend({
allocatedVCores: payload.info.YARN_CONTAINER_ALLOCATED_VCORE,
assignedNodeId: payload.info.YARN_CONTAINER_ALLOCATED_HOST,
priority: payload.info.YARN_CONTAINER_ALLOCATED_PRIORITY,
- startedTime: Converter.timeStampToDate(startedTime),
- finishedTime: Converter.timeStampToDate(finishedTime),
+ startedTime: Converter.timeStampToDate(payload.createdtime),
+ finishedTime: Converter.timeStampToDate(payload.info.YARN_CONTAINER_FINISHED_TIME),
nodeHttpAddress: payload.info.YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS,
- containerExitStatus: containerExitStatus,
- containerState: containerState
+ containerExitStatus: payload.info.YARN_CONTAINER_EXIT_STATUS,
+ containerState: payload.info.YARN_CONTAINER_STATE
}
};
return fixedPayload;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b5b045b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/date-utils.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/date-utils.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/date-utils.js
index 83dd200..6a9780c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/date-utils.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/date-utils.js
@@ -24,4 +24,4 @@ const getDefaultTimezone = () => {
};
export const convertTimestampWithTz = (timestamp, format = "YYYY/MM/DD") =>
- moment.tz(timestamp, getDefaultTimezone()).format(format);
+ moment.tz(parseInt(timestamp), getDefaultTimezone()).format(format);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-4511. Common scheduler
changes to support scheduler-specific oversubscription implementations.
Posted by ha...@apache.org.
YARN-4511. Common scheduler changes to support scheduler-specific oversubscription implementations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eba84365
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eba84365
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eba84365
Branch: refs/heads/YARN-1011
Commit: eba84365dbdf9724e5b4d058ee3a56c370aaf3f5
Parents: e7158d4
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Nov 2 09:12:19 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 17:43:50 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 6 +
.../resourcemanager/ResourceTrackerService.java | 3 +-
.../monitor/capacity/TempSchedulerNode.java | 2 +-
.../server/resourcemanager/rmnode/RMNode.java | 7 +
.../resourcemanager/rmnode/RMNodeImpl.java | 13 +-
.../scheduler/AbstractYarnScheduler.java | 4 +-
.../scheduler/ClusterNodeTracker.java | 6 +-
.../scheduler/SchedulerNode.java | 317 +++++++++++----
.../scheduler/SchedulerNodeReport.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../allocator/RegularContainerAllocator.java | 4 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../scheduler/fair/FSPreemptionThread.java | 2 +-
.../scheduler/fair/FSSchedulerNode.java | 9 +-
.../yarn/server/resourcemanager/MockNodes.java | 6 +
.../TestWorkPreservingRMRestart.java | 39 +-
...alCapacityPreemptionPolicyMockFramework.java | 2 +-
...alCapacityPreemptionPolicyMockFramework.java | 6 +-
.../scheduler/TestAbstractYarnScheduler.java | 4 +-
.../scheduler/TestSchedulerNode.java | 393 +++++++++++++++++++
.../capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerAsyncScheduling.java | 8 +-
.../scheduler/capacity/TestLeafQueue.java | 4 +-
.../TestNodeLabelContainerAllocation.java | 14 +-
.../fair/TestContinuousScheduling.java | 42 +-
.../scheduler/fair/TestFSSchedulerNode.java | 18 +-
.../scheduler/fair/TestFairScheduler.java | 14 +-
.../scheduler/fifo/TestFifoScheduler.java | 4 +-
30 files changed, 787 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 1016ce1..e334893 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -199,6 +200,11 @@ public class NodeInfo {
}
@Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
+ @Override
public long getUntrackedTimeStamp() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index fdad826..c6dc204 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -189,6 +190,11 @@ public class RMNodeWrapper implements RMNode {
}
@Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return node.getOverAllocationInfo();
+ }
+
+ @Override
public long getUntrackedTimeStamp() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 9d95f63..4aa099b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -395,7 +395,8 @@ public class ResourceTrackerService extends AbstractService implements
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
- resolve(host), capability, nodeManagerVersion, physicalResource);
+ resolve(host), capability, nodeManagerVersion, physicalResource,
+ request.getOverAllocationInfo());
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
index 320f262..25777af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
@@ -51,7 +51,7 @@ public class TempSchedulerNode {
public static TempSchedulerNode fromSchedulerNode(
FiCaSchedulerNode schedulerNode) {
TempSchedulerNode n = new TempSchedulerNode();
- n.totalResource = Resources.clone(schedulerNode.getTotalResource());
+ n.totalResource = Resources.clone(schedulerNode.getCapacity());
n.allocatedResource = Resources.clone(schedulerNode.getAllocatedResource());
n.runningContainers = schedulerNode.getCopiedListOfRunningContainers();
n.reservedContainer = schedulerNode.getReservedContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index a5615ef..5769409 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
/**
* Node managers information on available resources
@@ -114,6 +115,12 @@ public interface RMNode {
public ResourceUtilization getNodeUtilization();
/**
+ * Get the node overallocation threshold.
+ * @return the overallocation threshold
+ */
+ OverAllocationInfo getOverAllocationInfo();
+
+ /**
* the physical resources in the node.
* @return the physical resources in the node.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index da54eb9..3cac38f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
@@ -109,6 +110,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final WriteLock writeLock;
private final ConcurrentLinkedQueue<UpdatedContainerInfo> nodeUpdateQueue;
+ private final OverAllocationInfo overallocationInfo;
private volatile boolean nextHeartBeat = true;
private final NodeId nodeId;
@@ -364,12 +366,13 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
int cmPort, int httpPort, Node node, Resource capability,
String nodeManagerVersion) {
this(nodeId, context, hostName, cmPort, httpPort, node, capability,
- nodeManagerVersion, null);
+ nodeManagerVersion, null, null);
}
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
int cmPort, int httpPort, Node node, Resource capability,
- String nodeManagerVersion, Resource physResource) {
+ String nodeManagerVersion, Resource physResource,
+ OverAllocationInfo overAllocationInfo) {
this.nodeId = nodeId;
this.context = context;
this.hostName = hostName;
@@ -384,6 +387,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.nodeManagerVersion = nodeManagerVersion;
this.timeStamp = 0;
this.physicalResource = physResource;
+ this.overallocationInfo = overAllocationInfo;
this.latestNodeHeartBeatResponse.setResponseId(0);
@@ -533,6 +537,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
}
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return this.overallocationInfo;
+ }
+
public void setNodeUtilization(ResourceUtilization nodeUtilization) {
this.writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index c94c379..ca66055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -352,7 +352,7 @@ public abstract class AbstractYarnScheduler
}
application.containerLaunchedOnNode(containerId, node.getNodeID());
- node.containerStarted(containerId);
+ node.containerLaunched(containerId);
} finally {
readLock.unlock();
}
@@ -817,7 +817,7 @@ public abstract class AbstractYarnScheduler
writeLock.lock();
SchedulerNode node = getSchedulerNode(nm.getNodeID());
Resource newResource = resourceOption.getResource();
- Resource oldResource = node.getTotalResource();
+ Resource oldResource = node.getCapacity();
if (!oldResource.equals(newResource)) {
// Notify NodeLabelsManager about this change
rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 66d8810..3030373 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -90,7 +90,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
nodesList.add(node);
// Update cluster capacity
- Resources.addTo(clusterCapacity, node.getTotalResource());
+ Resources.addTo(clusterCapacity, node.getCapacity());
staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
@@ -175,7 +175,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
// Update cluster capacity
- Resources.subtractFrom(clusterCapacity, node.getTotalResource());
+ Resources.subtractFrom(clusterCapacity, node.getCapacity());
staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
@@ -237,7 +237,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
private void updateMaxResources(SchedulerNode node, boolean add) {
- Resource totalResource = node.getTotalResource();
+ Resource totalResource = node.getCapacity();
ResourceInformation[] totalResources;
if (totalResource != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 05dbf1e..349944e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -57,30 +58,39 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
+ private Resource capacity;
private Resource unallocatedResource = Resource.newInstance(0, 0);
- private Resource allocatedResource = Resource.newInstance(0, 0);
- private Resource totalResource;
+
private RMContainer reservedContainer;
- private volatile int numContainers;
private volatile ResourceUtilization containersUtilization =
ResourceUtilization.newInstance(0, 0, 0f);
private volatile ResourceUtilization nodeUtilization =
ResourceUtilization.newInstance(0, 0, 0f);
- /* set of containers that are allocated containers */
- private final Map<ContainerId, ContainerInfo> launchedContainers =
- new HashMap<>();
+ private final Map<ContainerId, ContainerInfo>
+ allocatedContainers = new HashMap<>();
+
+ private volatile int numGuaranteedContainers = 0;
+ private Resource allocatedResourceGuaranteed = Resource.newInstance(0, 0);
+
+ private volatile int numOpportunisticContainers = 0;
+ private Resource allocatedResourceOpportunistic = Resource.newInstance(0, 0);
private final RMNode rmNode;
private final String nodeName;
+ // The total amount of resources requested by containers that have been
+ // allocated but not yet launched on the node.
+ protected Resource resourceAllocatedPendingLaunch =
+ Resource.newInstance(0, 0);
+
private volatile Set<String> labels = null;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
Set<String> labels) {
this.rmNode = node;
this.unallocatedResource = Resources.clone(node.getTotalCapability());
- this.totalResource = Resources.clone(node.getTotalCapability());
+ this.capacity = Resources.clone(node.getTotalCapability());
if (usePortForNodeName) {
nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
} else {
@@ -102,9 +112,9 @@ public abstract class SchedulerNode {
* @param resource Total resources on the node.
*/
public synchronized void updateTotalResource(Resource resource){
- this.totalResource = resource;
- this.unallocatedResource = Resources.subtract(totalResource,
- this.allocatedResource);
+ this.capacity = Resources.clone(resource);
+ this.unallocatedResource = Resources.subtract(capacity,
+ this.allocatedResourceGuaranteed);
}
/**
@@ -163,17 +173,83 @@ public abstract class SchedulerNode {
protected synchronized void allocateContainer(RMContainer rmContainer,
boolean launchedOnNode) {
Container container = rmContainer.getContainer();
- if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
- deductUnallocatedResource(container.getResource());
- ++numContainers;
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ guaranteedContainerResourceAllocated(rmContainer, launchedOnNode);
+ } else {
+ opportunisticContainerResourceAllocated(rmContainer, launchedOnNode);
}
- launchedContainers.put(container.getId(),
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned container " + container.getId() + " of capacity "
+ + container.getResource() + " and type " +
+ container.getExecutionType() + " on host " + toString());
+ }
+ }
+
+ /**
+ * Handle an allocation of a GUARANTEED container.
+ * @param rmContainer the allocated GUARANTEED container
+ * @param launchedOnNode true if the container has been launched
+ */
+ private void guaranteedContainerResourceAllocated(
+ RMContainer rmContainer, boolean launchedOnNode) {
+ Container container = rmContainer.getContainer();
+
+ if (container.getExecutionType() != ExecutionType.GUARANTEED) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ allocatedContainers.put(container.getId(),
new ContainerInfo(rmContainer, launchedOnNode));
+
+ Resource resource = container.getResource();
+ if (containerResourceAllocated(resource, allocatedResourceGuaranteed)) {
+ Resources.subtractFrom(unallocatedResource, resource);
+ }
+
+ numGuaranteedContainers++;
}
/**
- * Get unallocated resources on the node.
+ * Handle an allocation of a OPPORTUNISTIC container.
+ * @param rmContainer the allocated OPPORTUNISTIC container
+ * @param launchedOnNode true if the container has been launched
+ */
+ private void opportunisticContainerResourceAllocated(
+ RMContainer rmContainer, boolean launchedOnNode) {
+ Container container = rmContainer.getContainer();
+
+ if (container.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ allocatedContainers.put(rmContainer.getContainerId(),
+ new ContainerInfo(rmContainer, launchedOnNode));
+ if (containerResourceAllocated(
+ container.getResource(), allocatedResourceOpportunistic)) {
+ // nothing to do here
+ }
+ numOpportunisticContainers++;
+ }
+
+ private boolean containerResourceAllocated(Resource allocated,
+ Resource aggregatedResources) {
+ if (allocated == null) {
+ LOG.error("Invalid deduction of null resource for "
+ + rmNode.getNodeAddress());
+ return false;
+ }
+ Resources.addTo(resourceAllocatedPendingLaunch, allocated);
+ Resources.addTo(aggregatedResources, allocated);
+ return true;
+ }
+
+
+ /**
+ * Get resources that are not allocated to GUARANTEED containers on the node.
* @return Unallocated resources on the node
*/
public synchronized Resource getUnallocatedResource() {
@@ -181,42 +257,57 @@ public abstract class SchedulerNode {
}
/**
- * Get allocated resources on the node.
- * @return Allocated resources on the node
+ * Get resources allocated to GUARANTEED containers on the node.
+ * @return Allocated resources to GUARANTEED containers on the node
*/
public synchronized Resource getAllocatedResource() {
- return this.allocatedResource;
+ return this.allocatedResourceGuaranteed;
+ }
+
+ /**
+ * Get resources allocated to OPPORTUNISTIC containers on the node.
+ * @return Allocated resources to OPPORTUNISTIC containers on the node
+ */
+ public synchronized Resource getOpportunisticResourceAllocated() {
+ return this.allocatedResourceOpportunistic;
+ }
+
+ @VisibleForTesting
+ public synchronized Resource getResourceAllocatedPendingLaunch() {
+ return this.resourceAllocatedPendingLaunch;
}
/**
* Get total resources on the node.
* @return Total resources on the node.
*/
- public synchronized Resource getTotalResource() {
- return this.totalResource;
+ public synchronized Resource getCapacity() {
+ return this.capacity;
}
/**
- * Check if a container is launched by this node.
+ * Check if a GUARANTEED container is launched by this node.
* @return If the container is launched by the node.
*/
- public synchronized boolean isValidContainer(ContainerId containerId) {
- if (launchedContainers.containsKey(containerId)) {
- return true;
- }
- return false;
+ @VisibleForTesting
+ public synchronized boolean isValidGuaranteedContainer(
+ ContainerId containerId) {
+ ContainerInfo containerInfo = allocatedContainers.get(containerId);
+ return containerInfo != null && ExecutionType.GUARANTEED ==
+ containerInfo.container.getExecutionType();
}
/**
- * Update the resources of the node when releasing a container.
- * @param container Container to release.
+ * Check if an OPPORTUNISTIC container is launched by this node.
+ * @param containerId id of the container to check
+ * @return If the container is launched by the node.
*/
- protected synchronized void updateResourceForReleasedContainer(
- Container container) {
- if (container.getExecutionType() == ExecutionType.GUARANTEED) {
- addUnallocatedResource(container.getResource());
- --numContainers;
- }
+ @VisibleForTesting
+ public synchronized boolean isValidOpportunisticContainer(
+ ContainerId containerId) {
+ ContainerInfo containerInfo = allocatedContainers.get(containerId);
+ return containerInfo != null && ExecutionType.OPPORTUNISTIC ==
+ containerInfo.container.getExecutionType();
}
/**
@@ -226,25 +317,34 @@ public abstract class SchedulerNode {
*/
public synchronized void releaseContainer(ContainerId containerId,
boolean releasedByNode) {
- ContainerInfo info = launchedContainers.get(containerId);
- if (info == null) {
+ RMContainer rmContainer = getContainer(containerId);
+ if (rmContainer == null) {
+ LOG.warn("Invalid container " + containerId + " is released.");
return;
}
- if (!releasedByNode && info.launchedOnNode) {
- // wait until node reports container has completed
+
+ if (!allocatedContainers.containsKey(containerId)) {
+ // do not process if the container is never allocated on the node
+ return;
+ }
+ if (!releasedByNode &&
+ allocatedContainers.get(containerId).launchedOnNode) {
+ // only process if the container has not been launched on a node
+ // yet or it is released by node.
return;
}
- launchedContainers.remove(containerId);
- Container container = info.container.getContainer();
- updateResourceForReleasedContainer(container);
+ Container container = rmContainer.getContainer();
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ guaranteedContainerReleased(container);
+ } else {
+ opportunisticContainerReleased(container);
+ }
if (LOG.isDebugEnabled()) {
- LOG.debug("Released container " + container.getId() + " of capacity "
- + container.getResource() + " on host " + rmNode.getNodeAddress()
- + ", which currently has " + numContainers + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available" + ", release resources=" + true);
+ LOG.debug("Released " + container.getExecutionType() + " container " +
+ containerId + " of " + "capacity " + container.getResource() +
+ " on node (" + toString() + ")" + ", release resources=" + true);
}
}
@@ -252,42 +352,75 @@ public abstract class SchedulerNode {
* Inform the node that a container has launched.
* @param containerId ID of the launched container
*/
- public synchronized void containerStarted(ContainerId containerId) {
- ContainerInfo info = launchedContainers.get(containerId);
- if (info != null) {
+ public synchronized void containerLaunched(ContainerId containerId) {
+ ContainerInfo info = allocatedContainers.get(containerId);
+ if (info != null && !info.launchedOnNode) {
info.launchedOnNode = true;
+ Resources.subtractFrom(resourceAllocatedPendingLaunch,
+ info.container.getContainer().getResource());
}
}
/**
- * Add unallocated resources to the node. This is used when unallocating a
- * container.
- * @param resource Resources to add.
+ * Handle the release of a GUARANTEED container.
+ * @param container Container to release.
*/
- private synchronized void addUnallocatedResource(Resource resource) {
- if (resource == null) {
- LOG.error("Invalid resource addition of null resource for "
- + rmNode.getNodeAddress());
- return;
+ protected synchronized void guaranteedContainerReleased(
+ Container container) {
+ if (container.getExecutionType() != ExecutionType.GUARANTEED) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ if (containerResourceReleased(container, allocatedResourceGuaranteed)) {
+ Resources.addTo(unallocatedResource, container.getResource());
}
- Resources.addTo(unallocatedResource, resource);
- Resources.subtractFrom(allocatedResource, resource);
+ // do not update allocated containers until the resources of
+ // the container are released because we need to check if we
+ // need to update resourceAllocatedPendingLaunch in case the
+ // container has not been launched on the node.
+ allocatedContainers.remove(container.getId());
+ numGuaranteedContainers--;
}
/**
- * Deduct unallocated resources from the node. This is used when allocating a
- * container.
- * @param resource Resources to deduct.
+ * Handle the release of an OPPORTUNISTIC container.
+ * @param container Container to release.
*/
- @VisibleForTesting
- public synchronized void deductUnallocatedResource(Resource resource) {
- if (resource == null) {
- LOG.error("Invalid deduction of null resource for "
+ private void opportunisticContainerReleased(
+ Container container) {
+ if (container.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ if (containerResourceReleased(container, allocatedResourceOpportunistic)) {
+ // nothing to do here
+ }
+ // do not update allocated containers until the resources of
+ // the container are released because we need to check if we
+ // need to update resourceAllocatedPendingLaunch in case the
+ // container has not been launched on the node.
+ allocatedContainers.remove(container.getId());
+ numOpportunisticContainers--;
+ }
+
+ private boolean containerResourceReleased(Container container,
+ Resource aggregatedResource) {
+ Resource released = container.getResource();
+ if (released == null) {
+ LOG.error("Invalid resource addition of null resource for "
+ rmNode.getNodeAddress());
- return;
+ return false;
}
- Resources.subtractFrom(unallocatedResource, resource);
- Resources.addTo(allocatedResource, resource);
+ Resources.subtractFrom(aggregatedResource, released);
+
+ if (!allocatedContainers.get(container.getId()).launchedOnNode) {
+ // update resourceAllocatedPendingLaunch if the container is has
+ // not yet been launched on the node
+ Resources.subtractFrom(resourceAllocatedPendingLaunch, released);
+ }
+ return true;
}
/**
@@ -307,17 +440,28 @@ public abstract class SchedulerNode {
@Override
public String toString() {
- return "host: " + rmNode.getNodeAddress() + " #containers="
- + getNumContainers() + " available=" + getUnallocatedResource()
- + " used=" + getAllocatedResource();
+ return "host: " + rmNode.getNodeAddress() + " #guaranteed containers=" +
+ getNumGuaranteedContainers() + " #opportunistic containers=" +
+ getNumOpportunisticContainers() + " available=" +
+ getUnallocatedResource() + " used by guaranteed containers=" +
+ allocatedResourceGuaranteed + " used by opportunistic containers=" +
+ allocatedResourceOpportunistic;
+ }
+
+ /**
+ * Get number of active GUARANTEED containers on the node.
+ * @return Number of active OPPORTUNISTIC containers on the node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numGuaranteedContainers;
}
/**
- * Get number of active containers on the node.
- * @return Number of active containers on the node.
+ * Get number of active OPPORTUNISTIC containers on the node.
+ * @return Number of active OPPORTUNISTIC containers on the node.
*/
- public int getNumContainers() {
- return numContainers;
+ public int getNumOpportunisticContainers() {
+ return numOpportunisticContainers;
}
/**
@@ -325,8 +469,8 @@ public abstract class SchedulerNode {
* @return A copy of containers running on the node.
*/
public synchronized List<RMContainer> getCopiedListOfRunningContainers() {
- List<RMContainer> result = new ArrayList<>(launchedContainers.size());
- for (ContainerInfo info : launchedContainers.values()) {
+ List<RMContainer> result = new ArrayList<>(allocatedContainers.size());
+ for (ContainerInfo info : allocatedContainers.values()) {
result.add(info.container);
}
return result;
@@ -336,12 +480,14 @@ public abstract class SchedulerNode {
* Get the containers running on the node with AM containers at the end.
* @return A copy of running containers with AM containers at the end.
*/
- public synchronized List<RMContainer> getRunningContainersWithAMsAtTheEnd() {
+ public synchronized List<RMContainer>
+ getRunningGuaranteedContainersWithAMsAtTheEnd() {
LinkedList<RMContainer> result = new LinkedList<>();
- for (ContainerInfo info : launchedContainers.values()) {
+ for (ContainerInfo info : allocatedContainers.values()) {
if(info.container.isAMContainer()) {
result.addLast(info.container);
- } else {
+ } else if (info.container.getExecutionType() ==
+ ExecutionType.GUARANTEED){
result.addFirst(info.container);
}
}
@@ -354,12 +500,9 @@ public abstract class SchedulerNode {
* @return The container for the specified container ID
*/
protected synchronized RMContainer getContainer(ContainerId containerId) {
- RMContainer container = null;
- ContainerInfo info = launchedContainers.get(containerId);
- if (info != null) {
- container = info.container;
- }
- return container;
+ ContainerInfo info = allocatedContainers.get(containerId);
+
+ return info != null ? info.container : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index fa71a25..ea30d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -31,11 +31,11 @@ public class SchedulerNodeReport {
private final Resource used;
private final Resource avail;
private final int num;
-
+
public SchedulerNodeReport(SchedulerNode node) {
this.used = node.getAllocatedResource();
this.avail = node.getUnallocatedResource();
- this.num = node.getNumContainers();
+ this.num = node.getNumGuaranteedContainers();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 99f4456..659290f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1719,7 +1719,7 @@ public class CapacityScheduler extends
// update this node to node label manager
if (labelManager != null) {
labelManager.activateNode(nodeManager.getNodeID(),
- schedulerNode.getTotalResource());
+ schedulerNode.getCapacity());
}
Resource clusterResource = getClusterResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 2642532..c69ea48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -502,13 +502,13 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
Resource capability = pendingAsk.getPerAllocationResource();
Resource available = node.getUnallocatedResource();
- Resource totalResource = node.getTotalResource();
+ Resource totalResource = node.getCapacity();
if (!Resources.lessThanOrEqual(rc, clusterResource,
capability, totalResource)) {
LOG.warn("Node : " + node.getNodeID()
+ " does not have sufficient resource for ask : " + pendingAsk
- + " node total capability : " + node.getTotalResource());
+ + " node total capability : " + node.getCapacity());
// Skip this locality request
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index d6ad292..cf651bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -978,7 +978,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
diagnosticMessageBldr.append(" ( Partition : ");
diagnosticMessageBldr.append(node.getLabels());
diagnosticMessageBldr.append(", Total resource : ");
- diagnosticMessageBldr.append(node.getTotalResource());
+ diagnosticMessageBldr.append(node.getCapacity());
diagnosticMessageBldr.append(", Available resource : ");
diagnosticMessageBldr.append(node.getUnallocatedResource());
diagnosticMessageBldr.append(" ).");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 7277779..52165a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -144,9 +144,9 @@ public class FiCaSchedulerNode extends SchedulerNode {
}
@Override
- protected synchronized void updateResourceForReleasedContainer(
+ protected synchronized void guaranteedContainerReleased(
Container container) {
- super.updateResourceForReleasedContainer(container);
+ super.guaranteedContainerReleased(container);
if (killableContainers.containsKey(container.getId())) {
Resources.subtractFrom(totalKillableResources, container.getResource());
killableContainers.remove(container.getId());
@@ -168,9 +168,10 @@ public class FiCaSchedulerNode extends SchedulerNode {
final Container container = rmContainer.getContainer();
LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + getRMNode().getNodeAddress()
- + ", which has " + getNumContainers() + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available after allocation");
+ + ", which has " + getNumGuaranteedContainers() + " guaranteed"
+ + " containers using " + getAllocatedResource() + ", "
+ + getNumOpportunisticContainers() + " opportunistic containers"
+ + " using " + getOpportunisticResourceAllocated());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index c05bff9..dd1653d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -173,7 +173,7 @@ class FSPreemptionThread extends Thread {
// Figure out list of containers to consider
List<RMContainer> containersToCheck =
- node.getRunningContainersWithAMsAtTheEnd();
+ node.getRunningGuaranteedContainersWithAMsAtTheEnd();
containersToCheck.removeAll(node.getContainersForPreemption());
// Initialize potential with unallocated but not reserved resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 44ec9c3..95490f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -242,11 +242,12 @@ public class FSSchedulerNode extends SchedulerNode {
super.allocateContainer(rmContainer, launchedOnNode);
if (LOG.isDebugEnabled()) {
final Container container = rmContainer.getContainer();
- LOG.debug("Assigned container " + container.getId() + " of capacity "
+ LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + getRMNode().getNodeAddress()
- + ", which has " + getNumContainers() + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available after allocation");
+ + ", which has " + getNumGuaranteedContainers() + " guaranteed "
+ + "containers using " + getAllocatedResource() + ", "
+ + getNumOpportunisticContainers() + " opportunistic containers "
+ + "using " + getOpportunisticResourceAllocated());
}
Resource allocated = rmContainer.getAllocatedResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index d6549b9..e81192e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -261,6 +262,11 @@ public class MockNodes {
return this.nodeUtilization;
}
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
return OpportunisticContainersStatus.newInstance();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index e4c83e3..5b49303 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -226,13 +226,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource =
Resource.newInstance(nm1.getMemory(), nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(schedulerNode1.isValidContainer(runningContainer
- .getContainerId()));
- assertFalse(schedulerNode1.isValidContainer(completedContainer
- .getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
@@ -383,13 +384,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource =
Resource.newInstance(nm1.getMemory(), nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(
- schedulerNode1.isValidContainer(runningContainer.getContainerId()));
- assertFalse(
- schedulerNode1.isValidContainer(completedContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
@@ -1650,13 +1652,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource = Resource.newInstance(nm1.getMemory(),
nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(
- schedulerNode1.isValidContainer(runningContainer.getContainerId()));
- assertFalse(
- schedulerNode1.isValidContainer(completedContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index ca43a95..631d707 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -480,7 +480,7 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
totalRes = parseResourceFromString(resSring);
}
}
- when(sn.getTotalResource()).thenReturn(totalRes);
+ when(sn.getCapacity()).thenReturn(totalRes);
when(sn.getUnallocatedResource()).thenReturn(Resources.clone(totalRes));
// TODO, add settings of killable resources when necessary
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
index 964a230..89fb846 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
@@ -237,17 +237,17 @@ public class TestProportionalCapacityPreemptionPolicyMockFramework
// Check host resources
Assert.assertEquals(3, this.cs.getAllNodes().size());
SchedulerNode node1 = cs.getSchedulerNode(NodeId.newInstance("n1", 1));
- Assert.assertEquals(100, node1.getTotalResource().getMemorySize());
+ Assert.assertEquals(100, node1.getCapacity().getMemorySize());
Assert.assertEquals(100, node1.getCopiedListOfRunningContainers().size());
Assert.assertNull(node1.getReservedContainer());
SchedulerNode node2 = cs.getSchedulerNode(NodeId.newInstance("n2", 1));
- Assert.assertEquals(0, node2.getTotalResource().getMemorySize());
+ Assert.assertEquals(0, node2.getCapacity().getMemorySize());
Assert.assertEquals(50, node2.getCopiedListOfRunningContainers().size());
Assert.assertNotNull(node2.getReservedContainer());
SchedulerNode node3 = cs.getSchedulerNode(NodeId.newInstance("n3", 1));
- Assert.assertEquals(30, node3.getTotalResource().getMemorySize());
+ Assert.assertEquals(30, node3.getCapacity().getMemorySize());
Assert.assertEquals(100, node3.getCopiedListOfRunningContainers().size());
Assert.assertNull(node3.getReservedContainer());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 979e68a..ef5a0c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -289,12 +289,12 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
SchedulerNode mockNode1 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
when(mockNode1.getUnallocatedResource()).thenReturn(emptyResource);
- when(mockNode1.getTotalResource()).thenReturn(fullResource1);
+ when(mockNode1.getCapacity()).thenReturn(fullResource1);
SchedulerNode mockNode2 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
when(mockNode2.getUnallocatedResource()).thenReturn(emptyResource);
- when(mockNode2.getTotalResource()).thenReturn(fullResource2);
+ when(mockNode2.getCapacity()).thenReturn(fullResource2);
verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
new file mode 100644
index 0000000..b04b277
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for SchedulerNode.
+ */
+public class TestSchedulerNode {
+ private final Resource nodeCapacity = Resource.newInstance(1024*10, 4);
+
+ @Test
+ public void testAllocateAndReleaseGuaranteedContainer() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ }
+
+ @Test
+ public void testAllocateAndReleaseOpportunisticContainer() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.OPPORTUNISTIC, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ nodeCapacity, schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidOpportunisticContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidOpportunisticContainer(containerId));
+ }
+
+ @Test
+ public void testAllocateAndReleaseContainers() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+
+ Resource guaranteedResource = Resource.newInstance(4096, 1);
+ RMContainer guaranteedContainer =
+ createRMContainer(0, guaranteedResource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId guaranteedContainerId = guaranteedContainer.getContainerId();
+
+ // allocate a guaranteed container on the node
+ schedulerNode.allocateContainer(guaranteedContainer);
+
+ Assert.assertEquals("The guaranteed container should have been allocated",
+ guaranteedResource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, guaranteedResource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The guaranteed container should have been allocated" +
+ " but not launched", guaranteedResource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(guaranteedContainerId));
+
+ Resource opportunisticResource = Resource.newInstance(8192, 4);
+ RMContainer opportunisticContainer =
+ createRMContainer(1, opportunisticResource,
+ ExecutionType.OPPORTUNISTIC, schedulerNode.getRMNode());
+ ContainerId opportunisticContainerId =
+ opportunisticContainer.getContainerId();
+
+ // allocate an opportunistic container on the node
+ schedulerNode.allocateContainer(opportunisticContainer);
+
+ Assert.assertEquals("The opportunistic container should have been" +
+ " allocated", opportunisticResource,
+ schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, guaranteedResource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The opportunistic container should also have been" +
+ " allocated but not launched",
+ Resources.add(guaranteedResource, opportunisticResource),
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidOpportunisticContainer(opportunisticContainerId));
+
+ // launch both containers on the node
+ schedulerNode.containerLaunched(guaranteedContainerId);
+ schedulerNode.containerLaunched(opportunisticContainerId);
+
+ Assert.assertEquals("Both containers should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release both containers
+ schedulerNode.releaseContainer(guaranteedContainerId, true);
+ schedulerNode.releaseContainer(opportunisticContainerId, true);
+
+ Assert.assertEquals("The guaranteed container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The opportunistic container should have been released",
+ 0, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertEquals("The guaranteed container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertEquals("The opportunistic container should have been released",
+ Resources.none(), schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertFalse("The guaranteed container should have been released",
+ schedulerNode.isValidGuaranteedContainer(guaranteedContainerId));
+ Assert.assertFalse("The opportunistic container should have been released",
+ schedulerNode.isValidOpportunisticContainer(opportunisticContainerId));
+ }
+
+ @Test
+ public void testReleaseLaunchedContainerNotAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, false);
+ Assert.assertEquals("The container should not have been released",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should not have been released",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertTrue("The container should not have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ }
+
+ @Test
+ public void testReleaseUnlaunchedContainerAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // make sure the container is not launched yet
+ Assert.assertEquals("The container should not be launched already",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+ }
+
+ @Test
+ public void testReleaseUnlaunchedContainerNotAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // make sure the container is not launched yet
+ Assert.assertEquals("The container should not have been launched",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, false);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+ }
+
+ private SchedulerNode createSchedulerNode(Resource capacity) {
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+
+ RMNode rmNode = mock(RMNode.class);
+ when(rmNode.getNodeID()).thenReturn(nodeId);
+ when(rmNode.getHostName()).thenReturn(nodeId.getHost());
+ when(rmNode.getTotalCapability()).thenReturn(capacity);
+ when(rmNode.getRackName()).thenReturn("/default");
+ when(rmNode.getHttpAddress()).thenReturn(nodeId.getHost());
+ when(rmNode.getNodeAddress()).thenReturn(nodeId.getHost());
+
+ return new SchedulerNodeForTest(rmNode);
+ }
+
+ private static RMContainerImpl createRMContainer(long containerId,
+ Resource resource, ExecutionType executionType, RMNode node) {
+ Container container =
+ createContainer(containerId, resource, executionType, node);
+
+ Dispatcher dispatcher = new AsyncDispatcher();
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ when(rmContext.getSystemMetricsPublisher()).
+ thenReturn(new NoOpSystemMetricPublisher());
+ when(rmContext.getYarnConfiguration()).
+ thenReturn(new YarnConfiguration());
+ when(rmContext.getContainerAllocationExpirer()).
+ thenReturn(new ContainerAllocationExpirer(dispatcher));
+ when(rmContext.getRMApplicationHistoryWriter()).
+ thenReturn(new RMApplicationHistoryWriter());
+
+ return new RMContainerImpl(container, null,
+ container.getId().getApplicationAttemptId(),
+ node.getNodeID(), "test", rmContext);
+ }
+
+ private static Container createContainer(long containerId, Resource resource,
+ ExecutionType executionType, RMNode node) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+ newInstance(ApplicationId.newInstance(0, 0), 0);
+ ContainerId cId =
+ ContainerId.newContainerId(appAttemptId, containerId);
+ Container container = Container.newInstance(
+ cId, node.getNodeID(), node.getNodeAddress(), resource,
+ Priority.newInstance(0), null, executionType);
+ return container;
+ }
+
+
+ /**
+ * A test implementation of SchedulerNode for the purpose of testing
+ * SchedulerNode only. Resource reservation is scheduler-dependent,
+ * and therefore not covered here.
+ */
+ private static final class SchedulerNodeForTest extends SchedulerNode {
+ SchedulerNodeForTest(RMNode node) {
+ super(node, false);
+ }
+
+ @Override
+ public void reserveResource(SchedulerApplicationAttempt attempt,
+ SchedulerRequestKey schedulerKey, RMContainer container) {
+ }
+
+ @Override
+ public void unreserveResource(SchedulerApplicationAttempt attempt) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 7628312..8ab63c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4156,7 +4156,7 @@ public class TestCapacityScheduler {
// Check total resource of scheduler node is also changed to 1 GB 1 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB,
totalResource.getMemorySize());
Assert.assertEquals("Total Resource Virtual Cores should be 1", 1,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 77596e2..647a1d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -218,8 +218,8 @@ public class TestCapacitySchedulerAsyncScheduling {
// nm1 runs 1 container(app1-container_01/AM)
// nm2 runs 1 container(app1-container_02)
- Assert.assertEquals(1, sn1.getNumContainers());
- Assert.assertEquals(1, sn2.getNumContainers());
+ Assert.assertEquals(1, sn1.getNumGuaranteedContainers());
+ Assert.assertEquals(1, sn2.getNumGuaranteedContainers());
// kill app attempt1
scheduler.handle(
@@ -314,8 +314,8 @@ public class TestCapacitySchedulerAsyncScheduling {
// nm1 runs 3 containers(app1-container_01/AM, app1-container_02,
// app2-container_01/AM)
// nm2 runs 1 container(app1-container_03)
- Assert.assertEquals(3, sn1.getNumContainers());
- Assert.assertEquals(1, sn2.getNumContainers());
+ Assert.assertEquals(3, sn1.getNumGuaranteedContainers());
+ Assert.assertEquals(1, sn2.getNumGuaranteedContainers());
// reserve 1 container(app1-container_04) for app1 on nm1
ResourceRequest rr2 = ResourceRequest
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba84365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index c45bdb4..2faba02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -385,7 +385,7 @@ public class TestLeafQueue {
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
assertEquals(
- (int)(node_0.getTotalResource().getMemorySize() * a.getCapacity()) - (1*GB),
+ (int)(node_0.getCapacity().getMemorySize() * a.getCapacity()) - (1*GB),
a.getMetrics().getAvailableMB());
}
@@ -684,7 +684,7 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0*GB, a.getMetrics().getAllocatedMB());
- assertEquals((int)(a.getCapacity() * node_0.getTotalResource().getMemorySize()),
+ assertEquals((int)(a.getCapacity() * node_0.getCapacity().getMemorySize()),
a.getMetrics().getAvailableMB());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-6670 Add separate NM
overallocation thresholds for cpu and memory (Haibo Chen)
Posted by ha...@apache.org.
YARN-6670 Add separate NM overallocation thresholds for cpu and memory (Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31a732ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31a732ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31a732ee
Branch: refs/heads/YARN-1011
Commit: 31a732ee515a243a086046502e7086adb7ca8d52
Parents: ab91ebd
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jul 10 09:55:42 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 17:43:50 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 36 +++++++++--
.../src/main/resources/yarn-default.xml | 42 ++++++++++--
.../server/api/records/ResourceThresholds.java | 11 +++-
.../monitor/ContainersMonitorImpl.java | 67 +++++++++++++++-----
4 files changed, 124 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31a732ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d8f4d34..ded0c1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1949,17 +1949,39 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
3000;
- /** Overallocation (= allocation based on utilization) configs. */
- public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD =
- NM_PREFIX + "overallocation.allocation-threshold";
- public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD
- = 0f;
+ /**
+ * General overallocation threshold if no resource-type-specific
+ * threshold is provided.
+ */
+ public static final String NM_OVERALLOCATION_GENERAL_THRESHOLD =
+ NM_PREFIX + "overallocation.general-utilization-threshold";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD = -1.0f;
+ /**
+ * The maximum value of utilization threshold for all resource types
+ * up to which the scheduler allocates OPPORTUNISTIC containers.
+ */
@Private
- public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f;
+ public static final float MAX_NM_OVERALLOCATION_THRESHOLD = 0.95f;
+
+ /**
+ * NM CPU utilization threshold up to which the scheduler allocates
+ * OPPORTUNISTIC containers after the node's capacity is fully allocated.
+ */
+ public static final String NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD =
+ NM_PREFIX + "overallocation.cpu-utilization-threshold";
+
+ /**
+ * NM memory utilization threshold up to which the scheduler allocates
+ * OPPORTUNISTIC containers after the node's capacity is fully allocated.
+ */
+ public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
+ NM_PREFIX + "overallocation.memory-utilization-threshold";
+
public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
NM_PREFIX + "overallocation.preemption-threshold";
public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
- = 0f;
+ = 0.96f;
/**
* Interval of time the linux container executor should try cleaning up
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31a732ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index fc5430a..27e1996 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1669,14 +1669,44 @@
<property>
<description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node that
+ applies to all resource types (expressed as a float between 0 and 0.95).
+ By default, over-allocation is turned off (value = -1). When turned on,
+ the node allows running OPPORTUNISTIC containers when the aggregate
+ utilization for each resource type is under the value specified here
+ multiplied by the node's advertised capacity. Note for each resource type,
+ it can be overridden by the type specific setting:
+ yarn.nodemanager.overallocation.cpu-utilization-threshold for CPU
+ yarn.nodemanager.overallocation.memory-utilization-threshold for memory
+ </description>
+ <name>yarn.nodemanager.overallocation.general-utilization-threshold</name>
+ <value>-1.0</value>
+ </property>
+
+ <property>
+ <description>The extent of over-allocation (container-allocation based on
current utilization instead of prior allocation) allowed on this node,
- expressed as a float between 0 and 0.95. By default, over-allocation is
- turned off (value = 0). When turned on, the node allows running
- OPPORTUNISTIC containers when the aggregate utilization is under the
+ in terms of the percentage of overall NM memory capacity utilized (
+ expressed as a float between 0 and 0.95). By default, over-allocation is
+ turned off (value = -1). When turned on, the node allows running
+ OPPORTUNISTIC containers only when the aggregate utilization is under the
value specified here multiplied by the node's advertised capacity.
</description>
- <name>yarn.nodemanager.overallocation.allocation-threshold</name>
- <value>0f</value>
+ <name>yarn.nodemanager.overallocation.memory-utilization-threshold</name>
+ <value>${yarn.nodemanager.overallocation.general-utilization-threshold}</value>
+ </property>
+
+ <property>
+ <description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node,
+ in terms of the percentage of overall NM CPU capacity utilized (
+ expressed as a float between 0 and 0.95). By default, over-allocation is
+ turned off (value = -1). When turned on, the node allows running
+ OPPORTUNISTIC containers only when the aggregate utilization is under the
+ value specified here multiplied by the node's advertised capacity.
+ </description>
+ <name>yarn.nodemanager.overallocation.cpu-utilization-threshold</name>
+ <value>${yarn.nodemanager.overallocation.general-utilization-threshold}</value>
</property>
<property>
@@ -1685,7 +1715,7 @@
beyond which OPPORTUNISTIC containers should start getting preempted.
</description>
<name>yarn.nodemanager.overallocation.preemption-threshold</name>
- <value>1</value>
+ <value>0.96</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31a732ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
index d57706a..c81e405 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
@@ -28,10 +28,15 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImp
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class ResourceThresholds {
- public static ResourceThresholds newInstance(float threshold) {
+ public static ResourceThresholds newInstance(float overallThreshold) {
+ return newInstance(overallThreshold, overallThreshold);
+ }
+
+ public static ResourceThresholds newInstance(float cpuThreshold,
+ float memoryThreshold) {
ResourceThresholds thresholds = new ResourceThresholdsPBImpl();
- thresholds.setMemoryThreshold(threshold);
- thresholds.setCpuThreshold(threshold);
+ thresholds.setCpuThreshold(cpuThreshold);
+ thresholds.setMemoryThreshold(memoryThreshold);
return thresholds;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31a732ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a8e29bb..65091f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -240,25 +240,60 @@ public class ContainersMonitorImpl extends AbstractService implements
}
private void initializeOverAllocation(Configuration conf) {
- float overAllocationTreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
- overAllocationTreshold = Math.min(overAllocationTreshold,
- YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
- overAllocationTreshold = Math.max(0, overAllocationTreshold);
-
- if (overAllocationTreshold > 0f) {
- ((NodeManager.NMContext) context).setOverAllocationInfo(
- OverAllocationInfo.newInstance(
- ResourceThresholds.newInstance(overAllocationTreshold)));
+ float generalResourceOverAllocationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_GENERAL_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD);
+
+ float overAllocationMemoryUtilizationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ generalResourceOverAllocationThreshold);
+ overAllocationMemoryUtilizationThreshold = Math.min(
+ overAllocationMemoryUtilizationThreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
+ if (overAllocationMemoryUtilizationThreshold <= 0) {
+ LOG.info("NodeManager oversubscription is disabled because the memory " +
+ "utilization threshold is no larger than zero.");
+ return;
+ }
- float preemptionThreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+ float overAllocationCpuUtilizationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ generalResourceOverAllocationThreshold);
+ overAllocationCpuUtilizationThreshold = Math.min(
+ overAllocationCpuUtilizationThreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
+ if (overAllocationCpuUtilizationThreshold <= 0) {
+ LOG.info("NodeManager oversubscription is disabled because the CPU " +
+ "utilization threshold is no larger than zero.");
+ return;
+ }
- this.overAllocationPreemptionThresholds =
- ResourceThresholds.newInstance(preemptionThreshold);
+ float preemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+ if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because preemption" +
+ "threshold is no larger than the cpu utilization threshold.");
+ return;
+ }
+ if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because preemption" +
+ "threshold is no larger than the memory utilization threshold.");
+ return;
}
+
+ ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
+ overAllocationCpuUtilizationThreshold,
+ overAllocationMemoryUtilizationThreshold);
+ ((NodeManager.NMContext) context).setOverAllocationInfo(
+ OverAllocationInfo.newInstance(resourceThresholds));
+ this.overAllocationPreemptionThresholds =
+ ResourceThresholds.newInstance(preemptionThreshold);
+
+ LOG.info("NodeManager oversubscription enabled with overallocation " +
+ "thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
+ ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
+ " threshold: " + preemptionThreshold);
}
private boolean isResourceCalculatorAvailable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-6921. Allow resource request
to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6921. Allow resource request to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f096a17e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f096a17e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f096a17e
Branch: refs/heads/YARN-1011
Commit: f096a17e8af071404bff2967179924698c2d3a04
Parents: 8390588
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Nov 22 09:03:05 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:23:37 2018 -0800
----------------------------------------------------------------------
.../scheduler/common/PendingAsk.java | 14 +++-
.../scheduler/fair/FSAppAttempt.java | 5 ++
.../LocalityAppPlacementAllocator.java | 27 ++++++-
.../scheduler/fair/FairSchedulerTestBase.java | 32 +++++++-
.../scheduler/fair/TestFairScheduler.java | 77 ++++++++++++++++++++
5 files changed, 148 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f096a17e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
index 85d8715..e9931a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
@@ -29,11 +29,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class PendingAsk {
private final Resource perAllocationResource;
private final int count;
- public final static PendingAsk ZERO = new PendingAsk(Resources.none(), 0);
+ private final boolean isGuaranteedTypeEnforced;
- public PendingAsk(Resource res, int num) {
+ public final static PendingAsk ZERO =
+ new PendingAsk(Resources.none(), 0, false);
+
+ public PendingAsk(Resource res, int num, boolean guaranteedTypeEnforced) {
this.perAllocationResource = res;
this.count = num;
+ this.isGuaranteedTypeEnforced = guaranteedTypeEnforced;
}
public Resource getPerAllocationResource() {
@@ -44,11 +48,17 @@ public class PendingAsk {
return count;
}
+ public boolean isGuaranteedTypeEnforced() {
+ return isGuaranteedTypeEnforced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("<per-allocation-resource=");
sb.append(getPerAllocationResource());
+ sb.append(", isGuaranteedEnforced=");
+ sb.append(isGuaranteedTypeEnforced());
sb.append(",repeat=");
sb.append(getCount());
sb.append(">");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f096a17e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 970ffcb..0719ca6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -860,6 +860,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
boolean reserved, boolean opportunistic,
SchedulerRequestKey schedulerKey) {
+ if (pendingAsk.isGuaranteedTypeEnforced() && opportunistic) {
+ // do not attempt to assign an OPPORTUNISTIC container to a resource
+ // request that has explicitly opted out of oversubscription
+ return Resources.none();
+ }
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f096a17e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index be1c1cc..dcc0f8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
@@ -165,13 +167,16 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
PendingAsk lastPendingAsk =
lastRequest == null ? null : new PendingAsk(
- lastRequest.getCapability(), lastRequest.getNumContainers());
+ lastRequest.getCapability(), lastRequest.getNumContainers(),
+ enforceGuaranteedExecutionType(lastRequest));
String lastRequestedNodePartition =
lastRequest == null ? null : lastRequest.getNodeLabelExpression();
updateResult = new PendingAskUpdateResult(lastPendingAsk,
new PendingAsk(request.getCapability(),
- request.getNumContainers()), lastRequestedNodePartition,
+ request.getNumContainers(),
+ enforceGuaranteedExecutionType(request)),
+ lastRequestedNodePartition,
request.getNodeLabelExpression());
}
}
@@ -198,8 +203,9 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
if (null == request) {
return PendingAsk.ZERO;
} else{
+ boolean guaranteedEnforced = enforceGuaranteedExecutionType(request);
return new PendingAsk(request.getCapability(),
- request.getNumContainers());
+ request.getNumContainers(), guaranteedEnforced);
}
} finally {
readLock.unlock();
@@ -207,6 +213,21 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
}
+ /**
+ * Check for a given ResourceRequest, if its guaranteed execution type
+ * needs to be enforced.
+ * @param request resource request
+ * @return true if its guaranteed execution type is to be enforced.
+ * false otherwise
+ */
+ private static boolean enforceGuaranteedExecutionType(
+ ResourceRequest request) {
+ ExecutionTypeRequest executionType = request.getExecutionTypeRequest();
+ return executionType != null &&
+ executionType.getExecutionType() == ExecutionType.GUARANTEED &&
+ executionType.getEnforceExecutionType();
+ }
+
@Override
public int getOutstandingAsksCount(String resourceName) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f096a17e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 5f29186..459b5b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -115,9 +117,15 @@ public class FairSchedulerTestBase {
relaxLocality);
}
+ protected ResourceRequest createResourceRequest(int memory, int vcores,
+ String host, int priority, int numContainers, boolean relaxLocality) {
+ return createResourceRequest(memory, vcores, host, priority,
+ numContainers, relaxLocality, false);
+ }
+
protected ResourceRequest createResourceRequest(
int memory, int vcores, String host, int priority, int numContainers,
- boolean relaxLocality) {
+ boolean relaxLocality, boolean guaranteedExecutionEnforced) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setCapability(BuilderUtils.newResource(memory, vcores));
request.setResourceName(host);
@@ -127,6 +135,11 @@ public class FairSchedulerTestBase {
request.setPriority(prio);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ if (guaranteedExecutionEnforced) {
+ ExecutionTypeRequest executionType = ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true);
+ request.setExecutionTypeRequest(executionType);
+ }
return request;
}
@@ -150,6 +163,13 @@ public class FairSchedulerTestBase {
}
protected ApplicationAttemptId createSchedulingRequest(
+ int memory, String queueId, String userId,
+ int numContainers, boolean guaranteedExecutionEnforced) {
+ return createSchedulingRequest(memory, 1, queueId,
+ userId, numContainers, 1, guaranteedExecutionEnforced);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers) {
return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1);
}
@@ -163,6 +183,13 @@ public class FairSchedulerTestBase {
protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers,
int priority) {
+ return createSchedulingRequest(memory, vcores, queueId, userId,
+ numContainers, priority, false);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
+ int memory, int vcores, String queueId, String userId, int numContainers,
+ int priority, boolean guaranteedExecutionEnforced) {
ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
this.ATTEMPT_ID++);
scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
@@ -174,7 +201,8 @@ public class FairSchedulerTestBase {
}
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request = createResourceRequest(memory, vcores,
- ResourceRequest.ANY, priority, numContainers, true);
+ ResourceRequest.ANY, priority, numContainers, true,
+ guaranteedExecutionEnforced);
ask.add(request);
RMApp rmApp = mock(RMApp.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f096a17e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 8756c7d..1f89ace 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2706,6 +2706,83 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getPriority().getPriority());
}
+ @Test
+ public void testResourceRequestOptOutOfOversubscription() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription and
+ // asks for more than what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public long getMemorySeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemorySeconds();
+ return getGuaranteedMemorySeconds();
}
@Override
+ @Deprecated
public long getVcoreSeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVcoreSeconds();
+ return getGuaranteedVcoreSeconds();
}
@Override
+ @Deprecated
public void setMemorySeconds(long memorySeconds) {
- maybeInitBuilder();
- builder.setMemorySeconds(memorySeconds);
+ setGuaranteedMemorySeconds(memorySeconds);
}
@Override
+ @Deprecated
public void setVcoreSeconds(long vcoreSeconds) {
+ setGuaranteedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public long getGuaranteedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public void setGuaranteedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public long getGuaranteedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcoreSeconds);
}
@Override
+ public long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
public long getPreemptedMemorySeconds() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ @Deprecated
+ public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ setGuaranteedResourceSecondsMap(resourceSecondsMap);
+ }
+
+ @Override
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
- p.getApplicationResourceUsageMapList());
- return this.resourceSecondsMap;
+ this.guaranteedResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationResourceUsageMapList());
+ return this.guaranteedResourceSecondsMap;
}
@Override
- public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ public void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
+
+ @Override
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
+ maybeInitBuilder();
+ builder.clearApplicationOpportunisticResourceUsageMap();
+ this.opportunisticResourceSecondsMap = resourceSecondsMap;
+ if (resourceSecondsMap != null) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 61936bb..53728c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -449,6 +449,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
+ LOG.info(applicationId.toString() + " started at " + this.startTime);
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
@@ -763,10 +764,12 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
- appUsageReport
- .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+ appUsageReport.setGuaranteedResourceSecondsMap(
+ rmAppMetrics.getGuaranteedResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
+ appUsageReport.setOpportunisticResourceSecondsMap(
+ rmAppMetrics.getOpportunisticResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@@ -1636,8 +1639,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+ Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+ Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
this.readLock.lock();
try {
for (RMAppAttempt attempt : attempts.values()) {
@@ -1653,20 +1657,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ guaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap(),
+ opportunsiticResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ attemptMetrics.getPreemptedResourceSecondsMap(),
+ preemptedSecondsMap);
}
}
} finally {
@@ -1674,7 +1673,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
- numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+ numAMContainerPreempted, guaranteedResourceSecondsMap,
+ preemptedSecondsMap, opportunsiticResourceSecondsMap);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
- private final Map<String, Long> resourceSecondsMap;
+ private final Map<String, Long> guaranteedResourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
}
public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
return numAMContainersPreempted;
}
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
}
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
public long getPreemptedMemorySeconds() {
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceSecondsMap() {
- return resourceSecondsMap;
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ return guaranteedResourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
@Private
public class AggregateAppResourceUsage {
- private Map<String, Long> resourceSecondsMap = new HashMap<>();
+ private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
- public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap.putAll(resourceSecondsMap);
+ public AggregateAppResourceUsage(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap.putAll(
+ guaranteedResourceSecondsMap);
+ this.opportunisticResourceSecondsMap.putAll(
+ opportunisticResourceSecondsMap);
}
/**
- * @return the memorySeconds
+ * @return the guaranteed memory seconds
*/
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
- * @return the vcoreSeconds
+ * @return the guaranteed vcore seconds
*/
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceUsageSecondsMap() {
- return resourceSecondsMap;
+ /**
+ * @return the opportunistic memory seconds
+ */
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+
+ /**
+ * @return the opportunistic vcore seconds
+ */
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index cf10be4..6f46b33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -934,15 +934,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock.lock();
try {
ApplicationResourceUsageReport report =
- scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+ scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
- report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+ report.setGuaranteedResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
+ report.setOpportunisticResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@@ -979,10 +982,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
- this.attemptMetrics
- .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+ attemptState.getGuaranteedResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+ attemptState.getOpportunisticResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1360,9 +1365,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
- getFinishTime(), resUsage.getResourceUsageSecondsMap(),
- this.attemptMetrics.getPreemptedResourceSecondsMap());
+ stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+ exitStatus, getFinishTime(),
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ this.attemptMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 015cff7..192722b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -53,8 +53,11 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
- private Map<String, AtomicLong> resourceUsageMap = new HashMap<>();
+ private Map<String, AtomicLong> guaranteedResourceUsageMap
+ = new HashMap<>();
private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+ private Map<String, AtomicLong> opportunisticResourceUsageMap
+ = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@@ -129,8 +132,10 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
- Map<String, Long> resourcesUsed =
- convertAtomicLongMaptoLongMap(resourceUsageMap);
+ Map<String, Long> guaranteedResourcesUsed =
+ convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+ Map<String, Long> opportunisticResourcesUsed =
+ convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -139,27 +144,25 @@ public class RMAppAttemptMetrics {
if (currentAttempt != null
&& currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
- rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+ rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
if (appResUsageReport != null) {
- Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
- for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- Long value = resourcesUsed.get(entry.getKey());
- if (value != null) {
- value += entry.getValue();
- } else {
- value = entry.getValue();
- }
- resourcesUsed.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getGuaranteedResourceSecondsMap(),
+ guaranteedResourcesUsed);
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getOpportunisticResourceSecondsMap(),
+ opportunisticResourcesUsed);
}
}
}
- return new AggregateAppResourceUsage(resourcesUsed);
+
+ return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+ opportunisticResourcesUsed);
}
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
- updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+ updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -167,9 +170,14 @@ public class RMAppAttemptMetrics {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
- public void updateAggregateAppResourceUsage(
+ public void updateAggregateAppGuaranteedResourceUsage(
Map<String, Long> resourceSecondsMap) {
- updateUsageMap(resourceSecondsMap, resourceUsageMap);
+ updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+ }
+
+ public void updateAggregateAppOpportunisticResourceUsage(
+ Map<String, Long> resourceSecondsMap) {
+ updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
@@ -199,10 +207,9 @@ public class RMAppAttemptMetrics {
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
-
}
resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.set(entry.getValue());
+ resourceUsed.addAndGet(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+ private final Map<String, Long> guaranteedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
+
+ public ContainerResourceUsageReport(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index f3cbf63..4c43c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
-import java.util.List;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -29,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -85,7 +82,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
ContainerState getContainerState();
ContainerReport createContainerReport();
-
+
+ ContainerResourceUsageReport getResourceUsageReport();
+
boolean isAMContainer();
ContainerRequest getContainerRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index e26689e..8f51d1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import java.util.Collections;
import java.util.EnumSet;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +178,7 @@ public class RMContainerImpl implements RMContainer {
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
private long creationTime;
- private long finishTime;
+ private long finishTime = -1L;
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private ContainerRequest containerRequestForRecovery;
@@ -189,6 +191,10 @@ public class RMContainerImpl implements RMContainer {
private boolean isExternallyAllocated;
private SchedulerRequestKey allocatedSchedulerKey;
+ // This is updated whenever the container instance is updated in
+ // the case of container increase/decrease/promotion/demotion
+ private long lastContainerUpdateTime;
+
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -230,6 +236,7 @@ public class RMContainerImpl implements RMContainer {
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
+ this.lastContainerUpdateTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -277,9 +284,87 @@ public class RMContainerImpl implements RMContainer {
}
public void setContainer(Container container) {
+ // containers are updated by resetting the underlying Container instance,
+ // always update RMContainer resource usage before the update.
+ updateAppAttemptMetrics(true);
this.container = container;
}
+ /**
+ * Collect the resource usage information of the current underlying Container
+ * instance and update its RMAppAttemptMetrics. This is called whenever the
+ * RMContainer is updated (by updating its underlying container) or finishes.
+ * @param containerUpdated if it is called upon container update event
+ */
+ private void updateAppAttemptMetrics(boolean containerUpdated) {
+ RMAppAttempt rmAttempt = rmContext.getRMApps()
+ .get(getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+
+ if (rmAttempt != null) {
+ // collect resource usage information of the current Container instance
+ ContainerResourceUsageReport resourceUsage =
+ getCurrentContainerResourceUsage(containerUpdated);
+
+ // If this is a preempted container, update preemption metrics
+ if (finishedStatus != null &&
+ ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics()
+ .updatePreemptionInfo(container.getResource(), this);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getOpportunisticResourceSecondsMap());
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppGuaranteedResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ }
+
+ /**
+ * Get resource usage of the current underlying Container instance.
+ * @param containerUpdated if this is called upon container update
+ * @return resource usage of the current <code>container</code>
+ */
+ private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+ boolean containerUpdated) {
+ ContainerResourceUsageReport report;
+
+ // A container generates usage until it finishes which is indicated
+ // by a positive finish timestamp that is set once it finishes
+ final long currentTimeMillis =
+ finishTime < 0 ? System.currentTimeMillis() : finishTime;
+ final long usedSeconds = (currentTimeMillis -
+ lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND;
+ Resource resource = container.getResource();
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ guaranteedResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+ Collections.emptyMap());
+ } else {
+ Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ opportunisticResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(Collections.emptyMap(),
+ opportunisticResourceSeconds);
+ }
+
+ if (containerUpdated) {
+ lastContainerUpdateTime = currentTimeMillis;
+ }
+ return report;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
@@ -683,7 +768,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateAttemptMetrics(container);
+ container.updateAppAttemptMetrics(false);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -705,27 +790,6 @@ public class RMContainerImpl implements RMContainer {
}
}
-
- private static void updateAttemptMetrics(RMContainerImpl container) {
- Resource resource = container.getContainer().getResource();
- RMAppAttempt rmAttempt = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
-
- if (rmAttempt != null) {
- long usedMillis = container.finishTime - container.creationTime;
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregateAppResourceUsage(resource, usedMillis);
- // If this is a preempted container, update preemption metrics
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
- }
- }
- }
}
private static final class KillTransition extends FinishedTransition {
@@ -764,6 +828,11 @@ public class RMContainerImpl implements RMContainer {
}
@Override
+ public ContainerResourceUsageReport getResourceUsageReport() {
+ return getCurrentContainerResourceUsage(false);
+ }
+
+ @Override
public String getNodeHttpAddress() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a755a4c..b78e93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -412,7 +412,7 @@ public abstract class AbstractYarnScheduler
}
@Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
+ public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId) {
SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
if (attempt == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f02f113..af4ef54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -60,12 +59,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -107,7 +106,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
- private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> lastGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ private Map<String, Long> lastOpportunisticResourceSecondsMap =
+ new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -1053,34 +1055,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+ new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
- long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
- Resource resource = rmContainer.getContainer().getResource();
- for (ResourceInformation entry : resource.getResources()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
- value += entry.getValue() * usedMillis
- / DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getName(), value);
- }
+ ContainerResourceUsageReport containerResourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ aggregatedGuaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+ aggregatedOpportunisticResourceSecondsMap);
}
-
+ lastGuaranteedResourceSecondsMap =
+ aggregatedGuaranteedResourceSecondsMap;
+ lastOpportunisticResourceSecondsMap =
+ aggregatedOpportunisticResourceSecondsMap;
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
- lastResourceSecondsMap = resourceSecondsMap;
}
- return new AggregateAppResourceUsage(lastResourceSecondsMap);
+ return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+ lastOpportunisticResourceSecondsMap);
}
+ /**
+ * Get the resources that are actively being used by the app attempt.
+ * @return active resource usage
+ */
public ApplicationResourceUsageReport getResourceUsageReport() {
try {
writeLock.lock();
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
- Resource usedResourceClone = Resources.clone(
+ Resource guaranteedResourceUsedClone = Resources.clone(
attemptResourceUsage.getAllUsed());
+ Resource opportunisticResourceUsedClone = Resources.clone(
+ attemptOpportunisticResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
+ Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+ Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
@@ -1095,18 +1110,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
float queueCapacityPerc = queue.getQueueInfo(false, false)
.getCapacity();
if (queueCapacityPerc != 0) {
- queueUsagePerc = calc.divide(cluster, usedResourceClone,
+ queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc =
- calc.divide(cluster, usedResourceClone, cluster) * 100;
+ calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
}
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
- usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
- clusterUsagePerc, preemptedResourceSecondsMaps);
+ guaranteedResourceUsedClone, reservedResourceClone,
+ neededResource,
+ runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+ queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+ opportunisticResourceUsedClone,
+ runningResourceUsage.getOpportunisticResourceSecondsMap());
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 93ca7c2..a8ec8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -171,7 +171,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
*/
@LimitedPrivate("yarn")
@Evolving
- ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index cf651bb..75b2f81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1017,7 +1017,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.getEffectiveCapacity(getAppAMNodePartitionName());
if (!effCap.equals(Resources.none())) {
queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(), effCap) * 100;
+ report.getGuaranteedResourcesUsed(), effCap) * 100;
}
report.setQueueUsagePercentage(queueUsagePerc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0719ca6..9ef4d31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedGuaranteedResource(containerResource);
+ if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
+ } else {
+ this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+ }
+
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index ac88f86..14dab9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -95,6 +95,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".runningcontainer", "Running Containers").
th(".allocatedCpu", "Allocated CPU VCores").
th(".allocatedMemory", "Allocated Memory MB").
+ th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU Vcores").
+ th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB").
th(".reservedCpu", "Reserved CPU VCores").
th(".reservedMemory", "Reserved Memory MB").
th(".progress", "Progress").
@@ -144,6 +148,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
.append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
.valueOf(appInfo.getAllocatedMB())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
.append(appInfo.getReservedVCores() == -1 ? "N/A" : String
.valueOf(appInfo.getReservedVCores())).append("\",\"")
.append(appInfo.getReservedMB() == -1 ? "N/A" : String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 8553d8c..6cd10f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
- StringHelper
- .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+ .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getGuaranteedResourceSecondsMap()))
+ .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getOpportunisticResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index d0dccab..0510d62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -69,6 +69,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU VCores")
+ .th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -155,6 +159,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 631c908..e7ac27e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -95,6 +95,8 @@ public class AppInfo {
private String masterNodeId;
private long allocatedMB;
private long allocatedVCores;
+ private long allocatedOpportunisticMB;
+ private long allocatedOpportunisticVCores;
private long reservedMB;
private long reservedVCores;
private int runningContainers;
@@ -103,6 +105,9 @@ public class AppInfo {
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
+ private long opportunisticMemorySeconds;
+ private long opportunisticVcoreSeconds;
+ private Map<String, Long> opportunisticResourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -202,10 +207,17 @@ public class AppInfo {
ApplicationResourceUsageReport resourceReport =
attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
- Resource usedResources = resourceReport.getUsedResources();
+ Resource guaranteedResourcesUsed =
+ resourceReport.getGuaranteedResourcesUsed();
+ Resource opportunisticResourceUsed =
+ resourceReport.getOpportunisticResourcesUsed();
Resource reservedResources = resourceReport.getReservedResources();
- allocatedMB = usedResources.getMemorySize();
- allocatedVCores = usedResources.getVirtualCores();
+ allocatedMB = guaranteedResourcesUsed.getMemorySize();
+ allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+ allocatedOpportunisticMB =
+ opportunisticResourceUsed.getMemorySize();
+ allocatedOpportunisticVCores =
+ opportunisticResourceUsed.getVirtualCores();
reservedMB = reservedResources.getMemorySize();
reservedVCores = reservedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
@@ -239,9 +251,13 @@ public class AppInfo {
numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
- memorySeconds = appMetrics.getMemorySeconds();
- vcoreSeconds = appMetrics.getVcoreSeconds();
- resourceSecondsMap = appMetrics.getResourceSecondsMap();
+ memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+ vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+ resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+ opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+ opportunisticResourceSecondsMap =
+ appMetrics.getOpportunisticResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -454,6 +470,13 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public long getAllocatedOpportunisticMB() {
+ return this.allocatedOpportunisticMB;
+ }
+
+ public long getAllocatedOpportunisticVCores() {
+ return this.allocatedOpportunisticVCores;
+ }
public long getReservedMB() {
return this.reservedMB;
}
@@ -478,18 +501,30 @@ public class AppInfo {
return numAMContainerPreempted;
}
- public long getMemorySeconds() {
+ public long getGuaranteedMemorySeconds() {
return memorySeconds;
}
- public long getVcoreSeconds() {
+ public long getGuaranteedVcoreSeconds() {
return vcoreSeconds;
}
- public Map<String, Long> getResourceSecondsMap() {
+ public long getOpportunisticMemorySeconds() {
+ return opportunisticMemorySeconds;
+ }
+
+ public long getOpportunisticVcoreSeconds() {
+ return opportunisticVcoreSeconds;
+ }
+
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
return resourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -582,6 +617,15 @@ public class AppInfo {
this.allocatedVCores = allocatedVCores;
}
+ public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+ this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+ }
+
+ public void setAllocatedOpportunisticVCores(
+ long allocatedOpportunisticVCores) {
+ this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+ }
+
public void setReservedMB(long reservedMB) {
this.reservedMB = reservedMB;
}
@@ -594,12 +638,20 @@ public class AppInfo {
this.runningContainers = runningContainers;
}
- public void setMemorySeconds(long memorySeconds) {
- this.memorySeconds = memorySeconds;
+ public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+ this.memorySeconds = guaranteedMemorySeconds;
+ }
+
+ public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+ this.vcoreSeconds = guaranteedVcoreSeconds;
+ }
+
+ public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+ this.opportunisticMemorySeconds = oppMemorySeconds;
}
- public void setVcoreSeconds(long vcoreSeconds) {
- this.vcoreSeconds = vcoreSeconds;
+ public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+ this.opportunisticVcoreSeconds = oppVcoreSeconds;
}
public void setAppId(String appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 39a56a8..1721b1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto {
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 140c15f..6ad6006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1031,12 +1031,20 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, resourceSecondsMap, new HashMap<>());
+ 10, 1, guaranteedResourceSecondsMap,
+ new HashMap<>(), opportunisticResourceSecondsMap);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1047,16 +1055,19 @@ public class TestAppManager{
Assert.assertFalse(msg.contains("\r"));
String escaped = "\\n\\n\\r\\r";
- assertTrue(msg.contains("Multiline" + escaped +"AppName"));
- assertTrue(msg.contains("Multiline" + escaped +"UserName"));
- assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- assertTrue(msg.contains("submitTime=1000"));
- assertTrue(msg.contains("memorySeconds=16384"));
- assertTrue(msg.contains("vcoreSeconds=64"));
- assertTrue(msg.contains("preemptedAMContainers=1"));
- assertTrue(msg.contains("preemptedNonAMContainers=10"));
- assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
- assertTrue(msg.contains("applicationType=MAPREDUCE"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+ Assert.assertTrue(msg.contains("submitTime=1000"));
+ Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(
+ msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
- -1, usageReport.getUsedResources().getMemorySize());
+ -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemorySize());
+ Assert.assertEquals("Enemy should not see app used opportunistic resources",
+ -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 1c50dd3..a66a91b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -370,8 +370,8 @@ public class TestClientRMService {
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
- Assert.assertEquals(10, usageReport.getMemorySeconds());
- Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
@@ -1402,26 +1402,28 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
- RMAppImpl app =
- spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+ ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ Resource.newInstance(1024, 1), 1);
+ RMAppImpl rmApp =
+ new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- Collections.singletonList(BuilderUtils.newResourceRequest(
- RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1))){
- @Override
- public ApplicationReport createAndGetApplicationReport(
- String clientUserName, boolean allowAccess) {
- ApplicationReport report = super.createAndGetApplicationReport(
- clientUserName, allowAccess);
- ApplicationResourceUsageReport usageReport =
- report.getApplicationResourceUsageReport();
- usageReport.setMemorySeconds(memorySeconds);
- usageReport.setVcoreSeconds(vcoreSeconds);
- report.setApplicationResourceUsageReport(usageReport);
- return report;
- }
- });
+ Collections.singletonList(resourceRequest)) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setGuaranteedMemorySeconds(memorySeconds);
+ usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ };
+ RMAppImpl app = spy(rmApp);
app.getAMResourceRequests().get(0)
.setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1474,7 +1476,8 @@ public class TestClientRMService {
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+ .thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-7491. Make sure AM is not
scheduled on an opportunistic container. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee49eb92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee49eb92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee49eb92
Branch: refs/heads/YARN-1011
Commit: ee49eb925a4c6f808855dd089b95cf0346d2de06
Parents: a2bb70c
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:23:37 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 130 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 63333ac..6abc617 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -349,15 +349,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 2983077..1e05fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -488,8 +488,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -520,7 +520,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 7c2ce64..140c15f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -691,15 +691,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -720,8 +750,14 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -742,8 +778,16 @@ public class TestAppManager{
RMApp app = testRMAppSubmit();
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -783,7 +827,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
@@ -1100,7 +1144,8 @@ public class TestAppManager{
req.getNodeLabelExpression() != null
? new String(req.getNodeLabelExpression()) : null,
ExecutionTypeRequest.newInstance(
- req.getExecutionTypeRequest().getExecutionType()));
+ req.getExecutionTypeRequest().getExecutionType(),
+ req.getExecutionTypeRequest().getEnforceExecutionType()));
}
private static List<ResourceRequest> cloneResourceRequests(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-7760. [UI2] Clicking 'Master
Node' or link next to 'AM Node Web UI' under application's appAttempt page
goes to OLD RM UI. (Vasudevan Skm / Sunil G via wangda)
Posted by ha...@apache.org.
YARN-7760. [UI2] Clicking 'Master Node' or link next to 'AM Node Web UI' under application's appAttempt page goes to OLD RM UI. (Vasudevan Skm / Sunil G via wangda)
Change-Id: Ia7e2b955d3551ca43c6852fe253fd01983b56565
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75618c0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75618c0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75618c0d
Branch: refs/heads/YARN-1011
Commit: 75618c0dec9199387fb55c8b4d63ac0003ddc1fe
Parents: e0cfb0a
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Jan 26 15:44:55 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Jan 26 15:44:55 2018 +0800
----------------------------------------------------------------------
.../resourcemanager/webapp/dao/AppInfo.java | 11 +++
.../webapp/TestRMWebServicesApps.java | 2 +-
.../src/main/webapp/app/models/yarn-app.js | 96 ++++++++++++--------
.../src/main/webapp/app/serializers/yarn-app.js | 1 +
.../src/main/webapp/app/templates/yarn-app.hbs | 2 +-
5 files changed, 70 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75618c0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 0711b45..631c908 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -92,6 +92,7 @@ public class AppInfo {
protected String amContainerLogs;
protected String amHostHttpAddress;
private String amRPCAddress;
+ private String masterNodeId;
private long allocatedMB;
private long allocatedVCores;
private long reservedMB;
@@ -162,6 +163,7 @@ public class AppInfo {
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
this.priority = 0;
+ this.masterNodeId = "";
if (app.getApplicationPriority() != null) {
this.priority = app.getApplicationPriority().getPriority();
@@ -192,6 +194,7 @@ public class AppInfo {
schemePrefix + masterContainer.getNodeHttpAddress(),
masterContainer.getId().toString(), app.getUser());
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
+ this.masterNodeId = masterContainer.getNodeId().toString();
}
this.amRPCAddress = getAmRPCAddressFromRMAppAttempt(attempt);
@@ -615,4 +618,12 @@ public class AppInfo {
public void setName(String name) {
this.name = name;
}
+
+ public String getMasterNodeId() {
+ return masterNodeId;
+ }
+
+ public void setMasterNodeId(String masterNodeId) {
+ this.masterNodeId = masterNodeId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75618c0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index f0704ac..a381ed4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1603,7 +1603,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75618c0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
index b9897be..7e3050f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
@@ -20,42 +20,43 @@ import Converter from 'yarn-ui/utils/converter';
import DS from 'ember-data';
export default DS.Model.extend({
- appName: DS.attr('string'),
- user: DS.attr('string'),
- queue: DS.attr('string'),
- state: DS.attr('string'),
- startTime: DS.attr('string'),
- elapsedTime: DS.attr('string'),
- finalStatus: DS.attr('string'),
- finishedTime: DS.attr('finishedTime'),
- progress: DS.attr('number'),
- diagnostics: DS.attr('string'),
- amContainerLogs: DS.attr('string'),
- amHostHttpAddress: DS.attr('string'),
- logAggregationStatus: DS.attr('string'),
- unmanagedApplication: DS.attr('boolean'),
- amNodeLabelExpression: DS.attr('string'),
- applicationTags: DS.attr('string'),
- applicationType: DS.attr('string'),
- priority: DS.attr('string'),
- allocatedMB: DS.attr('number'),
- allocatedVCores: DS.attr('number'),
- runningContainers: DS.attr('number'),
- memorySeconds: DS.attr('number'),
- vcoreSeconds: DS.attr('number'),
- preemptedResourceMB: DS.attr('number'),
- preemptedResourceVCores: DS.attr('number'),
- numNonAMContainerPreempted: DS.attr('number'),
- numAMContainerPreempted: DS.attr('number'),
- clusterUsagePercentage: DS.attr('number'),
- queueUsagePercentage: DS.attr('number'),
- currentAppAttemptId: DS.attr('string'),
- remainingTimeoutInSeconds: DS.attr('number'),
- applicationExpiryTime: DS.attr('string'),
- resourceRequests: DS.attr('array'),
+ appName: DS.attr("string"),
+ user: DS.attr("string"),
+ queue: DS.attr("string"),
+ state: DS.attr("string"),
+ startTime: DS.attr("string"),
+ elapsedTime: DS.attr("string"),
+ finalStatus: DS.attr("string"),
+ finishedTime: DS.attr("finishedTime"),
+ progress: DS.attr("number"),
+ diagnostics: DS.attr("string"),
+ amContainerLogs: DS.attr("string"),
+ amHostHttpAddress: DS.attr("string"),
+ masterNodeId: DS.attr("string"),
+ logAggregationStatus: DS.attr("string"),
+ unmanagedApplication: DS.attr("boolean"),
+ amNodeLabelExpression: DS.attr("string"),
+ applicationTags: DS.attr("string"),
+ applicationType: DS.attr("string"),
+ priority: DS.attr("string"),
+ allocatedMB: DS.attr("number"),
+ allocatedVCores: DS.attr("number"),
+ runningContainers: DS.attr("number"),
+ memorySeconds: DS.attr("number"),
+ vcoreSeconds: DS.attr("number"),
+ preemptedResourceMB: DS.attr("number"),
+ preemptedResourceVCores: DS.attr("number"),
+ numNonAMContainerPreempted: DS.attr("number"),
+ numAMContainerPreempted: DS.attr("number"),
+ clusterUsagePercentage: DS.attr("number"),
+ queueUsagePercentage: DS.attr("number"),
+ currentAppAttemptId: DS.attr("string"),
+ remainingTimeoutInSeconds: DS.attr("number"),
+ applicationExpiryTime: DS.attr("string"),
+ resourceRequests: DS.attr("array"),
isFailed: function() {
- return this.get('finalStatus') === "FAILED";
+ return this.get("finalStatus") === "FAILED";
}.property("finalStatus"),
validatedFinishedTs: function() {
@@ -70,27 +71,42 @@ export default DS.Model.extend({
}.property("hasFinishedTime"),
formattedElapsedTime: function() {
- return Converter.msToElapsedTimeUnit(this.get('elapsedTime'));
- }.property('elapsedTime'),
+ return Converter.msToElapsedTimeUnit(this.get("elapsedTime"));
+ }.property("elapsedTime"),
allocatedResource: function() {
- return Converter.resourceToString(this.get("allocatedMB"), this.get("allocatedVCores"));
+ return Converter.resourceToString(
+ this.get("allocatedMB"),
+ this.get("allocatedVCores")
+ );
}.property("allocatedMB", "allocatedVCores"),
preemptedResource: function() {
- return Converter.resourceToString(this.get("preemptedResourceMB"), this.get("preemptedResourceVCores"));
+ return Converter.resourceToString(
+ this.get("preemptedResourceMB"),
+ this.get("preemptedResourceVCores")
+ );
}.property("preemptedResourceMB", "preemptedResourceVCores"),
aggregatedResourceUsage: function() {
- return Converter.resourceToString(this.get("memorySeconds"), this.get("vcoreSeconds")) + " (× Secs)";
+ return (
+ Converter.resourceToString(
+ this.get("memorySeconds"),
+ this.get("vcoreSeconds")
+ ) + " (× Secs)"
+ );
}.property("memorySeconds", "vcoreSeconds"),
+ masterNodeURL: function() {
+ return `#/yarn-node/${this.get("masterNodeId")}/${this.get("amHostHttpAddress")}/info/`;
+ }.property("masterNodeId", "amHostHttpAddress"),
+
progressStyle: function() {
return "width: " + this.get("progress") + "%";
}.property("progress"),
runningContainersNumber: function() {
- if(this.get("runningContainers") < 0) {
+ if (this.get("runningContainers") < 0) {
return 0;
}
return this.get("runningContainers");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75618c0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
index 42d31ca..bd9e984 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
@@ -51,6 +51,7 @@ export default DS.JSONAPISerializer.extend({
diagnostics: (payload.diagnostics && payload.diagnostics !== 'null')? payload.diagnostics : '',
amContainerLogs: payload.amContainerLogs,
amHostHttpAddress: payload.amHostHttpAddress,
+ masterNodeId: payload.masterNodeId,
logAggregationStatus: payload.logAggregationStatus,
unmanagedApplication: payload.unmanagedApplication,
amNodeLabelExpression: payload.amNodeLabelExpression,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75618c0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs
index ca93428..992b979 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs
@@ -65,7 +65,7 @@
<div class="flex-right">
<div class="links">
<a href="{{model.app.amContainerLogs}}" target="_blank">Master Container Log</a> |
- <a href="{{amHostHttpAddressFormatted}}" target="_blank">Master Node</a> |
+ <a href="{{model.app.masterNodeURL}}">Master Node</a> |
{{#if isRunningService}}
<div class="btn-group">
<button type="button" class="btn btn-unstyled dropdown-toggle" title="Settings" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDFS-12772. RBF: Federation Router
State State Store internal API. Contributed by Inigo Goiri.
Posted by ha...@apache.org.
HDFS-12772. RBF: Federation Router State State Store internal API. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95743c67
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95743c67
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95743c67
Branch: refs/heads/YARN-1011
Commit: 95743c672e6b42b227a22dfa7cc16edc7bdb58bb
Parents: d95c137
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Jan 23 19:15:44 2018 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Jan 23 19:15:44 2018 -0800
----------------------------------------------------------------------
.../federation/router/FederationUtil.java | 20 ++
.../federation/router/RouterServiceState.java | 31 +++
.../server/federation/store/RouterStore.java | 78 +++++++
.../federation/store/impl/RouterStoreImpl.java | 93 +++++++++
.../protocol/GetRouterRegistrationRequest.java | 47 +++++
.../protocol/GetRouterRegistrationResponse.java | 44 ++++
.../protocol/GetRouterRegistrationsRequest.java | 34 ++++
.../GetRouterRegistrationsResponse.java | 55 +++++
.../store/protocol/RouterHeartbeatRequest.java | 50 +++++
.../store/protocol/RouterHeartbeatResponse.java | 49 +++++
.../pb/GetRouterRegistrationRequestPBImpl.java | 70 +++++++
.../pb/GetRouterRegistrationResponsePBImpl.java | 79 ++++++++
.../pb/GetRouterRegistrationsRequestPBImpl.java | 61 ++++++
.../GetRouterRegistrationsResponsePBImpl.java | 102 ++++++++++
.../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 +++++++
.../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 +++++++
.../federation/store/records/RouterState.java | 169 ++++++++++++++++
.../store/records/StateStoreVersion.java | 91 +++++++++
.../records/impl/pb/RouterStatePBImpl.java | 202 +++++++++++++++++++
.../impl/pb/StateStoreVersionPBImpl.java | 79 ++++++++
.../src/main/proto/FederationProtocol.proto | 44 ++++
.../store/driver/TestStateStoreDriverBase.java | 13 ++
.../store/records/TestRouterState.java | 85 ++++++++
23 files changed, 1644 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 224cac1..3f736d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.VersionInfo;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -106,6 +107,25 @@ public final class FederationUtil {
}
/**
+ * Fetch the compile timestamp for this jar.
+ *
+ * @return Date compiled.
+ */
+ public static String getBuildVersion() {
+ return VersionInfo.getBuildVersion();
+ }
+
+ /**
+ * Fetch the build/compile information for this jar.
+ *
+ * @return String Compilation info.
+ */
+ public static String getCompileInfo() {
+ return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
+ + VersionInfo.getBranch();
+ }
+
+ /**
* Create an instance of an interface with a constructor using a context.
*
* @param conf Configuration for the class names.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
new file mode 100644
index 0000000..25a6466
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+/**
+ * States of the Router.
+ */
+public enum RouterServiceState {
+ NONE,
+ INITIALIZING,
+ SAFEMODE,
+ RUNNING,
+ STOPPING,
+ SHUTDOWN,
+ EXPIRED;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
new file mode 100644
index 0000000..c6a0dad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * Management API for
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.RouterState
+ * RouterState} records in the state store. Accesses the data store via the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.
+ * StateStoreDriver StateStoreDriver} interface. No data is cached.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class RouterStore extends CachedRecordStore<RouterState> {
+
+ public RouterStore(StateStoreDriver driver) {
+ super(RouterState.class, driver, true);
+ }
+
+ /**
+ * Fetches the current router state object.
+ *
+ * @param request Fully populated request object.
+ * @return The matching router record or null if none exists.
+ * @throws IOException Throws exception if unable to query the data store or
+ * if more than one matching record is found.
+ */
+ public abstract GetRouterRegistrationResponse getRouterRegistration(
+ GetRouterRegistrationRequest request) throws IOException;
+
+ /**
+ * Fetches all router status objects.
+ *
+ * @param request Fully populated request object.
+ * @return List of Router records present in the data store.
+ * @throws IOException Throws exception if unable to query the data store
+ */
+ public abstract GetRouterRegistrationsResponse getRouterRegistrations(
+ GetRouterRegistrationsRequest request) throws IOException;
+
+ /**
+ * Update the state of this router in the State Store.
+ *
+ * @param request Fully populated request object.
+ * @return True if the update was successfully recorded, false otherwise.
+ * @throws IOException Throws exception if unable to query the data store
+ */
+ public abstract RouterHeartbeatResponse routerHeartbeat(
+ RouterHeartbeatRequest request) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
new file mode 100644
index 0000000..d58c288
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * Implementation of the {@link RouterStore} state store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class RouterStoreImpl extends RouterStore {
+
+ public RouterStoreImpl(StateStoreDriver driver) {
+ super(driver);
+ }
+
+ @Override
+ public GetRouterRegistrationResponse getRouterRegistration(
+ GetRouterRegistrationRequest request) throws IOException {
+
+ final RouterState partial = RouterState.newInstance();
+ partial.setAddress(request.getRouterId());
+ final Query<RouterState> query = new Query<RouterState>(partial);
+ RouterState record = getDriver().get(getRecordClass(), query);
+ if (record != null) {
+ overrideExpiredRecord(record);
+ }
+ GetRouterRegistrationResponse response =
+ GetRouterRegistrationResponse.newInstance();
+ response.setRouter(record);
+ return response;
+ }
+
+ @Override
+ public GetRouterRegistrationsResponse getRouterRegistrations(
+ GetRouterRegistrationsRequest request) throws IOException {
+
+ // Get all values from the cache
+ QueryResult<RouterState> recordsAndTimeStamp =
+ getCachedRecordsAndTimeStamp();
+ List<RouterState> records = recordsAndTimeStamp.getRecords();
+ long timestamp = recordsAndTimeStamp.getTimestamp();
+
+ // Generate response
+ GetRouterRegistrationsResponse response =
+ GetRouterRegistrationsResponse.newInstance();
+ response.setRouters(records);
+ response.setTimestamp(timestamp);
+ return response;
+ }
+
+ @Override
+ public RouterHeartbeatResponse routerHeartbeat(RouterHeartbeatRequest request)
+ throws IOException {
+
+ RouterState record = request.getRouter();
+ boolean status = getDriver().put(record, true, false);
+ RouterHeartbeatResponse response =
+ RouterHeartbeatResponse.newInstance(status);
+ return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
new file mode 100644
index 0000000..9ba5788
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for retrieving a single router registration present in the state
+ * store.
+ */
+public abstract class GetRouterRegistrationRequest {
+
+ public static GetRouterRegistrationRequest newInstance() {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationRequest.class);
+ }
+
+ public static GetRouterRegistrationRequest newInstance(String routerId) {
+ GetRouterRegistrationRequest request = newInstance();
+ request.setRouterId(routerId);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getRouterId();
+
+ @Public
+ @Unstable
+ public abstract void setRouterId(String routerId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
new file mode 100644
index 0000000..b569376
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API response for retrieving a single router registration present in the state
+ * store.
+ */
+public abstract class GetRouterRegistrationResponse {
+
+ public static GetRouterRegistrationResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract RouterState getRouter() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouter(RouterState router) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
new file mode 100644
index 0000000..b70cccf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for retrieving a all non-expired router registrations present in
+ * the state store.
+ */
+public abstract class GetRouterRegistrationsRequest {
+
+ public static GetRouterRegistrationsRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationsRequest.class);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
new file mode 100644
index 0000000..94e35c6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API response for retrieving a all non-expired router registrations present in
+ * the state store.
+ */
+public abstract class GetRouterRegistrationsResponse {
+
+ public static GetRouterRegistrationsResponse newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationsResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract List<RouterState> getRouters() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouters(List<RouterState> routers)
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract long getTimestamp();
+
+ @Public
+ @Unstable
+ public abstract void setTimestamp(long time);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
new file mode 100644
index 0000000..97d637b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API request for registering a router with the state store.
+ */
+public abstract class RouterHeartbeatRequest {
+
+ public static RouterHeartbeatRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RouterHeartbeatRequest.class);
+ }
+
+ public static RouterHeartbeatRequest newInstance(RouterState router)
+ throws IOException {
+ RouterHeartbeatRequest request = newInstance();
+ request.setRouter(router);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract RouterState getRouter() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouter(RouterState routerState);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
new file mode 100644
index 0000000..5912e9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for registering a router with the state store.
+ */
+public abstract class RouterHeartbeatResponse {
+
+ public static RouterHeartbeatResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RouterHeartbeatResponse.class);
+ }
+
+ public static RouterHeartbeatResponse newInstance(boolean status)
+ throws IOException {
+ RouterHeartbeatResponse response = newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
new file mode 100644
index 0000000..19bf4ec
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationRequest.
+ */
+public class GetRouterRegistrationRequestPBImpl
+ extends GetRouterRegistrationRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
+ Builder, GetRouterRegistrationRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
+ Builder, GetRouterRegistrationRequestProtoOrBuilder>(
+ GetRouterRegistrationRequestProto.class);
+
+ public GetRouterRegistrationRequestPBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getRouterId() {
+ return this.translator.getProtoOrBuilder().getRouterId();
+ }
+
+ @Override
+ public void setRouterId(String routerId) {
+ this.translator.getBuilder().setRouterId(routerId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
new file mode 100644
index 0000000..ae17e92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationResponse.
+ */
+public class GetRouterRegistrationResponsePBImpl
+ extends GetRouterRegistrationResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationResponseProto,
+ Builder, GetRouterRegistrationResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationResponseProto, Builder,
+ GetRouterRegistrationResponseProtoOrBuilder>(
+ GetRouterRegistrationResponseProto.class);
+
+ public GetRouterRegistrationResponsePBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public RouterState getRouter() {
+ RouterRecordProto proto = this.translator.getProtoOrBuilder().getRouter();
+ return new RouterStatePBImpl(proto);
+ }
+
+ @Override
+ public void setRouter(RouterState router) {
+ if (router instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerPB = (RouterStatePBImpl)router;
+ RouterRecordProto routerProto = routerPB.getProto();
+ this.translator.getBuilder().setRouter(routerProto);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
new file mode 100644
index 0000000..4b48682
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationsRequest.
+ */
+public class GetRouterRegistrationsRequestPBImpl
+ extends GetRouterRegistrationsRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationsRequestProto,
+ Builder, GetRouterRegistrationsRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationsRequestProto, Builder,
+ GetRouterRegistrationsRequestProtoOrBuilder>(
+ GetRouterRegistrationsRequestProto.class);
+
+ public GetRouterRegistrationsRequestPBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationsRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
new file mode 100644
index 0000000..2d597fb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationsResponse.
+ */
+public class GetRouterRegistrationsResponsePBImpl
+ extends GetRouterRegistrationsResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationsResponseProto,
+ Builder, GetRouterRegistrationsResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationsResponseProto, Builder,
+ GetRouterRegistrationsResponseProtoOrBuilder>(
+ GetRouterRegistrationsResponseProto.class);
+
+ public GetRouterRegistrationsResponsePBImpl() {
+
+ }
+
+ @Override
+ public GetRouterRegistrationsResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<RouterState> getRouters() throws IOException {
+
+ List<RouterState> ret = new ArrayList<RouterState>();
+ List<RouterRecordProto> memberships =
+ this.translator.getProtoOrBuilder().getRoutersList();
+ for (RouterRecordProto memberProto : memberships) {
+ RouterState membership = new RouterStatePBImpl(memberProto);
+ ret.add(membership);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setRouters(List<RouterState> records) throws IOException {
+
+ this.translator.getBuilder().clearRouters();
+ for (RouterState router : records) {
+ if (router instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerPB = (RouterStatePBImpl) router;
+ this.translator.getBuilder().addRouters(routerPB.getProto());
+ }
+ }
+ }
+
+ @Override
+ public long getTimestamp() {
+ return this.translator.getProtoOrBuilder().getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long time) {
+ this.translator.getBuilder().setTimestamp(time);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..cc2be04
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RouterHeartbeatRequest.
+ */
+public class RouterHeartbeatRequestPBImpl extends RouterHeartbeatRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterHeartbeatRequestProto, Builder,
+ RouterHeartbeatRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterHeartbeatRequestProto,
+ Builder, RouterHeartbeatRequestProtoOrBuilder>(
+ RouterHeartbeatRequestProto.class);
+
+ public RouterHeartbeatRequestPBImpl() {
+ }
+
+ @Override
+ public RouterHeartbeatRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public RouterState getRouter() throws IOException {
+ RouterRecordProto routerProto =
+ this.translator.getProtoOrBuilder().getRouter();
+ return new RouterStatePBImpl(routerProto);
+ }
+
+ @Override
+ public void setRouter(RouterState routerState) {
+ if (routerState instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerStatePB = (RouterStatePBImpl)routerState;
+ this.translator.getBuilder().setRouter(routerStatePB.getProto());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..ac534fd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RouterHeartbeatResponse.
+ */
+public class RouterHeartbeatResponsePBImpl extends RouterHeartbeatResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterHeartbeatResponseProto, Builder,
+ RouterHeartbeatResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterHeartbeatResponseProto,
+ Builder, RouterHeartbeatResponseProtoOrBuilder>(
+ RouterHeartbeatResponseProto.class);
+
+ public RouterHeartbeatResponsePBImpl() {
+ }
+
+ @Override
+ public RouterHeartbeatResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
new file mode 100644
index 0000000..ccdd392
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Entry to log the state of a
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * FederationStateStoreService}.
+ */
+public abstract class RouterState extends BaseRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RouterState.class);
+
+ /** Expiration time in ms for this entry. */
+ private static long expirationMs;
+
+ /**
+ * Constructors.
+ */
+ public RouterState() {
+ super();
+ }
+
+ public static RouterState newInstance() {
+ RouterState record = StateStoreSerializer.newRecord(RouterState.class);
+ record.init();
+ return record;
+ }
+
+ public static RouterState newInstance(String addr, long startTime,
+ RouterServiceState status) {
+ RouterState record = newInstance();
+ record.setDateStarted(startTime);
+ record.setAddress(addr);
+ record.setStatus(status);
+ record.setCompileInfo(FederationUtil.getCompileInfo());
+ record.setBuildVersion(FederationUtil.getBuildVersion());
+ return record;
+ }
+
+ public abstract void setAddress(String address);
+
+ public abstract void setDateStarted(long dateStarted);
+
+ public abstract String getAddress();
+
+ public abstract StateStoreVersion getStateStoreVersion() throws IOException;
+
+ public abstract void setStateStoreVersion(StateStoreVersion version);
+
+ public abstract RouterServiceState getStatus();
+
+ public abstract void setStatus(RouterServiceState newStatus);
+
+ public abstract String getBuildVersion();
+
+ public abstract void setBuildVersion(String version);
+
+ public abstract String getCompileInfo();
+
+ public abstract void setCompileInfo(String info);
+
+ public abstract long getDateStarted();
+
+ /**
+ * Get the identifier for the Router. It uses the address.
+ *
+ * @return Identifier for the Router.
+ */
+ public String getRouterId() {
+ return getAddress();
+ }
+
+ @Override
+ public boolean like(BaseRecord o) {
+ if (o instanceof RouterState) {
+ RouterState other = (RouterState)o;
+ if (getAddress() != null &&
+ !getAddress().equals(other.getAddress())) {
+ return false;
+ }
+ if (getStatus() != null &&
+ !getStatus().equals(other.getStatus())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getAddress() + " -> " + getStatus() + "," + getBuildVersion();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<>();
+ map.put("address", getAddress());
+ return map;
+ }
+
+ @Override
+ public boolean validate() {
+ boolean ret = super.validate();
+ if ((getAddress() == null || getAddress().length() == 0) &&
+ getStatus() != RouterServiceState.INITIALIZING) {
+ LOG.error("Invalid router entry, no address specified {}", this);
+ ret = false;
+ }
+ return ret;
+ }
+
+ @Override
+ public int compareTo(BaseRecord other) {
+ if (other == null) {
+ return -1;
+ } else if (other instanceof RouterState) {
+ RouterState router = (RouterState) other;
+ return this.getAddress().compareTo(router.getAddress());
+ } else {
+ return super.compareTo(other);
+ }
+ }
+
+ @Override
+ public boolean checkExpired(long currentTime) {
+ if (super.checkExpired(currentTime)) {
+ setStatus(RouterServiceState.EXPIRED);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return RouterState.expirationMs;
+ }
+
+ public static void setExpirationMs(long time) {
+ RouterState.expirationMs = time;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
new file mode 100644
index 0000000..ce86eb0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Entry to track the version of the State Store data stored in the State Store
+ * by a Router.
+ */
+public abstract class StateStoreVersion extends BaseRecord {
+
+ public static StateStoreVersion newInstance() {
+ return StateStoreSerializer.newRecord(StateStoreVersion.class);
+ }
+
+ public static StateStoreVersion newInstance(long membershipVersion,
+ long mountTableVersion) {
+ StateStoreVersion record = newInstance();
+ record.setMembershipVersion(membershipVersion);
+ record.setMountTableVersion(mountTableVersion);
+ return record;
+ }
+
+ public abstract long getMembershipVersion();
+
+ public abstract void setMembershipVersion(long version);
+
+ public abstract long getMountTableVersion();
+
+ public abstract void setMountTableVersion(long version);
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ // This record is not stored directly, no key needed
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ return map;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ // This record is not stored directly, no expiration needed
+ return -1;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateModified() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateCreated() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public String toString() {
+ return "Membership: " + getMembershipVersion() +
+ " Mount Table: " + getMountTableVersion();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
new file mode 100644
index 0000000..aebc7a8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the RouterState record.
+ */
+public class RouterStatePBImpl extends RouterState implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterRecordProto, Builder,
+ RouterRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterRecordProto, Builder,
+ RouterRecordProtoOrBuilder>(RouterRecordProto.class);
+
+ public RouterStatePBImpl() {
+ }
+
+ public RouterStatePBImpl(RouterRecordProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public RouterRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setAddress(String address) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (address == null) {
+ builder.clearAddress();
+ } else {
+ builder.setAddress(address);
+ }
+ }
+
+ @Override
+ public String getAddress() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasAddress()) {
+ return null;
+ }
+ return proto.getAddress();
+ }
+
+ @Override
+ public void setStateStoreVersion(StateStoreVersion version) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (version instanceof StateStoreVersionPBImpl) {
+ StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
+ StateStoreVersionRecordProto versionProto =
+ (StateStoreVersionRecordProto)versionPB.getProto();
+ builder.setStateStoreVersion(versionProto);
+ } else {
+ builder.clearStateStoreVersion();
+ }
+ }
+
+ @Override
+ public StateStoreVersion getStateStoreVersion() throws IOException {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasStateStoreVersion()) {
+ return null;
+ }
+ StateStoreVersionRecordProto versionProto = proto.getStateStoreVersion();
+ StateStoreVersion version =
+ StateStoreSerializer.newRecord(StateStoreVersion.class);
+ if (version instanceof StateStoreVersionPBImpl) {
+ StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
+ versionPB.setProto(versionProto);
+ return versionPB;
+ } else {
+ throw new IOException("Cannot get State Store version");
+ }
+ }
+
+ @Override
+ public RouterServiceState getStatus() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasStatus()) {
+ return null;
+ }
+ return RouterServiceState.valueOf(proto.getStatus());
+ }
+
+ @Override
+ public void setStatus(RouterServiceState newStatus) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (newStatus == null) {
+ builder.clearStatus();
+ } else {
+ builder.setStatus(newStatus.toString());
+ }
+ }
+
+ @Override
+ public String getBuildVersion() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasBuildVersion()) {
+ return null;
+ }
+ return proto.getBuildVersion();
+ }
+
+ @Override
+ public void setBuildVersion(String version) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (version == null) {
+ builder.clearBuildVersion();
+ } else {
+ builder.setBuildVersion(version);
+ }
+ }
+
+ @Override
+ public String getCompileInfo() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasCompileInfo()) {
+ return null;
+ }
+ return proto.getCompileInfo();
+ }
+
+ @Override
+ public void setCompileInfo(String info) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (info == null) {
+ builder.clearCompileInfo();
+ } else {
+ builder.setCompileInfo(info);
+ }
+ }
+
+ @Override
+ public void setDateStarted(long dateStarted) {
+ this.translator.getBuilder().setDateStarted(dateStarted);
+ }
+
+ @Override
+ public long getDateStarted() {
+ return this.translator.getProtoOrBuilder().getDateStarted();
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
new file mode 100644
index 0000000..7696136
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the StateStoreVersion record.
+ */
+public class StateStoreVersionPBImpl extends StateStoreVersion
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<StateStoreVersionRecordProto, Builder,
+ StateStoreVersionRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<StateStoreVersionRecordProto,
+ Builder, StateStoreVersionRecordProtoOrBuilder>(
+ StateStoreVersionRecordProto.class);
+
+ public StateStoreVersionPBImpl() {
+ }
+
+ @Override
+ public StateStoreVersionRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public long getMembershipVersion() {
+ return this.translator.getProtoOrBuilder().getMembershipVersion();
+ }
+
+ @Override
+ public void setMembershipVersion(long version) {
+ this.translator.getBuilder().setMembershipVersion(version);
+ }
+
+ @Override
+ public long getMountTableVersion() {
+ return this.translator.getProtoOrBuilder().getMountTableVersion();
+ }
+
+ @Override
+ public void setMountTableVersion(long version) {
+ this.translator.getBuilder().setMountTableVersion(version);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
index 5d9b9d4..f98d0ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
@@ -172,3 +172,47 @@ message GetMountTableEntriesResponseProto {
optional uint64 timestamp = 2;
}
+
+/////////////////////////////////////////////////
+// Routers
+/////////////////////////////////////////////////
+
+message StateStoreVersionRecordProto {
+ optional uint64 membershipVersion = 1;
+ optional uint64 mountTableVersion = 2;
+}
+
+message RouterRecordProto {
+ optional uint64 dateCreated = 1;
+ optional uint64 dateModified = 2;
+ optional string address = 3;
+ optional string status = 4;
+ optional StateStoreVersionRecordProto stateStoreVersion = 5;
+ optional string buildVersion = 6;
+ optional string compileInfo = 7;
+ optional uint64 dateStarted = 8;
+}
+
+message GetRouterRegistrationRequestProto {
+ optional string routerId = 1;
+}
+
+message GetRouterRegistrationResponseProto {
+ optional RouterRecordProto router = 1;
+}
+
+message GetRouterRegistrationsRequestProto {
+}
+
+message GetRouterRegistrationsResponseProto {
+ optional uint64 timestamp = 1;
+ repeated RouterRecordProto routers = 2;
+}
+
+message RouterHeartbeatRequestProto {
+ optional RouterRecordProto router = 1;
+}
+
+message RouterHeartbeatResponseProto {
+ optional bool status = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index 01fe149..1091c21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -37,6 +37,7 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
@@ -44,6 +45,8 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
import org.junit.AfterClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -94,6 +97,10 @@ public class TestStateStoreDriverBase {
return randomString;
}
+ private long generateRandomLong() {
+ return RANDOM.nextLong();
+ }
+
@SuppressWarnings("rawtypes")
private <T extends Enum> T generateRandomEnum(Class<T> enumClass) {
int x = RANDOM.nextInt(enumClass.getEnumConstants().length);
@@ -117,6 +124,12 @@ public class TestStateStoreDriverBase {
Map<String, String> destMap = Collections.singletonMap(
generateRandomString(), "/" + generateRandomString());
return (T) MountTable.newInstance(src, destMap);
+ } else if (recordClass == RouterState.class) {
+ RouterState routerState = RouterState.newInstance(generateRandomString(),
+ generateRandomLong(), generateRandomEnum(RouterServiceState.class));
+ StateStoreVersion version = generateFakeRecord(StateStoreVersion.class);
+ routerState.setStateStoreVersion(version);
+ return (T) routerState;
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95743c67/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
new file mode 100644
index 0000000..76e0b96
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.junit.Test;
+
+/**
+ * Test the Router State records.
+ */
+public class TestRouterState {
+
+ private static final String ADDRESS = "address";
+ private static final String BUILD_VERSION = "buildVersion";
+ private static final String COMPILE_INFO = "compileInfo";
+ private static final long START_TIME = 100;
+ private static final long DATE_MODIFIED = 200;
+ private static final long DATE_CREATED = 300;
+ private static final long FILE_RESOLVER_VERSION = 500;
+ private static final RouterServiceState STATE = RouterServiceState.RUNNING;
+
+
+ private RouterState generateRecord() throws IOException {
+ RouterState record = RouterState.newInstance(ADDRESS, START_TIME, STATE);
+ record.setBuildVersion(BUILD_VERSION);
+ record.setCompileInfo(COMPILE_INFO);
+ record.setDateCreated(DATE_CREATED);
+ record.setDateModified(DATE_MODIFIED);
+
+ StateStoreVersion version = StateStoreVersion.newInstance();
+ version.setMountTableVersion(FILE_RESOLVER_VERSION);
+ record.setStateStoreVersion(version);
+ return record;
+ }
+
+ private void validateRecord(RouterState record) throws IOException {
+ assertEquals(ADDRESS, record.getAddress());
+ assertEquals(START_TIME, record.getDateStarted());
+ assertEquals(STATE, record.getStatus());
+ assertEquals(COMPILE_INFO, record.getCompileInfo());
+ assertEquals(BUILD_VERSION, record.getBuildVersion());
+
+ StateStoreVersion version = record.getStateStoreVersion();
+ assertEquals(FILE_RESOLVER_VERSION, version.getMountTableVersion());
+ }
+
+ @Test
+ public void testGetterSetter() throws IOException {
+ RouterState record = generateRecord();
+ validateRecord(record);
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+
+ RouterState record = generateRecord();
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ RouterState newRecord =
+ serializer.deserialize(serializedString, RouterState.class);
+
+ validateRecord(newRecord);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-7765. Fixed an issue that
kerberos tgt not found when NM posting timeline events. Contributed by Rohith
Sharma K S
Posted by ha...@apache.org.
YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/443523f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/443523f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/443523f9
Branch: refs/heads/YARN-1011
Commit: 443523f9c0fcc4ba2503791090b1879c6031759b
Parents: 649ef7a
Author: Jian He <ji...@apache.org>
Authored: Sat Jan 27 22:18:51 2018 -0800
Committer: Jian He <ji...@apache.org>
Committed: Sat Jan 27 22:18:51 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/nodemanager/NodeManager.java | 11 ++++-------
.../nodemanager/timelineservice/NMTimelinePublisher.java | 1 +
.../timelineservice/storage/HBaseTimelineWriterImpl.java | 5 +++++
3 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 5cacd20..42b7b5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -454,18 +454,15 @@ public class NodeManager extends CompositeService
((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
nmStore.setNodeStatusUpdater(nodeStatusUpdater);
- super.serviceInit(conf);
- // TODO add local dirs to del
- }
-
- @Override
- protected void serviceStart() throws Exception {
+ // Do secure login before calling init for added services.
try {
doSecureLogin();
} catch (IOException e) {
throw new YarnRuntimeException("Failed NodeManager login", e);
}
- super.serviceStart();
+
+ super.serviceInit(conf);
+ // TODO add local dirs to del
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index b8192ca..2ce3c5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -100,6 +100,7 @@ public class NMTimelinePublisher extends CompositeService {
this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ?
UserGroupInformation.getLoginUser() :
UserGroupInformation.getCurrentUser();
+ LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI);
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 9e9134c..f938185 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -129,6 +129,11 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
new FlowActivityTable().getTableMutator(hbaseConf, conn);
subApplicationTable =
new SubApplicationTable().getTableMutator(hbaseConf, conn);
+
+ UserGroupInformation ugi = UserGroupInformation.isSecurityEnabled() ?
+ UserGroupInformation.getLoginUser() :
+ UserGroupInformation.getCurrentUser();
+ LOG.info("Initialized HBaseTimelineWriterImpl UGI to " + ugi);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-7810. Update
TestDockerContainerRuntime to test with current user credential. (contributed
by Shane Kumpf)
Posted by ha...@apache.org.
YARN-7810. Update TestDockerContainerRuntime to test with current user credential.
(contributed by Shane Kumpf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59828be1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59828be1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59828be1
Branch: refs/heads/YARN-1011
Commit: 59828be1978ec942dda38774a1d9f741efa96f71
Parents: 051ad9d
Author: Eric Yang <ey...@apache.org>
Authored: Wed Jan 24 21:24:19 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jan 24 21:24:19 2018 -0500
----------------------------------------------------------------------
.../runtime/TestDockerContainerRuntime.java | 228 ++++++++++---------
1 file changed, 126 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59828be1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index da1d1eb..a825828 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -117,7 +117,8 @@ public class TestDockerContainerRuntime {
private HashMap<String, String> env;
private String image;
private String uidGidPair;
- private String runAsUser;
+ private String runAsUser = System.getProperty("user.name");
+ private String[] groups = {};
private String user;
private String appId;
private String containerIdStr = containerId;
@@ -166,8 +167,37 @@ public class TestDockerContainerRuntime {
when(context.getEnvironment()).thenReturn(env);
when(container.getUser()).thenReturn(submittingUser);
- uidGidPair = "";
- runAsUser = "run_as_user";
+ // Get the running user's uid and gid for remap
+ String uid = "";
+ String gid = "";
+ Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-u", runAsUser});
+ Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-g", runAsUser});
+ Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", runAsUser});
+ try {
+ shexec1.execute();
+ // get rid of newline at the end
+ uid = shexec1.getOutput().replaceAll("\n$", "");
+ } catch (Exception e) {
+ LOG.info("Could not run id -u command: " + e);
+ }
+ try {
+ shexec2.execute();
+ // get rid of newline at the end
+ gid = shexec2.getOutput().replaceAll("\n$", "");
+ } catch (Exception e) {
+ LOG.info("Could not run id -g command: " + e);
+ }
+ try {
+ shexec3.execute();
+ groups = shexec3.getOutput().replace("\n", " ").split(" ");
+ } catch (Exception e) {
+ LOG.info("Could not run id -G command: " + e);
+ }
+ uidGidPair = uid + ":" + gid;
+
user = "user";
appId = "app_id";
containerIdStr = containerId;
@@ -336,7 +366,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 13;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -346,6 +376,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -361,7 +393,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
}
@@ -372,13 +404,6 @@ public class TestDockerContainerRuntime {
IOException {
conf.setBoolean(YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
true);
- Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
- new String[]{"whoami"});
- shexec.execute();
- // get rid of newline at the end
- runAsUser = shexec.getOutput().replaceAll("\n$", "");
- builder.setExecutionAttribute(RUN_AS_USER, runAsUser);
-
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, null);
@@ -388,37 +413,6 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
- String uid = "";
- String gid = "";
- String[] groups = {};
- Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
- new String[]{"id", "-u", runAsUser});
- Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
- new String[]{"id", "-g", runAsUser});
- Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor(
- new String[]{"id", "-G", runAsUser});
- try {
- shexec1.execute();
- // get rid of newline at the end
- uid = shexec1.getOutput().replaceAll("\n$", "");
- } catch (Exception e) {
- LOG.info("Could not run id -u command: " + e);
- }
- try {
- shexec2.execute();
- // get rid of newline at the end
- gid = shexec2.getOutput().replaceAll("\n$", "");
- } catch (Exception e) {
- LOG.info("Could not run id -g command: " + e);
- }
- try {
- shexec3.execute();
- groups = shexec3.getOutput().replace("\n", " ").split(" ");
- } catch (Exception e) {
- LOG.info("Could not run id -G command: " + e);
- }
- uidGidPair = uid + ":" + gid;
-
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@@ -540,7 +534,7 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 13;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -550,6 +544,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=test.hostname",
dockerCommands.get(counter++));
Assert
@@ -567,7 +563,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
}
@@ -606,7 +602,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 13;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -616,6 +612,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -631,7 +629,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
@@ -659,6 +657,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -675,7 +675,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
@@ -712,7 +712,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 13;
+ int expected = 14;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -763,7 +763,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -773,6 +773,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -789,7 +791,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
}
@@ -813,7 +815,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 13;
+ int expected = 14;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -922,7 +924,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -932,6 +934,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -948,7 +952,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
}
@@ -1039,33 +1043,39 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(14, dockerCommands.size());
- Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ int expected = 15;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
- dockerCommands.get(1));
- Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
- Assert.assertEquals(" detach=true", dockerCommands.get(3));
- Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
- Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert.assertEquals(" image=busybox:latest",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
- dockerCommands.get(7));
- Assert.assertEquals(" name=container_id", dockerCommands.get(8));
- Assert.assertEquals(" net=host", dockerCommands.get(9));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(
" ro-mounts=/test_local_dir/test_resource_file:test_mount",
- dockerCommands.get(10));
+ dockerCommands.get(counter++));
Assert.assertEquals(
" rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ "/test_filecache_dir:/test_filecache_dir,"
+ "/test_container_work_dir:/test_container_work_dir,"
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
- dockerCommands.get(11));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
- dockerCommands.get(13));
+ dockerCommands.get(counter++));
}
@Test
@@ -1109,34 +1119,40 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(14, dockerCommands.size());
- Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ int expected = 15;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
- dockerCommands.get(1));
- Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
- Assert.assertEquals(" detach=true", dockerCommands.get(3));
- Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
- Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert.assertEquals(" image=busybox:latest",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
- dockerCommands.get(7));
- Assert.assertEquals(" name=container_id", dockerCommands.get(8));
- Assert.assertEquals(" net=host", dockerCommands.get(9));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(
" ro-mounts=/test_local_dir/test_resource_file:test_mount1,"
+ "/test_local_dir/test_resource_file:test_mount2",
- dockerCommands.get(10));
+ dockerCommands.get(counter++));
Assert.assertEquals(
" rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ "/test_filecache_dir:/test_filecache_dir,"
+ "/test_container_work_dir:/test_container_work_dir,"
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
- dockerCommands.get(11));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
- dockerCommands.get(13));
+ dockerCommands.get(counter++));
}
@@ -1160,22 +1176,28 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(14, dockerCommands.size());
- Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ int expected = 15;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
- dockerCommands.get(1));
- Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
- Assert.assertEquals(" detach=true", dockerCommands.get(3));
- Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
- Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert.assertEquals(" image=busybox:latest",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
- dockerCommands.get(7));
- Assert.assertEquals(" name=container_id", dockerCommands.get(8));
- Assert.assertEquals(" net=host", dockerCommands.get(9));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" ro-mounts=/tmp/foo:/tmp/foo",
- dockerCommands.get(10));
+ dockerCommands.get(counter++));
Assert.assertEquals(
" rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ "/test_filecache_dir:/test_filecache_dir,"
@@ -1183,10 +1205,10 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir,"
+ "/tmp/bar:/tmp/bar",
- dockerCommands.get(11));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
- dockerCommands.get(13));
+ dockerCommands.get(counter++));
}
@Test
@@ -1265,7 +1287,7 @@ public class TestDockerContainerRuntime {
PrivilegedOperation op = capturePrivilegedOperation();
Assert.assertEquals(op.getOperationType(),
PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
- Assert.assertEquals("run_as_user", op.getArguments().get(0));
+ Assert.assertEquals(runAsUser, op.getArguments().get(0));
Assert.assertEquals("user", op.getArguments().get(1));
Assert.assertEquals("2", op.getArguments().get(2));
Assert.assertEquals("1234", op.getArguments().get(3));
@@ -1639,7 +1661,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 16;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1649,6 +1671,8 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
@@ -1666,7 +1690,7 @@ public class TestDockerContainerRuntime {
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
// Verify volume-driver is set to expected value.
Assert.assertEquals(" volume-driver=driver-1",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-7628. [Documentation]
Documenting the ability to disable elasticity at Capacity Scheduler's sub
queues. (Zian Chen via wangda)
Posted by ha...@apache.org.
YARN-7628. [Documentation] Documenting the ability to disable elasticity at Capacity Scheduler's sub queues. (Zian Chen via wangda)
Change-Id: I5c37df1c5bee9e3a78379775ef307ef94506ebc4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/051ad9d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/051ad9d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/051ad9d6
Branch: refs/heads/YARN-1011
Commit: 051ad9d63f41ae98be728a7cdbdad9b072316318
Parents: 0c559b2
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jan 25 07:24:57 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jan 25 07:24:57 2018 +0800
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/051ad9d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 2598997..87cfd39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -123,7 +123,7 @@ Configuration
| Property | Description |
|:---- |:---- |
| `yarn.scheduler.capacity.<queue-path>.capacity` | Queue *capacity* in percentage (%) as a float (e.g. 12.5) OR as absolute resource queue minimum capacity. The sum of capacities for all queues, at each level, must be equal to 100. However if absolute resource is configured, sum of absolute resources of child queues could be less than it's parent absolute resource capacity. Applications in the queue may consume more resources than the queue's capacity if there are free resources, providing elasticity. |
-| `yarn.scheduler.capacity.<queue-path>.maximum-capacity` | Maximum queue capacity in percentage (%) as a float OR as absolute resource queue maximum capacity. This limits the *elasticity* for applications in the queue. Defaults to -1 which disables it. |
+| `yarn.scheduler.capacity.<queue-path>.maximum-capacity` | Maximum queue capacity in percentage (%) as a float OR as absolute resource queue maximum capacity. This limits the *elasticity* for applications in the queue. 1) Value is between 0 and 100. 2) Admin needs to make sure absolute maximum capacity >= absolute capacity for each queue. Also, setting this value to -1 sets maximum capacity to 100%. |
| `yarn.scheduler.capacity.<queue-path>.minimum-user-limit-percent` | Each queue enforces a limit on the percentage of resources allocated to a user at any given time, if there is demand for resources. The user limit can vary between a minimum and maximum value. The former (the minimum value) is set to this property value and the latter (the maximum value) depends on the number of users who have submitted applications. For e.g., suppose the value of this property is 25. If two users have submitted applications to a queue, no single user can use more than 50% of the queue resources. If a third user submits an application, no single user can use more than 33% of the queue resources. With 4 or more users, no user can use more than 25% of the queues resources. A value of 100 implies no user limits are imposed. The default is 100. Value is specified as a integer. |
| `yarn.scheduler.capacity.<queue-path>.user-limit-factor` | The multiple of the queue capacity which can be configured to allow a single user to acquire more resources. By default this is set to 1 which ensures that a single user can never take more than the queue's configured capacity irrespective of how idle the cluster is. Value is specified as a float. |
| `yarn.scheduler.capacity.<queue-path>.maximum-allocation-mb` | The per queue maximum limit of memory to allocate to each container request at the Resource Manager. This setting overrides the cluster configuration `yarn.scheduler.maximum-allocation-mb`. This value must be smaller than or equal to the cluster maximum. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-6705 Add separate NM
preemption thresholds for cpu and memory (Haibo Chen)
Posted by ha...@apache.org.
YARN-6705 Add separate NM preemption thresholds for cpu and memory (Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7158d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7158d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7158d46
Branch: refs/heads/YARN-1011
Commit: e7158d46b41cb34abfcc26412ec104a355088e9d
Parents: 31a732e
Author: Haibo Chen <ha...@apache.org>
Authored: Wed Jul 12 12:32:13 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 17:43:50 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 31 +++++++++++++--
.../src/main/resources/yarn-default.xml | 34 ++++++++++++++--
.../monitor/ContainersMonitorImpl.java | 42 +++++++++++++-------
3 files changed, 85 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7158d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ded0c1f..ed8404c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1978,10 +1978,33 @@ public class YarnConfiguration extends Configuration {
public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
NM_PREFIX + "overallocation.memory-utilization-threshold";
- public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
- NM_PREFIX + "overallocation.preemption-threshold";
- public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
- = 0.96f;
+ /**
+ * The CPU utilization threshold, if went beyond for a few times in a row,
+ * OPPORTUNISTIC containers started due to overallocation should start
+ * getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.cpu";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
+
+ /**
+ * The number of times that CPU utilization must go over the CPU preemption
+ * threshold consecutively before preemption starts to kick in.
+ */
+ public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
+ NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
+ public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
+
+
+ /**
+ * The memory utilization threshold beyond which OPPORTUNISTIC containers
+ * started due to overallocation should start getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.memory";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
/**
* Interval of time the linux container executor should try cleaning up
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7158d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 27e1996..da31694 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1711,11 +1711,37 @@
<property>
<description>When a node is over-allocated to improve utilization by
- running OPPORTUNISTIC containers, this config captures the utilization
- beyond which OPPORTUNISTIC containers should start getting preempted.
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted. This is used in combination with
+ yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
+ only when the CPU utilization goes over this threshold consecutively for
+ a few times will preemption kicks in.
</description>
- <name>yarn.nodemanager.overallocation.preemption-threshold</name>
- <value>0.96</value>
+ <name>yarn.nodemanager.overallocation.preemption-threshold.cpu</name>
+ <value>0.99</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the number of
+ times that CPU utilization has to go above
+ ${yarn.nodemanager.overallocation.preemption-threshold.cpu}
+ consecutively for NM to start preempting OPPORTUNISTIC containers
+ started due to overallocation.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold-count.cpu</name>
+ <value>4</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold.memory</name>
+ <value>0.95</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7158d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 65091f7..4863ee2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -106,6 +106,7 @@ public class ContainersMonitorImpl extends AbstractService implements
private ResourceUtilization containersUtilization;
private ResourceThresholds overAllocationPreemptionThresholds;
+ private int overAlloctionPreemptionCpuCount = -1;
private volatile boolean stopped = false;
@@ -252,7 +253,7 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationMemoryUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the memory " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
@@ -264,36 +265,49 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationCpuUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the CPU " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
- float preemptionThreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
- if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the cpu utilization threshold.");
+ float cpuPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
+ if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the cpu " +
+ " preemption threshold is no larger than the cpu overallocation" +
+ " threshold.");
return;
}
- if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the memory utilization threshold.");
+
+ float memoryPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
+ if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the memory" +
+ " preemption threshold is no larger than the memory overallocation" +
+ " threshold.");
return;
}
+ this.overAlloctionPreemptionCpuCount = conf.getInt(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
+
ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
overAllocationCpuUtilizationThreshold,
overAllocationMemoryUtilizationThreshold);
((NodeManager.NMContext) context).setOverAllocationInfo(
OverAllocationInfo.newInstance(resourceThresholds));
- this.overAllocationPreemptionThresholds =
- ResourceThresholds.newInstance(preemptionThreshold);
+ this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
+ cpuPreemptionThreshold, memoryPreemptionThreshold);
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
- " threshold: " + preemptionThreshold);
+ " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
+ cpuPreemptionThreshold + ")");
}
private boolean isResourceCalculatorAvailable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-7460. Exclude findbugs
warnings on SchedulerNode.numGuaranteedContainers and
numOpportunisticContainers. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7460. Exclude findbugs warnings on SchedulerNode.numGuaranteedContainers and numOpportunisticContainers. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dcead07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dcead07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dcead07
Branch: refs/heads/YARN-1011
Commit: 7dcead07b2922d35c5de0f906f1668ed301a7c02
Parents: bdf3e66
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Mon Nov 20 13:27:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:08:11 2018 -0800
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dcead07/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 6a10312..26ba5b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -76,6 +76,16 @@
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numOpportunisticContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numGuaranteedContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl$AppRejectedTransition" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-7064. Use cgroup to get
container resource utilization. (Miklos Szegedi via Haibo Chen)
Posted by ha...@apache.org.
YARN-7064. Use cgroup to get container resource utilization. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/649ef7ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/649ef7ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/649ef7ac
Branch: refs/heads/YARN-1011
Commit: 649ef7ac334e63a7c676f8e7406f59d9466eb6f2
Parents: 6eef3d7
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Jan 26 16:27:31 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Jan 26 16:27:31 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/util/CpuTimeTracker.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +-
.../yarn/conf/TestYarnConfigurationFields.java | 2 -
.../yarn/util/ProcfsBasedProcessTree.java | 3 +
.../util/ResourceCalculatorProcessTree.java | 9 +
.../src/main/resources/yarn-default.xml | 34 +-
.../linux/resources/CGroupsHandler.java | 9 +-
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../CGroupsMemoryResourceHandlerImpl.java | 52 +--
.../resources/CGroupsResourceCalculator.java | 357 +++++++++++++++++++
.../resources/CombinedResourceCalculator.java | 108 ++++++
.../linux/resources/ResourceHandlerModule.java | 43 ++-
.../monitor/ContainersMonitorImpl.java | 33 +-
.../TestCGroupsMemoryResourceHandlerImpl.java | 45 +++
.../TestCGroupsResourceCalculator.java | 274 ++++++++++++++
.../TestCompareResourceCalculators.java | 227 ++++++++++++
.../resources/TestResourceHandlerModule.java | 47 ++-
17 files changed, 1183 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CpuTimeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CpuTimeTracker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CpuTimeTracker.java
index b4ebe86..4355367 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CpuTimeTracker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CpuTimeTracker.java
@@ -99,7 +99,7 @@ public class CpuTimeTracker {
public void updateElapsedJiffies(BigInteger elapsedJiffies, long newTime) {
BigInteger newValue = elapsedJiffies.multiply(jiffyLengthInMillis);
cumulativeCpuTime = newValue.compareTo(cumulativeCpuTime) >= 0 ?
- newValue : cumulativeCpuTime;
+ newValue : cumulativeCpuTime;
sampleTime = newTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f132683..bbbfc52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1357,22 +1357,20 @@ public class YarnConfiguration extends Configuration {
public static final String NM_MEMORY_RESOURCE_PREFIX = NM_PREFIX
+ "resource.memory.";
- @Private
public static final String NM_MEMORY_RESOURCE_ENABLED =
NM_MEMORY_RESOURCE_PREFIX + "enabled";
- @Private
public static final boolean DEFAULT_NM_MEMORY_RESOURCE_ENABLED = false;
- @Private
+ public static final String NM_MEMORY_RESOURCE_ENFORCED =
+ NM_MEMORY_RESOURCE_PREFIX + "enforced";
+ public static final boolean DEFAULT_NM_MEMORY_RESOURCE_ENFORCED = true;
+
public static final String NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS =
NM_MEMORY_RESOURCE_PREFIX + "cgroups.swappiness";
- @Private
public static final int DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS = 0;
- @Private
public static final String NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE =
NM_MEMORY_RESOURCE_PREFIX + "cgroups.soft-limit-percentage";
- @Private
public static final float
DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE =
90.0f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3976d2d..9fe4f88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -159,8 +159,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPrefixToSkipCompare
.add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED);
configurationPrefixToSkipCompare
- .add(YarnConfiguration.NM_MEMORY_RESOURCE_PREFIX);
- configurationPrefixToSkipCompare
.add(YarnConfiguration.NM_CPU_RESOURCE_ENABLED);
configurationPrefixToSkipCompare.add(
YarnConfiguration.NM_NETWORK_TAG_MAPPING_MANAGER);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 7431fdf..55be001 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -468,6 +468,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
@Override
public float getCpuUsagePercent() {
BigInteger processTotalJiffies = getTotalProcessJiffies();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Process " + pid + " jiffies:" + processTotalJiffies);
+ }
cpuTimeTracker.updateElapsedJiffies(processTotalJiffies,
clock.getTime());
return cpuTimeTracker.getCpuTrackerUsagePercent();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
index 7e5cf55..c581b83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* Interface class to obtain process resource usage
@@ -51,6 +52,13 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
}
/**
+ * Initialize the object.
+ * @throws YarnException Throws an exception on error.
+ */
+ public void initialize() throws YarnException {
+ }
+
+ /**
* Update the process-tree with latest state.
*
* Each call to this function should increment the age of the running
@@ -168,6 +176,7 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
Constructor <? extends ResourceCalculatorProcessTree> c = clazz.getConstructor(String.class);
ResourceCalculatorProcessTree rctree = c.newInstance(pid);
rctree.setConf(conf);
+ rctree.initialize();
return rctree;
} catch(Exception e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1348d6d..0bb4fca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1309,6 +1309,37 @@
</property>
<property>
+ <description>Whether YARN CGroups memory tracking is enabled.</description>
+ <name>yarn.nodemanager.resource.memory.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Whether YARN CGroups strict memory enforcement is enabled.
+ </description>
+ <name>yarn.nodemanager.resource.memory.enforced</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>If memory limit is enforced, this the percentage of soft limit
+ compared to the memory assigned to the container. If there is memory
+ pressure container memory usage will be pushed back to its soft limit
+ by swapping out memory.
+ </description>
+ <name>yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage</name>
+ <value>90.0</value>
+ </property>
+
+ <property>
+ <description>Container swappiness is the likelihood a page will be swapped
+ out compared to be kept in memory. Value is between 0-100.
+ </description>
+ <name>yarn.nodemanager.resource.memory.cgroups.swappiness</name>
+ <value>0</value>
+ </property>
+
+ <property>
<description>Whether physical memory limits will be enforced for
containers.</description>
<name>yarn.nodemanager.pmem-check-enabled</name>
@@ -1622,7 +1653,8 @@
or be allowed to consume spare resources if they need them. For example, turning the
flag on will restrict apps to use only their share of CPU, even if the node has spare
CPU cycles. The default value is false i.e. use available resources. Please note that
- turning this flag on may reduce job throughput on the cluster.</description>
+ turning this flag on may reduce job throughput on the cluster. This setting does
+ not apply to other subsystems like memory.</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
<value>false</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
index 5f4d3e4..e279504 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
@@ -54,7 +54,7 @@ public interface CGroupsHandler {
this.name = name;
}
- String getName() {
+ public String getName() {
return name;
}
@@ -113,6 +113,13 @@ public interface CGroupsHandler {
ResourceHandlerException;
/**
+ * Gets the absolute path to the specified cgroup controller.
+ * @param controller - controller type for the cgroup
+ * @return the root of the controller.
+ */
+ String getControllerPath(CGroupController controller);
+
+ /**
* Gets the relative path for the cgroup, independent of a controller, for a
* given cgroup id.
* @param cGroupId - id of the cgroup
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 619a65b..008f3d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -125,7 +125,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
initializeControllerPaths();
}
- private String getControllerPath(CGroupController controller) {
+ @Override
+ public String getControllerPath(CGroupController controller) {
try {
rwLock.readLock().lock();
return controllerPaths.get(controller);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index d3e787e..558751f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -52,6 +52,7 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
private static final int OPPORTUNISTIC_SOFT_LIMIT = 0;
private CGroupsHandler cGroupsHandler;
+ private boolean enforce = true;
private int swappiness = 0;
// multiplier to set the soft limit - value should be between 0 and 1
private float softLimit = 0.0f;
@@ -79,6 +80,9 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
throw new ResourceHandlerException(msg);
}
this.cGroupsHandler.initializeCGroupController(MEMORY);
+ enforce = conf.getBoolean(
+ YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED,
+ YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_ENFORCED);
swappiness = conf
.getInt(YarnConfiguration.NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS,
YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS);
@@ -124,31 +128,33 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
(long) (container.getResource().getMemorySize() * this.softLimit);
long containerHardLimit = container.getResource().getMemorySize();
cGroupsHandler.createCGroup(MEMORY, cgroupId);
- try {
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES,
- String.valueOf(containerHardLimit) + "M");
- ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
- if (id != null && id.getExecutionType() ==
- ExecutionType.OPPORTUNISTIC) {
+ if (enforce) {
+ try {
cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
- String.valueOf(OPPORTUNISTIC_SOFT_LIMIT) + "M");
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
- String.valueOf(OPPORTUNISTIC_SWAPPINESS));
- } else {
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
- String.valueOf(containerSoftLimit) + "M");
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
- String.valueOf(swappiness));
+ CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES,
+ String.valueOf(containerHardLimit) + "M");
+ ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
+ if (id != null && id.getExecutionType() ==
+ ExecutionType.OPPORTUNISTIC) {
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
+ String.valueOf(OPPORTUNISTIC_SOFT_LIMIT) + "M");
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
+ String.valueOf(OPPORTUNISTIC_SWAPPINESS));
+ } else {
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
+ String.valueOf(containerSoftLimit) + "M");
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
+ String.valueOf(swappiness));
+ }
+ } catch (ResourceHandlerException re) {
+ cGroupsHandler.deleteCGroup(MEMORY, cgroupId);
+ LOG.warn("Could not update cgroup for container", re);
+ throw re;
}
- } catch (ResourceHandlerException re) {
- cGroupsHandler.deleteCGroup(MEMORY, cgroupId);
- LOG.warn("Could not update cgroup for container", re);
- throw re;
}
List<PrivilegedOperation> ret = new ArrayList<>();
ret.add(new PrivilegedOperation(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java
new file mode 100644
index 0000000..50ce3ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java
@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.SysInfoLinux;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.SystemClock;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A cgroups file-system based Resource calculator without the process tree
+ * features.
+ *
+ * CGroups has its limitations. It can only be enabled, if both CPU and memory
+ * cgroups are enabled with yarn.nodemanager.resource.cpu.enabled and
+ * yarn.nodemanager.resource.memory.enabled respectively. This means that
+ * memory limits are enforced by default. You can turn this off and keep
+ * memory reporting only with yarn.nodemanager.resource.memory.enforced.
+ *
+ * Another limitation is virtual memory measurement. CGroups does not have the
+ * ability to measure virtual memory usage. This includes memory reserved but
+ * not used. CGroups measures used memory as sa sum of
+ * physical memory and swap usage. This will be returned in the virtual
+ * memory counters.
+ * If the real virtual memory is required please use the legacy procfs based
+ * resource calculator or CombinedResourceCalculator.
+ */
+public class CGroupsResourceCalculator extends ResourceCalculatorProcessTree {
+ enum Result {
+ Continue,
+ Exit
+ }
+ protected static final Log LOG = LogFactory
+ .getLog(CGroupsResourceCalculator.class);
+ private static final String PROCFS = "/proc";
+ static final String CGROUP = "cgroup";
+ static final String CPU_STAT = "cpuacct.stat";
+ static final String MEM_STAT = "memory.usage_in_bytes";
+ static final String MEMSW_STAT = "memory.memsw.usage_in_bytes";
+ private static final String USER = "user ";
+ private static final String SYSTEM = "system ";
+
+ private static final Pattern CGROUP_FILE_FORMAT = Pattern.compile(
+ "^(\\d+):([^:]+):/(.*)$");
+ private final String procfsDir;
+ private CGroupsHandler cGroupsHandler;
+
+ private String pid;
+ private File cpuStat;
+ private File memStat;
+ private File memswStat;
+
+ private BigInteger processTotalJiffies;
+ private long processPhysicalMemory;
+ private long processVirtualMemory;
+
+ private final long jiffyLengthMs;
+ private final CpuTimeTracker cpuTimeTracker;
+ private Clock clock;
+
+ /**
+ * Create resource calculator for all Yarn containers.
+ */
+ public CGroupsResourceCalculator()
+ throws YarnException {
+ this(null, PROCFS, ResourceHandlerModule.getCGroupsHandler(),
+ SystemClock.getInstance(), SysInfoLinux.JIFFY_LENGTH_IN_MILLIS);
+ }
+
+ /**
+ * Create resource calculator for the container that has the specified pid.
+ * @param pid A pid from the cgroup or null for all containers
+ */
+ public CGroupsResourceCalculator(String pid) {
+ this(pid, PROCFS, ResourceHandlerModule.getCGroupsHandler(),
+ SystemClock.getInstance(), SysInfoLinux.JIFFY_LENGTH_IN_MILLIS);
+ }
+
+ /**
+ * Create resource calculator for testing.
+ * @param pid A pid from the cgroup or null for all containers
+ * @param procfsDir Path to /proc or a mock /proc directory
+ * @param cGroupsHandler Initialized cgroups handler object
+ * @param clock A clock object
+ * @param jiffyLengthMs0 Jiffy length in milliseconds
+ */
+ @VisibleForTesting
+ CGroupsResourceCalculator(String pid, String procfsDir,
+ CGroupsHandler cGroupsHandler,
+ Clock clock,
+ long jiffyLengthMs0) {
+ super(pid);
+ this.procfsDir = procfsDir;
+ this.cGroupsHandler = cGroupsHandler;
+ this.pid = pid != null && pid.equals("0") ? "1" : pid;
+ this.jiffyLengthMs = jiffyLengthMs0;
+ this.cpuTimeTracker =
+ new CpuTimeTracker(this.jiffyLengthMs);
+ this.clock = clock;
+ this.processTotalJiffies = BigInteger.ZERO;
+ this.processPhysicalMemory = UNAVAILABLE;
+ this.processVirtualMemory = UNAVAILABLE;
+ }
+
+ @Override
+ public void initialize() throws YarnException {
+ if (!CGroupsResourceCalculator.isAvailable()) {
+ throw new YarnException("CGroupsResourceCalculator is not available");
+ }
+ setCGroupFilePaths();
+ }
+
+ @Override
+ public float getCpuUsagePercent() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Process " + pid + " jiffies:" + processTotalJiffies);
+ }
+ return cpuTimeTracker.getCpuTrackerUsagePercent();
+ }
+
+ @Override
+ public long getCumulativeCpuTime() {
+ if (jiffyLengthMs < 0) {
+ return UNAVAILABLE;
+ }
+ return processTotalJiffies.longValue() * jiffyLengthMs;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+ if (olderThanAge > 1) {
+ return UNAVAILABLE;
+ }
+ return processPhysicalMemory;
+ }
+
+ @Override
+ public long getVirtualMemorySize(int olderThanAge) {
+ if (olderThanAge > 1) {
+ return UNAVAILABLE;
+ }
+ return processVirtualMemory;
+ }
+
+ @Override
+ public void updateProcessTree() {
+ try {
+ this.processTotalJiffies = readTotalProcessJiffies();
+ cpuTimeTracker.updateElapsedJiffies(processTotalJiffies,
+ clock.getTime());
+ } catch (YarnException e) {
+ LOG.warn("Failed to parse " + pid, e);
+ }
+ processPhysicalMemory = getMemorySize(memStat);
+ if (memswStat.exists()) {
+ processVirtualMemory = getMemorySize(memswStat);
+ } else if(LOG.isDebugEnabled()) {
+ LOG.debug("Swap cgroups monitoring is not compiled into the kernel " +
+ memswStat.getAbsolutePath().toString());
+ }
+ }
+
+ @Override
+ public String getProcessTreeDump() {
+ // We do not have a process tree in cgroups return just the pid for tracking
+ return pid;
+ }
+
+ @Override
+ public boolean checkPidPgrpidForMatch() {
+ // We do not have a process tree in cgroups returning default ok
+ return true;
+ }
+
+ /**
+ * Checks if the CGroupsResourceCalculator is available on this system.
+ * This assumes that Linux container executor is already initialized.
+ *
+ * @return true if CGroupsResourceCalculator is available. False otherwise.
+ */
+ public static boolean isAvailable() {
+ try {
+ if (!Shell.LINUX) {
+ LOG.info("CGroupsResourceCalculator currently is supported only on "
+ + "Linux.");
+ return false;
+ }
+ if (ResourceHandlerModule.getCGroupsHandler() == null ||
+ ResourceHandlerModule.getCpuResourceHandler() == null ||
+ ResourceHandlerModule.getMemoryResourceHandler() == null) {
+ LOG.info("CGroupsResourceCalculator requires enabling CGroups" +
+ "cpu and memory");
+ return false;
+ }
+ } catch (SecurityException se) {
+ LOG.warn("Failed to get Operating System name. " + se);
+ return false;
+ }
+ return true;
+ }
+
+ private long getMemorySize(File cgroupUsageFile) {
+ long[] mem = new long[1];
+ try {
+ processFile(cgroupUsageFile, (String line) -> {
+ mem[0] = Long.parseLong(line);
+ return Result.Exit;
+ });
+ return mem[0];
+ } catch (YarnException e) {
+ LOG.warn("Failed to parse cgroups " + memswStat, e);
+ }
+ return UNAVAILABLE;
+ }
+
+ private BigInteger readTotalProcessJiffies() throws YarnException {
+ final BigInteger[] totalCPUTimeJiffies = new BigInteger[1];
+ totalCPUTimeJiffies[0] = BigInteger.ZERO;
+ processFile(cpuStat, (String line) -> {
+ if (line.startsWith(USER)) {
+ totalCPUTimeJiffies[0] = totalCPUTimeJiffies[0].add(
+ new BigInteger(line.substring(USER.length())));
+ }
+ if (line.startsWith(SYSTEM)) {
+ totalCPUTimeJiffies[0] = totalCPUTimeJiffies[0].add(
+ new BigInteger(line.substring(SYSTEM.length())));
+ }
+ return Result.Continue;
+ });
+ return totalCPUTimeJiffies[0];
+ }
+
+ private String getCGroupRelativePath(
+ CGroupsHandler.CGroupController controller)
+ throws YarnException {
+ if (pid == null) {
+ return cGroupsHandler.getRelativePathForCGroup("");
+ } else {
+ return getCGroupRelativePathForPid(controller);
+ }
+ }
+
+ private String getCGroupRelativePathForPid(
+ CGroupsHandler.CGroupController controller)
+ throws YarnException {
+ File pidCgroupFile = new File(new File(procfsDir, pid), CGROUP);
+ String[] result = new String[1];
+ processFile(pidCgroupFile, (String line)->{
+ Matcher m = CGROUP_FILE_FORMAT.matcher(line);
+ boolean mat = m.find();
+ if (mat) {
+ if (m.group(2).contains(controller.getName())) {
+ // Instead of returning the full path we compose it
+ // based on the last item as the container id
+ // This helps to avoid confusion within a privileged Docker container
+ // where the path is referred in /proc/<pid>/cgroup as
+ // /docker/<dcontainerid>/hadoop-yarn/<containerid>
+ // but it is /hadoop-yarn/<containerid> in the cgroups hierarchy
+ String cgroupPath = m.group(3);
+
+ if (cgroupPath != null) {
+ String cgroup =
+ new File(cgroupPath).toPath().getFileName().toString();
+ result[0] = cGroupsHandler.getRelativePathForCGroup(cgroup);
+ } else {
+ LOG.warn("Invalid cgroup path for " + pidCgroupFile);
+ }
+ return Result.Exit;
+ }
+ } else {
+ LOG.warn(
+ "Unexpected: cgroup file is not in the expected format"
+ + " for process with pid " + pid);
+ }
+ return Result.Continue;
+ });
+ if (result[0] == null) {
+ throw new YarnException(controller.getName() + " CGroup for pid " + pid +
+ " not found " + pidCgroupFile);
+ }
+ return result[0];
+ }
+
+ private void processFile(File file, Function<String, Result> processLine)
+ throws YarnException {
+ // Read "procfsDir/<pid>/stat" file - typically /proc/<pid>/stat
+ try (InputStreamReader fReader = new InputStreamReader(
+ new FileInputStream(file), Charset.forName("UTF-8"))) {
+ try (BufferedReader in = new BufferedReader(fReader)) {
+ try {
+ String str;
+ while ((str = in.readLine()) != null) {
+ Result result = processLine.apply(str);
+ if (result == Result.Exit) {
+ return;
+ }
+ }
+ } catch (IOException io) {
+ throw new YarnException("Error reading the stream " + io, io);
+ }
+ }
+ } catch (IOException f) {
+ throw new YarnException("The process vanished in the interim " + pid, f);
+ }
+ }
+
+ void setCGroupFilePaths() throws YarnException {
+ if (cGroupsHandler == null) {
+ throw new YarnException("CGroups handler is not initialized");
+ }
+ File cpuDir = new File(
+ cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.CPUACCT),
+ getCGroupRelativePath(CGroupsHandler.CGroupController.CPUACCT));
+ File memDir = new File(
+ cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.MEMORY),
+ getCGroupRelativePath(CGroupsHandler.CGroupController.MEMORY));
+ cpuStat = new File(cpuDir, CPU_STAT);
+ memStat = new File(memDir, MEM_STAT);
+ memswStat = new File(memDir, MEMSW_STAT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CombinedResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CombinedResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CombinedResourceCalculator.java
new file mode 100644
index 0000000..84b3ed0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CombinedResourceCalculator.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+/**
+ * CombinedResourceCalculator is a resource calculator that uses cgroups but
+ * it is backward compatible with procfs in terms of virtual memory usage.
+ */
+public class CombinedResourceCalculator extends ResourceCalculatorProcessTree {
+ protected static final Log LOG = LogFactory
+ .getLog(CombinedResourceCalculator.class);
+ private ProcfsBasedProcessTree procfs;
+ private CGroupsResourceCalculator cgroup;
+
+ public CombinedResourceCalculator(String pid) {
+ super(pid);
+ procfs = new ProcfsBasedProcessTree(pid);
+ cgroup = new CGroupsResourceCalculator(pid);
+ }
+
+ @Override
+ public void initialize() throws YarnException {
+ procfs.initialize();
+ cgroup.initialize();
+ }
+
+ @Override
+ public void updateProcessTree() {
+ procfs.updateProcessTree();
+ cgroup.updateProcessTree();
+ }
+
+ @Override
+ public String getProcessTreeDump() {
+ return procfs.getProcessTreeDump();
+ }
+
+ @Override
+ public float getCpuUsagePercent() {
+ float cgroupUsage = cgroup.getCpuUsagePercent();
+ if (LOG.isDebugEnabled()) {
+ float procfsUsage = procfs.getCpuUsagePercent();
+ LOG.debug("CPU Comparison:" + procfsUsage + " " + cgroupUsage);
+ LOG.debug("Jiffy Comparison:" +
+ procfs.getCumulativeCpuTime() + " " +
+ cgroup.getCumulativeCpuTime());
+ }
+
+ return cgroupUsage;
+ }
+
+ @Override
+ public boolean checkPidPgrpidForMatch() {
+ return procfs.checkPidPgrpidForMatch();
+ }
+
+ @Override
+ public long getCumulativeCpuTime() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CPU Comparison:" +
+ procfs.getCumulativeCpuTime() + " " +
+ cgroup.getCumulativeCpuTime());
+ }
+ return cgroup.getCumulativeCpuTime();
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("MEM Comparison:" +
+ procfs.getRssMemorySize(olderThanAge) + " " +
+ cgroup.getRssMemorySize(olderThanAge));
+ }
+ return cgroup.getRssMemorySize(olderThanAge);
+ }
+
+ @Override
+ public long getVirtualMemorySize(int olderThanAge) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("VMEM Comparison:" +
+ procfs.getVirtualMemorySize(olderThanAge) + " " +
+ cgroup.getVirtualMemorySize(olderThanAge));
+ }
+ return procfs.getVirtualMemorySize(olderThanAge);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index 921f920..a02204d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
@@ -101,7 +100,27 @@ public class ResourceHandlerModule {
return cGroupsHandler;
}
- private static CGroupsCpuResourceHandlerImpl getCGroupsCpuResourceHandler(
+ public static NetworkPacketTaggingHandlerImpl
+ getNetworkResourceHandler() {
+ return networkPacketTaggingHandlerImpl;
+ }
+
+ public static DiskResourceHandler
+ getDiskResourceHandler() {
+ return cGroupsBlkioResourceHandler;
+ }
+
+ public static MemoryResourceHandler
+ getMemoryResourceHandler() {
+ return cGroupsMemoryResourceHandler;
+ }
+
+ public static CpuResourceHandler
+ getCpuResourceHandler() {
+ return cGroupsCpuResourceHandler;
+ }
+
+ private static CGroupsCpuResourceHandlerImpl initCGroupsCpuResourceHandler(
Configuration conf) throws ResourceHandlerException {
boolean cgroupsCpuEnabled =
conf.getBoolean(YarnConfiguration.NM_CPU_RESOURCE_ENABLED,
@@ -150,7 +169,7 @@ public class ResourceHandlerModule {
}
}
- public static ResourceHandler getNetworkResourceHandler(Configuration conf)
+ public static ResourceHandler initNetworkResourceHandler(Configuration conf)
throws ResourceHandlerException {
boolean useNetworkTagHandler = conf.getBoolean(
YarnConfiguration.NM_NETWORK_TAG_HANDLER_ENABLED,
@@ -181,12 +200,12 @@ public class ResourceHandlerModule {
}
public static OutboundBandwidthResourceHandler
- getOutboundBandwidthResourceHandler(Configuration conf)
+ initOutboundBandwidthResourceHandler(Configuration conf)
throws ResourceHandlerException {
return getTrafficControlBandwidthHandler(conf);
}
- public static DiskResourceHandler getDiskResourceHandler(Configuration conf)
+ public static DiskResourceHandler initDiskResourceHandler(Configuration conf)
throws ResourceHandlerException {
if (conf.getBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED,
YarnConfiguration.DEFAULT_NM_DISK_RESOURCE_ENABLED)) {
@@ -210,7 +229,7 @@ public class ResourceHandlerModule {
return cGroupsBlkioResourceHandler;
}
- public static MemoryResourceHandler getMemoryResourceHandler(
+ public static MemoryResourceHandler initMemoryResourceHandler(
Configuration conf) throws ResourceHandlerException {
if (conf.getBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENABLED,
YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_ENABLED)) {
@@ -246,10 +265,14 @@ public class ResourceHandlerModule {
throws ResourceHandlerException {
ArrayList<ResourceHandler> handlerList = new ArrayList<>();
- addHandlerIfNotNull(handlerList, getNetworkResourceHandler(conf));
- addHandlerIfNotNull(handlerList, getDiskResourceHandler(conf));
- addHandlerIfNotNull(handlerList, getMemoryResourceHandler(conf));
- addHandlerIfNotNull(handlerList, getCGroupsCpuResourceHandler(conf));
+ addHandlerIfNotNull(handlerList,
+ initNetworkResourceHandler(conf));
+ addHandlerIfNotNull(handlerList,
+ initDiskResourceHandler(conf));
+ addHandlerIfNotNull(handlerList,
+ initMemoryResourceHandler(conf));
+ addHandlerIfNotNull(handlerList,
+ initCGroupsCpuResourceHandler(conf));
addHandlersFromConfiguredResourcePlugins(handlerList, conf, nmContext);
resourceHandlerChain = new ResourceHandlerChain(handlerList);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 23c89c0..33986a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -215,15 +215,25 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.DEFAULT_NM_CONTAINER_MONITOR_ENABLED);
}
+ /**
+ * Get the best process tree calculator.
+ * @param pId container process id
+ * @return process tree calculator
+ */
+ private ResourceCalculatorProcessTree
+ getResourceCalculatorProcessTree(String pId) {
+ return ResourceCalculatorProcessTree.
+ getResourceCalculatorProcessTree(
+ pId, processTreeClass, conf);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
.getClass().getName() + " is disabled.");
return false;
}
- if (ResourceCalculatorProcessTree
- .getResourceCalculatorProcessTree("0", processTreeClass, conf)
- == null) {
+ if (getResourceCalculatorProcessTree("0") == null) {
LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
+ this.getClass().getName() + " is disabled.");
return false;
@@ -535,9 +545,7 @@ public class ContainersMonitorImpl extends AbstractService implements
LOG.debug("Tracking ProcessTree " + pId + " for the first time");
}
ResourceCalculatorProcessTree pt =
- ResourceCalculatorProcessTree.
- getResourceCalculatorProcessTree(
- pId, processTreeClass, conf);
+ getResourceCalculatorProcessTree(pId);
ptInfo.setPid(pId);
ptInfo.setProcessTree(pt);
@@ -599,11 +607,14 @@ public class ContainersMonitorImpl extends AbstractService implements
long pmemLimit = ptInfo.getPmemLimit();
if (AUDITLOG.isDebugEnabled()) {
AUDITLOG.debug(String.format(
- "Memory usage of ProcessTree %s for container-id %s: ",
- pId, containerId.toString()) +
- formatUsageString(
- currentVmemUsage, vmemLimit,
- currentPmemUsage, pmemLimit));
+ "Resource usage of ProcessTree %s for container-id %s:" +
+ " %s CPU:%f CPU/core:%f",
+ pId, containerId.toString(),
+ formatUsageString(
+ currentVmemUsage, vmemLimit,
+ currentPmemUsage, pmemLimit),
+ cpuUsagePercentPerCore,
+ cpuUsageTotalCoresPercentage));
}
// Add resource utilization for this container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
index 8fd5a9d..78ccc61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
@@ -149,6 +149,51 @@ public class TestCGroupsMemoryResourceHandlerImpl {
}
@Test
+ public void testPreStartNonEnforced() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED, false);
+ cGroupsMemoryResourceHandler.bootstrap(conf);
+ String id = "container_01_01";
+ String path = "test-path/" + id;
+ ContainerId mockContainerId = mock(ContainerId.class);
+ when(mockContainerId.toString()).thenReturn(id);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getContainerId()).thenReturn(mockContainerId);
+ when(mockCGroupsHandler
+ .getPathForCGroupTasks(CGroupsHandler.CGroupController.MEMORY, id))
+ .thenReturn(path);
+ int memory = 1024;
+ when(mockContainer.getResource())
+ .thenReturn(Resource.newInstance(memory, 1));
+ List<PrivilegedOperation> ret =
+ cGroupsMemoryResourceHandler.preStart(mockContainer);
+ verify(mockCGroupsHandler, times(1))
+ .createCGroup(CGroupsHandler.CGroupController.MEMORY, id);
+ verify(mockCGroupsHandler, times(0))
+ .updateCGroupParam(CGroupsHandler.CGroupController.MEMORY, id,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES,
+ String.valueOf(memory) + "M");
+ verify(mockCGroupsHandler, times(0))
+ .updateCGroupParam(CGroupsHandler.CGroupController.MEMORY, id,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
+ String.valueOf((int) (memory * 0.9)) + "M");
+ verify(mockCGroupsHandler, times(0))
+ .updateCGroupParam(CGroupsHandler.CGroupController.MEMORY, id,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS, String.valueOf(0));
+ Assert.assertNotNull(ret);
+ Assert.assertEquals(1, ret.size());
+ PrivilegedOperation op = ret.get(0);
+ Assert.assertEquals(PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP,
+ op.getOperationType());
+ List<String> args = op.getArguments();
+ Assert.assertEquals(1, args.size());
+ Assert.assertEquals(PrivilegedOperation.CGROUP_ARG_PREFIX + path,
+ args.get(0));
+ }
+
+ @Test
public void testReacquireContainer() throws Exception {
ContainerId containerIdMock = mock(ContainerId.class);
Assert.assertNull(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java
new file mode 100644
index 0000000..a2ad11f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.mockito.Mockito.*;
+
+/**
+ * Unit test for CGroupsResourceCalculator.
+ */
+public class TestCGroupsResourceCalculator {
+
+ private ControlledClock clock = new ControlledClock();
+ private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ private String basePath = "/tmp/" + this.getClass().getName();
+
+ public TestCGroupsResourceCalculator() {
+ when(cGroupsHandler.getRelativePathForCGroup("container_1"))
+ .thenReturn("/yarn/container_1");
+ when(cGroupsHandler.getRelativePathForCGroup("")).thenReturn("/yarn/");
+ }
+
+ @Test(expected = YarnException.class)
+ public void testPidNotFound() throws Exception {
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ "1234", ".", cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+ Assert.assertEquals("Expected exception", null, calculator);
+ }
+
+ @Test(expected = YarnException.class)
+ public void testNoMemoryCGgroupMount() throws Exception {
+ File procfs = new File(basePath + "/1234");
+ Assert.assertTrue("Setup error", procfs.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(procfs, CGroupsResourceCalculator.CGROUP),
+ "7:devices:/yarn/container_1\n" +
+ "6:cpuacct,cpu:/yarn/container_1\n" +
+ "5:pids:/yarn/container_1\n");
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ "1234", basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+ Assert.assertEquals("Expected exception", null, calculator);
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+
+ @Test
+ public void testCGgroupNotFound() throws Exception {
+ File procfs = new File(basePath + "/1234");
+ Assert.assertTrue("Setup error", procfs.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(procfs, CGroupsResourceCalculator.CGROUP),
+ "7:devices:/yarn/container_1\n" +
+ "6:cpuacct,cpu:/yarn/container_1\n" +
+ "5:pids:/yarn/container_1\n" +
+ "4:memory:/yarn/container_1\n");
+
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ "1234", basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+ calculator.updateProcessTree();
+ Assert.assertEquals("cgroups should be missing",
+ (long)ResourceCalculatorProcessTree.UNAVAILABLE,
+ calculator.getRssMemorySize(0));
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+
+ @Test
+ public void testCPUParsing() throws Exception {
+ File cgcpuacctDir =
+ new File(basePath + "/cgcpuacct");
+ File cgcpuacctContainerDir =
+ new File(cgcpuacctDir, "/yarn/container_1");
+ File procfs = new File(basePath + "/1234");
+ when(cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.CPUACCT)).
+ thenReturn(cgcpuacctDir.getAbsolutePath());
+ Assert.assertTrue("Setup error", procfs.mkdirs());
+ Assert.assertTrue("Setup error", cgcpuacctContainerDir.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(procfs, CGroupsResourceCalculator.CGROUP),
+ "7:devices:/yarn/container_1\n" +
+ "6:cpuacct,cpu:/yarn/container_1\n" +
+ "5:pids:/yarn/container_1\n" +
+ "4:memory:/yarn/container_1\n");
+ FileUtils.writeStringToFile(
+ new File(cgcpuacctContainerDir, CGroupsResourceCalculator.CPU_STAT),
+ "Can you handle this?\n" +
+ "user 5415\n" +
+ "system 3632");
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ "1234", basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+ calculator.updateProcessTree();
+ Assert.assertEquals("Incorrect CPU usage",
+ 90470,
+ calculator.getCumulativeCpuTime());
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+
+ @Test
+ public void testMemoryParsing() throws Exception {
+ File cgcpuacctDir =
+ new File(basePath + "/cgcpuacct");
+ File cgcpuacctContainerDir =
+ new File(cgcpuacctDir, "/yarn/container_1");
+ File cgmemoryDir =
+ new File(basePath + "/memory");
+ File cgMemoryContainerDir =
+ new File(cgmemoryDir, "/yarn/container_1");
+ File procfs = new File(basePath + "/1234");
+ when(cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.MEMORY)).
+ thenReturn(cgmemoryDir.getAbsolutePath());
+ Assert.assertTrue("Setup error", procfs.mkdirs());
+ Assert.assertTrue("Setup error", cgcpuacctContainerDir.mkdirs());
+ Assert.assertTrue("Setup error", cgMemoryContainerDir.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(procfs, CGroupsResourceCalculator.CGROUP),
+ "6:cpuacct,cpu:/yarn/container_1\n" +
+ "4:memory:/yarn/container_1\n");
+ FileUtils.writeStringToFile(
+ new File(cgMemoryContainerDir, CGroupsResourceCalculator.MEM_STAT),
+ "418496512\n");
+
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ "1234", basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+
+ calculator.updateProcessTree();
+ // Test the case where memsw is not available (Ubuntu)
+ Assert.assertEquals("Incorrect memory usage",
+ 418496512,
+ calculator.getRssMemorySize());
+ Assert.assertEquals("Incorrect swap usage",
+ (long)ResourceCalculatorProcessTree.UNAVAILABLE,
+ calculator.getVirtualMemorySize());
+
+ // Test the case where memsw is available
+ FileUtils.writeStringToFile(
+ new File(cgMemoryContainerDir, CGroupsResourceCalculator.MEMSW_STAT),
+ "418496513\n");
+ calculator.updateProcessTree();
+ Assert.assertEquals("Incorrect swap usage",
+ 418496513,
+ calculator.getVirtualMemorySize());
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+
+ @Test
+ public void testCPUParsingRoot() throws Exception {
+ File cgcpuacctDir =
+ new File(basePath + "/cgcpuacct");
+ File cgcpuacctRootDir =
+ new File(cgcpuacctDir, "/yarn");
+ when(cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.CPUACCT)).
+ thenReturn(cgcpuacctDir.getAbsolutePath());
+ Assert.assertTrue("Setup error", cgcpuacctRootDir.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(cgcpuacctRootDir, CGroupsResourceCalculator.CPU_STAT),
+ "user 5415\n" +
+ "system 3632");
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ null, basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+ calculator.updateProcessTree();
+ Assert.assertEquals("Incorrect CPU usage",
+ 90470,
+ calculator.getCumulativeCpuTime());
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+
+ @Test
+ public void testMemoryParsingRoot() throws Exception {
+ File cgcpuacctDir =
+ new File(basePath + "/cgcpuacct");
+ File cgcpuacctRootDir =
+ new File(cgcpuacctDir, "/yarn");
+ File cgmemoryDir =
+ new File(basePath + "/memory");
+ File cgMemoryRootDir =
+ new File(cgmemoryDir, "/yarn");
+ File procfs = new File(basePath + "/1234");
+ when(cGroupsHandler.getControllerPath(
+ CGroupsHandler.CGroupController.MEMORY)).
+ thenReturn(cgmemoryDir.getAbsolutePath());
+ Assert.assertTrue("Setup error", procfs.mkdirs());
+ Assert.assertTrue("Setup error", cgcpuacctRootDir.mkdirs());
+ Assert.assertTrue("Setup error", cgMemoryRootDir.mkdirs());
+ try {
+ FileUtils.writeStringToFile(
+ new File(cgMemoryRootDir, CGroupsResourceCalculator.MEM_STAT),
+ "418496512\n");
+
+ CGroupsResourceCalculator calculator =
+ new CGroupsResourceCalculator(
+ null, basePath,
+ cGroupsHandler, clock, 10);
+ calculator.setCGroupFilePaths();
+
+ calculator.updateProcessTree();
+
+ // Test the case where memsw is not available (Ubuntu)
+ Assert.assertEquals("Incorrect memory usage",
+ 418496512,
+ calculator.getRssMemorySize());
+ Assert.assertEquals("Incorrect swap usage",
+ (long)ResourceCalculatorProcessTree.UNAVAILABLE,
+ calculator.getVirtualMemorySize());
+
+ // Test the case where memsw is available
+ FileUtils.writeStringToFile(
+ new File(cgMemoryRootDir, CGroupsResourceCalculator.MEMSW_STAT),
+ "418496513\n");
+ calculator.updateProcessTree();
+ Assert.assertEquals("Incorrect swap usage",
+ 418496513,
+ calculator.getVirtualMemorySize());
+ } finally {
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCompareResourceCalculators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCompareResourceCalculators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCompareResourceCalculators.java
new file mode 100644
index 0000000..8be0590
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCompareResourceCalculators.java
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Random;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Functional test for CGroupsResourceCalculator to compare two resource
+ * calculators. It is OS dependent.
+ * Ignored in automated tests due to flakiness by design.
+ */
+public class TestCompareResourceCalculators {
+ private Process target = null;
+ private String cgroup = null;
+ private String cgroupCPU = null;
+ private String cgroupMemory = null;
+ public static final long SHMEM_KB = 100 * 1024;
+
+ @Before
+ public void setup() throws IOException, YarnException {
+ Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "TestCompareResourceCalculators");
+ conf.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
+ conf.setStrings(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH,
+ "/sys/fs/cgroup");
+ conf.setBoolean(YarnConfiguration.NM_CPU_RESOURCE_ENABLED, true);
+ ResourceHandlerChain module = null;
+ try {
+ module = ResourceHandlerModule.getConfiguredResourceHandlerChain(conf,
+ mock(Context.class));
+ } catch (ResourceHandlerException e) {
+ throw new YarnException("Cannot access cgroups", e);
+ }
+ Assume.assumeNotNull(module);
+ Assume.assumeNotNull(
+ ResourceHandlerModule.getCGroupsHandler()
+ .getControllerPath(CGroupsHandler.CGroupController.CPU));
+ Assume.assumeNotNull(
+ ResourceHandlerModule.getCGroupsHandler()
+ .getControllerPath(CGroupsHandler.CGroupController.MEMORY));
+
+ Random random = new Random(System.currentTimeMillis());
+ cgroup = Long.toString(random.nextLong());
+ cgroupCPU = ResourceHandlerModule.getCGroupsHandler()
+ .getPathForCGroup(CGroupsHandler.CGroupController.CPU, cgroup);
+ cgroupMemory = ResourceHandlerModule.getCGroupsHandler()
+ .getPathForCGroup(CGroupsHandler.CGroupController.MEMORY, cgroup);
+ }
+
+ @After
+ public void tearDown() throws YarnException {
+ stopTestProcess();
+ }
+
+
+ // Ignored in automated tests due to flakiness by design
+ @Ignore
+ @Test
+ public void testCompareResults()
+ throws YarnException, InterruptedException, IOException {
+
+ startTestProcess();
+
+ ProcfsBasedProcessTree legacyCalculator =
+ new ProcfsBasedProcessTree(Long.toString(getPid()));
+ CGroupsResourceCalculator cgroupsCalculator =
+ new CGroupsResourceCalculator(Long.toString(getPid()));
+ cgroupsCalculator.setCGroupFilePaths();
+
+ for (int i = 0; i < 5; ++i) {
+ Thread.sleep(3000);
+ compareMetrics(legacyCalculator, cgroupsCalculator);
+ }
+
+ stopTestProcess();
+
+ ensureCleanedUp(legacyCalculator, cgroupsCalculator);
+ }
+
+ private void ensureCleanedUp(
+ ResourceCalculatorProcessTree metric1,
+ ResourceCalculatorProcessTree metric2) {
+ metric1.updateProcessTree();
+ metric2.updateProcessTree();
+ long pmem1 = metric1.getRssMemorySize(0);
+ long pmem2 = metric2.getRssMemorySize(0);
+ System.out.println(pmem1 + " " + pmem2);
+ Assert.assertTrue("pmem should be invalid " + pmem1 + " " + pmem2,
+ pmem1 == ResourceCalculatorProcessTree.UNAVAILABLE &&
+ pmem2 == ResourceCalculatorProcessTree.UNAVAILABLE);
+ long vmem1 = metric1.getRssMemorySize(0);
+ long vmem2 = metric2.getRssMemorySize(0);
+ System.out.println(vmem1 + " " + vmem2);
+ Assert.assertTrue("vmem Error outside range " + vmem1 + " " + vmem2,
+ vmem1 == ResourceCalculatorProcessTree.UNAVAILABLE &&
+ vmem2 == ResourceCalculatorProcessTree.UNAVAILABLE);
+ float cpu1 = metric1.getCpuUsagePercent();
+ float cpu2 = metric2.getCpuUsagePercent();
+ // TODO ProcfsBasedProcessTree may report negative on process exit
+ Assert.assertTrue("CPU% Error outside range " + cpu1 + " " + cpu2,
+ cpu1 == 0 && cpu2 == 0);
+ }
+
+ private void compareMetrics(
+ ResourceCalculatorProcessTree metric1,
+ ResourceCalculatorProcessTree metric2) {
+ metric1.updateProcessTree();
+ metric2.updateProcessTree();
+ long pmem1 = metric1.getRssMemorySize(0);
+ long pmem2 = metric2.getRssMemorySize(0);
+ // TODO The calculation is different and cgroup
+ // can report a small amount after process stop
+ // This is not an issue since the cgroup is deleted
+ System.out.println(pmem1 + " " + (pmem2 - SHMEM_KB * 1024));
+ Assert.assertTrue("pmem Error outside range " + pmem1 + " " + pmem2,
+ Math.abs(pmem1 - (pmem2 - SHMEM_KB * 1024)) < 5000000);
+ long vmem1 = metric1.getRssMemorySize(0);
+ long vmem2 = metric2.getRssMemorySize(0);
+ System.out.println(vmem1 + " " + (vmem2 - SHMEM_KB * 1024));
+ // TODO The calculation is different and cgroup
+ // can report a small amount after process stop
+ // This is not an issue since the cgroup is deleted
+ Assert.assertTrue("vmem Error outside range " + vmem1 + " " + vmem2,
+ Math.abs(vmem1 - (vmem2 - SHMEM_KB * 1024)) < 5000000);
+ float cpu1 = metric1.getCpuUsagePercent();
+ float cpu2 = metric2.getCpuUsagePercent();
+ if (cpu1 > 0) {
+ // TODO ProcfsBasedProcessTree may report negative on process exit
+ Assert.assertTrue("CPU% Error outside range " + cpu1 + " " + cpu2,
+ Math.abs(cpu2 - cpu1) < 10);
+ }
+ }
+
+ private void startTestProcess() throws IOException {
+ ProcessBuilder builder = new ProcessBuilder();
+ String script =
+ "mkdir -p " + cgroupCPU + ";" +
+ "echo $$ >" + cgroupCPU + "/tasks;" +
+ "mkdir -p " + cgroupMemory + ";" +
+ "echo $$ >" + cgroupMemory + "/tasks;" +
+ "dd if=/dev/zero of=/dev/shm/" +
+ cgroup + " bs=1k count=" + SHMEM_KB + ";" +
+ "dd if=/dev/zero of=/dev/null bs=1k &" +
+ "echo $! >/tmp/\" + cgroup + \".pid;" +
+ //"echo while [ -f /tmp/" + cgroup + ".pid ]; do sleep 1; done;" +
+ "sleep 10000;" +
+ "echo kill $(jobs -p);";
+ builder.command("bash", "-c", script);
+ builder.redirectError(new File("/tmp/a.txt"));
+ builder.redirectOutput(new File("/tmp/b.txt"));
+ target = builder.start();
+ }
+
+ private void stopTestProcess() throws YarnException {
+ if (target != null) {
+ target.destroyForcibly();
+ target = null;
+ }
+ try {
+ ProcessBuilder builder = new ProcessBuilder();
+ String script =
+ "rm -f /dev/shm/" + cgroup + ";" +
+ "cat " + cgroupCPU + "/tasks | xargs kill;" +
+ "rm -f /tmp/" + cgroup + ".pid;" +
+ "sleep 4;" +
+ "rmdir " + cgroupCPU + ";" +
+ "rmdir " + cgroupMemory + ";";
+ builder.command("bash", "-c", script);
+ Process cleanup = builder.start();
+ cleanup.waitFor();
+ } catch (IOException|InterruptedException e) {
+ throw new YarnException("Could not clean up", e);
+ }
+ }
+
+ private long getPid() throws YarnException {
+ Class processClass = target.getClass();
+ if (processClass.getName().equals("java.lang.UNIXProcess")) {
+ try {
+ Field pidField = processClass.getDeclaredField("pid");
+ pidField.setAccessible(true);
+ long pid = pidField.getLong(target);
+ pidField.setAccessible(false);
+ return pid;
+ } catch (NoSuchFieldException|IllegalAccessException e) {
+ throw new YarnException("Reflection error", e);
+ }
+ } else {
+ throw new YarnException("Not Unix " + processClass.getName());
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649ef7ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
index 0563694..9456303 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
@@ -36,8 +36,8 @@ import static org.mockito.Mockito.mock;
public class TestResourceHandlerModule {
private static final Logger LOG =
LoggerFactory.getLogger(TestResourceHandlerModule.class);
- Configuration emptyConf;
- Configuration networkEnabledConf;
+ private Configuration emptyConf;
+ private Configuration networkEnabledConf;
@Before
public void setup() throws Exception {
@@ -55,23 +55,28 @@ public class TestResourceHandlerModule {
//This resourceHandler should be non-null only if network as a resource
//is explicitly enabled
OutboundBandwidthResourceHandler resourceHandler = ResourceHandlerModule
- .getOutboundBandwidthResourceHandler(emptyConf);
+ .initOutboundBandwidthResourceHandler(emptyConf);
Assert.assertNull(resourceHandler);
//When network as a resource is enabled this should be non-null
resourceHandler = ResourceHandlerModule
- .getOutboundBandwidthResourceHandler(networkEnabledConf);
+ .initOutboundBandwidthResourceHandler(networkEnabledConf);
Assert.assertNotNull(resourceHandler);
//Ensure that outbound bandwidth resource handler is present in the chain
ResourceHandlerChain resourceHandlerChain = ResourceHandlerModule
- .getConfiguredResourceHandlerChain(networkEnabledConf, mock(Context.class));
- List<ResourceHandler> resourceHandlers = resourceHandlerChain
- .getResourceHandlerList();
- //Exactly one resource handler in chain
- Assert.assertEquals(resourceHandlers.size(), 1);
- //Same instance is expected to be in the chain.
- Assert.assertTrue(resourceHandlers.get(0) == resourceHandler);
+ .getConfiguredResourceHandlerChain(networkEnabledConf,
+ mock(Context.class));
+ if (resourceHandlerChain != null) {
+ List<ResourceHandler> resourceHandlers = resourceHandlerChain
+ .getResourceHandlerList();
+ //Exactly one resource handler in chain
+ Assert.assertEquals(resourceHandlers.size(), 1);
+ //Same instance is expected to be in the chain.
+ Assert.assertTrue(resourceHandlers.get(0) == resourceHandler);
+ } else {
+ Assert.fail("Null returned");
+ }
} catch (ResourceHandlerException e) {
Assert.fail("Unexpected ResourceHandlerException: " + e);
}
@@ -81,23 +86,27 @@ public class TestResourceHandlerModule {
public void testDiskResourceHandler() throws Exception {
DiskResourceHandler handler =
- ResourceHandlerModule.getDiskResourceHandler(emptyConf);
+ ResourceHandlerModule.initDiskResourceHandler(emptyConf);
Assert.assertNull(handler);
Configuration diskConf = new YarnConfiguration();
diskConf.setBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED, true);
- handler = ResourceHandlerModule.getDiskResourceHandler(diskConf);
+ handler = ResourceHandlerModule.initDiskResourceHandler(diskConf);
Assert.assertNotNull(handler);
ResourceHandlerChain resourceHandlerChain =
ResourceHandlerModule.getConfiguredResourceHandlerChain(diskConf,
mock(Context.class));
- List<ResourceHandler> resourceHandlers =
- resourceHandlerChain.getResourceHandlerList();
- // Exactly one resource handler in chain
- Assert.assertEquals(resourceHandlers.size(), 1);
- // Same instance is expected to be in the chain.
- Assert.assertTrue(resourceHandlers.get(0) == handler);
+ if (resourceHandlerChain != null) {
+ List<ResourceHandler> resourceHandlers =
+ resourceHandlerChain.getResourceHandlerList();
+ // Exactly one resource handler in chain
+ Assert.assertEquals(resourceHandlers.size(), 1);
+ // Same instance is expected to be in the chain.
+ Assert.assertTrue(resourceHandlers.get(0) == handler);
+ } else {
+ Assert.fail("Null returned");
+ }
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-13063. Fix the incorrect
spelling in HDFSHighAvailabilityWithQJM.md
Posted by ha...@apache.org.
HDFS-13063. Fix the incorrect spelling in HDFSHighAvailabilityWithQJM.md
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb2dd086
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb2dd086
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb2dd086
Branch: refs/heads/YARN-1011
Commit: eb2dd0868864da994612f1dd1452ea38d31f5e85
Parents: 59828be
Author: Jianfei Jiang <ji...@gmail.com>
Authored: Thu Jan 25 15:27:28 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Jan 25 15:30:42 2018 +0900
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2dd086/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index b28790f..8df3209 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -49,7 +49,7 @@ Architecture
In a typical HA cluster, two or more separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the others are in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standbys are simply acting as workers, maintaining enough state to provide a fast failover if necessary.
-In order for the Standby node to keep its state synchronized with the Active node, both nodes communicate with a group of separate daemons called "JournalNodes" (JNs). When any namespace modification is performed by the Active node, it durably logs a record of the modification to a majority of these JNs. The Standby node is capable of reading the edits from the JNs, and is constantly watching them for changes to the edit log. As the Standby Node sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the JounalNodes before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.
+In order for the Standby node to keep its state synchronized with the Active node, both nodes communicate with a group of separate daemons called "JournalNodes" (JNs). When any namespace modification is performed by the Active node, it durably logs a record of the modification to a majority of these JNs. The Standby node is capable of reading the edits from the JNs, and is constantly watching them for changes to the edit log. As the Standby Node sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the JournalNodes before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.
In order to provide a fast failover, it is also necessary that the Standby node have up-to-date information regarding the location of blocks in the cluster. In order to achieve this, the DataNodes are configured with the location of all NameNodes, and send block location information and heartbeats to all.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-7337. Expose per-node
over-allocation info in Node Report Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8390588a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8390588a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8390588a
Branch: refs/heads/YARN-1011
Commit: 8390588a225a180cbf7ebcde02b64b20adcb0278
Parents: 7dcead0
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Nov 22 08:39:03 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:17:45 2018 -0800
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++-
.../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++-----
.../src/main/proto/yarn_protos.proto | 7 +-
.../applications/distributedshell/Client.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 49 +++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++---
.../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++----
.../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++----
.../server/resourcemanager/ClientRMService.java | 21 +++--
.../resourcemanager/DefaultAMSProcessor.java | 22 +++--
.../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++----
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/SchedulerNodeReport.java | 50 +++++++---
.../webapp/dao/FifoSchedulerInfo.java | 9 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 17 ++--
.../server/resourcemanager/NodeManager.java | 4 +-
.../capacity/TestApplicationPriority.java | 66 ++++++++-----
.../capacity/TestCapacityScheduler.java | 32 ++++---
.../TestNodeLabelContainerAllocation.java | 40 ++++----
.../scheduler/fifo/TestFifoScheduler.java | 50 ++++++----
.../webapp/TestRMWebServicesNodes.java | 13 +--
21 files changed, 491 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index efea709..76b5493 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -124,11 +124,17 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").asInt());
+ n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+ 0, n.get("NumContainers").asInt());
+ Assert.assertEquals(n.get("NodeId") +
+ ": Unexpected number of opportunistic containers used",
+ 0, n.get("NumOpportunisticContainers").asInt());
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected amount of used memory",
+ n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
0, n.get("UsedMemoryMB").asInt());
+ Assert.assertEquals(
+ n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+ 0, n.get("UsedOpportunisticMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
+ Assert.assertNull(n.get("NumOpportunisticContainers"));
+ Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 3a80641..53e9f92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link NodeId} of the node.</li>
* <li>HTTP Tracking URL of the node.</li>
* <li>Rack name for the node.</li>
- * <li>Used {@link Resource} on the node.</li>
+ * <li>Used guaranteed {@link Resource} on the node.</li>
+ * <li>Used opportunistic {@link Resource} on the node.</li>
* <li>Total available {@link Resource} of the node.</li>
- * <li>Number of running containers on the node.</li>
+ * <li>Number of total running containers on the node.</li>
+ * <li>Number of running guaranteed containers on the node.</li>
+ * <li>Number of running opportunistic containers on the node.</li>
* </ul>
*
* @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime) {
+ return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+ capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+ null, null, null, null, 0);
}
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+ Resource opportunisticUsed, int numOpportunisticContainers) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setDecommissioningTimeout(decommissioningTimeout);
nodeReport.setNodeUpdateType(nodeUpdateType);
+ nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
return nodeReport;
}
@@ -129,17 +136,43 @@ public abstract class NodeReport {
public abstract void setRackName(String rackName);
/**
- * Get <em>used</em> <code>Resource</code> on the node.
- * @return <em>used</em> <code>Resource</code> on the node
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsed();
@Private
@Unstable
+ @Deprecated
public abstract void setUsed(Resource used);
-
+
+ /**
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+ /**
+ * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+ * @return <em>opportunistic</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
/**
* Get the <em>total</em> <code>Resource</code> on the node.
* @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
public abstract void setCapability(Resource capability);
/**
- * Get the <em>number of allocated containers</em> on the node.
- * @return <em>number of allocated containers</em> on the node
+ * Get the <em>number of guaranteed containers</em> allocated on the node.
+ * @return <em>number of guaranteed containers</em> allocated on the node
*/
@Private
@Unstable
- public abstract int getNumContainers();
+ public abstract int getNumGuaranteedContainers();
@Private
@Unstable
- public abstract void setNumContainers(int numContainers);
-
+ public abstract void setNumGuaranteedContainers(int numContainers);
- /**
+ /**
+ * Get the <em>number of opportunistic containers</em> allocated on the node.
+ * @return <em>number of opportunistic containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public abstract int getNumOpportunisticContainers();
+
+ @Private
+ @Unstable
+ public abstract void setNumOpportunisticContainers(int numContainers);
+
+ /**
+ * Get the <em>number of containers</em> allocated on the node.
+ * @return <em>number of containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public int getNumTotalContainers() {
+ return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+ }
+
+ /**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b6ea5f9..44e1b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -342,9 +342,9 @@ message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
optional string rackName = 3;
- optional ResourceProto used = 4;
+ optional ResourceProto guaranteedResourceUsed = 4;
optional ResourceProto capability = 5;
- optional int32 numContainers = 6;
+ optional int32 numGuaranteedContainers = 6;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
@@ -353,6 +353,9 @@ message NodeReportProto {
optional ResourceUtilizationProto node_utilization = 12;
optional uint32 decommissioning_timeout = 13;
optional NodeUpdateTypeProto node_update_type = 14;
+ optional int32 numOpportunisticContainers = 15;
+ optional ResourceProto opportunisticResourceUsed = 16;
+ optional int32 numTotalContainers = 17;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index ef635d3..7520d30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -600,7 +600,7 @@ public class Client {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
- + ", nodeNumContainers=" + node.getNumContainers());
+ + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
}
QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 288a5d2..be466fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
@Private
@Unstable
@@ -185,7 +185,7 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
- .getNumContainers());
+ .getNumGuaranteedContainers());
}
writer.flush();
}
@@ -210,13 +210,18 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
- nodeReport.getNumContainers());
+ nodeReport.getNumGuaranteedContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
- writer.print("\tAllocated Resources : ");
- if (nodeReport.getUsed() != null) {
- writer.print(nodeReport.getUsed());
+ writer.print("\tAllocated Guaranteed Resources : ");
+ if (nodeReport.getGuaranteedResourceUsed() != null) {
+ writer.print(nodeReport.getGuaranteedResourceUsed());
+ }
+ writer.println();
+ writer.print("\tAllocated Opportunistic Resources : ");
+ if (nodeReport.getOpportunisticResourceUsed() != null) {
+ writer.print(nodeReport.getOpportunisticResourceUsed());
}
writer.println();
@@ -287,16 +292,32 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getHealthReport());
- nodeReportStr.print("\tContainers : ");
- nodeReportStr.println(nodeReport.getNumContainers());
- nodeReportStr.print("\tMemory-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ nodeReportStr.print("\tGuaranteed Containers : ");
+ nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+ nodeReportStr.print("\tOpportunistic Containers : ");
+ nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+ Resource guaranteedResourceUsed =
+ nodeReport.getGuaranteedResourceUsed();
+ Resource opportunisticResourceUsed =
+ nodeReport.getOpportunisticResourceUsed();
+ nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0MB" :
+ (guaranteedResourceUsed.getMemorySize() + "MB"));
+ nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0MB" :
+ (opportunisticResourceUsed.getMemorySize() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
- nodeReportStr.print("\tCPU-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
- : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0 vcores" :
+ (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0 vcores" :
+ (opportunisticResourceUsed.getVirtualCores() + " vcores"));
nodeReportStr.print("\tCPU-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.print("\tNode-Labels : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index fdd3fc8..e172acd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1352,7 +1352,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1360,7 +1361,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1561,10 +1563,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1597,10 +1602,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1633,10 +1641,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : ");
@@ -1989,10 +2000,12 @@ public class TestYarnCLI {
// ordered
nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
}
- NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
- "rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+ "rack1", Records.newRecord(Resource.class),
+ Records.newRecord(Resource.class), 0, "",
+ 0, nodeLabels, null, null,
+ Records.newRecord(Resource.class), 0);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index ced588d..0960eb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -45,7 +45,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeReportProto.Builder builder = null;
private boolean viaProto = false;
private NodeId nodeId;
- private Resource used;
+ private Resource guaranteedResourceUsed;
+ private Resource opportunisticResourceUsed;
private Resource capability;
private ResourceUtilization containersUtilization = null;
private ResourceUtilization nodeUtilization = null;
@@ -109,9 +110,17 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public int getNumContainers() {
+ public int getNumGuaranteedContainers() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+ return (p.hasNumGuaranteedContainers()) ?
+ p.getNumGuaranteedContainers() : 0;
+ }
+
+ @Override
+ public int getNumOpportunisticContainers() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNumOpportunisticContainers()) ?
+ p.getNumOpportunisticContainers() : 0;
}
@Override
@@ -120,18 +129,25 @@ public class NodeReportPBImpl extends NodeReport {
return (p.hasRackName()) ? p.getRackName() : null;
}
+ @Deprecated
@Override
public Resource getUsed() {
- if (this.used != null) {
- return this.used;
+ return getGuaranteedResourceUsed();
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsed() {
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
}
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUsed()) {
+ if (!p.hasOpportunisticResourceUsed()) {
return null;
}
- this.used = convertFromProtoFormat(p.getUsed());
- return this.used;
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getOpportunisticResourceUsed());
+ return this.opportunisticResourceUsed;
}
@Override
@@ -196,13 +212,24 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public void setNumContainers(int numContainers) {
+ public void setNumGuaranteedContainers(int numContainers) {
+ maybeInitBuilder();
+ if (numContainers == 0) {
+ builder.clearNumGuaranteedContainers();
+ return;
+ }
+ builder.setNumGuaranteedContainers(numContainers);
+ }
+
+ @Override
+ public void setNumOpportunisticContainers(int numContainers) {
maybeInitBuilder();
if (numContainers == 0) {
- builder.clearNumContainers();
+ builder.clearNumOpportunisticContainers();
return;
}
- builder.setNumContainers(numContainers);
+ builder.setNumOpportunisticContainers(numContainers);
+
}
@Override
@@ -215,13 +242,43 @@ public class NodeReportPBImpl extends NodeReport {
builder.setRackName(rackName);
}
+ @Deprecated
@Override
public void setUsed(Resource used) {
+ setGuaranteedResourceUsed(used);
+ }
+
+ @Override
+ public Resource getGuaranteedResourceUsed() {
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasGuaranteedResourceUsed()) {
+ return null;
+ }
+ this.guaranteedResourceUsed =
+ convertFromProtoFormat(p.getGuaranteedResourceUsed());
+ return this.guaranteedResourceUsed;
+ }
+
+ @Override
+ public void setGuaranteedResourceUsed(Resource guaranteed) {
+ maybeInitBuilder();
+ if (guaranteedResourceUsed == null) {
+ builder.clearGuaranteedResourceUsed();
+ }
+ this.guaranteedResourceUsed = guaranteed;
+ }
+
+ @Override
+ public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
maybeInitBuilder();
- if (used == null) {
- builder.clearUsed();
+ if (opportunisticUsed == null) {
+ builder.clearOpportunisticResourceUsed();
}
- this.used = used;
+ this.opportunisticResourceUsed = opportunisticUsed;
}
public NodeReportProto getProto() {
@@ -258,8 +315,13 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null) {
- builder.setUsed(convertToProtoFormat(this.used));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setGuaranteedResourceUsed(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setOpportunisticResourceUsed(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 83f912f..63333ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -185,26 +185,23 @@ public class BuilderUtils {
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainer,
+ String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+ guaranteedResourceUsed, capability, numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+ lastHealthReportTime, nodeLabels, null, null,
+ decommissioningTimeout, nodeUpdateType);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null, decommissioningTimeout, nodeUpdateType);
- }
-
- public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainers,
+ String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
NodeUpdateType nodeUpdateType) {
@@ -213,9 +210,11 @@ public class BuilderUtils {
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
+ nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+ nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 55a3f0b..7b560fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1027,17 +1027,26 @@ public class ClientRMService extends AbstractService implements
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed,
+ rmNode.getTotalCapability(), numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 713947f..cea00c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -340,17 +340,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ guaranteedResourceUsed =
+ schedulerNodeReport.getGuaranteedResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeId nodeId = rmNode.getNodeID();
NodeReport report =
BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed, rmNode.getTotalCapability(),
+ numGuaranteedContainers, opportunisticResourceUsed,
+ numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
rmNodeEntry.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..849db5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
for (final RMNode ni : nodes) {
- SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
- InfoMap info = new InfoMap();
- info.put("HostName", ni.getHostName());
- info.put("Rack", ni.getRackName());
- info.put("State", ni.getState().toString());
- info.put("NodeId", ni.getNodeID());
- info.put("NodeHTTPAddress", ni.getHttpAddress());
- info.put("LastHealthUpdate",
- ni.getLastHealthReportTime());
- info.put("HealthReport",
- ni.getHealthReport());
- info.put("NodeManagerVersion",
- ni.getNodeManagerVersion());
- if(report != null) {
- info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
- info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemorySize());
- }
+ SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+ InfoMap info = new InfoMap();
+ info.put("HostName", ni.getHostName());
+ info.put("Rack", ni.getRackName());
+ info.put("State", ni.getState().toString());
+ info.put("NodeId", ni.getNodeID());
+ info.put("NodeHTTPAddress", ni.getHttpAddress());
+ info.put("LastHealthUpdate",
+ ni.getLastHealthReportTime());
+ info.put("HealthReport",
+ ni.getHealthReport());
+ info.put("NodeManagerVersion",
+ ni.getNodeManagerVersion());
+ if(report != null) {
+ info.put("NumContainers", report.getNumGuaranteedContainers());
+ info.put("NumOpportunisticContainers",
+ report.getNumOpportunisticContainers());
+ info.put("UsedMemoryMB",
+ report.getGuaranteedResourceUsed().getMemorySize());
+ info.put("UsedOpportunisticMemoryMB",
+ report.getOpportunisticResourceUsed().getMemorySize());
+ info.put("AvailableMemoryMB",
+ report.getAvailableGuaranteedResource().getMemorySize());
+ }
- nodesInfo.add(info);
+ nodesInfo.add(info);
}
return JSON.toString(nodesInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 3cac38f..9e2781f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -139,7 +139,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/** Physical resources in the node. */
private volatile Resource physicalResource;
- /* Container Queue Information for the node.. Used by Distributed Scheduler */
+ /* Container Queue Information for the node..*/
private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index ea30d78..25621a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Stable
public class SchedulerNodeReport {
- private final Resource used;
- private final Resource avail;
- private final int num;
+ private final Resource guaranteedResourceUsage;
+ private final Resource opportunisticResourceUsage;
+ private final Resource guaranteedResourceAvail;
+ private final int numOfGuaranteedContainers;
+ private final int numOfOpportunisticContainers;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getAllocatedResource();
- this.avail = node.getUnallocatedResource();
- this.num = node.getNumGuaranteedContainers();
+ this.guaranteedResourceUsage = node.getAllocatedResource();
+ this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+ this.guaranteedResourceAvail = node.getUnallocatedResource();
+ this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+ this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
}
/**
- * @return the amount of resources currently used by the node.
+ * @return the amount of guaranteed resources currently used by the node.
*/
- public Resource getUsedResource() {
- return used;
+ public Resource getGuaranteedResourceUsed() {
+ return guaranteedResourceUsage;
}
/**
- * @return the amount of resources currently available on the node
+ * @return the amount of opportunistic resources currently used by the node.
*/
- public Resource getAvailableResource() {
- return avail;
+ public Resource getOpportunisticResourceUsed() {
+ return opportunisticResourceUsage;
}
/**
- * @return the number of containers currently running on this node.
+ * @return the amount of guaranteed resources currently available on the node
*/
- public int getNumContainers() {
- return num;
+ public Resource getAvailableGuaranteedResource() {
+ return guaranteedResourceAvail;
+ }
+
+ /**
+ * @return the number of guaranteed containers currently running on
+ * this node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numOfGuaranteedContainers;
+ }
+
+ /**
+ * @return the number of opportunistic containers currently running on
+ * this node.
+ */
+ public int getNumOpportunisticContainers() {
+ return numOfOpportunisticContainers;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
index 1752546..f5d8d07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
for (RMNode ni : rmContext.getRMNodes().values()) {
SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
- this.usedNodeCapacity += report.getUsedResource().getMemorySize();
- this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+ this.usedNodeCapacity +=
+ report.getGuaranteedResourceUsed().getMemorySize();
+ this.availNodeCapacity +=
+ report.getAvailableGuaranteedResource().getMemorySize();
this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
- this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ this.numContainers +=
+ fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 3cec215..2c67dfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -71,14 +71,17 @@ public class NodeInfo {
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null) {
- this.numContainers = report.getNumContainers();
- this.usedMemoryMB = report.getUsedResource().getMemorySize();
- this.availMemoryMB = report.getAvailableResource().getMemorySize();
- this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.numContainers = report.getNumGuaranteedContainers();
+ this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+ this.availMemoryMB =
+ report.getAvailableGuaranteedResource().getMemorySize();
+ this.usedVirtualCores =
+ report.getGuaranteedResourceUsed().getVirtualCores();
this.availableVirtualCores =
- report.getAvailableResource().getVirtualCores();
- this.usedResource = new ResourceInfo(report.getUsedResource());
- this.availableResource = new ResourceInfo(report.getAvailableResource());
+ report.getAvailableGuaranteedResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+ this.availableResource =
+ new ResourceInfo(report.getAvailableGuaranteedResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index ee974e3..96613fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getAvailableResource().getMemorySize());
+ this.nodeId).getAvailableGuaranteedResource().getMemorySize());
Assert.assertEquals(used.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getUsedResource().getMemorySize());
+ this.nodeId).getGuaranteedResourceUsed().getMemorySize());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index cad0151..9c317e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -176,8 +176,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// send updated request for App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
// check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(8 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 7
Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
// check node report, 1 GB used and 7 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
Assert.assertEquals(2, allocated2.size());
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// kill 1 more
counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
// check node report, 14 GB used and 2 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(14 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8ab63c1..849a2c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1267,8 +1267,10 @@ public class TestCapacityScheduler {
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1289,8 +1291,10 @@ public class TestCapacityScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1309,7 +1313,7 @@ public class TestCapacityScheduler {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+ if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1318,8 +1322,10 @@ public class TestCapacityScheduler {
}
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1335,9 +1341,11 @@ public class TestCapacityScheduler {
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2837,8 +2845,10 @@ public class TestCapacityScheduler {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 02c0cc5..8023457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(5 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(3 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(7 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(14 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Try to launch app2 in a2, asked 2GB, should success
// app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a2
cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
cs.getApplicationAttempt(am3.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a1
cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 63da0c3..0640004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -697,7 +697,8 @@ public class TestFifoScheduler {
am1.registerAppAttempt();
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -707,7 +708,8 @@ public class TestFifoScheduler {
am2.registerAppAttempt();
SchedulerNodeReport report_nm2 =
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -743,11 +745,15 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getAvailableGuaranteedResource().getMemorySize());
- Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -765,7 +771,8 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -822,7 +829,8 @@ public class TestFifoScheduler {
int checkAlloc =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -1113,8 +1121,10 @@ public class TestFifoScheduler {
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1135,8 +1145,10 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1155,7 +1167,7 @@ public class TestFifoScheduler {
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
if (null != report_nm1 &&
- report_nm1.getAvailableResource().getMemorySize() != 0) {
+ report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1165,8 +1177,10 @@ public class TestFifoScheduler {
// Now, the used resource is still 4 GB, and available resource is minus
// value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource
// over-commitment.
@@ -1184,9 +1198,11 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index fb597fc..f972681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -811,15 +811,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
if (report != null) {
assertEquals("numContainers doesn't match: " + numContainers,
- report.getNumContainers(), numContainers);
+ report.getNumGuaranteedContainers(), numContainers);
assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
- .getUsedResource().getMemorySize(), usedMemoryMB);
+ .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
- .getAvailableResource().getMemorySize(), availMemoryMB);
+ .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
- .getUsedResource().getVirtualCores(), usedVirtualCores);
- assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
- .getAvailableResource().getVirtualCores(), availVirtualCores);
+ .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+ report.getAvailableGuaranteedResource().getVirtualCores(),
+ availVirtualCores);
}
if (opportunisticStatus != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDFS-13049. RBF: Inconsistent
Router OPTS config in branch-2 and branch-3. Contributed by Wei Yan.
Posted by ha...@apache.org.
HDFS-13049. RBF: Inconsistent Router OPTS config in branch-2 and branch-3. Contributed by Wei Yan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c139d5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c139d5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c139d5b
Branch: refs/heads/YARN-1011
Commit: 0c139d5bcfbcd62fc69111ee6204926c57d57bf1
Parents: 82cc6f6
Author: Wei Yan <we...@apache.org>
Authored: Thu Jan 25 11:36:28 2018 -0800
Committer: Wei Yan <we...@apache.org>
Committed: Thu Jan 25 11:36:40 2018 -0800
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/hadoop-env.cmd | 9 +++++++++
.../hadoop-common/src/main/conf/hadoop-env.sh | 9 +++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c139d5b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
index 5dbd635..159840a 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
@@ -70,6 +70,15 @@ set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
@rem Where log files are stored in the secure data environment.
set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+@rem
+@rem Router-based HDFS Federation specific parameters
+@rem Specify the JVM options to be used when starting the RBF Routers.
+@rem These options will be appended to the options specified as HADOOP_OPTS
+@rem and therefore may override any similar flags set in HADOOP_OPTS
+@rem
+@rem set HADOOP_DFSROUTER_OPTS=""
+@rem
+
@rem The directory where pid files are stored. /tmp by default.
@rem NOTE: this should be set to a directory that can only be written to by
@rem the user that will run the hadoop daemons. Otherwise there is the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c139d5b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index bef4dab..24aacdf 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -396,6 +396,15 @@ esac
# export HDFS_MOVER_OPTS=""
###
+# Router-based HDFS Federation specific parameters
+# Specify the JVM options to be used when starting the RBF Routers.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# export HDFS_DFSROUTER_OPTS=""
+###
+
+###
# Advanced Users Only!
###
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org