You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/12/19 01:36:36 UTC
[15/20] hadoop git commit: YARN-4418. AM Resource Limit per partition
can be updated to ResourceUsage as well. (Sunil G via wangda)
YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda)
(cherry picked from commit 07b0fb996a32020678bd2ce482b672f0434651f0)
(cherry picked from commit c915a658c8a5481ca88d8b2f6ce9a72f2e1f5b6a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d1c6c1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d1c6c1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d1c6c1f
Branch: refs/heads/branch-2.8
Commit: 1d1c6c1f8dbb221f41bc5bb87cbea37f71771a41
Parents: c7e10d7
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Dec 14 11:24:30 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:44 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 62 ++++++++++++++++++--
.../scheduler/capacity/LeafQueue.java | 1 +
.../scheduler/capacity/QueueCapacities.java | 9 +++
.../scheduler/TestResourceUsage.java | 22 +++++--
.../scheduler/capacity/TestQueueCapacities.java | 3 +-
6 files changed, 88 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6d3f519..155ba2a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.8.0 - UNRELEASED
YARN-4309. Add container launch related debug information to container logs
when a container fails. (Varun Vasudev via wangda)
+ YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
+ (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 88e93c1..2857379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -61,7 +61,7 @@ public class ResourceUsage {
//CACHED_USED and CACHED_PENDING may be read by anyone, but must only
//be written by ordering policies
USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
- CACHED_PENDING(5);
+ CACHED_PENDING(5), AMLIMIT(6);
private int idx;
@@ -92,6 +92,7 @@ public class ResourceUsage {
sb.append("pending=" + resArr[1] + "%, ");
sb.append("am_used=" + resArr[2] + "%, ");
sb.append("reserved=" + resArr[3] + "%}");
+ sb.append("am_limit=" + resArr[6] + "%, ");
return sb.toString();
}
}
@@ -106,11 +107,19 @@ public class ResourceUsage {
public Resource getUsed(String label) {
return _get(label, ResourceType.USED);
}
-
+
+ public Resource getCachedUsed() {
+ return _get(NL, ResourceType.CACHED_USED);
+ }
+
public Resource getCachedUsed(String label) {
return _get(label, ResourceType.CACHED_USED);
}
-
+
+ public Resource getCachedPending() {
+ return _get(NL, ResourceType.CACHED_PENDING);
+ }
+
public Resource getCachedPending(String label) {
return _get(label, ResourceType.CACHED_PENDING);
}
@@ -149,15 +158,23 @@ public class ResourceUsage {
public void setUsed(String label, Resource res) {
_set(label, ResourceType.USED, res);
}
-
+
public void setCachedUsed(String label, Resource res) {
_set(label, ResourceType.CACHED_USED, res);
}
-
+
+ public void setCachedUsed(Resource res) {
+ _set(NL, ResourceType.CACHED_USED, res);
+ }
+
public void setCachedPending(String label, Resource res) {
_set(label, ResourceType.CACHED_PENDING, res);
}
+ public void setCachedPending(Resource res) {
+ _set(NL, ResourceType.CACHED_PENDING, res);
+ }
+
/*
* Pending
*/
@@ -263,6 +280,41 @@ public class ResourceUsage {
_set(label, ResourceType.AMUSED, res);
}
+ /*
+ * AM-Resource Limit
+ */
+ public Resource getAMLimit() {
+ return getAMLimit(NL);
+ }
+
+ public Resource getAMLimit(String label) {
+ return _get(label, ResourceType.AMLIMIT);
+ }
+
+ public void incAMLimit(String label, Resource res) {
+ _inc(label, ResourceType.AMLIMIT, res);
+ }
+
+ public void incAMLimit(Resource res) {
+ incAMLimit(NL, res);
+ }
+
+ public void decAMLimit(Resource res) {
+ decAMLimit(NL, res);
+ }
+
+ public void decAMLimit(String label, Resource res) {
+ _dec(label, ResourceType.AMLIMIT, res);
+ }
+
+ public void setAMLimit(Resource res) {
+ setAMLimit(NL, res);
+ }
+
+ public void setAMLimit(String label, Resource res) {
+ _set(label, ResourceType.AMLIMIT, res);
+ }
+
private static Resource normalize(Resource res) {
if (res == null) {
return Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f58aba2..b74ce45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -605,6 +605,7 @@ public class LeafQueue extends AbstractCSQueue {
minimumAllocation);
metrics.setAMResouceLimit(amResouceLimit);
+ queueUsage.setAMLimit(nodePartition, amResouceLimit);
return amResouceLimit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 65100f6..f2c2632 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -217,6 +217,11 @@ public class QueueCapacities {
}
/* Absolute Maximum AM resource percentage Getter and Setter */
+
+ public float getMaxAMResourcePercentage() {
+ return _get(NL, CapacityType.MAX_AM_PERC);
+ }
+
public float getMaxAMResourcePercentage(String label) {
return _get(label, CapacityType.MAX_AM_PERC);
}
@@ -225,6 +230,10 @@ public class QueueCapacities {
_set(label, CapacityType.MAX_AM_PERC, value);
}
+ public void setMaxAMResourcePercentage(float value) {
+ _set(NL, CapacityType.MAX_AM_PERC, value);
+ }
+
/**
* Clear configurable fields, like
* (absolute)capacity/(absolute)maximum-capacity, this will be used by queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
index f0bf892..1ddeb0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
@@ -37,8 +37,8 @@ public class TestResourceUsage {
@Parameterized.Parameters
public static Collection<String[]> getParameters() {
- return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
- { "Reserved" }, { "AMUsed" } });
+ return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"},
+ {"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}});
}
public TestResourceUsage(String suffix) {
@@ -112,16 +112,26 @@ public class TestResourceUsage {
check(0, 0, res);
// Add 1,1 should returns 1,1
- inc(usage, suffix, Resource.newInstance(1, 1), label);
- check(1, 1, get(usage, suffix, label));
+ try {
+ inc(usage, suffix, Resource.newInstance(1, 1), label);
+ check(1, 1, get(usage, suffix, label));
+ } catch (NoSuchMethodException e) {
+ // Few operations need not have to be verified as some resources doesn't
+ // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+ }
// Set 2,2
set(usage, suffix, Resource.newInstance(2, 2), label);
check(2, 2, get(usage, suffix, label));
// dec 2,2
- dec(usage, suffix, Resource.newInstance(2, 2), label);
- check(0, 0, get(usage, suffix, label));
+ try {
+ dec(usage, suffix, Resource.newInstance(2, 2), label);
+ check(0, 0, get(usage, suffix, label));
+ } catch (NoSuchMethodException e) {
+ // Few operations need not have to be verified, as some resources doesn't
+ // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+ }
}
void check(int mem, int cpu, Resource res) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
index d543f80..9d2fa15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -43,7 +43,8 @@ public class TestQueueCapacities {
{ "UsedCapacity" },
{ "AbsoluteUsedCapacity" },
{ "MaximumCapacity" },
- { "AbsoluteMaximumCapacity" } });
+ { "AbsoluteMaximumCapacity" },
+ { "MaxAMResourcePercentage" } });
}
public TestQueueCapacities(String suffix) {