You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/01 19:55:27 UTC
[26/39] hadoop git commit: YARN-4844. Add
getMemorySize/getVirtualCoresSize to o.a.h.y.api.records.Resource.
Contributed by Wangda Tan.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index b5c9967..8f97fab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -54,15 +54,15 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
if (isInvalidDivisor(clusterResource)) {
- if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
+ if ((lhs.getMemorySize() < rhs.getMemorySize() && lhs.getVirtualCores() > rhs
.getVirtualCores())
- || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs
+ || (lhs.getMemorySize() > rhs.getMemorySize() && lhs.getVirtualCores() < rhs
.getVirtualCores())) {
return 0;
- } else if (lhs.getMemory() > rhs.getMemory()
+ } else if (lhs.getMemorySize() > rhs.getMemorySize()
|| lhs.getVirtualCores() > rhs.getVirtualCores()) {
return 1;
- } else if (lhs.getMemory() < rhs.getMemory()
+ } else if (lhs.getMemorySize() < rhs.getMemorySize()
|| lhs.getVirtualCores() < rhs.getVirtualCores()) {
return -1;
}
@@ -100,20 +100,20 @@ public class DominantResourceCalculator extends ResourceCalculator {
// Just use 'dominant' resource
return (dominant) ?
Math.max(
- (float)resource.getMemory() / clusterResource.getMemory(),
+ (float)resource.getMemorySize() / clusterResource.getMemorySize(),
(float)resource.getVirtualCores() / clusterResource.getVirtualCores()
)
:
Math.min(
- (float)resource.getMemory() / clusterResource.getMemory(),
+ (float)resource.getMemorySize() / clusterResource.getMemorySize(),
(float)resource.getVirtualCores() / clusterResource.getVirtualCores()
);
}
@Override
- public int computeAvailableContainers(Resource available, Resource required) {
+ public long computeAvailableContainers(Resource available, Resource required) {
return Math.min(
- available.getMemory() / required.getMemory(),
+ available.getMemorySize() / required.getMemorySize(),
available.getVirtualCores() / required.getVirtualCores());
}
@@ -127,7 +127,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean isInvalidDivisor(Resource r) {
- if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+ if (r.getMemorySize() == 0.0f || r.getVirtualCores() == 0.0f) {
return true;
}
return false;
@@ -136,15 +136,15 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public float ratio(Resource a, Resource b) {
return Math.max(
- (float)a.getMemory()/b.getMemory(),
+ (float)a.getMemorySize()/b.getMemorySize(),
(float)a.getVirtualCores()/b.getVirtualCores()
);
}
@Override
- public Resource divideAndCeil(Resource numerator, int denominator) {
+ public Resource divideAndCeil(Resource numerator, long denominator) {
return Resources.createResource(
- divideAndCeil(numerator.getMemory(), denominator),
+ divideAndCeil(numerator.getMemorySize(), denominator),
divideAndCeil(numerator.getVirtualCores(), denominator)
);
}
@@ -152,12 +152,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
- int normalizedMemory = Math.min(
+ long normalizedMemory = Math.min(
roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
- stepFactor.getMemory()),
- maximumResource.getMemory());
- int normalizedCores = Math.min(
+ Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
+ stepFactor.getMemorySize()),
+ maximumResource.getMemorySize());
+ long normalizedCores = Math.min(
roundUp(
Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()),
stepFactor.getVirtualCores()),
@@ -169,7 +169,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemory(), stepFactor.getMemory()),
+ roundUp(r.getMemorySize(), stepFactor.getMemorySize()),
roundUp(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
@@ -177,7 +177,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource roundDown(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundDown(r.getMemory(), stepFactor.getMemory()),
+ roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
roundDown(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
@@ -187,7 +187,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource stepFactor) {
return Resources.createResource(
roundUp(
- (int)Math.ceil(r.getMemory() * by), stepFactor.getMemory()),
+ (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()),
roundUp(
(int)Math.ceil(r.getVirtualCores() * by),
stepFactor.getVirtualCores())
@@ -199,8 +199,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource stepFactor) {
return Resources.createResource(
roundDown(
- (int)(r.getMemory() * by),
- stepFactor.getMemory()
+ (int)(r.getMemorySize() * by),
+ stepFactor.getMemorySize()
),
roundDown(
(int)(r.getVirtualCores() * by),
@@ -212,7 +212,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean fitsIn(Resource cluster,
Resource smaller, Resource bigger) {
- return smaller.getMemory() <= bigger.getMemory()
+ return smaller.getMemorySize() <= bigger.getMemorySize()
&& smaller.getVirtualCores() <= bigger.getVirtualCores();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 3a31225..3fe8ade 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -31,18 +31,18 @@ public abstract class ResourceCalculator {
public abstract int
compare(Resource clusterResource, Resource lhs, Resource rhs);
- public static int divideAndCeil(int a, int b) {
+ public static long divideAndCeil(long a, long b) {
if (b == 0) {
return 0;
}
return (a + (b - 1)) / b;
}
- public static int roundUp(int a, int b) {
+ public static long roundUp(long a, long b) {
return divideAndCeil(a, b) * b;
}
- public static int roundDown(int a, int b) {
+ public static long roundDown(long a, long b) {
return (a / b) * b;
}
@@ -54,7 +54,7 @@ public abstract class ResourceCalculator {
* @param required required resources
* @return number of containers which can be allocated
*/
- public abstract int computeAvailableContainers(
+ public abstract long computeAvailableContainers(
Resource available, Resource required);
/**
@@ -169,7 +169,7 @@ public abstract class ResourceCalculator {
* @param denominator denominator
* @return resultant resource
*/
- public abstract Resource divideAndCeil(Resource numerator, int denominator);
+ public abstract Resource divideAndCeil(Resource numerator, long denominator);
/**
* Check if a smaller resource can be contained by bigger resource.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 558f96c..e944e11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -31,12 +31,18 @@ public class Resources {
private static final Resource NONE = new Resource() {
@Override
+ @SuppressWarnings("deprecation")
public int getMemory() {
return 0;
}
@Override
- public void setMemory(int memory) {
+ public long getMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public void setMemory(long memory) {
throw new RuntimeException("NONE cannot be modified!");
}
@@ -46,17 +52,22 @@ public class Resources {
}
@Override
- public void setVirtualCores(int cores) {
+ public long getVirtualCoresSize() {
+ return 0;
+ }
+
+ @Override
+ public void setVirtualCores(long cores) {
throw new RuntimeException("NONE cannot be modified!");
}
@Override
public int compareTo(Resource o) {
- int diff = 0 - o.getMemory();
+ long diff = 0 - o.getMemorySize();
if (diff == 0) {
diff = 0 - o.getVirtualCores();
}
- return diff;
+ return Long.signum(diff);
}
};
@@ -64,12 +75,18 @@ public class Resources {
private static final Resource UNBOUNDED = new Resource() {
@Override
+ @SuppressWarnings("deprecation")
public int getMemory() {
return Integer.MAX_VALUE;
}
@Override
- public void setMemory(int memory) {
+ public long getMemorySize() {
+ return Long.MAX_VALUE;
+ }
+
+ @Override
+ public void setMemory(long memory) {
throw new RuntimeException("UNBOUNDED cannot be modified!");
}
@@ -79,26 +96,31 @@ public class Resources {
}
@Override
- public void setVirtualCores(int cores) {
+ public long getVirtualCoresSize() {
+ return Long.MAX_VALUE;
+ }
+
+ @Override
+ public void setVirtualCores(long cores) {
throw new RuntimeException("UNBOUNDED cannot be modified!");
}
@Override
public int compareTo(Resource o) {
- int diff = Integer.MAX_VALUE - o.getMemory();
+ long diff = Long.MAX_VALUE - o.getMemorySize();
if (diff == 0) {
- diff = Integer.MAX_VALUE - o.getVirtualCores();
+ diff = Long.MAX_VALUE - o.getVirtualCoresSize();
}
- return diff;
+ return Long.signum(diff);
}
};
- public static Resource createResource(int memory) {
+ public static Resource createResource(long memory) {
return createResource(memory, (memory > 0) ? 1 : 0);
}
- public static Resource createResource(int memory, int cores) {
+ public static Resource createResource(long memory, long cores) {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(cores);
@@ -114,11 +136,11 @@ public class Resources {
}
public static Resource clone(Resource res) {
- return createResource(res.getMemory(), res.getVirtualCores());
+ return createResource(res.getMemorySize(), res.getVirtualCores());
}
public static Resource addTo(Resource lhs, Resource rhs) {
- lhs.setMemory(lhs.getMemory() + rhs.getMemory());
+ lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize());
lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
return lhs;
}
@@ -128,7 +150,7 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- lhs.setMemory(lhs.getMemory() - rhs.getMemory());
+ lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize());
lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores());
return lhs;
}
@@ -142,7 +164,7 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- lhs.setMemory((int)(lhs.getMemory() * by));
+ lhs.setMemory((int)(lhs.getMemorySize() * by));
lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
return lhs;
}
@@ -157,7 +179,7 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- lhs.setMemory(lhs.getMemory() + (int)(rhs.getMemory() * by));
+ lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by));
lhs.setVirtualCores(lhs.getVirtualCores()
+ (int)(rhs.getVirtualCores() * by));
return lhs;
@@ -175,7 +197,7 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- out.setMemory((int)(lhs.getMemory() * by));
+ out.setMemory((int)(lhs.getMemorySize() * by));
out.setVirtualCores((int)(lhs.getVirtualCores() * by));
return out;
}
@@ -264,7 +286,7 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- return smaller.getMemory() <= bigger.getMemory() &&
+ return smaller.getMemorySize() <= bigger.getMemorySize() &&
smaller.getVirtualCores() <= bigger.getVirtualCores();
}
@@ -274,12 +296,12 @@ public class Resources {
}
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
- return createResource(Math.min(lhs.getMemory(), rhs.getMemory()),
+ return createResource(Math.min(lhs.getMemorySize(), rhs.getMemorySize()),
Math.min(lhs.getVirtualCores(), rhs.getVirtualCores()));
}
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
- return createResource(Math.max(lhs.getMemory(), rhs.getMemory()),
+ return createResource(Math.max(lhs.getMemorySize(), rhs.getMemorySize()),
Math.max(lhs.getVirtualCores(), rhs.getVirtualCores()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index d38ffdc..7396adc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue;
public class TestResources {
- public Resource createResource(int memory, int vCores) {
+ public Resource createResource(long memory, long vCores) {
return Resource.newInstance(memory, vCores);
}
@Test(timeout=1000)
public void testCompareToWithUnboundedResource() {
assertTrue(Resources.unbounded().compareTo(
- createResource(Integer.MAX_VALUE, Integer.MAX_VALUE)) == 0);
+ createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0);
assertTrue(Resources.unbounded().compareTo(
- createResource(Integer.MAX_VALUE, 0)) > 0);
+ createResource(Long.MAX_VALUE, 0)) > 0);
assertTrue(Resources.unbounded().compareTo(
- createResource(0, Integer.MAX_VALUE)) > 0);
+ createResource(0, Long.MAX_VALUE)) > 0);
}
@Test(timeout=1000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index b97f935..22e45fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -442,7 +442,7 @@ public class BuilderUtils {
return report;
}
- public static Resource newResource(int memory, int vCores) {
+ public static Resource newResource(long memory, long vCores) {
Resource resource = recordFactory.newRecordInstance(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(vCores);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index cad3b2e..ff4519d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -59,8 +59,8 @@ public class AppInfo {
protected long elapsedTime;
protected String applicationTags;
protected int priority;
- private int allocatedCpuVcores;
- private int allocatedMemoryMB;
+ private long allocatedCpuVcores;
+ private long allocatedMemoryMB;
protected boolean unmanagedApplication;
private String appNodeLabelExpression;
private String amNodeLabelExpression;
@@ -100,7 +100,7 @@ public class AppInfo {
allocatedCpuVcores = app.getApplicationResourceUsageReport()
.getUsedResources().getVirtualCores();
allocatedMemoryMB = app.getApplicationResourceUsageReport()
- .getUsedResources().getMemory();
+ .getUsedResources().getMemorySize();
}
}
progress = app.getProgress() * 100; // in percent
@@ -152,11 +152,11 @@ public class AppInfo {
return runningContainers;
}
- public int getAllocatedCpuVcores() {
+ public long getAllocatedCpuVcores() {
return allocatedCpuVcores;
}
- public int getAllocatedMemoryMB() {
+ public long getAllocatedMemoryMB() {
return allocatedMemoryMB;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
index f127f9c..1a5ee85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.yarn.util.Times;
public class ContainerInfo {
protected String containerId;
- protected int allocatedMB;
- protected int allocatedVCores;
+ protected long allocatedMB;
+ protected long allocatedVCores;
protected String assignedNodeId;
protected int priority;
protected long startedTime;
@@ -57,7 +57,7 @@ public class ContainerInfo {
public ContainerInfo(ContainerReport container) {
containerId = container.getContainerId().toString();
if (container.getAllocatedResource() != null) {
- allocatedMB = container.getAllocatedResource().getMemory();
+ allocatedMB = container.getAllocatedResource().getMemorySize();
allocatedVCores = container.getAllocatedResource().getVirtualCores();
}
if (container.getAssignedNode() != null) {
@@ -79,11 +79,11 @@ public class ContainerInfo {
return containerId;
}
- public int getAllocatedMB() {
+ public long getAllocatedMB() {
return allocatedMB;
}
- public int getAllocatedVCores() {
+ public long getAllocatedVCores() {
return allocatedVCores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index c9427dd..e25547d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -180,7 +180,7 @@ public class TestYarnServerApiClasses {
assertEquals(1, copy.getContainersToDecrease().get(0)
.getId().getContainerId());
assertEquals(1024, copy.getContainersToDecrease().get(1)
- .getResource().getMemory());
+ .getResource().getMemorySize());
}
/**
@@ -201,7 +201,7 @@ public class TestYarnServerApiClasses {
assertEquals(8080, copy.getHttpPort());
assertEquals(9090, copy.getNodeId().getPort());
- assertEquals(10000, copy.getResource().getMemory());
+ assertEquals(10000, copy.getResource().getMemorySize());
assertEquals(2, copy.getResource().getVirtualCores());
}
@@ -273,7 +273,7 @@ public class TestYarnServerApiClasses {
assertEquals(1, copy.getIncreasedContainers().get(0)
.getId().getContainerId());
assertEquals(4096, copy.getIncreasedContainers().get(1)
- .getResource().getMemory());
+ .getResource().getMemorySize());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index d08ee67..98171af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -405,7 +405,7 @@ public abstract class ContainerExecutor implements Configurable {
.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
- memory = resource.getMemory();
+ memory = (int) resource.getMemorySize();
}
if (conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 2ff8e7c..304488e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -215,7 +215,7 @@ public class ContainerImpl implements Container {
if (recoveredCapability != null
&& !this.resource.equals(recoveredCapability)) {
// resource capability had been updated before NM was down
- this.resource = Resource.newInstance(recoveredCapability.getMemory(),
+ this.resource = Resource.newInstance(recoveredCapability.getMemorySize(),
recoveredCapability.getVirtualCores());
}
this.remainingRetryAttempts = rcs.getRemainingRetryAttempts();
@@ -611,7 +611,7 @@ public class ContainerImpl implements Container {
long launchDuration = clock.getTime() - containerLaunchStartTime;
metrics.addContainerLaunchDuration(launchDuration);
- long pmemBytes = getResource().getMemory() * 1024 * 1024L;
+ long pmemBytes = getResource().getMemorySize() * 1024 * 1024L;
float pmemRatio = daemonConf.getFloat(
YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index 78113e5..4630c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -115,8 +115,8 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
String cgroupId = container.getContainerId().toString();
//memory is in MB
long containerSoftLimit =
- (long) (container.getResource().getMemory() * this.softLimit);
- long containerHardLimit = container.getResource().getMemory();
+ (long) (container.getResource().getMemorySize() * this.softLimit);
+ long containerHardLimit = container.getResource().getMemorySize();
cGroupsHandler.createCGroup(MEMORY, cgroupId);
try {
cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 8ca744d..3fbae10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -646,7 +646,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ChangeMonitoringContainerResourceEvent changeEvent =
(ChangeMonitoringContainerResourceEvent) monitoringEvent;
Resource resource = changeEvent.getResource();
- pmemLimitMBs = resource.getMemory();
+ pmemLimitMBs = (int) resource.getMemorySize();
vmemLimitMBs = (int) (pmemLimitMBs * vmemRatio);
cpuVcores = resource.getVirtualCores();
usageMetrics.recordResourceLimit(
@@ -822,7 +822,7 @@ public class ContainersMonitorImpl extends AbstractService implements
}
LOG.info("Changing resource-monitoring for " + containerId);
updateContainerMetrics(monitoringEvent);
- long pmemLimit = changeEvent.getResource().getMemory() * 1024L * 1024L;
+ long pmemLimit = changeEvent.getResource().getMemorySize() * 1024L * 1024L;
long vmemLimit = (long) (pmemLimit * vmemRatio);
int cpuVcores = changeEvent.getResource().getVirtualCores();
processTreeInfo.setResourceLimit(pmemLimit, vmemLimit, cpuVcores);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
index a750d93..1ce3356 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
@@ -600,7 +600,7 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
private ProcessTreeInfo createProcessTreeInfo(ContainerId containerId,
Resource resource, Configuration conf) {
- long pmemBytes = resource.getMemory() * 1024 * 1024L;
+ long pmemBytes = resource.getMemorySize() * 1024 * 1024L;
float pmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
long vmemBytes = (long) (pmemRatio * pmemBytes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index a38d0b7..11e8684 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -115,9 +115,9 @@ public class NodeManagerMetrics {
public void allocateContainer(Resource res) {
allocatedContainers.incr();
- allocatedMB = allocatedMB + res.getMemory();
+ allocatedMB = allocatedMB + res.getMemorySize();
allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
- availableMB = availableMB - res.getMemory();
+ availableMB = availableMB - res.getMemorySize();
availableGB.set((int)Math.floor(availableMB/1024d));
allocatedVCores.incr(res.getVirtualCores());
availableVCores.decr(res.getVirtualCores());
@@ -125,16 +125,16 @@ public class NodeManagerMetrics {
public void releaseContainer(Resource res) {
allocatedContainers.decr();
- allocatedMB = allocatedMB - res.getMemory();
+ allocatedMB = allocatedMB - res.getMemorySize();
allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
- availableMB = availableMB + res.getMemory();
+ availableMB = availableMB + res.getMemorySize();
availableGB.set((int)Math.floor(availableMB/1024d));
allocatedVCores.decr(res.getVirtualCores());
availableVCores.incr(res.getVirtualCores());
}
public void changeContainer(Resource before, Resource now) {
- int deltaMB = now.getMemory() - before.getMemory();
+ long deltaMB = now.getMemorySize() - before.getMemorySize();
int deltaVCores = now.getVirtualCores() - before.getVirtualCores();
allocatedMB = allocatedMB + deltaMB;
allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
@@ -145,7 +145,7 @@ public class NodeManagerMetrics {
}
public void addResource(Resource res) {
- availableMB = availableMB + res.getMemory();
+ availableMB = availableMB + res.getMemorySize();
availableGB.incr((int)Math.floor(availableMB/1024d));
availableVCores.incr(res.getVirtualCores());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
index cf022b9..10a8156 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
@@ -81,7 +81,7 @@ public class ContainerInfo {
this.user = container.getUser();
Resource res = container.getResource();
if (res != null) {
- this.totalMemoryNeededMB = res.getMemory();
+ this.totalMemoryNeededMB = res.getMemorySize();
this.totalVCoresNeeded = res.getVirtualCores();
}
this.containerLogsShortLink = ujoin("containerlogs", this.id,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 689a327..7975f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -190,7 +190,7 @@ public class TestNodeStatusUpdater {
InetSocketAddress expected = NetUtils.getConnectAddress(
conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString());
- Assert.assertEquals(5 * 1024, resource.getMemory());
+ Assert.assertEquals(5 * 1024, resource.getMemorySize());
registeredNodes.add(nodeId);
RegisterNodeManagerResponse response = recordFactory
@@ -918,7 +918,7 @@ public class TestNodeStatusUpdater {
conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
Assert.assertEquals(NetUtils.getHostPortString(expected),
nodeId.toString());
- Assert.assertEquals(5 * 1024, resource.getMemory());
+ Assert.assertEquals(5 * 1024, resource.getMemorySize());
registeredNodes.add(nodeId);
RegisterNodeManagerResponse response = recordFactory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index 008f7ee..94145e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -228,7 +228,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
+ Resource r = BuilderUtils.newResource(0, 0);
ContainerTokenIdentifier containerIdentifier =
new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index ef4a0d4..f6af030 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -94,9 +94,9 @@ public class RMNMInfo implements RMNMInfoBeans {
ni.getNodeManagerVersion());
if(report != null) {
info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemory());
+ info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemory());
+ report.getAvailableResource().getMemorySize());
}
nodesInfo.add(info);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 1318d58..95fdb05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -142,7 +142,7 @@ public class RMServerUtils {
// example, you cannot request target resource of a <10G, 10> container to
// <20G, 8>
if (increase) {
- if (originalResource.getMemory() > targetResource.getMemory()
+ if (originalResource.getMemorySize() > targetResource.getMemorySize()
|| originalResource.getVirtualCores() > targetResource
.getVirtualCores()) {
String msg =
@@ -153,7 +153,7 @@ public class RMServerUtils {
throw new InvalidResourceRequestException(msg);
}
} else {
- if (originalResource.getMemory() < targetResource.getMemory()
+ if (originalResource.getMemorySize() < targetResource.getMemorySize()
|| originalResource.getVirtualCores() < targetResource
.getVirtualCores()) {
String msg =
@@ -243,15 +243,15 @@ public class RMServerUtils {
return;
}
for (ContainerResourceChangeRequest request : requests) {
- if (request.getCapability().getMemory() < 0
- || request.getCapability().getMemory() > maximumAllocation
- .getMemory()) {
+ if (request.getCapability().getMemorySize() < 0
+ || request.getCapability().getMemorySize() > maximumAllocation
+ .getMemorySize()) {
throw new InvalidResourceRequestException("Invalid "
+ (increase ? "increase" : "decrease") + " request"
+ ", requested memory < 0"
+ ", or requested memory > max configured" + ", requestedMemory="
- + request.getCapability().getMemory() + ", maxMemory="
- + maximumAllocation.getMemory());
+ + request.getCapability().getMemorySize() + ", maxMemory="
+ + maximumAllocation.getMemorySize());
}
if (request.getCapability().getVirtualCores() < 0
|| request.getCapability().getVirtualCores() > maximumAllocation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f8c9258..c55884e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -362,7 +362,7 @@ public class ResourceTrackerService extends AbstractService implements
}
// Check if this node has minimum allocations
- if (capability.getMemory() < minAllocMb
+ if (capability.getMemorySize() < minAllocMb
|| capability.getVirtualCores() < minAllocVcores) {
String message =
"NodeManager from " + host
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 0e3802b..b0b976d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -480,7 +480,7 @@ public class SystemMetricsPublisher extends CompositeService {
TimelineEntity entity = createContainerEntity(event.getContainerId());
Map<String, Object> entityInfo = new HashMap<String, Object>();
entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
- event.getAllocatedResource().getMemory());
+ event.getAllocatedResource().getMemorySize());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
event.getAllocatedResource().getVirtualCores());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 116cd22..04ed135 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -258,17 +258,17 @@ public class TempQueuePerPartition {
void appendLogString(StringBuilder sb) {
sb.append(queueName).append(", ")
- .append(current.getMemory()).append(", ")
+ .append(current.getMemorySize()).append(", ")
.append(current.getVirtualCores()).append(", ")
- .append(pending.getMemory()).append(", ")
+ .append(pending.getMemorySize()).append(", ")
.append(pending.getVirtualCores()).append(", ")
- .append(getGuaranteed().getMemory()).append(", ")
+ .append(getGuaranteed().getMemorySize()).append(", ")
.append(getGuaranteed().getVirtualCores()).append(", ")
- .append(idealAssigned.getMemory()).append(", ")
+ .append(idealAssigned.getMemorySize()).append(", ")
.append(idealAssigned.getVirtualCores()).append(", ")
- .append(toBePreempted.getMemory()).append(", ")
+ .append(toBePreempted.getMemorySize()).append(", ")
.append(toBePreempted.getVirtualCores() ).append(", ")
- .append(actuallyToBePreempted.getMemory()).append(", ")
+ .append(actuallyToBePreempted.getMemorySize()).append(", ")
.append(actuallyToBePreempted.getVirtualCores());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
index eaf2902..90357e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
@@ -160,7 +160,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
Resource capToAssign = res.getResourcesAtTime(now);
float targetCapacity = 0f;
- if (planResources.getMemory() > 0
+ if (planResources.getMemorySize() > 0
&& planResources.getVirtualCores() > 0) {
if (shouldResize) {
capToAssign =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
index f8b68e3..07bee99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
@@ -246,7 +246,7 @@ public class CapacityOverTimePolicy implements SharingPolicy {
long vcores;
public IntegralResource(Resource resource) {
- this.memory = resource.getMemory();
+ this.memory = resource.getMemorySize();
this.vcores = resource.getVirtualCores();
}
@@ -256,12 +256,12 @@ public class CapacityOverTimePolicy implements SharingPolicy {
}
public void add(Resource r) {
- memory += r.getMemory();
+ memory += r.getMemorySize();
vcores += r.getVirtualCores();
}
public void subtract(Resource r) {
- memory -= r.getMemory();
+ memory -= r.getMemorySize();
vcores -= r.getVirtualCores();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java
index aba4822..81d32c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java
@@ -106,7 +106,7 @@ public final class ReservationSystemUtil {
public static ResourceProto convertToProtoFormat(Resource e) {
return YarnProtos.ResourceProto.newBuilder()
- .setMemory(e.getMemory())
+ .setMemory(e.getMemorySize())
.setVirtualCores(e.getVirtualCores())
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java
index 5a46a4e..43d6584 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java
@@ -88,7 +88,7 @@ public class StageEarliestStartByDemand implements StageEarliestStart {
// Weight = total memory consumption of stage
protected double calcWeight(ReservationRequest stage) {
- return (stage.getDuration() * stage.getCapability().getMemory())
+ return (stage.getDuration() * stage.getCapability().getMemorySize())
* (stage.getNumContainers());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 973a7db..5121493 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -712,7 +712,7 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
}
long usedMillis = container.finishTime - container.creationTime;
- long memorySeconds = resource.getMemory()
+ long memorySeconds = resource.getMemorySize()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
long vcoreSeconds = resource.getVirtualCores()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 34b4267..feb071f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -56,8 +56,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Resource staleClusterCapacity = null;
// Max allocation
- private int maxNodeMemory = -1;
- private int maxNodeVCores = -1;
+ private long maxNodeMemory = -1;
+ private long maxNodeVCores = -1;
private Resource configuredMaxAllocation;
private boolean forceConfiguredMaxAllocation = true;
private long configuredMaxAllocationWaitTime;
@@ -211,7 +211,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
return Resources.createResource(
- Math.min(configuredMaxAllocation.getMemory(), maxNodeMemory),
+ Math.min(configuredMaxAllocation.getMemorySize(), maxNodeMemory),
Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores)
);
} finally {
@@ -224,7 +224,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
if (add) { // added node
- int nodeMemory = totalResource.getMemory();
+ long nodeMemory = totalResource.getMemorySize();
if (nodeMemory > maxNodeMemory) {
maxNodeMemory = nodeMemory;
}
@@ -233,7 +233,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
maxNodeVCores = nodeVCores;
}
} else { // removed node
- if (maxNodeMemory == totalResource.getMemory()) {
+ if (maxNodeMemory == totalResource.getMemorySize()) {
maxNodeMemory = -1;
}
if (maxNodeVCores == totalResource.getVirtualCores()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index d9c7283..69c3e8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterInt;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -59,8 +60,8 @@ public class QueueMetrics implements MetricsSource {
@Metric("# of apps killed") MutableCounterInt appsKilled;
@Metric("# of apps failed") MutableCounterInt appsFailed;
- @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB;
- @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores;
+ @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB;
+ @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores;
@Metric("# of allocated containers") MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of allocated node-local containers")
@@ -70,13 +71,13 @@ public class QueueMetrics implements MetricsSource {
@Metric("Aggregate # of allocated off-switch containers")
MutableCounterLong aggregateOffSwitchContainersAllocated;
@Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased;
- @Metric("Available memory in MB") MutableGaugeInt availableMB;
- @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores;
- @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB;
- @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores;
+ @Metric("Available memory in MB") MutableGaugeLong availableMB;
+ @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores;
+ @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB;
+ @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores;
@Metric("# of pending containers") MutableGaugeInt pendingContainers;
- @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB;
- @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores;
+ @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB;
+ @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores;
@Metric("# of reserved containers") MutableGaugeInt reservedContainers;
@Metric("# of active users") MutableGaugeInt activeUsers;
@Metric("# of active applications") MutableGaugeInt activeApplications;
@@ -325,7 +326,7 @@ public class QueueMetrics implements MetricsSource {
* @param limit resource limit
*/
public void setAvailableResourcesToQueue(Resource limit) {
- availableMB.set(limit.getMemory());
+ availableMB.set(limit.getMemorySize());
availableVCores.set(limit.getVirtualCores());
}
@@ -362,8 +363,8 @@ public class QueueMetrics implements MetricsSource {
private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers);
- pendingMB.incr(res.getMemory() * containers);
- pendingVCores.incr(res.getVirtualCores() * containers);
+ pendingMB.incr(res.getMemorySize() * containers);
+ pendingVCores.incr(res.getVirtualCoresSize() * containers);
}
public void decrPendingResources(String user, int containers, Resource res) {
@@ -379,8 +380,8 @@ public class QueueMetrics implements MetricsSource {
private void _decrPendingResources(int containers, Resource res) {
pendingContainers.decr(containers);
- pendingMB.decr(res.getMemory() * containers);
- pendingVCores.decr(res.getVirtualCores() * containers);
+ pendingMB.decr(res.getMemorySize() * containers);
+ pendingVCores.decr(res.getVirtualCoresSize() * containers);
}
public void incrNodeTypeAggregations(String user, NodeType type) {
@@ -407,8 +408,8 @@ public class QueueMetrics implements MetricsSource {
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
- allocatedMB.incr(res.getMemory() * containers);
- allocatedVCores.incr(res.getVirtualCores() * containers);
+ allocatedMB.incr(res.getMemorySize() * containers);
+ allocatedVCores.incr(res.getVirtualCoresSize() * containers);
if (decrPending) {
_decrPendingResources(containers, res);
}
@@ -428,10 +429,10 @@ public class QueueMetrics implements MetricsSource {
* @param res
*/
public void allocateResources(String user, Resource res) {
- allocatedMB.incr(res.getMemory());
+ allocatedMB.incr(res.getMemorySize());
allocatedVCores.incr(res.getVirtualCores());
- pendingMB.decr(res.getMemory());
+ pendingMB.decr(res.getMemorySize());
pendingVCores.decr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
@@ -446,8 +447,8 @@ public class QueueMetrics implements MetricsSource {
public void releaseResources(String user, int containers, Resource res) {
allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
- allocatedMB.decr(res.getMemory() * containers);
- allocatedVCores.decr(res.getVirtualCores() * containers);
+ allocatedMB.decr(res.getMemorySize() * containers);
+ allocatedVCores.decr(res.getVirtualCoresSize() * containers);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res);
@@ -464,7 +465,7 @@ public class QueueMetrics implements MetricsSource {
* @param res
*/
public void releaseResources(String user, Resource res) {
- allocatedMB.decr(res.getMemory());
+ allocatedMB.decr(res.getMemorySize());
allocatedVCores.decr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
@@ -477,7 +478,7 @@ public class QueueMetrics implements MetricsSource {
public void reserveResource(String user, Resource res) {
reservedContainers.incr();
- reservedMB.incr(res.getMemory());
+ reservedMB.incr(res.getMemorySize());
reservedVCores.incr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
@@ -490,7 +491,7 @@ public class QueueMetrics implements MetricsSource {
public void unreserveResource(String user, Resource res) {
reservedContainers.decr();
- reservedMB.decr(res.getMemory());
+ reservedMB.decr(res.getMemorySize());
reservedVCores.decr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
@@ -563,11 +564,11 @@ public class QueueMetrics implements MetricsSource {
return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value());
}
- public int getAllocatedMB() {
+ public long getAllocatedMB() {
return allocatedMB.value();
}
- public int getAllocatedVirtualCores() {
+ public long getAllocatedVirtualCores() {
return allocatedVCores.value();
}
@@ -575,19 +576,19 @@ public class QueueMetrics implements MetricsSource {
return allocatedContainers.value();
}
- public int getAvailableMB() {
+ public long getAvailableMB() {
return availableMB.value();
}
- public int getAvailableVirtualCores() {
+ public long getAvailableVirtualCores() {
return availableVCores.value();
}
- public int getPendingMB() {
+ public long getPendingMB() {
return pendingMB.value();
}
- public int getPendingVirtualCores() {
+ public long getPendingVirtualCores() {
return pendingVCores.value();
}
@@ -595,11 +596,11 @@ public class QueueMetrics implements MetricsSource {
return pendingContainers.value();
}
- public int getReservedMB() {
+ public long getReservedMB() {
return reservedMB.value();
}
- public int getReservedVirtualCores() {
+ public long getReservedVirtualCores() {
return reservedVCores.value();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 7308e22..ffb8657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -444,7 +444,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
*/
public synchronized Resource getHeadroom() {
// Corner case to deal with applications being slightly over-limit
- if (resourceLimit.getMemory() < 0) {
+ if (resourceLimit.getMemorySize() < 0) {
resourceLimit.setMemory(0);
}
@@ -480,7 +480,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
if (requests != null) {
LOG.debug("showRequests:" + " application=" + getApplicationId()
+ " headRoom=" + getHeadroom() + " currentConsumption="
- + attemptResourceUsage.getUsed().getMemory());
+ + attemptResourceUsage.getUsed().getMemorySize());
for (ResourceRequest request : requests.values()) {
LOG.debug("showRequests:" + " application=" + getApplicationId()
+ " request=" + request);
@@ -682,7 +682,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
for (RMContainer rmContainer : this.liveContainers.values()) {
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
Resource resource = rmContainer.getContainer().getResource();
- memorySeconds += resource.getMemory() * usedMillis /
+ memorySeconds += resource.getMemorySize() * usedMillis /
DateUtils.MILLIS_PER_SECOND;
vcoreSeconds += resource.getVirtualCores() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index b460964..c999e26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -274,13 +274,13 @@ public class SchedulerUtils {
private static void validateResourceRequest(ResourceRequest resReq,
Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
- if (resReq.getCapability().getMemory() < 0 ||
- resReq.getCapability().getMemory() > maximumResource.getMemory()) {
+ if (resReq.getCapability().getMemorySize() < 0 ||
+ resReq.getCapability().getMemorySize() > maximumResource.getMemorySize()) {
throw new InvalidResourceRequestException("Invalid resource request"
+ ", requested memory < 0"
+ ", or requested memory > max configured"
- + ", requestedMemory=" + resReq.getCapability().getMemory()
- + ", maxMemory=" + maximumResource.getMemory());
+ + ", requestedMemory=" + resReq.getCapability().getMemorySize()
+ + ", maxMemory=" + maximumResource.getMemorySize());
}
if (resReq.getCapability().getVirtualCores() < 0 ||
resReq.getCapability().getVirtualCores() >
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 51d39ff..58c3984 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -32,37 +32,37 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
public class CSQueueMetrics extends QueueMetrics {
@Metric("AM memory limit in MB")
- MutableGaugeInt AMResourceLimitMB;
+ MutableGaugeLong AMResourceLimitMB;
@Metric("AM CPU limit in virtual cores")
- MutableGaugeInt AMResourceLimitVCores;
+ MutableGaugeLong AMResourceLimitVCores;
@Metric("Used AM memory limit in MB")
- MutableGaugeInt usedAMResourceMB;
+ MutableGaugeLong usedAMResourceMB;
@Metric("Used AM CPU limit in virtual cores")
- MutableGaugeInt usedAMResourceVCores;
+ MutableGaugeLong usedAMResourceVCores;
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
super(ms, queueName, parent, enableUserMetrics, conf);
}
- public int getAMResourceLimitMB() {
+ public long getAMResourceLimitMB() {
return AMResourceLimitMB.value();
}
- public int getAMResourceLimitVCores() {
+ public long getAMResourceLimitVCores() {
return AMResourceLimitVCores.value();
}
- public int getUsedAMResourceMB() {
+ public long getUsedAMResourceMB() {
return usedAMResourceMB.value();
}
- public int getUsedAMResourceVCores() {
+ public long getUsedAMResourceVCores() {
return usedAMResourceVCores.value();
}
public void setAMResouceLimit(Resource res) {
- AMResourceLimitMB.set(res.getMemory());
+ AMResourceLimitMB.set(res.getMemorySize());
AMResourceLimitVCores.set(res.getVirtualCores());
}
@@ -74,7 +74,7 @@ public class CSQueueMetrics extends QueueMetrics {
}
public void incAMUsed(String user, Resource res) {
- usedAMResourceMB.incr(res.getMemory());
+ usedAMResourceMB.incr(res.getMemorySize());
usedAMResourceVCores.incr(res.getVirtualCores());
CSQueueMetrics userMetrics = (CSQueueMetrics) getUserMetrics(user);
if (userMetrics != null) {
@@ -83,7 +83,7 @@ public class CSQueueMetrics extends QueueMetrics {
}
public void decAMUsed(String user, Resource res) {
- usedAMResourceMB.decr(res.getMemory());
+ usedAMResourceMB.decr(res.getMemorySize());
usedAMResourceVCores.decr(res.getVirtualCores());
CSQueueMetrics userMetrics = (CSQueueMetrics) getUserMetrics(user);
if (userMetrics != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
index 95a12dc..49892f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
@@ -65,7 +65,7 @@ public class CapacityHeadroomProvider {
}
}
// Corner case to deal with applications being slightly over-limit
- if (headroom.getMemory() < 0) {
+ if (headroom.getMemorySize() < 0) {
headroom.setMemory(0);
}
return headroom;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index d5bca66..d5d1374 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -677,7 +677,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
*/
public Resource getMaximumAllocationPerQueue(String queue) {
String queuePrefix = getQueuePrefix(queue);
- int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
+ long maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
(int)UNDEFINED);
int maxAllocationVcoresPerQueue = getInt(
queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
@@ -690,7 +690,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
Resource clusterMax = getMaximumAllocation();
if (maxAllocationMbPerQueue == (int)UNDEFINED) {
LOG.info("max alloc mb per queue for " + queue + " is undefined");
- maxAllocationMbPerQueue = clusterMax.getMemory();
+ maxAllocationMbPerQueue = clusterMax.getMemorySize();
}
if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
LOG.info("max alloc vcore per queue for " + queue + " is undefined");
@@ -698,7 +698,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
Resource result = Resources.createResource(maxAllocationMbPerQueue,
maxAllocationVcoresPerQueue);
- if (maxAllocationMbPerQueue > clusterMax.getMemory()
+ if (maxAllocationMbPerQueue > clusterMax.getMemorySize()
|| maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) {
throw new IllegalArgumentException(
"Queue maximum allocation cannot be larger than the cluster setting"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index d562f34..6dcafec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -449,7 +449,7 @@ public class LeafQueue extends AbstractCSQueue {
// since we have already told running AM's the size
Resource oldMax = getMaximumAllocation();
Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
- if (newMax.getMemory() < oldMax.getMemory()
+ if (newMax.getMemorySize() < oldMax.getMemorySize()
|| newMax.getVirtualCores() < oldMax.getVirtualCores()) {
throw new IOException(
"Trying to reinitialize "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index a5ca2d8..b2d4bbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -442,7 +442,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
priority, capability);
// Can we allocate a container on this node?
- int availableContainers =
+ long availableContainers =
rc.computeAvailableContainers(available, capability);
// How much need to unreserve equals to:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f90ab8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 82d665c..5b83c9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -193,7 +193,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resources.subtractFrom(availableResources,
node.getUnallocatedResource());
}
- if (availableResources.getMemory() < 0) {
+ if (availableResources.getMemorySize() < 0) {
availableResources.setMemory(0);
}
if (availableResources.getVirtualCores() < 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org