You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/06/01 05:24:03 UTC
[7/7] hadoop git commit: Revert "YARn-4844. Add
getMemoryLong/getVirtualCoreLong to o.a.h.y.api.records.Resource. Contributed
by Wangda Tan."
Revert "YARn-4844. Add getMemoryLong/getVirtualCoreLong to o.a.h.y.api.records.Resource. Contributed by Wangda Tan."
This reverts commit 457884737f75c796413ce860b1859a31cc5292ca.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f3e1d96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f3e1d96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f3e1d96
Branch: refs/heads/branch-2.8
Commit: 2f3e1d965d188c286b68222a0351d353bb9e1207
Parents: 6fe1c67
Author: Wangda Tan <wa...@apache.org>
Authored: Tue May 31 22:16:53 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue May 31 22:16:53 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 4 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 1 -
.../v2/app/rm/RMContainerAllocator.java | 16 +-
.../v2/app/rm/ResourceCalculatorUtils.java | 17 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +-
.../v2/app/job/impl/TestTaskAttempt.java | 19 +-
.../v2/app/rm/TestRMContainerAllocator.java | 2 +-
.../v2/app/rm/TestResourceCalculatorUtils.java | 4 +-
.../apache/hadoop/mapreduce/TypeConverter.java | 6 +-
.../org/apache/hadoop/mapreduce/JobStatus.java | 18 +-
.../jobhistory/NormalizedResourceEvent.java | 6 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 7 +-
.../apache/hadoop/mapred/JobClientUnitTest.java | 6 +-
.../sls/scheduler/FairSchedulerMetrics.java | 60 +--
.../sls/scheduler/ResourceSchedulerWrapper.java | 36 +-
.../hadoop/yarn/api/records/Resource.java | 34 +-
.../src/main/proto/yarn_protos.proto | 4 +-
.../distributedshell/ApplicationMaster.java | 6 +-
.../applications/distributedshell/Client.java | 4 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 16 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 4 +-
.../yarn/client/api/impl/TestYarnClient.java | 2 +-
.../api/records/impl/pb/ResourcePBImpl.java | 30 +-
.../resource/DefaultResourceCalculator.java | 36 +-
.../resource/DominantResourceCalculator.java | 46 +-
.../yarn/util/resource/ResourceCalculator.java | 10 +-
.../hadoop/yarn/util/resource/Resources.java | 62 +--
.../yarn/util/resource/TestResources.java | 8 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 2 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 10 +-
.../yarn/server/webapp/dao/ContainerInfo.java | 10 +-
.../hadoop/yarn/TestYarnServerApiClasses.java | 6 +-
.../server/nodemanager/ContainerExecutor.java | 2 +-
.../container/ContainerImpl.java | 4 +-
.../monitor/ContainersMonitorImpl.java | 4 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 12 +-
.../nodemanager/webapp/dao/ContainerInfo.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 4 +-
.../monitor/TestContainersMonitor.java | 2 +-
.../yarn/server/resourcemanager/RMNMInfo.java | 4 +-
.../server/resourcemanager/RMServerUtils.java | 14 +-
.../resourcemanager/ResourceTrackerService.java | 2 +-
.../metrics/SystemMetricsPublisher.java | 2 +-
.../ProportionalCapacityPreemptionPolicy.java | 12 +-
.../AbstractSchedulerPlanFollower.java | 2 +-
.../reservation/CapacityOverTimePolicy.java | 6 +-
.../reservation/ReservationSystemUtil.java | 2 +-
.../planning/StageEarliestStartByDemand.java | 2 +-
.../rmcontainer/RMContainerImpl.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 14 +-
.../resourcemanager/scheduler/QueueMetrics.java | 61 ++-
.../scheduler/SchedulerApplicationAttempt.java | 6 +-
.../scheduler/SchedulerUtils.java | 8 +-
.../scheduler/capacity/CSQueueMetrics.java | 24 +-
.../capacity/CapacityHeadroomProvider.java | 2 +-
.../CapacitySchedulerConfiguration.java | 6 +-
.../scheduler/capacity/LeafQueue.java | 2 +-
.../allocator/RegularContainerAllocator.java | 2 +-
.../scheduler/fair/FSParentQueue.java | 2 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 12 +-
.../scheduler/fair/FSQueueMetrics.java | 43 +-
.../scheduler/fair/FairScheduler.java | 6 +-
.../fair/policies/ComputeFairShares.java | 18 +-
.../DominantResourceFairnessPolicy.java | 10 +-
.../fair/policies/FairSharePolicy.java | 20 +-
.../scheduler/fair/policies/FifoPolicy.java | 8 +-
.../scheduler/fifo/FifoScheduler.java | 21 +-
.../scheduler/policy/FairOrderingPolicy.java | 5 +-
.../webapp/DefaultSchedulerPage.java | 4 +-
.../webapp/FairSchedulerAppsBlock.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 8 +-
.../resourcemanager/webapp/dao/AppInfo.java | 20 +-
.../webapp/dao/FairSchedulerInfo.java | 4 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 14 +-
.../webapp/dao/FifoSchedulerInfo.java | 18 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 4 +-
.../webapp/dao/ResourceInfo.java | 10 +-
.../server/resourcemanager/Application.java | 2 +-
.../yarn/server/resourcemanager/MockNM.java | 12 +-
.../yarn/server/resourcemanager/MockNodes.java | 4 +-
.../server/resourcemanager/NodeManager.java | 8 +-
.../ReservationACLsTestBase.java | 2 +-
.../resourcemanager/TestApplicationACLs.java | 6 +-
.../resourcemanager/TestApplicationCleanup.java | 2 +-
.../TestContainerResourceUsage.java | 2 +-
.../yarn/server/resourcemanager/TestRM.java | 8 +-
.../yarn/server/resourcemanager/TestRMHA.java | 4 +-
.../resourcemanager/TestRMNodeTransitions.java | 22 +-
.../TestReservationSystemWithRMHA.java | 2 +-
.../TestResourceTrackerService.java | 2 +-
.../TestWorkPreservingRMRestart.java | 29 +-
.../metrics/TestSystemMetricsPublisher.java | 10 +-
...estProportionalCapacityPreemptionPolicy.java | 24 +-
...pacityPreemptionPolicyForNodePartitions.java | 2 +-
.../nodelabels/TestRMNodeLabelsManager.java | 2 +-
.../reservation/ReservationSystemTestUtil.java | 2 +-
.../TestRLESparseResourceAllocation.java | 18 +-
.../planning/TestSimpleCapacityReplanner.java | 4 +-
.../scheduler/TestAbstractYarnScheduler.java | 16 +-
.../scheduler/TestQueueMetrics.java | 8 +-
.../scheduler/TestResourceUsage.java | 2 +-
.../scheduler/TestSchedulerUtils.java | 20 +-
.../capacity/TestApplicationLimits.java | 6 +-
.../capacity/TestApplicationPriority.java | 52 +-
.../capacity/TestCapacityScheduler.java | 124 +++--
.../TestCapacitySchedulerNodeLabelUpdate.java | 52 +-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 10 +-
.../capacity/TestContainerResizing.java | 152 +++---
.../capacity/TestIncreaseAllocationExpirer.java | 36 +-
.../scheduler/capacity/TestLeafQueue.java | 493 +++++++++----------
.../TestNodeLabelContainerAllocation.java | 4 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestQueueCapacities.java | 2 +-
.../scheduler/capacity/TestReservations.java | 410 +++++++--------
...TestWorkPreservingRMRestartForNodeLabel.java | 4 +-
.../scheduler/fair/TestComputeFairShares.java | 2 +-
.../fair/TestContinuousScheduling.java | 4 +-
.../scheduler/fair/TestFSAppAttempt.java | 38 +-
.../scheduler/fair/TestFSLeafQueue.java | 15 +-
.../scheduler/fair/TestFairScheduler.java | 341 +++++++------
.../fair/TestFairSchedulerFairShare.java | 32 +-
.../fair/TestFairSchedulerPreemption.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 63 ++-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../webapp/TestRMWebServicesNodes.java | 4 +-
127 files changed, 1470 insertions(+), 1531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d9025b8..0af8f14 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -689,9 +689,9 @@ public class JobHistoryEventHandler extends AbstractService
NormalizedResourceEvent normalizedResourceEvent =
(NormalizedResourceEvent) event;
if (normalizedResourceEvent.getTaskType() == TaskType.MAP) {
- summary.setResourcesPerMap((int) normalizedResourceEvent.getMemory());
+ summary.setResourcesPerMap(normalizedResourceEvent.getMemory());
} else if (normalizedResourceEvent.getTaskType() == TaskType.REDUCE) {
- summary.setResourcesPerReduce((int) normalizedResourceEvent.getMemory());
+ summary.setResourcesPerReduce(normalizedResourceEvent.getMemory());
}
break;
case JOB_INITED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 9d8a439..a6ae5d1 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -1436,7 +1436,6 @@ public abstract class TaskAttemptImpl implements
TaskAttemptImpl taskAttempt) {
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
-
int mbRequired =
taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 9b244fd..1d63fb6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -357,10 +357,10 @@ public class RMContainerAllocator extends RMContainerRequestor
eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent(
org.apache.hadoop.mapreduce.TaskType.MAP, mapResourceRequest
- .getMemorySize())));
+ .getMemory())));
LOG.info("mapResourceRequest:" + mapResourceRequest);
- if (mapResourceRequest.getMemorySize() > supportedMaxContainerCapability
- .getMemorySize()
+ if (mapResourceRequest.getMemory() > supportedMaxContainerCapability
+ .getMemory()
|| mapResourceRequest.getVirtualCores() > supportedMaxContainerCapability
.getVirtualCores()) {
String diagMsg =
@@ -374,7 +374,7 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
// set the resources
- reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize());
+ reqEvent.getCapability().setMemory(mapResourceRequest.getMemory());
reqEvent.getCapability().setVirtualCores(
mapResourceRequest.getVirtualCores());
scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
@@ -384,10 +384,10 @@ public class RMContainerAllocator extends RMContainerRequestor
eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent(
org.apache.hadoop.mapreduce.TaskType.REDUCE,
- reduceResourceRequest.getMemorySize())));
+ reduceResourceRequest.getMemory())));
LOG.info("reduceResourceRequest:" + reduceResourceRequest);
- if (reduceResourceRequest.getMemorySize() > supportedMaxContainerCapability
- .getMemorySize()
+ if (reduceResourceRequest.getMemory() > supportedMaxContainerCapability
+ .getMemory()
|| reduceResourceRequest.getVirtualCores() > supportedMaxContainerCapability
.getVirtualCores()) {
String diagMsg =
@@ -402,7 +402,7 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
// set the resources
- reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize());
+ reqEvent.getCapability().setMemory(reduceResourceRequest.getMemory());
reqEvent.getCapability().setVirtualCores(
reduceResourceRequest.getVirtualCores());
if (reqEvent.getEarlierAttemptFailed()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
index 4748dfd..39cb22e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
@@ -20,42 +20,43 @@ package org.apache.hadoop.mapreduce.v2.app.rm;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.util.Records;
import java.util.EnumSet;
public class ResourceCalculatorUtils {
- public static int divideAndCeil(long a, long b) {
+ public static int divideAndCeil(int a, int b) {
if (b == 0) {
return 0;
}
- return (int) ((a + (b - 1)) / b);
+ return (a + (b - 1)) / b;
}
public static int computeAvailableContainers(Resource available,
Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
return Math.min(
- calculateRatioOrMaxValue(available.getMemorySize(), required.getMemorySize()),
+ calculateRatioOrMaxValue(available.getMemory(), required.getMemory()),
calculateRatioOrMaxValue(available.getVirtualCores(), required
.getVirtualCores()));
}
return calculateRatioOrMaxValue(
- available.getMemorySize(), required.getMemorySize());
+ available.getMemory(), required.getMemory());
}
public static int divideAndCeilContainers(Resource required, Resource factor,
EnumSet<SchedulerResourceTypes> resourceTypes) {
if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
- return Math.max(divideAndCeil(required.getMemorySize(), factor.getMemorySize()),
+ return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
}
- return divideAndCeil(required.getMemorySize(), factor.getMemorySize());
+ return divideAndCeil(required.getMemory(), factor.getMemory());
}
- private static int calculateRatioOrMaxValue(long numerator, long denominator) {
+ private static int calculateRatioOrMaxValue(int numerator, int denominator) {
if (denominator == 0) {
return Integer.MAX_VALUE;
}
- return (int) (numerator / denominator);
+ return numerator / denominator;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
index 7c60992..0e07bcf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
@@ -1789,7 +1789,7 @@ public class TestRecovery {
int appAttemptId = 3;
MRAppMetrics metrics = mock(MRAppMetrics.class);
Resource minContainerRequirements = mock(Resource.class);
- when(minContainerRequirements.getMemorySize()).thenReturn(1000L);
+ when(minContainerRequirements.getMemory()).thenReturn(1000);
ClusterInfo clusterInfo = mock(ClusterInfo.class);
AppContext appContext = mock(AppContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index ac7c49c..df59991 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -305,7 +305,6 @@ public class TestTaskAttempt{
Assert.assertEquals(rta.getLaunchTime(), 10);
Counters counters = job.getAllCounters();
Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
-
counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
@@ -567,7 +566,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl =
@@ -625,7 +624,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl =
@@ -689,7 +688,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl =
@@ -759,7 +758,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
@@ -816,7 +815,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl =
@@ -884,7 +883,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
@@ -1044,7 +1043,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
jobFile, 1, splits, jobConf, taListener, new Token(),
@@ -1098,7 +1097,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
jobFile, 1, splits, jobConf, taListener, new Token(),
@@ -1155,7 +1154,7 @@ public class TestTaskAttempt{
ClusterInfo clusterInfo = mock(ClusterInfo.class);
Resource resource = mock(Resource.class);
when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- when(resource.getMemorySize()).thenReturn(1024L);
+ when(resource.getMemory()).thenReturn(1024);
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
jobFile, 1, splits, jobConf, taListener, new Token(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 755f8ee..5c05189 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -1771,7 +1771,7 @@ public class TestRMContainerAllocator {
when(excessC.getId()).thenReturn(containerId);
when(excessC.getPriority()).thenReturn(RMContainerAllocator.PRIORITY_REDUCE);
Resource mockR = mock(Resource.class);
- when(mockR.getMemorySize()).thenReturn(2048L);
+ when(mockR.getMemory()).thenReturn(2048);
when(excessC.getResource()).thenReturn(mockR);
NodeId nId = mock(NodeId.class);
when(nId.getHost()).thenReturn("local");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java
index cab8f54..d87f6db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java
@@ -47,8 +47,8 @@ public class TestResourceCalculatorUtils {
Integer.MAX_VALUE,
expectedNumberOfContainersForCPU);
- Resource zeroCpuResource = Resource.newInstance(
- nonZeroResource.getMemorySize(), 0);
+ Resource zeroCpuResource = Resource.newInstance(nonZeroResource.getMemory(),
+ 0);
verifyDifferentResourceTypes(clusterAvailableResources, zeroCpuResource,
expectedNumberOfContainersForMemory,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index 5979eda..4af5b89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -520,13 +520,13 @@ public class TypeConverter {
application.getApplicationResourceUsageReport();
if (resourceUsageReport != null) {
jobStatus.setNeededMem(
- resourceUsageReport.getNeededResources().getMemorySize());
+ resourceUsageReport.getNeededResources().getMemory());
jobStatus.setNumReservedSlots(
resourceUsageReport.getNumReservedContainers());
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
- resourceUsageReport.getReservedResources().getMemorySize());
- jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemorySize());
+ resourceUsageReport.getReservedResources().getMemory());
+ jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
}
return jobStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
index d0c5f10..df7f747 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
@@ -95,9 +95,9 @@ public class JobStatus implements Writable, Cloneable {
private String trackingUrl ="";
private int numUsedSlots;
private int numReservedSlots;
- private long usedMem;
- private long reservedMem;
- private long neededMem;
+ private int usedMem;
+ private int reservedMem;
+ private int neededMem;
private boolean isUber;
/**
@@ -552,42 +552,42 @@ public class JobStatus implements Writable, Cloneable {
/**
* @return the used memory
*/
- public long getUsedMem() {
+ public int getUsedMem() {
return usedMem;
}
/**
* @param m the used memory
*/
- public void setUsedMem(long m) {
+ public void setUsedMem(int m) {
this.usedMem = m;
}
/**
* @return the reserved memory
*/
- public long getReservedMem() {
+ public int getReservedMem() {
return reservedMem;
}
/**
* @param r the reserved memory
*/
- public void setReservedMem(long r) {
+ public void setReservedMem(int r) {
this.reservedMem = r;
}
/**
* @return the needed memory
*/
- public long getNeededMem() {
+ public int getNeededMem() {
return neededMem;
}
/**
* @param n the needed memory
*/
- public void setNeededMem(long n) {
+ public void setNeededMem(int n) {
this.neededMem = n;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
index eead9cf..b8f049c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.mapreduce.TaskType;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class NormalizedResourceEvent implements HistoryEvent {
- private long memory;
+ private int memory;
private TaskType taskType;
/**
@@ -36,7 +36,7 @@ public class NormalizedResourceEvent implements HistoryEvent {
* @param taskType the tasktype of the request.
* @param memory the normalized memory requirements.
*/
- public NormalizedResourceEvent(TaskType taskType, long memory) {
+ public NormalizedResourceEvent(TaskType taskType, int memory) {
this.memory = memory;
this.taskType = taskType;
}
@@ -53,7 +53,7 @@ public class NormalizedResourceEvent implements HistoryEvent {
* the normalized memory
* @return the normalized memory
*/
- public long getMemory() {
+ public int getMemory() {
return this.memory;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index 5af3c79..79fea2c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -666,10 +666,9 @@ public class CLI extends Configured implements Tool {
for (JobStatus job : jobs) {
int numUsedSlots = job.getNumUsedSlots();
int numReservedSlots = job.getNumReservedSlots();
-
- long usedMem = job.getUsedMem();
- long rsvdMem = job.getReservedMem();
- long neededMem = job.getNeededMem();
+ int usedMem = job.getUsedMem();
+ int rsvdMem = job.getReservedMem();
+ int neededMem = job.getNeededMem();
writer.printf(dataPattern,
job.getJobID().toString(), job.getState(), job.getStartTime(),
job.getUsername(), job.getQueue(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
index 4895a5b..84b76bf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
@@ -170,9 +170,9 @@ public class JobClientUnitTest {
when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL);
when(mockJobStatus.getNumUsedSlots()).thenReturn(1);
when(mockJobStatus.getNumReservedSlots()).thenReturn(1);
- when(mockJobStatus.getUsedMem()).thenReturn(1024L);
- when(mockJobStatus.getReservedMem()).thenReturn(512L);
- when(mockJobStatus.getNeededMem()).thenReturn(2048L);
+ when(mockJobStatus.getUsedMem()).thenReturn(1024);
+ when(mockJobStatus.getReservedMem()).thenReturn(512);
+ when(mockJobStatus.getNeededMem()).thenReturn(2048);
when(mockJobStatus.getSchedulingInfo()).thenReturn("NA");
Job mockJob = mock(Job.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 3b539fa..d100e1d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -67,10 +67,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
FairScheduler fair = (FairScheduler) scheduler;
final FSAppAttempt app = fair.getSchedulerApp(appAttemptId);
metrics.register("variable.app." + oldAppId + ".demand.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return app.getDemand().getMemorySize();
+ public Integer getValue() {
+ return app.getDemand().getMemory();
}
}
);
@@ -83,10 +83,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.app." + oldAppId + ".usage.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return app.getResourceUsage().getMemorySize();
+ public Integer getValue() {
+ return app.getResourceUsage().getMemory();
}
}
);
@@ -99,26 +99,26 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.app." + oldAppId + ".minshare.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return app.getMinShare().getMemorySize();
+ public Integer getValue() {
+ return app.getMinShare().getMemory();
}
}
);
metrics.register("variable.app." + oldAppId + ".minshare.vcores",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return app.getMinShare().getMemorySize();
+ public Integer getValue() {
+ return app.getMinShare().getMemory();
}
}
);
metrics.register("variable.app." + oldAppId + ".maxshare.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return Math.min(app.getMaxShare().getMemorySize(), totalMemoryMB);
+ public Integer getValue() {
+ return Math.min(app.getMaxShare().getMemory(), totalMemoryMB);
}
}
);
@@ -154,10 +154,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
FairScheduler fair = (FairScheduler) scheduler;
final FSQueue queue = fair.getQueueManager().getQueue(queueName);
metrics.register("variable.queue." + queueName + ".demand.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return queue.getDemand().getMemorySize();
+ public Integer getValue() {
+ return queue.getDemand().getMemory();
}
}
);
@@ -170,10 +170,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.queue." + queueName + ".usage.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return queue.getResourceUsage().getMemorySize();
+ public Integer getValue() {
+ return queue.getResourceUsage().getMemory();
}
}
);
@@ -186,10 +186,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.queue." + queueName + ".minshare.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return queue.getMinShare().getMemorySize();
+ public Integer getValue() {
+ return queue.getMinShare().getMemory();
}
}
);
@@ -202,9 +202,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.queue." + queueName + ".maxshare.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
+ public Integer getValue() {
if (! maxReset &&
SLSRunner.simulateInfoMap.containsKey("Number of nodes") &&
SLSRunner.simulateInfoMap.containsKey("Node memory (MB)") &&
@@ -221,7 +221,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
maxReset = false;
}
- return Math.min(queue.getMaxShare().getMemorySize(), totalMemoryMB);
+ return Math.min(queue.getMaxShare().getMemory(), totalMemoryMB);
}
}
);
@@ -234,10 +234,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
metrics.register("variable.queue." + queueName + ".fairshare.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
- return queue.getFairShare().getMemorySize();
+ public Integer getValue() {
+ return queue.getFairShare().getMemory();
}
}
);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 1a936ab..8835deb 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -267,7 +267,7 @@ public class ResourceSchedulerWrapper
// should have one container which is AM container
RMContainer rmc = app.getLiveContainers().iterator().next();
updateQueueMetrics(queue,
- rmc.getContainer().getResource().getMemorySize(),
+ rmc.getContainer().getResource().getMemory(),
rmc.getContainer().getResource().getVirtualCores());
}
}
@@ -323,7 +323,7 @@ public class ResourceSchedulerWrapper
if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
for (RMContainer rmc : app.getLiveContainers()) {
if (rmc.getContainerId() == containerId) {
- releasedMemory += rmc.getContainer().getResource().getMemorySize();
+ releasedMemory += rmc.getContainer().getResource().getMemory();
releasedVCores += rmc.getContainer()
.getResource().getVirtualCores();
break;
@@ -332,7 +332,7 @@ public class ResourceSchedulerWrapper
} else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
if (preemptionContainerMap.containsKey(containerId)) {
Resource preResource = preemptionContainerMap.get(containerId);
- releasedMemory += preResource.getMemorySize();
+ releasedMemory += preResource.getMemory();
releasedVCores += preResource.getVirtualCores();
preemptionContainerMap.remove(containerId);
}
@@ -423,9 +423,9 @@ public class ResourceSchedulerWrapper
"counter.queue." + queueName + ".pending.cores",
"counter.queue." + queueName + ".allocated.memory",
"counter.queue." + queueName + ".allocated.cores"};
- long values[] = new long[]{pendingResource.getMemorySize(),
+ int values[] = new int[]{pendingResource.getMemory(),
pendingResource.getVirtualCores(),
- allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()};
+ allocatedResource.getMemory(), allocatedResource.getVirtualCores()};
for (int i = names.length - 1; i >= 0; i --) {
if (! counterMap.containsKey(names[i])) {
metrics.counter(names[i]);
@@ -531,11 +531,11 @@ public class ResourceSchedulerWrapper
private void registerClusterResourceMetrics() {
metrics.register("variable.cluster.allocated.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
+ public Integer getValue() {
if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
+ return 0;
} else {
return scheduler.getRootQueueMetrics().getAllocatedMB();
}
@@ -543,11 +543,11 @@ public class ResourceSchedulerWrapper
}
);
metrics.register("variable.cluster.allocated.vcores",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
+ public Integer getValue() {
if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
+ return 0;
} else {
return scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
}
@@ -555,11 +555,11 @@ public class ResourceSchedulerWrapper
}
);
metrics.register("variable.cluster.available.memory",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
+ public Integer getValue() {
if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
+ return 0;
} else {
return scheduler.getRootQueueMetrics().getAvailableMB();
}
@@ -567,11 +567,11 @@ public class ResourceSchedulerWrapper
}
);
metrics.register("variable.cluster.available.vcores",
- new Gauge<Long>() {
+ new Gauge<Integer>() {
@Override
- public Long getValue() {
+ public Integer getValue() {
if(scheduler == null || scheduler.getRootQueueMetrics() == null) {
- return 0L;
+ return 0;
} else {
return scheduler.getRootQueueMetrics().getAvailableVirtualCores();
}
@@ -749,7 +749,7 @@ public class ResourceSchedulerWrapper
}
private void updateQueueMetrics(String queue,
- long releasedMemory, int releasedVCores) {
+ int releasedMemory, int releasedVCores) {
// update queue counters
SortedMap<String, Counter> counterMap = metrics.getCounters();
if (releasedMemory != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index e2d8472..88b57f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,9 +18,7 @@
package org.apache.hadoop.yarn.api.records;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -55,7 +53,7 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Stable
- public static Resource newInstance(long memory, long vCores) {
+ public static Resource newInstance(int memory, int vCores) {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(vCores);
@@ -63,23 +61,12 @@ public abstract class Resource implements Comparable<Resource> {
}
/**
- * This method is DEPRECATED:
- * Use {@link Resource#getMemorySize()} instead
- *
* Get <em>memory</em> of the resource.
* @return <em>memory</em> of the resource
*/
@Public
- @Deprecated
+ @Stable
public abstract int getMemory();
-
- /**
- * Get <em>memory</em> of the resource.
- * @return <em>memory</em> of the resource
- */
- @Private
- @Unstable
- public abstract long getMemorySize();
/**
* Set <em>memory</em> of the resource.
@@ -87,7 +74,7 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Stable
- public abstract void setMemory(long memory);
+ public abstract void setMemory(int memory);
/**
@@ -103,10 +90,6 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Evolving
public abstract int getVirtualCores();
-
- @Public
- @Unstable
- public abstract long getVirtualCoresSize();
/**
* Set <em>number of virtual cpu cores</em> of the resource.
@@ -120,14 +103,13 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract void setVirtualCores(long vCores);
+ public abstract void setVirtualCores(int vCores);
@Override
public int hashCode() {
final int prime = 263167;
-
- int result = (int) (939769357
- + getMemorySize()); // prime * result = 939769357 initially
+ int result = 3571;
+ result = 939769357 + getMemory(); // prime * result = 939769357 initially
result = prime * result + getVirtualCores();
return result;
}
@@ -141,7 +123,7 @@ public abstract class Resource implements Comparable<Resource> {
if (!(obj instanceof Resource))
return false;
Resource other = (Resource) obj;
- if (getMemorySize() != other.getMemorySize() ||
+ if (getMemory() != other.getMemory() ||
getVirtualCores() != other.getVirtualCores()) {
return false;
}
@@ -150,6 +132,6 @@ public abstract class Resource implements Comparable<Resource> {
@Override
public String toString() {
- return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">";
+ return "<memory:" + getMemory() + ", vCores:" + getVirtualCores() + ">";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index a3741f1..02fb28b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -54,8 +54,8 @@ message ContainerIdProto {
}
message ResourceProto {
- optional int64 memory = 1;
- optional int64 virtual_cores = 2;
+ optional int32 memory = 1;
+ optional int32 virtual_cores = 2;
}
message ResourceUtilizationProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 0c670df..23920d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -221,7 +221,7 @@ public class ApplicationMaster {
@VisibleForTesting
protected int numTotalContainers = 1;
// Memory to request for the container on which the shell command will run
- private long containerMemory = 10;
+ private int containerMemory = 10;
// VirtualCores to request for the container on which the shell command will run
private int containerVirtualCores = 1;
// Priority of the request
@@ -593,7 +593,7 @@ public class ApplicationMaster {
appMasterTrackingUrl);
// Dump out information about cluster capability as seen by the
// resource manager
- long maxMem = response.getMaximumResourceCapability().getMemorySize();
+ int maxMem = response.getMaximumResourceCapability().getMemory();
LOG.info("Max mem capability of resources in this cluster " + maxMem);
int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
@@ -823,7 +823,7 @@ public class ApplicationMaster {
+ ":" + allocatedContainer.getNodeId().getPort()
+ ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress()
+ ", containerResourceMemory"
- + allocatedContainer.getResource().getMemorySize()
+ + allocatedContainer.getResource().getMemory()
+ ", containerResourceVirtualCores"
+ allocatedContainer.getResource().getVirtualCores());
// + ", containerToken"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index c632794..68d2bde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -126,7 +126,7 @@ public class Client {
// Queue for App master
private String amQueue = "";
// Amt. of memory resource to request for to run the App Master
- private long amMemory = 10;
+ private int amMemory = 10;
// Amt. of virtual core resource to request for to run the App Master
private int amVCores = 1;
@@ -487,7 +487,7 @@ public class Client {
// the required resources from the RM for the app master
// Memory ask has to be a multiple of min and less than max.
// Dump out information about cluster capability as seen by the resource manager
- long maxMem = appResponse.getMaximumResourceCapability().getMemorySize();
+ int maxMem = appResponse.getMaximumResourceCapability().getMemory();
LOG.info("Max mem capability of resources in this cluster " + maxMem);
// A resource ask cannot exceed the max.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 4366c25..46ddc4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -122,10 +122,10 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
class ResourceReverseMemoryThenCpuComparator implements Comparator<Resource> {
@Override
public int compare(Resource arg0, Resource arg1) {
- long mem0 = arg0.getMemorySize();
- long mem1 = arg1.getMemorySize();
- long cpu0 = arg0.getVirtualCores();
- long cpu1 = arg1.getVirtualCores();
+ int mem0 = arg0.getMemory();
+ int mem1 = arg1.getMemory();
+ int cpu0 = arg0.getVirtualCores();
+ int cpu1 = arg1.getVirtualCores();
if(mem0 == mem1) {
if(cpu0 == cpu1) {
return 0;
@@ -143,10 +143,10 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
static boolean canFit(Resource arg0, Resource arg1) {
- long mem0 = arg0.getMemorySize();
- long mem1 = arg1.getMemorySize();
- long cpu0 = arg0.getVirtualCores();
- long cpu1 = arg1.getVirtualCores();
+ int mem0 = arg0.getMemory();
+ int mem1 = arg1.getMemory();
+ int cpu0 = arg0.getVirtualCores();
+ int cpu1 = arg1.getVirtualCores();
return (mem0 <= mem1 && cpu0 <= cpu1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index a89551f..c112857 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -275,9 +275,9 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.println(nodeReport.getNumContainers());
nodeReportStr.print("\tMemory-Used : ");
nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ : (nodeReport.getUsed().getMemory() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
- nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
+ nodeReportStr.println(nodeReport.getCapability().getMemory() + "MB");
nodeReportStr.print("\tCPU-Used : ");
nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
: (nodeReport.getUsed().getVirtualCores() + " vcores"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index 157cf51..79d451a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -155,7 +155,7 @@ public class TopCLI extends YarnCLI {
displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores));
usedMemory =
appReport.getApplicationResourceUsageReport().getUsedResources()
- .getMemorySize() / 1024;
+ .getMemory() / 1024;
displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G");
reservedVirtualCores =
appReport.getApplicationResourceUsageReport().getReservedResources()
@@ -164,7 +164,7 @@ public class TopCLI extends YarnCLI {
String.valueOf(reservedVirtualCores));
reservedMemory =
appReport.getApplicationResourceUsageReport().getReservedResources()
- .getMemorySize() / 1024;
+ .getMemory() / 1024;
displayStringsMap.put(Columns.RMEM, String.valueOf(reservedMemory) + "G");
attempts = appReport.getCurrentApplicationAttemptId().getAttemptId();
nodes = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index da8cb6c..c018c4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1206,7 +1206,7 @@ public class TestYarnClient {
for (attempts = 10; attempts > 0; attempts--) {
if (cluster.getResourceManager().getRMContext().getReservationSystem()
.getPlan(ReservationSystemTestUtil.reservationQ).getTotalCapacity()
- .getMemorySize() > 6000) {
+ .getMemory() > 6000) {
break;
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 937941a..a28c6ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -53,49 +53,39 @@ public class ResourcePBImpl extends Resource {
}
viaProto = false;
}
-
+
+
@Override
- @SuppressWarnings("deprecation")
public int getMemory() {
- return (int) getMemorySize();
- }
-
- @Override
- public long getMemorySize() {
ResourceProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemory();
+ return (p.getMemory());
}
@Override
- public void setMemory(long memory) {
+ public void setMemory(int memory) {
maybeInitBuilder();
- builder.setMemory(memory);
+ builder.setMemory((memory));
}
@Override
public int getVirtualCores() {
- return (int) getVirtualCoresSize();
- }
-
- @Override
- public long getVirtualCoresSize() {
ResourceProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVirtualCores();
+ return (p.getVirtualCores());
}
@Override
- public void setVirtualCores(long vCores) {
+ public void setVirtualCores(int vCores) {
maybeInitBuilder();
- builder.setVirtualCores(vCores);
+ builder.setVirtualCores((vCores));
}
@Override
public int compareTo(Resource other) {
- long diff = this.getMemorySize() - other.getMemorySize();
+ int diff = this.getMemory() - other.getMemory();
if (diff == 0) {
diff = this.getVirtualCores() - other.getVirtualCores();
}
- return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
+ return diff;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index b98cf0c..2fdf214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -28,13 +28,13 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public int compare(Resource unused, Resource lhs, Resource rhs) {
// Only consider memory
- return Long.compare(lhs.getMemorySize(), rhs.getMemorySize());
+ return lhs.getMemory() - rhs.getMemory();
}
@Override
- public long computeAvailableContainers(Resource available, Resource required) {
+ public int computeAvailableContainers(Resource available, Resource required) {
// Only consider memory
- return available.getMemorySize() / required.getMemorySize();
+ return available.getMemory() / required.getMemory();
}
@Override
@@ -44,7 +44,7 @@ public class DefaultResourceCalculator extends ResourceCalculator {
}
public boolean isInvalidDivisor(Resource r) {
- if (r.getMemorySize() == 0.0f) {
+ if (r.getMemory() == 0.0f) {
return true;
}
return false;
@@ -52,23 +52,23 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public float ratio(Resource a, Resource b) {
- return (float)a.getMemorySize() / b.getMemorySize();
+ return (float)a.getMemory() / b.getMemory();
}
@Override
- public Resource divideAndCeil(Resource numerator, long denominator) {
+ public Resource divideAndCeil(Resource numerator, int denominator) {
return Resources.createResource(
- divideAndCeil(numerator.getMemorySize(), denominator));
+ divideAndCeil(numerator.getMemory(), denominator));
}
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
- long normalizedMemory = Math.min(
+ int normalizedMemory = Math.min(
roundUp(
- Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
- stepFactor.getMemorySize()),
- maximumResource.getMemorySize());
+ Math.max(r.getMemory(), minimumResource.getMemory()),
+ stepFactor.getMemory()),
+ maximumResource.getMemory());
return Resources.createResource(normalizedMemory);
}
@@ -81,22 +81,22 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemorySize(), stepFactor.getMemorySize())
+ roundUp(r.getMemory(), stepFactor.getMemory())
);
}
@Override
public Resource roundDown(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundDown(r.getMemorySize(), stepFactor.getMemorySize()));
+ roundDown(r.getMemory(), stepFactor.getMemory()));
}
@Override
public Resource multiplyAndNormalizeUp(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
- roundUp((long) (r.getMemorySize() * by + 0.5),
- stepFactor.getMemorySize()));
+ roundUp((int)(r.getMemory() * by + 0.5), stepFactor.getMemory())
+ );
}
@Override
@@ -104,8 +104,8 @@ public class DefaultResourceCalculator extends ResourceCalculator {
Resource stepFactor) {
return Resources.createResource(
roundDown(
- (long)(r.getMemorySize() * by),
- stepFactor.getMemorySize()
+ (int)(r.getMemory() * by),
+ stepFactor.getMemory()
)
);
}
@@ -113,6 +113,6 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public boolean fitsIn(Resource cluster,
Resource smaller, Resource bigger) {
- return smaller.getMemorySize() <= bigger.getMemorySize();
+ return smaller.getMemory() <= bigger.getMemory();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 8f97fab..b5c9967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -54,15 +54,15 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
if (isInvalidDivisor(clusterResource)) {
- if ((lhs.getMemorySize() < rhs.getMemorySize() && lhs.getVirtualCores() > rhs
+ if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
.getVirtualCores())
- || (lhs.getMemorySize() > rhs.getMemorySize() && lhs.getVirtualCores() < rhs
+ || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs
.getVirtualCores())) {
return 0;
- } else if (lhs.getMemorySize() > rhs.getMemorySize()
+ } else if (lhs.getMemory() > rhs.getMemory()
|| lhs.getVirtualCores() > rhs.getVirtualCores()) {
return 1;
- } else if (lhs.getMemorySize() < rhs.getMemorySize()
+ } else if (lhs.getMemory() < rhs.getMemory()
|| lhs.getVirtualCores() < rhs.getVirtualCores()) {
return -1;
}
@@ -100,20 +100,20 @@ public class DominantResourceCalculator extends ResourceCalculator {
// Just use 'dominant' resource
return (dominant) ?
Math.max(
- (float)resource.getMemorySize() / clusterResource.getMemorySize(),
+ (float)resource.getMemory() / clusterResource.getMemory(),
(float)resource.getVirtualCores() / clusterResource.getVirtualCores()
)
:
Math.min(
- (float)resource.getMemorySize() / clusterResource.getMemorySize(),
+ (float)resource.getMemory() / clusterResource.getMemory(),
(float)resource.getVirtualCores() / clusterResource.getVirtualCores()
);
}
@Override
- public long computeAvailableContainers(Resource available, Resource required) {
+ public int computeAvailableContainers(Resource available, Resource required) {
return Math.min(
- available.getMemorySize() / required.getMemorySize(),
+ available.getMemory() / required.getMemory(),
available.getVirtualCores() / required.getVirtualCores());
}
@@ -127,7 +127,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean isInvalidDivisor(Resource r) {
- if (r.getMemorySize() == 0.0f || r.getVirtualCores() == 0.0f) {
+ if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
return true;
}
return false;
@@ -136,15 +136,15 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public float ratio(Resource a, Resource b) {
return Math.max(
- (float)a.getMemorySize()/b.getMemorySize(),
+ (float)a.getMemory()/b.getMemory(),
(float)a.getVirtualCores()/b.getVirtualCores()
);
}
@Override
- public Resource divideAndCeil(Resource numerator, long denominator) {
+ public Resource divideAndCeil(Resource numerator, int denominator) {
return Resources.createResource(
- divideAndCeil(numerator.getMemorySize(), denominator),
+ divideAndCeil(numerator.getMemory(), denominator),
divideAndCeil(numerator.getVirtualCores(), denominator)
);
}
@@ -152,12 +152,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
- long normalizedMemory = Math.min(
+ int normalizedMemory = Math.min(
roundUp(
- Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
- stepFactor.getMemorySize()),
- maximumResource.getMemorySize());
- long normalizedCores = Math.min(
+ Math.max(r.getMemory(), minimumResource.getMemory()),
+ stepFactor.getMemory()),
+ maximumResource.getMemory());
+ int normalizedCores = Math.min(
roundUp(
Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()),
stepFactor.getVirtualCores()),
@@ -169,7 +169,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemorySize(), stepFactor.getMemorySize()),
+ roundUp(r.getMemory(), stepFactor.getMemory()),
roundUp(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
@@ -177,7 +177,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource roundDown(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
+ roundDown(r.getMemory(), stepFactor.getMemory()),
roundDown(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
@@ -187,7 +187,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource stepFactor) {
return Resources.createResource(
roundUp(
- (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()),
+ (int)Math.ceil(r.getMemory() * by), stepFactor.getMemory()),
roundUp(
(int)Math.ceil(r.getVirtualCores() * by),
stepFactor.getVirtualCores())
@@ -199,8 +199,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource stepFactor) {
return Resources.createResource(
roundDown(
- (int)(r.getMemorySize() * by),
- stepFactor.getMemorySize()
+ (int)(r.getMemory() * by),
+ stepFactor.getMemory()
),
roundDown(
(int)(r.getVirtualCores() * by),
@@ -212,7 +212,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean fitsIn(Resource cluster,
Resource smaller, Resource bigger) {
- return smaller.getMemorySize() <= bigger.getMemorySize()
+ return smaller.getMemory() <= bigger.getMemory()
&& smaller.getVirtualCores() <= bigger.getVirtualCores();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 3fe8ade..3a31225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -31,18 +31,18 @@ public abstract class ResourceCalculator {
public abstract int
compare(Resource clusterResource, Resource lhs, Resource rhs);
- public static long divideAndCeil(long a, long b) {
+ public static int divideAndCeil(int a, int b) {
if (b == 0) {
return 0;
}
return (a + (b - 1)) / b;
}
- public static long roundUp(long a, long b) {
+ public static int roundUp(int a, int b) {
return divideAndCeil(a, b) * b;
}
- public static long roundDown(long a, long b) {
+ public static int roundDown(int a, int b) {
return (a / b) * b;
}
@@ -54,7 +54,7 @@ public abstract class ResourceCalculator {
* @param required required resources
* @return number of containers which can be allocated
*/
- public abstract long computeAvailableContainers(
+ public abstract int computeAvailableContainers(
Resource available, Resource required);
/**
@@ -169,7 +169,7 @@ public abstract class ResourceCalculator {
* @param denominator denominator
* @return resultant resource
*/
- public abstract Resource divideAndCeil(Resource numerator, long denominator);
+ public abstract Resource divideAndCeil(Resource numerator, int denominator);
/**
* Check if a smaller resource can be contained by bigger resource.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 9d9b670..b05d021 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -31,18 +31,12 @@ public class Resources {
private static final Resource NONE = new Resource() {
@Override
- @SuppressWarnings("deprecation")
public int getMemory() {
return 0;
}
@Override
- public long getMemorySize() {
- return 0;
- }
-
- @Override
- public void setMemory(long memory) {
+ public void setMemory(int memory) {
throw new RuntimeException("NONE cannot be modified!");
}
@@ -52,22 +46,17 @@ public class Resources {
}
@Override
- public long getVirtualCoresSize() {
- return 0;
- }
-
- @Override
- public void setVirtualCores(long cores) {
+ public void setVirtualCores(int cores) {
throw new RuntimeException("NONE cannot be modified!");
}
@Override
public int compareTo(Resource o) {
- long diff = 0 - o.getMemorySize();
+ int diff = 0 - o.getMemory();
if (diff == 0) {
diff = 0 - o.getVirtualCores();
}
- return Long.signum(diff);
+ return diff;
}
};
@@ -75,18 +64,12 @@ public class Resources {
private static final Resource UNBOUNDED = new Resource() {
@Override
- @SuppressWarnings("deprecation")
public int getMemory() {
return Integer.MAX_VALUE;
}
@Override
- public long getMemorySize() {
- return Long.MAX_VALUE;
- }
-
- @Override
- public void setMemory(long memory) {
+ public void setMemory(int memory) {
throw new RuntimeException("UNBOUNDED cannot be modified!");
}
@@ -96,31 +79,26 @@ public class Resources {
}
@Override
- public long getVirtualCoresSize() {
- return Long.MAX_VALUE;
- }
-
- @Override
- public void setVirtualCores(long cores) {
+ public void setVirtualCores(int cores) {
throw new RuntimeException("UNBOUNDED cannot be modified!");
}
@Override
public int compareTo(Resource o) {
- long diff = Long.MAX_VALUE - o.getMemorySize();
+ int diff = Integer.MAX_VALUE - o.getMemory();
if (diff == 0) {
- diff = Long.MAX_VALUE - o.getVirtualCoresSize();
+ diff = Integer.MAX_VALUE - o.getVirtualCores();
}
- return Long.signum(diff);
+ return diff;
}
};
- public static Resource createResource(long memory) {
+ public static Resource createResource(int memory) {
return createResource(memory, (memory > 0) ? 1 : 0);
}
- public static Resource createResource(long memory, long cores) {
+ public static Resource createResource(int memory, int cores) {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(cores);
@@ -136,11 +114,11 @@ public class Resources {
}
public static Resource clone(Resource res) {
- return createResource(res.getMemorySize(), res.getVirtualCores());
+ return createResource(res.getMemory(), res.getVirtualCores());
}
public static Resource addTo(Resource lhs, Resource rhs) {
- lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize());
+ lhs.setMemory(lhs.getMemory() + rhs.getMemory());
lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
return lhs;
}
@@ -150,7 +128,7 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize());
+ lhs.setMemory(lhs.getMemory() - rhs.getMemory());
lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores());
return lhs;
}
@@ -164,7 +142,7 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- lhs.setMemory((int)(lhs.getMemorySize() * by));
+ lhs.setMemory((int)(lhs.getMemory() * by));
lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
return lhs;
}
@@ -179,7 +157,7 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by));
+ lhs.setMemory(lhs.getMemory() + (int)(rhs.getMemory() * by));
lhs.setVirtualCores(lhs.getVirtualCores()
+ (int)(rhs.getVirtualCores() * by));
return lhs;
@@ -197,7 +175,7 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- out.setMemory((int)(lhs.getMemorySize() * by));
+ out.setMemory((int)(lhs.getMemory() * by));
out.setVirtualCores((int)(lhs.getVirtualCores() * by));
return out;
}
@@ -286,7 +264,7 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- return smaller.getMemorySize() <= bigger.getMemorySize() &&
+ return smaller.getMemory() <= bigger.getMemory() &&
smaller.getVirtualCores() <= bigger.getVirtualCores();
}
@@ -296,12 +274,12 @@ public class Resources {
}
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
- return createResource(Math.min(lhs.getMemorySize(), rhs.getMemorySize()),
+ return createResource(Math.min(lhs.getMemory(), rhs.getMemory()),
Math.min(lhs.getVirtualCores(), rhs.getVirtualCores()));
}
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
- return createResource(Math.max(lhs.getMemorySize(), rhs.getMemorySize()),
+ return createResource(Math.max(lhs.getMemory(), rhs.getMemory()),
Math.max(lhs.getVirtualCores(), rhs.getVirtualCores()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 7396adc..d38ffdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue;
public class TestResources {
- public Resource createResource(long memory, long vCores) {
+ public Resource createResource(int memory, int vCores) {
return Resource.newInstance(memory, vCores);
}
@Test(timeout=1000)
public void testCompareToWithUnboundedResource() {
assertTrue(Resources.unbounded().compareTo(
- createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0);
+ createResource(Integer.MAX_VALUE, Integer.MAX_VALUE)) == 0);
assertTrue(Resources.unbounded().compareTo(
- createResource(Long.MAX_VALUE, 0)) > 0);
+ createResource(Integer.MAX_VALUE, 0)) > 0);
assertTrue(Resources.unbounded().compareTo(
- createResource(0, Long.MAX_VALUE)) > 0);
+ createResource(0, Integer.MAX_VALUE)) > 0);
}
@Test(timeout=1000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f3e1d96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 1d1b186..4fdd43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -425,7 +425,7 @@ public class BuilderUtils {
return report;
}
- public static Resource newResource(long memory, long vCores) {
+ public static Resource newResource(int memory, int vCores) {
Resource resource = recordFactory.newRecordInstance(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(vCores);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org