You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/12/19 01:36:41 UTC
[20/20] hadoop git commit: YARN-4440.
FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler
time. Contributed by Lin Yiqun
YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler time. Contributed by Lin Yiqun
(cherry picked from commit 2aaed1032719e978660d1f14a8ed299e83b5f65e)
(cherry picked from commit be50df687bf800d35f418128d0c3f05ab6c429c8)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4f638d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4f638d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4f638d8
Branch: refs/heads/branch-2.8
Commit: e4f638d81524ef4ecf69ba809c5ac6ca10c52014
Parents: 1d1c6c1
Author: Zhihai Xu <zx...@apache.org>
Authored: Tue Dec 15 00:17:21 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:45 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/FSAppAttempt.java | 7 +++
.../scheduler/fair/TestFairScheduler.java | 60 ++++++++++++++++++++
3 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 155ba2a..4c171d1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1075,6 +1075,9 @@ Release 2.8.0 - UNRELEASED
YARN-4392. ApplicationCreatedEvent event time resets after RM restart/failover.
(Naganarasimha G R and Xuan Gong via xgong)
+ YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the
+ lastScheduler time. (Lin Yiqun via zxu)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index abbf77a..3778cba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -286,6 +286,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// default level is NODE_LOCAL
if (! allowedLocalityLevel.containsKey(priority)) {
+ // add the initial time of priority to prevent comparing with FsApp
+ // startTime and allowedLocalityLevel degrade
+ lastScheduledContainer.put(priority, currentTimeMs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Init the lastScheduledContainer time, priority: " + priority
+ + ", time: " + currentTimeMs);
+ }
allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL);
return NodeType.NODE_LOCAL;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index e5402d0..f37d7dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5151,4 +5151,64 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("root.user1", resourceManager.getRMContext().getRMApps()
.get(attId3.getApplicationId()).getQueue());
}
+
+ @Test
+ public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
+ int DELAY_THRESHOLD_TIME_MS = 1000;
+ conf.set(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, "true");
+ conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_NODE_MS,
+ String.valueOf(DELAY_THRESHOLD_TIME_MS));
+ conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_RACK_MS,
+ String.valueOf(DELAY_THRESHOLD_TIME_MS));
+
+ ControlledClock clock = new ControlledClock();
+ scheduler.setClock(clock);
+ scheduler.init(conf);
+ scheduler.start();
+
+ int priorityValue;
+ Priority priority;
+ FSAppAttempt fsAppAttempt;
+ ResourceRequest request1;
+ ResourceRequest request2;
+ ApplicationAttemptId id11;
+
+ priorityValue = 1;
+ id11 = createAppAttemptId(1, 1);
+ createMockRMApp(id11);
+ priority = Priority.newInstance(priorityValue);
+ scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1",
+ false);
+ scheduler.addApplicationAttempt(id11, false, false);
+ fsAppAttempt = scheduler.getApplicationAttempt(id11);
+
+ String hostName = "127.0.0.1";
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(16 * 1024, 16), 1,
+ hostName);
+ List<ResourceRequest> ask1 = new ArrayList<>();
+ request1 =
+ createResourceRequest(1024, 8, node1.getRackName(), priorityValue, 1,
+ true);
+ request2 =
+ createResourceRequest(1024, 8, ResourceRequest.ANY, priorityValue, 1,
+ true);
+ ask1.add(request1);
+ ask1.add(request2);
+ scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null,
+ null, null);
+
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ FSSchedulerNode node =
+ (FSSchedulerNode) scheduler.getSchedulerNode(node1.getNodeID());
+ // Tick the time and let the fsApp startTime different from initScheduler
+ // time
+ clock.tickSec(DELAY_THRESHOLD_TIME_MS / 1000);
+ scheduler.attemptScheduling(node);
+ Map<Priority, Long> lastScheduledContainer =
+ fsAppAttempt.getLastScheduledContainer();
+ long initSchedulerTime = lastScheduledContainer.get(priority);
+ assertEquals(DELAY_THRESHOLD_TIME_MS, initSchedulerTime);
+ }
}