You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/01/12 05:32:17 UTC
[18/23] hadoop git commit: YARN-4537. Pull out priority comparison
from fifocomparator and use compound comparator for FifoOrdering policy.
Contributed by Rohith Sharma K S
YARN-4537. Pull out priority comparison from fifocomparator and use compound comparator for FifoOrdering policy. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8942be8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8942be8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8942be8
Branch: refs/heads/HDFS-1312
Commit: b8942be888e1f6f7049398f3d0322602a58ee312
Parents: de37f37
Author: Jian He <ji...@apache.org>
Authored: Mon Jan 11 16:44:28 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Jan 11 16:44:28 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../dev-support/findbugs-exclude.xml | 4 +++
.../scheduler/policy/FifoComparator.java | 7 -----
.../scheduler/policy/FifoOrderingPolicy.java | 8 +++--
.../policy/TestFifoOrderingPolicy.java | 33 +++++++++++++++++---
5 files changed, 41 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e0c81b0..e58f15b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.8.0 - UNRELEASED
YARN-4544. All the log messages about rolling monitoring interval are
shown with WARN level. (Takashi Ohnishi via aajisaka)
+ YARN-4537. Pull out priority comparison from fifocomparator and use compound
+ comparator for FifoOrdering policy. (Rohith Sharma K S via jianhe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c79a35e..6fb3945 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -159,6 +159,10 @@
<Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.PriorityComparator" />
+ <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PartitionedQueueComparator" />
<Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
index 1045386..112c50f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
@@ -20,9 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
import java.util.*;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
-
/**
* A Comparator which orders SchedulableEntities by input order
*/
@@ -31,10 +28,6 @@ public class FifoComparator
@Override
public int compare(SchedulableEntity r1, SchedulableEntity r2) {
- if (r1.getPriority() != null
- && !r1.getPriority().equals(r2.getPriority())) {
- return r1.getPriority().compareTo(r2.getPriority());
- }
int res = r1.compareInputOrderTo(r2);
return res;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
index 74a422c..10f8eeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
import java.util.*;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
/**
* An OrderingPolicy which orders SchedulableEntities by input order
@@ -28,8 +27,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
public class FifoOrderingPolicy<S extends SchedulableEntity> extends AbstractComparatorOrderingPolicy<S> {
public FifoOrderingPolicy() {
- this.comparator = new FifoComparator();
+ List<Comparator<SchedulableEntity>> comparators =
+ new ArrayList<Comparator<SchedulableEntity>>();
+ comparators.add(new PriorityComparator());
+ comparators.add(new FifoComparator());
+ this.comparator = new CompoundComparator(comparators);
this.schedulableEntities = new TreeSet<S>(comparator);
+
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
index e05ef26..447618f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
@@ -24,11 +24,6 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
public class TestFifoOrderingPolicy {
@@ -80,4 +75,32 @@ public class TestFifoOrderingPolicy {
}
}
+ @Test
+ public void testFifoOrderingPolicyAlongWithPriorty() {
+ FifoOrderingPolicy<MockSchedulableEntity> policy =
+ new FifoOrderingPolicy<MockSchedulableEntity>();
+ MockSchedulableEntity r1 = new MockSchedulableEntity();
+ MockSchedulableEntity r2 = new MockSchedulableEntity();
+
+ Priority p1 = Priority.newInstance(1);
+ Priority p2 = Priority.newInstance(0);
+
+ // Both r1 and r1 priority is null
+ Assert.assertEquals(0, policy.getComparator().compare(r1, r2));
+
+ // r1 is null and r2 is not null
+ r2.setApplicationPriority(p2);
+ Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+
+ // r1 is not null and r2 is null
+ r2.setApplicationPriority(null);
+ r1.setApplicationPriority(p1);
+ Assert.assertEquals(1, policy.getComparator().compare(r1, r2));
+
+ // r1 is not null and r2 is not null
+ r1.setApplicationPriority(p1);
+ r2.setApplicationPriority(p2);
+ Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+ }
+
}