You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2015/03/10 02:20:47 UTC
incubator-aurora git commit: Improving NearestFit reporting accuracy.
Repository: incubator-aurora
Updated Branches:
refs/heads/master e55113daf -> 1b1931cc9
Improving NearestFit reporting accuracy.
Bugs closed: AURORA-1159
Reviewed at https://reviews.apache.org/r/31525/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/1b1931cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/1b1931cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/1b1931cc
Branch: refs/heads/master
Commit: 1b1931cc97f22882642c833ccf64761e98678a13
Parents: e55113d
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Mon Mar 9 18:17:58 2015 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Mon Mar 9 18:17:58 2015 -0700
----------------------------------------------------------------------
.../scheduler/filter/SchedulingFilter.java | 50 ++++++++++++----
.../scheduler/filter/SchedulingFilterImpl.java | 48 ++++++++-------
.../aurora/scheduler/metadata/NearestFit.java | 34 +----------
.../filter/SchedulingFilterImplTest.java | 13 ++--
.../scheduler/metadata/NearestFitTest.java | 63 ++++++++------------
5 files changed, 100 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1b1931cc/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
index 3313bd2..93fc05e 100644
--- a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
+++ b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
@@ -60,29 +60,42 @@ public interface SchedulingFilter {
/**
* Not enough resources to satisfy a proposed scheduling assignment.
*/
- INSUFFICIENT_RESOURCES("Insufficient: %s", VetoGroup.STATIC),
+ INSUFFICIENT_RESOURCES("Insufficient: %s", VetoGroup.STATIC, (int) Math.pow(10, 3)),
/**
* Unable to satisfy proposed scheduler assignment constraints.
*/
- CONSTRAINT_MISMATCH("Constraint not satisfied: %s", VetoGroup.STATIC),
+ CONSTRAINT_MISMATCH("Constraint not satisfied: %s", VetoGroup.STATIC, (int) Math.pow(10, 4)),
/**
* Constraint limit is not satisfied for a proposed scheduling assignment.
*/
- LIMIT_NOT_SATISFIED("Limit not satisfied: %s", VetoGroup.DYNAMIC),
+ LIMIT_NOT_SATISFIED("Limit not satisfied: %s", VetoGroup.DYNAMIC, (int) Math.pow(10, 4)),
/**
* Unable to satisfy a proposed scheduler assignment due to cluster maintenance.
*/
- MAINTENANCE("Host %s for maintenance", VetoGroup.STATIC);
+ MAINTENANCE("Host %s for maintenance", VetoGroup.STATIC, (int) Math.pow(10, 5)),
+
+ /**
+ * Unable to satisfy proposed scheduler assignment dedicated host constraint.
+ */
+ DEDICATED_CONSTRAINT_MISMATCH("Host is dedicated", VetoGroup.STATIC, (int) Math.pow(10, 6));
private final String reasonFormat;
private final VetoGroup group;
+ private final int score;
- VetoType(String reasonFormat, VetoGroup group) {
+ VetoType(String reasonFormat, VetoGroup group, int score) {
this.reasonFormat = reasonFormat;
this.group = group;
+
+ // Score ranges are chosen under assumption that no more than 9 same group score vetoes
+ // are applied per task/offer evaluation (e.g. 9 insufficient resource vetoes < 1 value
+ // constraint mismatch). This is a fair assumption given current "break early" constraint
+ // evaluation rules and a limited number of resource types (4). Should this ever change,
+ // consider more spread between score groups to ensure correct NearestFit operation.
+ this.score = score;
}
String formatReason(String reason) {
@@ -92,6 +105,10 @@ public interface SchedulingFilter {
VetoGroup getGroup() {
return group;
}
+
+ int getScore() {
+ return score;
+ }
}
/**
@@ -101,8 +118,6 @@ public interface SchedulingFilter {
* a scheduling assignment is 'weakest'.
*/
final class Veto {
- public static final int MAX_SCORE = 1000;
-
private final VetoType vetoType;
private final String reason;
private final int score;
@@ -110,7 +125,20 @@ public interface SchedulingFilter {
private Veto(VetoType vetoType, String reasonParameter, int score) {
this.vetoType = vetoType;
this.reason = vetoType.formatReason(reasonParameter);
- this.score = Math.min(MAX_SCORE, score);
+ this.score = score;
+ }
+
+ /**
+ * Creates a veto that represents an dedicated host constraint mismatch between the server
+ * and task's configuration for an attribute.
+ *
+ * @return A dedicated host constraint mismatch veto.
+ */
+ public static Veto dedicatedHostConstraintMismatch() {
+ return new Veto(
+ VetoType.DEDICATED_CONSTRAINT_MISMATCH,
+ "",
+ VetoType.DEDICATED_CONSTRAINT_MISMATCH.getScore());
}
/**
@@ -121,7 +149,7 @@ public interface SchedulingFilter {
* @return A constraint mismatch veto.
*/
public static Veto constraintMismatch(String constraint) {
- return new Veto(CONSTRAINT_MISMATCH, constraint, MAX_SCORE);
+ return new Veto(CONSTRAINT_MISMATCH, constraint, CONSTRAINT_MISMATCH.getScore());
}
/**
@@ -132,7 +160,7 @@ public interface SchedulingFilter {
* @return A unsatisfied limit veto.
*/
public static Veto unsatisfiedLimit(String limit) {
- return new Veto(LIMIT_NOT_SATISFIED, limit, MAX_SCORE);
+ return new Veto(LIMIT_NOT_SATISFIED, limit, LIMIT_NOT_SATISFIED.getScore());
}
/**
@@ -155,7 +183,7 @@ public interface SchedulingFilter {
* @return A maintenance veto.
*/
public static Veto maintenance(String maintenanceMode) {
- return new Veto(MAINTENANCE, maintenanceMode, MAX_SCORE);
+ return new Veto(MAINTENANCE, maintenanceMode, MAINTENANCE.getScore());
}
public String getReason() {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1b1931cc/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
index a020ce5..54ffd8e 100644
--- a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
@@ -50,10 +50,6 @@ import static org.apache.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVe
* fulfilled, and that tasks are allowed to run on the given machine.
*/
public class SchedulingFilterImpl implements SchedulingFilter {
-
- @VisibleForTesting static final Veto DEDICATED_HOST_VETO =
- Veto.constraintMismatch("Host is dedicated");
-
private static final Optional<Veto> NO_VETO = Optional.absent();
private static final Set<MaintenanceMode> VETO_MODES = EnumSet.of(DRAINING, DRAINED);
@@ -90,7 +86,9 @@ public class SchedulingFilterImpl implements SchedulingFilter {
}
private static int scale(double value, int range) {
- return Math.min(Veto.MAX_SCORE, (int) (Veto.MAX_SCORE * value) / range);
+ return Math.min(
+ VetoType.INSUFFICIENT_RESOURCES.getScore(),
+ (int) (VetoType.INSUFFICIENT_RESOURCES.getScore() * value) / range);
}
@VisibleForTesting
@@ -143,25 +141,20 @@ public class SchedulingFilterImpl implements SchedulingFilter {
this.executorSettings = requireNonNull(executorSettings);
}
- private Set<Veto> getConstraintVetoes(
+ private Optional<Veto> getConstraintVeto(
Iterable<IConstraint> taskConstraints,
AttributeAggregate jobState,
Iterable<IAttribute> offerAttributes) {
- ImmutableSet.Builder<Veto> vetoes = ImmutableSet.builder();
for (IConstraint constraint : VALUES_FIRST.sortedCopy(taskConstraints)) {
Optional<Veto> veto = ConstraintMatcher.getVeto(jobState, offerAttributes, constraint);
if (veto.isPresent()) {
- vetoes.add(veto.get());
- if (isValueConstraint(constraint)) {
- // Break when a value constraint mismatch is found to avoid other
- // potentially-expensive operations to satisfy other constraints.
- break;
- }
+ // Break early to avoid potentially-expensive operations to satisfy other constraints.
+ return veto;
}
}
- return vetoes.build();
+ return Optional.absent();
}
private Optional<Veto> getMaintenanceVeto(MaintenanceMode mode) {
@@ -178,27 +171,36 @@ public class SchedulingFilterImpl implements SchedulingFilter {
@Override
public Set<Veto> filter(UnusedResource resource, ResourceRequest request) {
+ // Apply veto filtering rules from higher to lower score making sure we cut over and return
+ // early any time a veto from a score group is applied. This helps to more accurately report
+ // a veto reason in the NearestFit.
+
+ // 1. Dedicated constraint check (highest score).
if (!ConfigurationManager.isDedicated(request.getConstraints())
&& isDedicated(resource.getAttributes())) {
- return ImmutableSet.of(DEDICATED_HOST_VETO);
+ return ImmutableSet.of(Veto.dedicatedHostConstraintMismatch());
}
+ // 2. Host maintenance check.
Optional<Veto> maintenanceVeto = getMaintenanceVeto(resource.getAttributes().getMode());
if (maintenanceVeto.isPresent()) {
return maintenanceVeto.asSet();
}
- Set<Veto> resourceVetoes = getResourceVetoes(
- resource.getResourceSlot(),
- ResourceSlot.from(request.getTask(), executorSettings));
- if (!resourceVetoes.isEmpty()) {
- return resourceVetoes;
- }
-
- return getConstraintVetoes(
+ // 3. Value and limit constraint check.
+ Optional<Veto> constraintVeto = getConstraintVeto(
request.getConstraints(),
request.getJobState(),
resource.getAttributes().getAttributes());
+
+ if (constraintVeto.isPresent()) {
+ return constraintVeto.asSet();
+ }
+
+ // 4. Resource check (lowest score).
+ return getResourceVetoes(
+ resource.getResourceSlot(),
+ ResourceSlot.from(request.getTask(), executorSettings));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1b1931cc/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
index 785b574..2b194e5 100644
--- a/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
+++ b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
@@ -21,7 +21,6 @@ import javax.inject.Inject;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Functions;
-import com.google.common.base.Predicate;
import com.google.common.base.Ticker;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -39,7 +38,6 @@ import org.apache.aurora.scheduler.events.PubsubEvent.EventSubscriber;
import org.apache.aurora.scheduler.events.PubsubEvent.TaskStateChange;
import org.apache.aurora.scheduler.events.PubsubEvent.TasksDeleted;
import org.apache.aurora.scheduler.events.PubsubEvent.Vetoed;
-import org.apache.aurora.scheduler.filter.SchedulingFilter;
import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
@@ -116,17 +114,8 @@ public class NearestFit implements EventSubscriber {
}
}
- private static final Predicate<Veto> IS_CONSTRAINT_MISMATCH = new Predicate<Veto>() {
- @Override
- public boolean apply(Veto veto) {
- return veto.getVetoType() == SchedulingFilter.VetoType.CONSTRAINT_MISMATCH;
- }
- };
-
/**
* Records a task veto event.
- * This will ignore any veto events with a type of
- * {@link SchedulingFilter.VetoType#CONSTRAINT_MISMATCH}
*
* @param vetoEvent Veto event.
*/
@@ -155,17 +144,8 @@ public class NearestFit implements EventSubscriber {
* Updates the nearest fit if the provided vetoes represents a closer fit than the current
* best fit.
* <p>
- * There are two classes of vetoes: those with and without constraint mismatches. A set of
- * vetoes without a constraint mismatch is always a better fit than a set with constraint
- * mismatches.
- * <p>
- * If two sets are equivalent in that they do or do not have constraint mismatches, they are
- * compared by the following criteria:
- * <ul>
- * <li> the one with fewer vetoes is a better fit, irrespective of scores
- * <li> if the veto count is equal, the one with the smaller aggregate score is a better fit
- * </ul>
- *
+ * Vetoes with a lower aggregate score are considered a better fit regardless of the total veto
+ * count. See {@link Veto} for more details on scoring differences.
* @param newVetoes The vetoes for the scheduling assignment with {@code newHost}.
*/
void maybeUpdate(Set<Veto> newVetoes) {
@@ -174,15 +154,7 @@ public class NearestFit implements EventSubscriber {
return;
}
- boolean valueMismatchOld = Iterables.any(vetoes, IS_CONSTRAINT_MISMATCH);
- boolean valueMismatchNew = Iterables.any(newVetoes, IS_CONSTRAINT_MISMATCH);
- if (valueMismatchOld == valueMismatchNew) {
- if (newVetoes.size() < vetoes.size()) {
- update(newVetoes);
- } else if (newVetoes.size() == vetoes.size() && score(newVetoes) < score(vetoes)) {
- update(newVetoes);
- }
- } else if (valueMismatchOld) {
+ if (score(newVetoes) < score(vetoes)) {
update(newVetoes);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1b1931cc/src/test/java/org/apache/aurora/scheduler/filter/SchedulingFilterImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/filter/SchedulingFilterImplTest.java b/src/test/java/org/apache/aurora/scheduler/filter/SchedulingFilterImplTest.java
index b006684..d06b89c 100644
--- a/src/test/java/org/apache/aurora/scheduler/filter/SchedulingFilterImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/filter/SchedulingFilterImplTest.java
@@ -42,6 +42,7 @@ import org.apache.aurora.scheduler.filter.SchedulingFilter.ResourceRequest;
import org.apache.aurora.scheduler.filter.SchedulingFilter.UnusedResource;
import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
import org.apache.aurora.scheduler.filter.SchedulingFilter.VetoGroup;
+import org.apache.aurora.scheduler.filter.SchedulingFilter.VetoType;
import org.apache.aurora.scheduler.mesos.Offers;
import org.apache.aurora.scheduler.mesos.TaskExecutors;
import org.apache.aurora.scheduler.storage.AttributeStore;
@@ -53,7 +54,6 @@ import org.junit.Before;
import org.junit.Test;
import static org.apache.aurora.scheduler.configuration.ConfigurationManager.DEDICATED_ATTRIBUTE;
-import static org.apache.aurora.scheduler.filter.SchedulingFilterImpl.DEDICATED_HOST_VETO;
import static org.apache.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.CPU;
import static org.apache.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.DISK;
import static org.apache.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.PORTS;
@@ -183,7 +183,7 @@ public class SchedulingFilterImplTest extends EasyMockTest {
IHostAttributes hostA = hostAttributes(HOST_A, dedicated(ROLE_A));
checkConstraint(hostA, DEDICATED_ATTRIBUTE, true, ROLE_A);
- assertVetoes(makeTask(OWNER_B, JOB_B), hostA, DEDICATED_HOST_VETO);
+ assertVetoes(makeTask(OWNER_B, JOB_B), hostA, Veto.dedicatedHostConstraintMismatch());
}
@Test
@@ -437,10 +437,11 @@ public class SchedulingFilterImplTest extends EasyMockTest {
public void testVetoScaling() {
control.replay();
- assertEquals((int) (Veto.MAX_SCORE * 1.0 / CPU.getRange()), CPU.veto(1).getScore());
- assertEquals(Veto.MAX_SCORE, CPU.veto(CPU.getRange() * 10).getScore());
- assertEquals((int) (Veto.MAX_SCORE * 2.0 / RAM.getRange()), RAM.veto(2).getScore());
- assertEquals((int) (Veto.MAX_SCORE * 200.0 / DISK.getRange()), DISK.veto(200).getScore());
+ int maxScore = VetoType.INSUFFICIENT_RESOURCES.getScore();
+ assertEquals((int) (maxScore * 1.0 / CPU.getRange()), CPU.veto(1).getScore());
+ assertEquals(maxScore, CPU.veto(CPU.getRange() * 10).getScore());
+ assertEquals((int) (maxScore * 2.0 / RAM.getRange()), RAM.veto(2).getScore());
+ assertEquals((int) (maxScore * 200.0 / DISK.getRange()), DISK.veto(200).getScore());
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1b1931cc/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java b/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
index e0f9d8b..210bafd 100644
--- a/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
@@ -16,6 +16,7 @@ package org.apache.aurora.scheduler.metadata;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
+
import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Time;
import com.twitter.common.util.testing.FakeTicker;
@@ -37,11 +38,18 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class NearestFitTest {
-
- private static final Veto ALMOST = Veto.insufficientResources("Almost", 1);
- private static final Veto NOPE = Veto.insufficientResources("Nope", 5);
- private static final Veto NO_CHANCE = Veto.insufficientResources("No chance", 1000);
- private static final Veto KERNEL = Veto.constraintMismatch("2.6.39");
+ private static final int RESOURCE_MAX_SCORE = 1000;
+ private static final Veto SEVERITY_1 = Veto.dedicatedHostConstraintMismatch();
+ private static final Veto SEVERITY_2 = Veto.maintenance("maintenance");
+ private static final Veto SEVERITY_3 = Veto.constraintMismatch("constraint");
+ private static final Veto SEVERITY_4_CPU =
+ Veto.insufficientResources("cpu", RESOURCE_MAX_SCORE);
+ private static final Veto SEVERITY_4_RAM =
+ Veto.insufficientResources("ram", RESOURCE_MAX_SCORE);
+ private static final Veto SEVERITY_4_DISK =
+ Veto.insufficientResources("disk", RESOURCE_MAX_SCORE);
+ private static final Veto SEVERITY_4_PORTS =
+ Veto.insufficientResources("ports", RESOURCE_MAX_SCORE);
private static final ITaskConfig TASK = ITaskConfig.build(new TaskConfig().setNumCpus(1.0));
private static final TaskGroupKey GROUP_KEY = TaskGroupKey.from(TASK);
@@ -61,29 +69,20 @@ public class NearestFitTest {
}
@Test
- public void testMultipleVetoes() {
- vetoed(ALMOST, NOPE);
- assertNearest(ALMOST, NOPE);
- // Even though the aggregate score for NO_CHANCE is higher than ALMOST and NOPE,
- // NO_CHANCE becomes the pending reason since we consider one vector smaller than two
- // (regardless of magnitude).
- vetoed(NO_CHANCE);
- assertNearest(NO_CHANCE);
- }
-
- @Test
public void testScoring() {
- vetoed(NO_CHANCE);
- assertNearest(NO_CHANCE);
- vetoed(ALMOST);
- assertNearest(ALMOST);
- vetoed(NO_CHANCE);
- assertNearest(ALMOST);
+ vetoed(SEVERITY_1);
+ assertNearest(SEVERITY_1);
+ vetoed(SEVERITY_2);
+ assertNearest(SEVERITY_2);
+ vetoed(SEVERITY_3);
+ assertNearest(SEVERITY_3);
+ vetoed(SEVERITY_4_CPU, SEVERITY_4_RAM, SEVERITY_4_DISK, SEVERITY_4_PORTS);
+ assertNearest(SEVERITY_4_CPU, SEVERITY_4_RAM, SEVERITY_4_DISK, SEVERITY_4_PORTS);
}
@Test
public void testRemove() {
- vetoed(NO_CHANCE);
+ vetoed(SEVERITY_1);
nearest.remove(new TasksDeleted(ImmutableSet.of(makeTask())));
assertNearest();
}
@@ -95,8 +94,8 @@ public class NearestFitTest {
@Test
public void testExpiration() {
- vetoed(ALMOST);
- assertNearest(ALMOST);
+ vetoed(SEVERITY_2);
+ assertNearest(SEVERITY_2);
ticker.advance(NearestFit.EXPIRATION);
ticker.advance(Amount.of(1L, Time.SECONDS));
assertNearest();
@@ -104,8 +103,8 @@ public class NearestFitTest {
@Test
public void testStateChanged() {
- vetoed(ALMOST);
- assertNearest(ALMOST);
+ vetoed(SEVERITY_2);
+ assertNearest(SEVERITY_2);
IScheduledTask task = IScheduledTask.build(new ScheduledTask()
.setStatus(ScheduleStatus.ASSIGNED)
.setAssignedTask(new AssignedTask().setTask(TASK.newBuilder())));
@@ -113,16 +112,6 @@ public class NearestFitTest {
assertNearest();
}
- @Test
- public void testConstraintMismatch() {
- vetoed(KERNEL);
- assertNearest(KERNEL);
- vetoed(ALMOST);
- assertNearest(ALMOST);
- vetoed(KERNEL);
- assertNearest(ALMOST);
- }
-
private Set<Veto> vetoes(Veto... vetoes) {
return ImmutableSet.<Veto>builder().add(vetoes).build();
}