You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2015/03/06 02:57:41 UTC
incubator-aurora git commit: Pending reason for all tasks. Part 2 -
NearestFit changes.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 21661a7c5 -> 8e71b5887
Pending reason for all tasks. Part 2 - NearestFit changes.
Bugs closed: AURORA-911
Reviewed at https://reviews.apache.org/r/31652/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/8e71b588
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/8e71b588
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/8e71b588
Branch: refs/heads/master
Commit: 8e71b58879a8271a88f0dda4b550e38450f7c9b9
Parents: 21661a7
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Thu Mar 5 17:54:33 2015 -0800
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Thu Mar 5 17:54:33 2015 -0800
----------------------------------------------------------------------
.../events/NotifyingSchedulingFilter.java | 3 +-
.../aurora/scheduler/events/PubsubEvent.java | 17 ++++++-----
.../aurora/scheduler/metadata/NearestFit.java | 31 +++++++++++++-------
.../scheduler/thrift/ReadOnlySchedulerImpl.java | 9 ++++--
.../apache/aurora/scheduler/TaskVarsTest.java | 3 +-
.../events/NotifyingSchedulingFilterTest.java | 5 ++--
.../scheduler/metadata/NearestFitTest.java | 14 +++++----
.../thrift/ReadOnlySchedulerImplTest.java | 29 ++++++++++++------
8 files changed, 72 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java b/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
index edaf2f4..f6c759f 100644
--- a/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
+++ b/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
@@ -20,6 +20,7 @@ import java.util.Set;
import javax.inject.Inject;
import javax.inject.Qualifier;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.events.PubsubEvent.Vetoed;
import org.apache.aurora.scheduler.filter.SchedulingFilter;
@@ -57,7 +58,7 @@ class NotifyingSchedulingFilter implements SchedulingFilter {
public Set<Veto> filter(UnusedResource resource, ResourceRequest request) {
Set<Veto> vetoes = delegate.filter(resource, request);
if (!vetoes.isEmpty()) {
- eventSink.post(new Vetoed(request.getTaskId(), vetoes));
+ eventSink.post(new Vetoed(TaskGroupKey.from(request.getTask()), vetoes));
}
return vetoes;
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java b/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
index c103472..0063aea 100644
--- a/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
+++ b/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
@@ -19,6 +19,7 @@ import java.util.Set;
import com.google.common.base.Optional;
import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
@@ -198,16 +199,16 @@ public interface PubsubEvent {
* Event sent when a scheduling assignment was vetoed.
*/
class Vetoed implements PubsubEvent {
- private final String taskId;
+ private final TaskGroupKey groupKey;
private final Set<Veto> vetoes;
- public Vetoed(String taskId, Set<Veto> vetoes) {
- this.taskId = requireNonNull(taskId);
+ public Vetoed(TaskGroupKey groupKey, Set<Veto> vetoes) {
+ this.groupKey = requireNonNull(groupKey);
this.vetoes = requireNonNull(vetoes);
}
- public String getTaskId() {
- return taskId;
+ public TaskGroupKey getGroupKey() {
+ return groupKey;
}
public Set<Veto> getVetoes() {
@@ -221,19 +222,19 @@ public interface PubsubEvent {
}
Vetoed other = (Vetoed) o;
- return Objects.equals(taskId, other.taskId)
+ return Objects.equals(groupKey, other.groupKey)
&& Objects.equals(vetoes, other.vetoes);
}
@Override
public int hashCode() {
- return Objects.hash(taskId, vetoes);
+ return Objects.hash(groupKey, vetoes);
}
@Override
public String toString() {
return com.google.common.base.Objects.toStringHelper(this)
- .add("taskId", taskId)
+ .add("groupKey", groupKey)
.add("vetoes", vetoes)
.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
index c3097e4..785b574 100644
--- a/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
+++ b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
@@ -19,6 +19,8 @@ import java.util.Set;
import javax.inject.Inject;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
import com.google.common.base.Predicate;
import com.google.common.base.Ticker;
import com.google.common.cache.CacheBuilder;
@@ -31,6 +33,7 @@ import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Time;
import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.events.PubsubEvent.EventSubscriber;
import org.apache.aurora.scheduler.events.PubsubEvent.TaskStateChange;
@@ -38,6 +41,7 @@ import org.apache.aurora.scheduler.events.PubsubEvent.TasksDeleted;
import org.apache.aurora.scheduler.events.PubsubEvent.Vetoed;
import org.apache.aurora.scheduler.filter.SchedulingFilter;
import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
+import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
/**
* Tracks vetoes against scheduling decisions and maintains the closest fit among all the vetoes
@@ -50,16 +54,16 @@ public class NearestFit implements EventSubscriber {
@VisibleForTesting
static final ImmutableSet<Veto> NO_VETO = ImmutableSet.of();
- private final LoadingCache<String, Fit> fitByTask;
+ private final LoadingCache<TaskGroupKey, Fit> fitByGroupKey;
@VisibleForTesting
NearestFit(Ticker ticker) {
- fitByTask = CacheBuilder.newBuilder()
+ fitByGroupKey = CacheBuilder.newBuilder()
.expireAfterWrite(EXPIRATION.getValue(), EXPIRATION.getUnit().getTimeUnit())
.ticker(ticker)
- .build(new CacheLoader<String, Fit>() {
+ .build(new CacheLoader<TaskGroupKey, Fit>() {
@Override
- public Fit load(String taskId) {
+ public Fit load(TaskGroupKey groupKey) {
return new Fit();
}
});
@@ -73,12 +77,12 @@ public class NearestFit implements EventSubscriber {
/**
* Gets the vetoes that represent the nearest fit for the given task.
*
- * @param taskId The task to look up.
+ * @param groupKey The task group key to look up.
* @return The nearest fit vetoes for the given task. This will return an empty set if
* no vetoes have been recorded for the task.
*/
- public synchronized ImmutableSet<Veto> getNearestFit(String taskId) {
- Fit fit = fitByTask.getIfPresent(taskId);
+ public synchronized ImmutableSet<Veto> getNearestFit(TaskGroupKey groupKey) {
+ Fit fit = fitByGroupKey.getIfPresent(groupKey);
return (fit == null) ? NO_VETO : fit.vetoes;
}
@@ -89,7 +93,14 @@ public class NearestFit implements EventSubscriber {
*/
@Subscribe
public synchronized void remove(TasksDeleted deletedEvent) {
- fitByTask.invalidateAll(Tasks.ids(deletedEvent.getTasks()));
+ fitByGroupKey.invalidateAll(Iterables.transform(deletedEvent.getTasks(), Functions.compose(
+ new Function<ITaskConfig, TaskGroupKey>() {
+ @Override
+ public TaskGroupKey apply(ITaskConfig task) {
+ return TaskGroupKey.from(task);
+ }
+ },
+ Tasks.SCHEDULED_TO_INFO)));
}
/**
@@ -101,7 +112,7 @@ public class NearestFit implements EventSubscriber {
@Subscribe
public synchronized void stateChanged(TaskStateChange event) {
if (event.isTransition() && event.getOldState().get() == ScheduleStatus.PENDING) {
- fitByTask.invalidate(event.getTaskId());
+ fitByGroupKey.invalidate(TaskGroupKey.from(event.getTask().getAssignedTask().getTask()));
}
}
@@ -122,7 +133,7 @@ public class NearestFit implements EventSubscriber {
@Subscribe
public synchronized void vetoed(Vetoed vetoEvent) {
Objects.requireNonNull(vetoEvent);
- fitByTask.getUnchecked(vetoEvent.getTaskId()).maybeUpdate(vetoEvent.getVetoes());
+ fitByGroupKey.getUnchecked(vetoEvent.getGroupKey()).maybeUpdate(vetoEvent.getVetoes());
}
private static class Fit {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/main/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImpl.java b/src/main/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImpl.java
index 53582c6..3876290 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImpl.java
@@ -69,6 +69,7 @@ import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.base.JobKeys;
import org.apache.aurora.scheduler.base.Jobs;
import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
import org.apache.aurora.scheduler.configuration.SanitizedConfiguration;
@@ -187,9 +188,11 @@ class ReadOnlySchedulerImpl implements ReadOnlyScheduler.Iface {
.transform(new Function<ScheduledTask, PendingReason>() {
@Override
public PendingReason apply(ScheduledTask scheduledTask) {
- String taskId = scheduledTask.getAssignedTask().getTaskId();
+ TaskGroupKey groupKey = TaskGroupKey.from(
+ ITaskConfig.build(scheduledTask.getAssignedTask().getTask()));
+
String reason = Joiner.on(',').join(Iterables.transform(
- nearestFit.getNearestFit(taskId),
+ nearestFit.getNearestFit(groupKey),
new Function<Veto, String>() {
@Override
public String apply(Veto veto) {
@@ -198,7 +201,7 @@ class ReadOnlySchedulerImpl implements ReadOnlyScheduler.Iface {
}));
return new PendingReason()
- .setTaskId(taskId)
+ .setTaskId(scheduledTask.getAssignedTask().getTaskId())
.setReason(reason);
}
}).toSet();
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/test/java/org/apache/aurora/scheduler/TaskVarsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/TaskVarsTest.java b/src/test/java/org/apache/aurora/scheduler/TaskVarsTest.java
index 13520eb..256ee77 100644
--- a/src/test/java/org/apache/aurora/scheduler/TaskVarsTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/TaskVarsTest.java
@@ -32,6 +32,7 @@ import org.apache.aurora.gen.JobKey;
import org.apache.aurora.gen.ScheduleStatus;
import org.apache.aurora.gen.ScheduledTask;
import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.events.PubsubEvent;
import org.apache.aurora.scheduler.events.PubsubEvent.TaskStateChange;
@@ -126,7 +127,7 @@ public class TaskVarsTest extends EasyMockTest {
private void applyVeto(IScheduledTask task, Veto... vetoes) {
vars.taskVetoed(new PubsubEvent.Vetoed(
- task.getAssignedTask().getTaskId(),
+ TaskGroupKey.from(task.getAssignedTask().getTask()),
ImmutableSet.copyOf(vetoes)));
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/test/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilterTest.java b/src/test/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilterTest.java
index ab7817f..61cea32 100644
--- a/src/test/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilterTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilterTest.java
@@ -23,6 +23,7 @@ import org.apache.aurora.gen.HostAttributes;
import org.apache.aurora.gen.MaintenanceMode;
import org.apache.aurora.gen.TaskConfig;
import org.apache.aurora.scheduler.ResourceSlot;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.events.PubsubEvent.Vetoed;
import org.apache.aurora.scheduler.filter.AttributeAggregate;
import org.apache.aurora.scheduler.filter.SchedulingFilter;
@@ -46,7 +47,7 @@ public class NotifyingSchedulingFilterTest extends EasyMockTest {
.setNumCpus(1)
.setRamMb(1024)
.setDiskMb(1024));
- private static final String TASK_ID = "taskId";
+ private static final TaskGroupKey GROUP_KEY = TaskGroupKey.from(TASK);
private static final UnusedResource RESOURCE = new UnusedResource(
ResourceSlot.from(TASK, TaskExecutors.NO_OVERHEAD_EXECUTOR),
IHostAttributes.build(new HostAttributes().setHost("host").setMode(MaintenanceMode.NONE)));
@@ -74,7 +75,7 @@ public class NotifyingSchedulingFilterTest extends EasyMockTest {
public void testEvents() {
Set<Veto> vetoes = ImmutableSet.of(VETO_1, VETO_2);
expect(delegate.filter(RESOURCE, request)).andReturn(vetoes);
- eventSink.post(new Vetoed(TASK_ID, vetoes));
+ eventSink.post(new Vetoed(GROUP_KEY, vetoes));
control.replay();
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java b/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
index 78a236c..e0f9d8b 100644
--- a/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/metadata/NearestFitTest.java
@@ -23,11 +23,14 @@ import com.twitter.common.util.testing.FakeTicker;
import org.apache.aurora.gen.AssignedTask;
import org.apache.aurora.gen.ScheduleStatus;
import org.apache.aurora.gen.ScheduledTask;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.events.PubsubEvent.TaskStateChange;
import org.apache.aurora.scheduler.events.PubsubEvent.TasksDeleted;
import org.apache.aurora.scheduler.events.PubsubEvent.Vetoed;
import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import org.junit.Before;
import org.junit.Test;
@@ -40,7 +43,8 @@ public class NearestFitTest {
private static final Veto NO_CHANCE = Veto.insufficientResources("No chance", 1000);
private static final Veto KERNEL = Veto.constraintMismatch("2.6.39");
- private static final String TASK = "taskId";
+ private static final ITaskConfig TASK = ITaskConfig.build(new TaskConfig().setNumCpus(1.0));
+ private static final TaskGroupKey GROUP_KEY = TaskGroupKey.from(TASK);
private FakeTicker ticker;
private NearestFit nearest;
@@ -86,7 +90,7 @@ public class NearestFitTest {
private IScheduledTask makeTask() {
return IScheduledTask.build(
- new ScheduledTask().setAssignedTask(new AssignedTask().setTaskId(TASK)));
+ new ScheduledTask().setAssignedTask(new AssignedTask().setTask(TASK.newBuilder())));
}
@Test
@@ -104,7 +108,7 @@ public class NearestFitTest {
assertNearest(ALMOST);
IScheduledTask task = IScheduledTask.build(new ScheduledTask()
.setStatus(ScheduleStatus.ASSIGNED)
- .setAssignedTask(new AssignedTask().setTaskId(TASK)));
+ .setAssignedTask(new AssignedTask().setTask(TASK.newBuilder())));
nearest.stateChanged(TaskStateChange.transition(task, ScheduleStatus.PENDING));
assertNearest();
}
@@ -124,10 +128,10 @@ public class NearestFitTest {
}
private void vetoed(Veto... vetoes) {
- nearest.vetoed(new Vetoed(TASK, ImmutableSet.<Veto>builder().add(vetoes).build()));
+ nearest.vetoed(new Vetoed(GROUP_KEY, ImmutableSet.<Veto>builder().add(vetoes).build()));
}
private void assertNearest(Veto... vetoes) {
- assertEquals(vetoes(vetoes), nearest.getNearestFit(TASK));
+ assertEquals(vetoes(vetoes), nearest.getNearestFit(GROUP_KEY));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8e71b588/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java
index ce5a626..3cf8dfd 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java
@@ -54,6 +54,7 @@ import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.base.JobKeys;
import org.apache.aurora.scheduler.base.Query;
import org.apache.aurora.scheduler.base.Query.Builder;
+import org.apache.aurora.scheduler.base.TaskGroupKey;
import org.apache.aurora.scheduler.configuration.SanitizedConfiguration;
import org.apache.aurora.scheduler.cron.CronPredictor;
import org.apache.aurora.scheduler.cron.CrontabEntry;
@@ -69,6 +70,7 @@ import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery;
import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
import org.junit.Before;
import org.junit.Test;
@@ -219,25 +221,34 @@ public class ReadOnlySchedulerImplTest extends EasyMockTest {
public void testGetPendingReason() throws Exception {
Builder query = Query.unscoped().byJob(JOB_KEY);
Builder filterQuery = Query.unscoped().byJob(JOB_KEY).byStatus(ScheduleStatus.PENDING);
- String taskId = "task_id_test";
+ String taskId1 = "task_id_test1";
+ String taskId2 = "task_id_test2";
ImmutableSet<Veto> result = ImmutableSet.of(
Veto.constraintMismatch("first"),
Veto.constraintMismatch("second"));
- IScheduledTask pendingTask = IScheduledTask.build(new ScheduledTask()
+ ITaskConfig taskConfig = ITaskConfig.build(defaultTask(true));
+ IScheduledTask pendingTask1 = IScheduledTask.build(new ScheduledTask()
.setAssignedTask(new AssignedTask()
- .setTask(defaultTask(true))
- .setTaskId(taskId))
+ .setTaskId(taskId1)
+ .setTask(taskConfig.newBuilder()))
.setStatus(ScheduleStatus.PENDING));
- storageUtil.expectTaskFetch(filterQuery, pendingTask);
- expect(nearestFit.getNearestFit(taskId)).andReturn(result);
+ IScheduledTask pendingTask2 = IScheduledTask.build(new ScheduledTask()
+ .setAssignedTask(new AssignedTask()
+ .setTaskId(taskId2)
+ .setTask(taskConfig.newBuilder()))
+ .setStatus(ScheduleStatus.PENDING));
+
+ storageUtil.expectTaskFetch(filterQuery, pendingTask1, pendingTask2);
+ expect(nearestFit.getNearestFit(TaskGroupKey.from(taskConfig))).andReturn(result).times(2);
control.replay();
- Set<PendingReason> expected = ImmutableSet.of(new PendingReason()
- .setTaskId(taskId)
- .setReason("Constraint not satisfied: first,Constraint not satisfied: second"));
+ String reason = "Constraint not satisfied: first,Constraint not satisfied: second";
+ Set<PendingReason> expected = ImmutableSet.of(
+ new PendingReason().setTaskId(taskId1).setReason(reason),
+ new PendingReason().setTaskId(taskId2).setReason(reason));
Response response = assertOkResponse(thrift.getPendingReason(query.get()));
assertEquals(expected, response.getResult().getGetPendingReasonResult().getReasons());