You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2016/05/19 15:27:17 UTC
aurora git commit: Fixing resource slot counting.
Repository: aurora
Updated Branches:
refs/heads/master 3cbff4117 -> 9ad153f0f
Fixing resource slot counting.
Reviewed at https://reviews.apache.org/r/47561/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9ad153f0
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9ad153f0
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9ad153f0
Branch: refs/heads/master
Commit: 9ad153f0f474649669b0c9fa2debd516a9a4db5a
Parents: 3cbff41
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Thu May 19 08:27:00 2016 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Thu May 19 08:27:00 2016 -0700
----------------------------------------------------------------------
.../org/apache/aurora/benchmark/Offers.java | 4 +--
.../aurora/benchmark/SchedulingBenchmarks.java | 3 +++
.../java/org/apache/aurora/benchmark/Tasks.java | 10 +++++++
.../aurora/scheduler/resources/ResourceBag.java | 28 +++++++++++++-------
.../scheduler/resources/ResourceBagTest.java | 17 +++++++++---
.../scheduler/stats/SlotSizeCounterTest.java | 22 +++++++++++++++
6 files changed, 70 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Offers.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/Offers.java b/src/jmh/java/org/apache/aurora/benchmark/Offers.java
index 1dabe15..144f47e 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/Offers.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/Offers.java
@@ -99,8 +99,8 @@ final class Offers {
Protos.Offer offer = Protos.Offer.newBuilder()
.addAllResources(ImmutableSet.of(
makeScalar(CPUS, cpu),
- makeScalar(RAM_MB, ram.getValue()),
- makeScalar(DISK_MB, disk.getValue()),
+ makeScalar(RAM_MB, ram.as(Data.MB)),
+ makeScalar(DISK_MB, disk.as(Data.MB)),
makeRange(
PORTS,
IntStream.range(1, ports).boxed().collect(Collectors.toSet()))))
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
index 1aa484c..65f5edc 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
@@ -33,6 +33,7 @@ import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.common.stats.StatsProvider;
import org.apache.aurora.common.util.Clock;
import org.apache.aurora.common.util.testing.FakeClock;
+import org.apache.aurora.gen.ServerInfo;
import org.apache.aurora.scheduler.HostOffer;
import org.apache.aurora.scheduler.TaskIdGenerator;
import org.apache.aurora.scheduler.TierModule;
@@ -60,6 +61,7 @@ import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
import org.apache.aurora.scheduler.storage.db.DbUtil;
import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.IServerInfo;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@@ -157,6 +159,7 @@ public class SchedulingBenchmarks {
bind(Clock.class).toInstance(clock);
bind(StatsProvider.class).toInstance(new FakeStatsProvider());
bind(EventSink.class).toInstance(eventBus::post);
+ bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo("jmh", "")));
}
}
);
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
index 4c4b03d..e548a09 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
@@ -15,6 +15,7 @@ package org.apache.aurora.benchmark;
import java.util.Set;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -32,6 +33,10 @@ import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import static org.apache.aurora.gen.Resource.diskMb;
+import static org.apache.aurora.gen.Resource.numCpus;
+import static org.apache.aurora.gen.Resource.ramMb;
+
/**
* Task factory.
*/
@@ -140,12 +145,17 @@ final class Tasks {
builder.getAssignedTask()
.setInstanceId(i)
.setTaskId(taskId);
+ builder.getAssignedTask().setAssignedPorts(ImmutableMap.of());
builder.getAssignedTask().getTask()
.setConstraints(constraints.build())
.setNumCpus(cpu)
.setRamMb(ram.as(Data.MB))
.setDiskMb(disk.as(Data.MB))
.setProduction(isProduction)
+ .setResources(ImmutableSet.of(
+ numCpus(cpu),
+ ramMb(ram.as(Data.MB)),
+ diskMb(disk.as(Data.MB))))
.setRequestedPorts(ImmutableSet.of());
tasks.add(IScheduledTask.build(builder));
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
index 3cff946..211f1fc 100644
--- a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
+++ b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
@@ -15,12 +15,14 @@ package org.apache.aurora.scheduler.resources;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.function.BinaryOperator;
import java.util.function.Predicate;
import java.util.stream.Stream;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
import static java.util.stream.Collectors.toMap;
@@ -132,7 +134,9 @@ public class ResourceBag {
/**
* Divides this by other bag contents.
- *
+ * <p>
+ * Note: any missing or zero resource values in {@code other} will result in
+ * {@code java.lang.Double.POSITIVE_INFINITY}.
* @param other Other bag to divide by.
* @return Result of division.
*/
@@ -173,16 +177,22 @@ public class ResourceBag {
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue)));
}
+ /**
+ * Applies {@code operator} to this and {@code other} resource values. Any missing resource type
+ * values on either side will get substituted with 0.0 before applying the operator.
+ *
+ * @param other Other ResourceBag.
+ * @param operator Operator to apply.
+ * @return Operation result.
+ */
private ResourceBag binaryOp(ResourceBag other, BinaryOperator<Double> operator) {
ImmutableMap.Builder<ResourceType, Double> builder = ImmutableMap.builder();
- for (Map.Entry<ResourceType, Double> entry : resourceVectors.entrySet()) {
- // Apply binary operator only on matching keys from the other. If there is no match, keep the
- // current value unchanged.
- builder.put(
- entry.getKey(),
- other.getResourceVectors().containsKey(entry.getKey())
- ? operator.apply(entry.getValue(), other.getResourceVectors().get(entry.getKey()))
- : entry.getValue());
+ Set<ResourceType> resourceTypes =
+ Sets.union(resourceVectors.keySet(), other.getResourceVectors().keySet());
+ for (ResourceType type : resourceTypes) {
+ Double left = resourceVectors.getOrDefault(type, 0.0);
+ Double right = other.getResourceVectors().getOrDefault(type, 0.0);
+ builder.put(type, operator.apply(left, right));
}
return new ResourceBag(builder.build());
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
index b8e6474..c163826 100644
--- a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
@@ -17,6 +17,8 @@ import com.google.common.collect.ImmutableMap;
import org.junit.Test;
+import static java.lang.Double.POSITIVE_INFINITY;
+
import static org.apache.aurora.scheduler.resources.ResourceBag.IS_NEGATIVE;
import static org.apache.aurora.scheduler.resources.ResourceBag.LARGE;
import static org.apache.aurora.scheduler.resources.ResourceBag.MEDIUM;
@@ -58,12 +60,21 @@ public class ResourceBagTest {
@Test
public void testKeyMismatch() {
assertEquals(
- new ResourceBag(ImmutableMap.of(CPUS, 9.0)),
+ new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)),
new ResourceBag(ImmutableMap.of(CPUS, 1.0)).add(LARGE));
+ // Check add with mismatched keys is still commutative.
+ assertEquals(
+ new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)),
+ LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0))));
+
+ // Check divide with missing values on the right.
assertEquals(
- LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0))),
- new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)));
+ new ResourceBag(ImmutableMap.of(
+ CPUS, 1.0,
+ RAM_MB, POSITIVE_INFINITY,
+ DISK_MB, POSITIVE_INFINITY)),
+ LARGE.divide(new ResourceBag(ImmutableMap.of(CPUS, 8.0))));
}
@Test
http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
index e529e67..6c532e6 100644
--- a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
@@ -142,6 +142,28 @@ public class SlotSizeCounterTest extends EasyMockTest {
}
@Test
+ public void testMissingResourceVector() {
+ expectStatExport();
+ expectGetSlots(
+ new MachineResource(
+ ResourceTestUtil.bag(ImmutableMap.of(RAM_MB, 65536.0, DISK_MB, 65536.0)),
+ false,
+ false));
+
+ control.replay();
+
+ slotCounter.run();
+ assertEquals(0, smallCounter.get());
+ assertEquals(0, smallDedicatedCounter.get());
+ assertEquals(0, smallRevocableCounter.get());
+ assertEquals(0, smallDedicatedRevocableCounter.get());
+ assertEquals(0, largeCounter.get());
+ assertEquals(0, largeDedicatedCounter.get());
+ assertEquals(0, largeRevocableCounter.get());
+ assertEquals(0, largeDedicatedRevocableCounter.get());
+ }
+
+ @Test
public void testCountSlots() {
expectStatExport();
expectGetSlots(