You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2015/08/08 00:38:57 UTC
aurora git commit: Resources: moving calculation methods from
Resources
Repository: aurora
Updated Branches:
refs/heads/master e8a61f68c -> 92fc094f8
Resources: moving calculation methods from Resources
Bugs closed: AURORA-1415
Reviewed at https://reviews.apache.org/r/37139/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/92fc094f
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/92fc094f
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/92fc094f
Branch: refs/heads/master
Commit: 92fc094f8267e680fe2dc29d5a42fb56cf8cbbff
Parents: e8a61f6
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Fri Aug 7 15:36:20 2015 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Fri Aug 7 15:36:20 2015 -0700
----------------------------------------------------------------------
.../apache/aurora/scheduler/ResourceSlot.java | 79 +++++++++++---
.../scheduler/configuration/Resources.java | 107 -------------------
.../aurora/scheduler/ResourceSlotTest.java | 24 +++++
.../scheduler/configuration/ResourcesTest.java | 46 --------
.../mesos/MesosTaskFactoryImplTest.java | 4 +-
5 files changed, 93 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
index ecadb3e..eb14ffe 100644
--- a/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
+++ b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
@@ -18,8 +18,10 @@ import java.util.List;
import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Ordering;
import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Data;
@@ -65,7 +67,7 @@ public final class ResourceSlot {
private static ResourceSlot from(Resources resources, ExecutorSettings executorSettings) {
// Apply a flat 'tax' of executor overhead resources to the task.
- Resources requiredTaskResources = Resources.sum(
+ Resources requiredTaskResources = sum(
resources,
executorSettings.getExecutorOverhead());
@@ -135,18 +137,37 @@ public final class ResourceSlot {
}
public static ResourceSlot sum(Iterable<ResourceSlot> rs) {
- Resources r = Resources.sum(Iterables.transform(rs, new Function<ResourceSlot, Resources>() {
- @Override
- public Resources apply(ResourceSlot input) {
- return input.resources;
- }
- }));
+ Resources sum = Resources.NONE;
+
+ for (ResourceSlot r : rs) {
+ double numCpus = sum.getNumCpus() + r.getNumCpus();
+ Amount<Long, Data> disk =
+ Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES);
+ Amount<Long, Data> ram =
+ Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES);
+ int ports = sum.getNumPorts() + r.getNumPorts();
+ sum = new Resources(numCpus, ram, disk, ports);
+ }
+
+ return new ResourceSlot(sum);
+ }
- return new ResourceSlot(r);
+ @VisibleForTesting
+ public static Resources sum(Resources a, Resources b) {
+ return sum(ImmutableList.of(new ResourceSlot(a), new ResourceSlot(b))).resources;
}
public static ResourceSlot subtract(ResourceSlot a, Resources b) {
- return new ResourceSlot(Resources.subtract(a.resources, b));
+ return new ResourceSlot(subtract(a.resources, b));
+ }
+
+ @VisibleForTesting
+ static Resources subtract(Resources a, Resources b) {
+ return new Resources(
+ a.getNumCpus() - b.getNumCpus(),
+ Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB),
+ Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB),
+ a.getNumPorts() - b.getNumPorts());
}
public List<Protos.Resource> toResourceList(Set<Integer> selectedPorts) {
@@ -156,7 +177,41 @@ public final class ResourceSlot {
public static final Ordering<ResourceSlot> ORDER = new Ordering<ResourceSlot>() {
@Override
public int compare(ResourceSlot left, ResourceSlot right) {
- return Resources.RESOURCE_ORDER.compare(left.resources, right.resources);
+ return RESOURCE_ORDER.compare(left.resources, right.resources);
}
};
+
+ /**
+ * A Resources object is greater than another iff _all_ of its resource components are greater
+ * or equal. A Resources object compares as equal if some but not all components are greater than
+ * or equal to the other.
+ */
+ public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() {
+ @Override
+ public int compare(Resources left, Resources right) {
+ int diskC = left.getDisk().compareTo(right.getDisk());
+ int ramC = left.getRam().compareTo(right.getRam());
+ int portC = Integer.compare(left.getNumPorts(), right.getNumPorts());
+ int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus());
+
+ FluentIterable<Integer> vector =
+ FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC));
+
+ if (vector.allMatch(IS_ZERO)) {
+ return 0;
+ }
+
+ if (vector.filter(Predicates.not(IS_ZERO)).allMatch(e -> e > 0)) {
+ return 1;
+ }
+
+ if (vector.filter(Predicates.not(IS_ZERO)).allMatch(e -> e < 0)) {
+ return -1;
+ }
+
+ return 0;
+ }
+ };
+
+ private static final Predicate<Integer> IS_ZERO = e -> e == 0;
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
index ea7ce24..6db3efb 100644
--- a/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
@@ -21,15 +21,12 @@ import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
import com.google.common.collect.ContiguousSet;
import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Data;
@@ -87,20 +84,6 @@ public class Resources {
}
/**
- * Tests whether this bundle of resources is greater than or equal to another bundle of resources.
- *
- * @param other Resources being compared to.
- * @return {@code true} if all resources in this bundle are greater than or equal to the
- * equivalents from {@code other}, otherwise {@code false}.
- */
- public boolean greaterThanOrEqual(Resources other) {
- return numCpus >= other.numCpus
- && disk.as(Data.MB) >= other.disk.as(Data.MB)
- && ram.as(Data.MB) >= other.ram.as(Data.MB)
- && numPorts >= other.numPorts;
- }
-
- /**
* Adapts this resources object to a list of mesos resources.
*
* @param selectedPorts The ports selected, to be applied as concrete task ranges.
@@ -207,43 +190,6 @@ public class Resources {
public static final Resources NONE =
new Resources(0, Amount.of(0L, Data.BITS), Amount.of(0L, Data.BITS), 0);
- /**
- * a - b.
- */
- public static Resources subtract(Resources a, Resources b) {
- return new Resources(
- a.getNumCpus() - b.getNumCpus(),
- Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB),
- Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB),
- a.getNumPorts() - b.getNumPorts());
- }
-
- /**
- * sum(a, b).
- */
- public static Resources sum(Resources a, Resources b) {
- return sum(ImmutableList.of(a, b));
- }
-
- /**
- * sum(rs).
- */
- public static Resources sum(Iterable<Resources> rs) {
- Resources sum = NONE;
-
- for (Resources r : rs) {
- double numCpus = sum.getNumCpus() + r.getNumCpus();
- Amount<Long, Data> disk =
- Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES);
- Amount<Long, Data> ram =
- Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES);
- int ports = sum.getNumPorts() + r.getNumPorts();
- sum = new Resources(numCpus, ram, disk, ports);
- }
-
- return sum;
- }
-
private static int getNumAvailablePorts(List<Resource> resource) {
int offeredPorts = 0;
for (Range range : getPortRanges(resource)) {
@@ -401,57 +347,4 @@ public class Resources {
Collections.shuffle(availablePorts);
return ImmutableSet.copyOf(availablePorts.subList(0, numPorts));
}
-
- /**
- * A Resources object is greater than another iff _all_ of its resource components are greater
- * or equal. A Resources object compares as equal if some but not all components are greater than
- * or equal to the other.
- */
- public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() {
- @Override
- public int compare(Resources left, Resources right) {
- int diskC = left.getDisk().compareTo(right.getDisk());
- int ramC = left.getRam().compareTo(right.getRam());
- int portC = Integer.compare(left.getNumPorts(), right.getNumPorts());
- int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus());
-
- FluentIterable<Integer> vector =
- FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC));
-
- if (vector.allMatch(IS_ZERO)) {
- return 0;
- }
-
- if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_POSITIVE)) {
- return 1;
- }
-
- if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_NEGATIVE)) {
- return -1;
- }
-
- return 0;
- }
- };
-
- private static final Predicate<Integer> IS_POSITIVE = new Predicate<Integer>() {
- @Override
- public boolean apply(Integer input) {
- return input > 0;
- }
- };
-
- private static final Predicate<Integer> IS_NEGATIVE = new Predicate<Integer>() {
- @Override
- public boolean apply(Integer input) {
- return input < 0;
- }
- };
-
- private static final Predicate<Integer> IS_ZERO = new Predicate<Integer>() {
- @Override
- public boolean apply(Integer input) {
- return input == 0;
- }
- };
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java
index b4dbab7..896833d 100644
--- a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java
@@ -23,6 +23,15 @@ import static org.junit.Assert.assertEquals;
public class ResourceSlotTest {
+ private static final Resources NEGATIVE_ONE =
+ new Resources(-1.0, Amount.of(-1L, Data.MB), Amount.of(-1L, Data.MB), -1);
+ private static final Resources ONE =
+ new Resources(1.0, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 1);
+ private static final Resources TWO =
+ new Resources(2.0, Amount.of(2L, Data.MB), Amount.of(2L, Data.MB), 2);
+ private static final Resources THREE =
+ new Resources(3.0, Amount.of(3L, Data.MB), Amount.of(3L, Data.MB), 3);
+
@Test
public void testMaxElements() {
Resources highRAM = new Resources(1, Amount.of(8L, Data.GB), Amount.of(10L, Data.MB), 0);
@@ -34,4 +43,19 @@ public class ResourceSlotTest {
assertEquals(result.getDisk(), Amount.of(10L, Data.GB));
assertEquals(result.getNumPorts(), 1);
}
+
+ @Test
+ public void testSubtract() {
+ assertEquals(ONE, ResourceSlot.subtract(TWO, ONE));
+ assertEquals(TWO, ResourceSlot.subtract(THREE, ONE));
+ assertEquals(NEGATIVE_ONE, ResourceSlot.subtract(ONE, TWO));
+ assertEquals(NEGATIVE_ONE, ResourceSlot.subtract(TWO, THREE));
+ }
+
+ @Test
+ public void testSum() {
+ assertEquals(TWO, ResourceSlot.sum(ONE, ONE));
+ assertEquals(THREE, ResourceSlot.sum(ONE, TWO));
+ assertEquals(THREE, ResourceSlot.sum(TWO, ONE));
+ }
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java
index 115dc08..9335141 100644
--- a/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java
@@ -33,9 +33,7 @@ import org.apache.mesos.Protos.Value.Type;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class ResourcesTest {
@@ -87,30 +85,6 @@ public class ResourcesTest {
assertEquals(ImmutableSet.of(), Resources.getPorts(createOffer(portsResource), 0));
}
- private static final Resources NEGATIVE_ONE =
- new Resources(-1.0, Amount.of(-1L, Data.MB), Amount.of(-1L, Data.MB), -1);
- private static final Resources ONE =
- new Resources(1.0, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 1);
- private static final Resources TWO =
- new Resources(2.0, Amount.of(2L, Data.MB), Amount.of(2L, Data.MB), 2);
- private static final Resources THREE =
- new Resources(3.0, Amount.of(3L, Data.MB), Amount.of(3L, Data.MB), 3);
-
- @Test
- public void testSum() {
- assertEquals(TWO, Resources.sum(ONE, ONE));
- assertEquals(THREE, Resources.sum(ONE, TWO));
- assertEquals(THREE, Resources.sum(TWO, ONE));
- }
-
- @Test
- public void testSubtract() {
- assertEquals(ONE, Resources.subtract(TWO, ONE));
- assertEquals(TWO, Resources.subtract(THREE, ONE));
- assertEquals(NEGATIVE_ONE, Resources.subtract(ONE, TWO));
- assertEquals(NEGATIVE_ONE, Resources.subtract(TWO, THREE));
- }
-
@Test(expected = Resources.InsufficientResourcesException.class)
public void testPortRangeScarcity() {
Resource portsResource = createPortRange(Pair.of(1, 2));
@@ -177,11 +151,6 @@ public class ResourcesTest {
.setDiskMb(2048)
.setRequestedPorts(ImmutableSet.of("http", "debug")));
- private static void assertLeftIsLarger(Resources left, Resources right) {
- assertTrue(left.greaterThanOrEqual(right));
- assertFalse(right.greaterThanOrEqual(left));
- }
-
@Test
public void testAccessors() {
Resources resources = Resources.from(TASK);
@@ -189,21 +158,6 @@ public class ResourcesTest {
assertEquals(Amount.of(TASK.getRamMb(), Data.MB), resources.getRam());
assertEquals(Amount.of(TASK.getDiskMb(), Data.MB), resources.getDisk());
assertEquals(TASK.getRequestedPorts().size(), resources.getNumPorts());
-
- assertTrue(resources.greaterThanOrEqual(resources));
- assertLeftIsLarger(
- resources,
- Resources.from(ITaskConfig.build(TASK.newBuilder().setNumCpus(0.5))));
- assertLeftIsLarger(
- resources,
- Resources.from(ITaskConfig.build(TASK.newBuilder().setRamMb(512))));
- assertLeftIsLarger(
- resources,
- Resources.from(ITaskConfig.build(TASK.newBuilder().setDiskMb(1024))));
- assertLeftIsLarger(
- resources,
- Resources.from(
- ITaskConfig.build(TASK.newBuilder().setRequestedPorts(ImmutableSet.of("http")))));
}
@Test
http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
index 7f18bbc..d19f5af 100644
--- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
@@ -166,7 +166,7 @@ public class MesosTaskFactoryImplTest {
private void checkTaskResources(ITaskConfig task, TaskInfo taskInfo) {
assertEquals(
- Resources.sum(Resources.from(task), config.getExecutorOverhead()),
+ ResourceSlot.sum(Resources.from(task), config.getExecutorOverhead()),
getTotalTaskResources(taskInfo));
}
@@ -238,6 +238,6 @@ public class MesosTaskFactoryImplTest {
private static Resources getTotalTaskResources(TaskInfo task) {
Resources taskResources = Resources.from(task.getResourcesList());
Resources executorResources = Resources.from(task.getExecutor().getResourcesList());
- return Resources.sum(taskResources, executorResources);
+ return ResourceSlot.sum(taskResources, executorResources);
}
}