You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2014/06/25 21:37:06 UTC
git commit: Add test coverage for ResourceCounter.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 9d211f620 -> 4be190f60
Add test coverage for ResourceCounter.
Reviewed at https://reviews.apache.org/r/22942/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/4be190f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/4be190f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/4be190f6
Branch: refs/heads/master
Commit: 4be190f60f3fc758c4637227bd4ed30e477c4a26
Parents: 9d211f6
Author: Bill Farner <wf...@apache.org>
Authored: Wed Jun 25 12:36:54 2014 -0700
Committer: Bill Farner <wf...@apache.org>
Committed: Wed Jun 25 12:36:54 2014 -0700
----------------------------------------------------------------------
.../aurora/scheduler/http/Utilization.java | 23 ++-
.../aurora/scheduler/stats/ResourceCounter.java | 73 ++++++-
.../scheduler/stats/ResourceCounterTest.java | 201 +++++++++++++++++++
3 files changed, 283 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4be190f6/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Utilization.java b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
index 06dc846..d6dfcf1 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
@@ -17,6 +17,7 @@ import java.io.StringWriter;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import javax.annotation.Nullable;
import javax.inject.Inject;
@@ -30,7 +31,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import com.google.common.base.Function;
-import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.collect.FluentIterable;
import com.twitter.common.base.Closure;
@@ -103,7 +103,7 @@ public class Utilization {
@Override
public int hashCode() {
- return Objects.hashCode(title, link);
+ return Objects.hash(title, link);
}
@Override
@@ -113,7 +113,7 @@ public class Utilization {
}
Display other = (Display) o;
- return Objects.equal(title, other.title) && Objects.equal(link, other.link);
+ return Objects.equals(title, other.title) && Objects.equals(link, other.link);
}
}
@@ -133,6 +133,23 @@ public class Utilization {
public String getLink() {
return display.link;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DisplayMetric)) {
+ return false;
+ }
+
+ DisplayMetric other = (DisplayMetric) o;
+
+ return super.equals(o)
+ && display.equals(other.display);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), display);
+ }
}
private static final Function<GlobalMetric, DisplayMetric> TO_DISPLAY =
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4be190f6/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java b/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
index 60dfc56..e72bc48 100644
--- a/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
+++ b/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
@@ -16,9 +16,11 @@ package org.apache.aurora.scheduler.stats;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import javax.inject.Inject;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@@ -26,6 +28,7 @@ import com.google.common.base.Predicates;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import org.apache.aurora.scheduler.base.Query;
@@ -55,6 +58,14 @@ public class ResourceCounter {
Tasks.SCHEDULED_TO_INFO);
}
+ private static final Function<MetricType, GlobalMetric> TO_GLOBAL_METRIC =
+ new Function<MetricType, GlobalMetric>() {
+ @Override
+ public GlobalMetric apply(MetricType type) {
+ return new GlobalMetric(type);
+ }
+ };
+
/**
* Computes totals for each of the {@link MetricType}s.
*
@@ -62,11 +73,9 @@ public class ResourceCounter {
* @throws StorageException if there was a problem fetching tasks from storage.
*/
public List<GlobalMetric> computeConsumptionTotals() throws StorageException {
- List<GlobalMetric> counts = Arrays.asList(
- new GlobalMetric(MetricType.TOTAL_CONSUMED),
- new GlobalMetric(MetricType.DEDICATED_CONSUMED),
- new GlobalMetric(MetricType.QUOTA_CONSUMED),
- new GlobalMetric(MetricType.FREE_POOL_CONSUMED));
+ List<GlobalMetric> counts = FluentIterable.from(Arrays.asList(MetricType.values()))
+ .transform(TO_GLOBAL_METRIC)
+ .toList();
for (ITaskConfig task : getTasks(Query.unscoped().active())) {
for (GlobalMetric count : counts) {
@@ -155,6 +164,12 @@ public class ResourceCounter {
public final MetricType type;
public GlobalMetric(MetricType type) {
+ this(type, 0, 0, 0);
+ }
+
+ @VisibleForTesting
+ GlobalMetric(MetricType type, long cpu, long ramMb, long diskMb) {
+ super(cpu, ramMb, diskMb);
this.type = type;
}
@@ -164,6 +179,22 @@ public class ResourceCounter {
super.accumulate(task);
}
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof GlobalMetric)) {
+ return false;
+ }
+
+ GlobalMetric other = (GlobalMetric) o;
+ return super.equals(other)
+ && Objects.equals(other.type, this.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), type);
+ }
}
public static class Metric {
@@ -172,15 +203,18 @@ public class ResourceCounter {
private long diskMb = 0;
public Metric() {
- this.cpu = 0;
- this.ramMb = 0;
- this.diskMb = 0;
+ this(0, 0, 0);
}
public Metric(Metric copy) {
- this.cpu = copy.cpu;
- this.ramMb = copy.ramMb;
- this.diskMb = copy.diskMb;
+ this(copy.cpu, copy.ramMb, copy.diskMb);
+ }
+
+ @VisibleForTesting
+ Metric(long cpu, long ramMb, long diskMb) {
+ this.cpu = cpu;
+ this.ramMb = ramMb;
+ this.diskMb = diskMb;
}
protected void accumulate(ITaskConfig task) {
@@ -206,5 +240,22 @@ public class ResourceCounter {
public long getDiskGb() {
return diskMb / 1024;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Metric)) {
+ return false;
+ }
+
+ Metric other = (Metric) o;
+ return getCpu() == other.getCpu()
+ && getRamGb() == other.getRamGb()
+ && getDiskGb() == other.getDiskGb();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cpu, ramMb, diskMb);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4be190f6/src/test/java/org/apache/aurora/scheduler/stats/ResourceCounterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/stats/ResourceCounterTest.java b/src/test/java/org/apache/aurora/scheduler/stats/ResourceCounterTest.java
new file mode 100644
index 0000000..fc12933
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/stats/ResourceCounterTest.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.stats;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.aurora.gen.AssignedTask;
+import org.apache.aurora.gen.Constraint;
+import org.apache.aurora.gen.Identity;
+import org.apache.aurora.gen.ResourceAggregate;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.gen.ScheduledTask;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.TaskConstraint;
+import org.apache.aurora.gen.ValueConstraint;
+import org.apache.aurora.scheduler.base.JobKeys;
+import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.configuration.ConfigurationManager;
+import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.entities.IJobKey;
+import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
+import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
+import org.apache.aurora.scheduler.storage.mem.MemStorage;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.aurora.gen.ScheduleStatus.ASSIGNED;
+import static org.apache.aurora.gen.ScheduleStatus.FAILED;
+import static org.apache.aurora.gen.ScheduleStatus.FINISHED;
+import static org.apache.aurora.gen.ScheduleStatus.KILLING;
+import static org.apache.aurora.gen.ScheduleStatus.PENDING;
+import static org.apache.aurora.gen.ScheduleStatus.RESTARTING;
+import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
+import static org.apache.aurora.scheduler.base.Tasks.INFO_TO_JOB_KEY;
+import static org.apache.aurora.scheduler.base.Tasks.IS_PRODUCTION;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.GlobalMetric;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.Metric;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.MetricType.DEDICATED_CONSUMED;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.MetricType.FREE_POOL_CONSUMED;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.MetricType.QUOTA_CONSUMED;
+import static org.apache.aurora.scheduler.stats.ResourceCounter.MetricType.TOTAL_CONSUMED;
+import static org.junit.Assert.assertEquals;
+
+public class ResourceCounterTest {
+
+ private static final Metric ZERO = new Metric(0, 0, 0);
+ private static final long GB = 1024;
+ private static final Optional<String> NOT_DEDICATED = Optional.absent();
+
+ private static final boolean PRODUCTION = true;
+ private static final boolean NONPRODUCTION = false;
+
+ private Storage storage;
+ private ResourceCounter resourceCounter;
+
+ @Before
+ public void setUp() throws Exception {
+ storage = MemStorage.newEmptyStorage();
+ resourceCounter = new ResourceCounter(storage);
+ }
+
+ @Test
+ public void testNoTasks() {
+ assertEquals(
+ ZERO,
+ resourceCounter.computeQuotaAllocationTotals());
+
+ Map<IJobKey, Metric> aggregates = resourceCounter.computeAggregates(
+ Query.unscoped(),
+ Predicates.<ITaskConfig>alwaysTrue(),
+ INFO_TO_JOB_KEY);
+ assertEquals(ImmutableMap.<IJobKey, Metric>of(), aggregates);
+
+ for (Metric metric : resourceCounter.computeConsumptionTotals()) {
+ assertEquals(ZERO, metric);
+ }
+ }
+
+ @Test
+ public void testComputeConsumptionTotals() {
+ insertTasks(
+ task("bob", "jobA", "a", 1, GB, GB, PRODUCTION, RUNNING, NOT_DEDICATED),
+ task("bob", "jobB", "b", 1, GB, GB, PRODUCTION, RUNNING, NOT_DEDICATED),
+ task("tim", "jobC", "c", 1, GB, GB, PRODUCTION, PENDING, NOT_DEDICATED),
+ task("tim", "jobD", "d", 1, GB, GB, PRODUCTION, KILLING, NOT_DEDICATED),
+ task("bob", "jobE", "e", 1, GB, GB, NONPRODUCTION, ASSIGNED, NOT_DEDICATED),
+ task("tom", "jobF", "f", 1, GB, GB, NONPRODUCTION, RUNNING, NOT_DEDICATED),
+ task("tom", "jobG", "g", 1, GB, GB, NONPRODUCTION, RESTARTING, Optional.of("database")),
+ task("lil", "jobH", "h", 1, GB, GB, PRODUCTION, RUNNING, Optional.of("queue")),
+ task("lil", "jobI", "i", 1, GB, GB, PRODUCTION, FINISHED, NOT_DEDICATED)
+ );
+
+ Set<GlobalMetric> expected = ImmutableSet.of(
+ new GlobalMetric(TOTAL_CONSUMED, 8, 8 * GB, 8 * GB),
+ new GlobalMetric(DEDICATED_CONSUMED, 2, 2 * GB, 2 * GB),
+ new GlobalMetric(QUOTA_CONSUMED, 5, 5 * GB, 5 * GB),
+ new GlobalMetric(FREE_POOL_CONSUMED, 2, 2 * GB, 2 * GB)
+ );
+
+ assertEquals(expected, ImmutableSet.copyOf(resourceCounter.computeConsumptionTotals()));
+ }
+
+ @Test
+ public void testComputeQuotaAllocationTotals() {
+ storage.write(new Storage.MutateWork.NoResult.Quiet() {
+ @Override
+ protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getQuotaStore()
+ .saveQuota("a", IResourceAggregate.build(new ResourceAggregate(1, 1, 1)));
+ storeProvider.getQuotaStore()
+ .saveQuota("b", IResourceAggregate.build(new ResourceAggregate(2, 3, 4)));
+ }
+ });
+
+ assertEquals(new Metric(3, 4, 5), resourceCounter.computeQuotaAllocationTotals());
+ }
+
+ @Test
+ public void testComputeAggregates() {
+ insertTasks(
+ task("bob", "jobA", "a", 1, GB, GB, PRODUCTION, RUNNING, NOT_DEDICATED),
+ task("bob", "jobB", "b", 1, GB, GB, PRODUCTION, FAILED, NOT_DEDICATED),
+ task("bob", "jobB", "b2", 1, GB, GB, PRODUCTION, FAILED, NOT_DEDICATED),
+ task("bob", "jobC", "c", 1, GB, GB, NONPRODUCTION, RUNNING, NOT_DEDICATED),
+ task("tim", "jobD", "d", 1, GB, GB, PRODUCTION, RUNNING, NOT_DEDICATED),
+ task("bob", "jobE", "e", 1, GB, GB, NONPRODUCTION, ASSIGNED, NOT_DEDICATED),
+ task("lil", "jobF", "f", 1, GB, GB, PRODUCTION, RUNNING, Optional.of("queue")),
+ task("lil", "jobG", "g", 1, GB, GB, PRODUCTION, FINISHED, NOT_DEDICATED)
+ );
+
+ assertEquals(
+ ImmutableMap.of(
+ JobKeys.from("bob", "test", "jobA"), new Metric(1, 1 * GB, 1 * GB),
+ JobKeys.from("bob", "test", "jobB"), new Metric(2, 2 * GB, 2 * GB)
+ ),
+ resourceCounter.computeAggregates(Query.roleScoped("bob"), IS_PRODUCTION, INFO_TO_JOB_KEY)
+ );
+ }
+
+ private static IScheduledTask task(
+ String role,
+ String job,
+ String id,
+ int numCpus,
+ long ramMb,
+ long diskMb,
+ boolean production,
+ ScheduleStatus status,
+ Optional<String> dedicated) {
+
+ TaskConfig task = new TaskConfig()
+ .setOwner(new Identity().setRole(role))
+ .setEnvironment("test")
+ .setJobName(job)
+ .setNumCpus(numCpus)
+ .setRamMb(ramMb)
+ .setDiskMb(diskMb)
+ .setProduction(production);
+
+ if (dedicated.isPresent()) {
+ task.addToConstraints(new Constraint(
+ ConfigurationManager.DEDICATED_ATTRIBUTE,
+ TaskConstraint.value(new ValueConstraint(false, ImmutableSet.of(dedicated.get())))));
+ }
+
+ return IScheduledTask.build(new ScheduledTask()
+ .setStatus(status)
+ .setAssignedTask(
+ new AssignedTask()
+ .setTaskId(id)
+ .setTask(task)));
+ }
+
+ private void insertTasks(final IScheduledTask... tasks) {
+ storage.write(new Storage.MutateWork.NoResult.Quiet() {
+ @Override
+ protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getUnsafeTaskStore()
+ .saveTasks(ImmutableSet.<IScheduledTask>builder().add(tasks).build());
+ }
+ });
+ }
+}