You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2016/02/10 04:12:36 UTC
aurora git commit: Improving job update query performance.
Repository: aurora
Updated Branches:
refs/heads/master ec0f38a75 -> 86b6d7919
Improving job update query performance.
Reapplying a patch reverted in https://reviews.apache.org/r/42922/
Original RB: https://reviews.apache.org/r/42882/
Bugs closed: AURORA-1600
Reviewed at https://reviews.apache.org/r/43396/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/86b6d791
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/86b6d791
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/86b6d791
Branch: refs/heads/master
Commit: 86b6d79196e372e848d6fb0d98eb3f1a9dff0be6
Parents: ec0f38a
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Tue Feb 9 19:12:22 2016 -0800
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Tue Feb 9 19:12:22 2016 -0800
----------------------------------------------------------------------
.../org/apache/aurora/benchmark/JobUpdates.java | 31 ++++++-
.../aurora/benchmark/UpdateStoreBenchmarks.java | 92 +++++++++++++++-----
.../storage/db/JobUpdateDetailsMapper.xml | 57 ++++++------
3 files changed, 128 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/86b6d791/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
index 0c9dbae..50044e1 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
@@ -47,6 +47,7 @@ final class JobUpdates {
private static final String USER = "user";
private int numEvents = 1;
private int numInstanceEvents = 5000;
+ private int numInstanceOverrides = 1;
Builder setNumEvents(int newCount) {
numEvents = newCount;
@@ -58,6 +59,11 @@ final class JobUpdates {
return this;
}
+ Builder setNumInstanceOverrides(int newCount) {
+ numInstanceOverrides = newCount;
+ return this;
+ }
+
Set<IJobUpdateDetails> build(int count) {
ImmutableSet.Builder<IJobUpdateDetails> result = ImmutableSet.builder();
for (int i = 0; i < count; i++) {
@@ -78,10 +84,9 @@ final class JobUpdates {
.setMaxPerInstanceFailures(1)
.setMinWaitInInstanceRunningMs(1)
.setRollbackOnFailure(true)
- .setWaitForBatchCompletion(false))
- .setInitialState(ImmutableSet.of(new InstanceTaskConfig()
- .setTask(task)
- .setInstances(ImmutableSet.of(new Range(0, 10)))))
+ .setWaitForBatchCompletion(false)
+ .setUpdateOnlyTheseInstances(ranges(numInstanceOverrides)))
+ .setInitialState(instanceConfigs(task, numInstanceOverrides))
.setDesiredState(new InstanceTaskConfig()
.setTask(task)
.setInstances(ImmutableSet.of(new Range(0, numInstanceEvents)))));
@@ -107,6 +112,24 @@ final class JobUpdates {
return result.build();
}
+ private static Set<InstanceTaskConfig> instanceConfigs(TaskConfig task, int count) {
+ ImmutableSet.Builder<InstanceTaskConfig> result = ImmutableSet.builder();
+ for (int i = 0; i < count; i++) {
+ result.add(new InstanceTaskConfig(task, ImmutableSet.of(new Range(i, i))));
+ }
+
+ return result.build();
+ }
+
+ private static Set<Range> ranges(int count) {
+ ImmutableSet.Builder<Range> result = ImmutableSet.builder();
+ for (int i = 0; i < count; i++) {
+ result.add(new Range(i, i));
+ }
+
+ return result.build();
+ }
+
private static String string(int numChars) {
char[] chars = new char[numChars];
Arrays.fill(chars, 'a');
http://git-wip-us.apache.org/repos/asf/aurora/blob/86b6d791/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
index 1d8986b..92849d9 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
@@ -48,6 +48,7 @@ import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
public class UpdateStoreBenchmarks {
+ private static final String USER = "user";
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
@@ -56,7 +57,6 @@ public class UpdateStoreBenchmarks {
@Fork(1)
@State(Scope.Thread)
public static class JobDetailsBenchmark {
- private static final String USER = "user";
private Storage storage;
private Set<IJobUpdateKey> keys;
@@ -71,29 +71,10 @@ public class UpdateStoreBenchmarks {
@Setup(Level.Iteration)
public void setUpIteration() {
storage.write((NoResult.Quiet) storeProvider -> {
- JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
Set<IJobUpdateDetails> updates =
new JobUpdates.Builder().setNumInstanceEvents(instances).build(1);
- ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder();
- for (IJobUpdateDetails details : updates) {
- IJobUpdateKey key = details.getUpdate().getSummary().getKey();
- keyBuilder.add(key);
- String lockToken = UUID.randomUUID().toString();
- storeProvider.getLockStore().saveLock(
- ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER, 0L)));
-
- updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken));
-
- for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) {
- updateStore.saveJobUpdateEvent(key, updateEvent);
- }
-
- for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) {
- updateStore.saveJobInstanceUpdateEvent(key, instanceEvent);
- }
- }
- keys = keyBuilder.build();
+ keys = saveToStore(updates, storeProvider);
});
}
@@ -111,4 +92,73 @@ public class UpdateStoreBenchmarks {
Iterables.getOnlyElement(keys)).get());
}
}
+
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.SECONDS)
+ @Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+ @Fork(1)
+ @State(Scope.Thread)
+ public static class JobInstructionsBenchmark {
+ private Storage storage;
+ private Set<IJobUpdateKey> keys;
+
+ @Param({"1", "10", "100", "1000"})
+ private int instanceOverrides;
+
+ @Setup(Level.Trial)
+ public void setUp() {
+ storage = DbUtil.createStorage();
+ }
+
+ @Setup(Level.Iteration)
+ public void setUpIteration() {
+ storage.write((NoResult.Quiet) storeProvider -> {
+ Set<IJobUpdateDetails> updates =
+ new JobUpdates.Builder().setNumInstanceOverrides(instanceOverrides).build(1);
+
+ keys = saveToStore(updates, storeProvider);
+ });
+ }
+
+ @TearDown(Level.Iteration)
+ public void tearDownIteration() {
+ storage.write((NoResult.Quiet) storeProvider -> {
+ storeProvider.getJobUpdateStore().deleteAllUpdatesAndEvents();
+ storeProvider.getLockStore().deleteLocks();
+ });
+ }
+
+ @Benchmark
+ public IJobUpdateDetails run() throws TException {
+ return storage.read(store -> store.getJobUpdateStore().fetchJobUpdateDetails(
+ Iterables.getOnlyElement(keys)).get());
+ }
+ }
+
+ private static Set<IJobUpdateKey> saveToStore(
+ Set<IJobUpdateDetails> updates,
+ Storage.MutableStoreProvider storeProvider) {
+
+ JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
+ ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder();
+ for (IJobUpdateDetails details : updates) {
+ IJobUpdateKey key = details.getUpdate().getSummary().getKey();
+ keyBuilder.add(key);
+ String lockToken = UUID.randomUUID().toString();
+ storeProvider.getLockStore().saveLock(
+ ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER, 0L)));
+
+ updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken));
+
+ for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) {
+ updateStore.saveJobUpdateEvent(key, updateEvent);
+ }
+
+ for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) {
+ updateStore.saveJobInstanceUpdateEvent(key, instanceEvent);
+ }
+ }
+ return keyBuilder.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/86b6d791/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
index fba7d4f..8563630 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
@@ -170,9 +170,9 @@
<resultMap id="jobUpdateSettingsMap" type="org.apache.aurora.gen.JobUpdateSettings">
<id column="id" />
<collection property="updateOnlyTheseInstances"
- resultMap="rangeMap"
- columnPrefix="r_"
- notNullColumn="id" />
+ select="selectInstanceOverrides"
+ column="id"
+ foreignColumn="update_row_id" />
</resultMap>
<resultMap
@@ -182,9 +182,9 @@
<association property="desiredState" resultMap="instanceConfigMap" columnPrefix="ditc_" />
<association property="settings" resultMap="jobUpdateSettingsMap" columnPrefix="juse_"/>
<collection property="initialState"
- resultMap="instanceConfigMap"
- columnPrefix="iitc_"
- notNullColumn="id" />
+ select="selectInstanceConfigs"
+ column="id"
+ foreignColumn="update_row_id" />
</resultMap>
<resultMap id="jobUpdateMap" type="org.apache.aurora.scheduler.storage.db.views.DbJobUpdate">
@@ -350,23 +350,12 @@
cn.task_config_row_id AS jui_ditc_task_config_row_id,
di.id AS jui_ditc_r_id,
di.first AS jui_ditc_r_first,
- di.last AS jui_ditc_r_last,
- co.id AS jui_iitc_id,
- co.task_config_row_id AS jui_iitc_task_config_row_id,
- ci.id AS jui_iitc_r_id,
- ci.first AS jui_iitc_r_first,
- ci.last AS jui_iitc_r_last,
- io.id AS jui_juse_r_id,
- io.first AS jui_juse_r_first,
- io.last AS jui_juse_r_last
+ di.last AS jui_ditc_r_last
</sql>
<sql id="job_update_outer_joins">
LEFT OUTER JOIN job_update_configs AS cn ON cn.update_row_id = u.id AND cn.is_new = TRUE
- LEFT OUTER JOIN job_update_configs AS co ON co.update_row_id = u.id AND co.is_new = FALSE
- LEFT OUTER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id
LEFT OUTER JOIN job_updates_to_desired_instances AS di ON di.update_row_id = u.id
- LEFT OUTER JOIN job_updates_to_instance_overrides AS io ON io.update_row_id = u.id
</sql>
<sql id="lock_outer_join">
@@ -402,15 +391,7 @@
cn.task_config_row_id AS ditc_task_config_row_id,
di.id AS ditc_r_id,
di.first AS ditc_r_first,
- di.last AS ditc_r_last,
- co.id AS iitc_id,
- co.task_config_row_id AS iitc_task_config_row_id,
- ci.id AS iitc_r_id,
- ci.first AS iitc_r_first,
- ci.last AS iitc_r_last,
- io.id AS juse_r_id,
- io.first AS juse_r_first,
- io.last AS juse_r_last
+ di.last AS ditc_r_last
FROM job_updates AS u
<include refid="job_key_inner_join" />
<include refid="job_update_outer_joins" />
@@ -450,6 +431,28 @@
WHERE <include refid="filter_by_update_key"/>
</select>
+ <select id="selectInstanceConfigs" resultMap="instanceConfigMap">
+ SELECT
+ co.id,
+ co.task_config_row_id AS task_config_row_id,
+ ci.id AS r_id,
+ ci.first AS r_first,
+ ci.last AS r_last
+ FROM job_update_configs as co
+ INNER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id
+ WHERE co.update_row_id = #{id}
+ AND co.is_new = FALSE
+ </select>
+
+ <select id="selectInstanceOverrides" resultMap="rangeMap">
+ SELECT
+ o.id,
+ o.first,
+ o.last
+ FROM job_updates_to_instance_overrides as o
+ WHERE update_row_id = #{id}
+ </select>
+
<select id="selectInstanceUpdateEvents" resultMap="jobInstanceUpdateMap">
SELECT
e.id,