You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2015/04/10 01:23:40 UTC
aurora git commit: Improving JobUpdateDetails fetch performance.
Repository: aurora
Updated Branches:
refs/heads/master 02717ec6a -> 8b9a3649a
Improving JobUpdateDetails fetch performance.
Bugs closed: AURORA-1263
Reviewed at https://reviews.apache.org/r/32990/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/8b9a3649
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/8b9a3649
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/8b9a3649
Branch: refs/heads/master
Commit: 8b9a3649abe5e6cd57d5cd01313932ff7c6cbcef
Parents: 02717ec
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Thu Apr 9 16:20:17 2015 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Thu Apr 9 16:20:17 2015 -0700
----------------------------------------------------------------------
.../aurora/benchmark/UpdateStoreBenchmarks.java | 174 +++++++++++++++++++
.../storage/db/JobUpdateDetailsMapper.xml | 61 ++++---
2 files changed, 211 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/8b9a3649/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
new file mode 100644
index 0000000..c7456aa
--- /dev/null
+++ b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.benchmark;
+
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.twitter.common.inject.Bindings;
+import com.twitter.common.util.Clock;
+
+import org.apache.aurora.gen.ExecutorConfig;
+import org.apache.aurora.gen.InstanceTaskConfig;
+import org.apache.aurora.gen.JobInstanceUpdateEvent;
+import org.apache.aurora.gen.JobKey;
+import org.apache.aurora.gen.JobUpdate;
+import org.apache.aurora.gen.JobUpdateAction;
+import org.apache.aurora.gen.JobUpdateEvent;
+import org.apache.aurora.gen.JobUpdateInstructions;
+import org.apache.aurora.gen.JobUpdateKey;
+import org.apache.aurora.gen.JobUpdateSettings;
+import org.apache.aurora.gen.JobUpdateStatus;
+import org.apache.aurora.gen.JobUpdateSummary;
+import org.apache.aurora.gen.Lock;
+import org.apache.aurora.gen.LockKey;
+import org.apache.aurora.gen.Range;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.scheduler.storage.JobUpdateStore;
+import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.db.DbModule;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
+import org.apache.aurora.scheduler.storage.entities.ILock;
+import org.apache.aurora.scheduler.storage.mem.MemStorage;
+import org.apache.aurora.scheduler.storage.mem.MemStorageModule;
+import org.apache.thrift.TException;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+public class UpdateStoreBenchmarks {
+
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.SECONDS)
+ @Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+ @Fork(1)
+ @State(Scope.Thread)
+ public static class JobDetailsBenchmark {
+ private static final String USER = "user";
+ private Storage storage;
+ private IJobUpdateKey key;
+
+ @Param({"1000", "5000", "10000"})
+ private int instances;
+
+ @Setup(Level.Trial)
+ public void setUp() {
+ Injector injector = Guice.createInjector(
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(Clock.class).toInstance(Clock.SYSTEM_CLOCK);
+ }
+ },
+ new MemStorageModule(Bindings.KeyFactory.PLAIN),
+ new DbModule(Bindings.annotatedKeyFactory(MemStorage.Delegated.class)));
+ storage = injector.getInstance(Storage.class);
+ storage.prepare();
+ }
+
+ @Setup(Level.Iteration)
+ public void setUpIteration() {
+ final JobKey job = new JobKey("role", "env", UUID.randomUUID().toString());
+ key = IJobUpdateKey.build(new JobUpdateKey()
+ .setJob(job)
+ .setId(UUID.randomUUID().toString()));
+
+ TaskConfig task = new TaskConfig()
+ .setJob(job)
+ .setExecutorConfig(new ExecutorConfig("cfg", string(10000)));
+
+ final JobUpdate update = new JobUpdate()
+ .setSummary(new JobUpdateSummary()
+ .setKey(key.newBuilder())
+ .setUpdateId(key.getId())
+ .setJobKey(job)
+ .setUser(USER))
+ .setInstructions(new JobUpdateInstructions()
+ .setSettings(new JobUpdateSettings()
+ .setUpdateGroupSize(100)
+ .setMaxFailedInstances(1)
+ .setMaxPerInstanceFailures(1)
+ .setMaxWaitToInstanceRunningMs(1)
+ .setMinWaitInInstanceRunningMs(1)
+ .setRollbackOnFailure(true)
+ .setWaitForBatchCompletion(false))
+ .setInitialState(ImmutableSet.of(new InstanceTaskConfig()
+ .setTask(task)
+ .setInstances(ImmutableSet.of(new Range(0, 10)))))
+ .setDesiredState(new InstanceTaskConfig()
+ .setTask(task)
+ .setInstances(ImmutableSet.of(new Range(0, instances)))));
+
+ storage.write(new Storage.MutateWork.NoResult.Quiet() {
+ @Override
+ protected void execute(Storage.MutableStoreProvider storeProvider) {
+ String lockToken = UUID.randomUUID().toString();
+ storeProvider.getLockStore().saveLock(
+ ILock.build(new Lock(LockKey.job(job), lockToken, USER, 0L)));
+
+ JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
+ updateStore.saveJobUpdate(IJobUpdate.build(update), Optional.of(lockToken));
+ updateStore.saveJobUpdateEvent(
+ key,
+ IJobUpdateEvent.build(new JobUpdateEvent(JobUpdateStatus.ROLLING_FORWARD, 0L)
+ .setUser(USER)
+ .setMessage("message")));
+
+ for (int i = 0; i < instances; i++) {
+ updateStore.saveJobInstanceUpdateEvent(
+ key,
+ IJobInstanceUpdateEvent.build(
+ new JobInstanceUpdateEvent(i, 0L, JobUpdateAction.INSTANCE_UPDATING)));
+ }
+ }
+ });
+ }
+
+ private static String string(int numChars) {
+ char[] chars = new char[numChars];
+ Arrays.fill(chars, 'a');
+ return new String(chars);
+ }
+
+ @Benchmark
+ public IJobUpdateDetails run() throws TException {
+ return storage.read(new Storage.Work.Quiet<IJobUpdateDetails>() {
+ @Override
+ public IJobUpdateDetails apply(Storage.StoreProvider store) {
+ return store.getJobUpdateStore().fetchJobUpdateDetails(key).get();
+ }
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/aurora/blob/8b9a3649/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
index 4536cd3..f76f9a9 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
@@ -206,24 +206,26 @@
typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateActionTypeHandler"/>
</resultMap>
+ <resultMap id="jobUpdateEventMap" type="org.apache.aurora.gen.JobUpdateEvent">
+ <id column="id" />
+ <result property="status"
+ column="status"
+ typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateStatusTypeHandler"/>
+ </resultMap>
+
+
<resultMap id="jobUpdateDetailsMap" type="org.apache.aurora.gen.storage.StoredJobUpdateDetails">
<id column="u_id" />
<association property="details.update" resultMap="jobUpdateMap" />
- <!--Using notNullColumn attribute is required below as LEFT JOIN with empty right side
- will produce an empty row.-->
<collection property="details.updateEvents"
- ofType="org.apache.aurora.gen.JobUpdateEvent"
- columnPrefix="e_"
- notNullColumn="id">
- <id column="id" />
- <result property="status"
- column="status"
- typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateStatusTypeHandler" />
+ select="selectUpdateEvents"
+ column="u_id"
+ foreignColumn="update_row_id">
</collection>
<collection property="details.instanceEvents"
- resultMap="jobInstanceUpdateMap"
- columnPrefix="i_"
- notNullColumn="id">
+ select="selectInstanceEvents"
+ column="u_id"
+ foreignColumn="update_row_id">
</collection>
</resultMap>
@@ -393,22 +395,11 @@
<sql id="unscoped_details_select">
SELECT
<include refid="job_update_columns" />,
- e.id AS e_id,
- e.status AS e_status,
- e.user AS e_user,
- e.timestamp_ms AS e_timestamp_ms,
- e.message AS e_message,
- i.id AS i_id,
- i.action AS i_action,
- i.instance_id AS i_instance_id,
- i.timestamp_ms AS i_timestamp_ms,
l.lock_token AS lock_token
FROM job_updates AS u
<include refid="job_key_inner_join" />
<include refid="timestamps_inner_joins" />
<include refid="job_update_outer_joins" />
- LEFT OUTER JOIN job_update_events AS e ON e.update_row_id = u.id
- LEFT OUTER JOIN job_instance_update_events AS i ON i.update_row_id = u.id
<include refid="lock_outer_join" />
</sql>
@@ -459,7 +450,6 @@
<select id="selectDetails" resultMap="jobUpdateDetailsMap">
<include refid="unscoped_details_select"/>
WHERE <include refid="filter_by_update_key"/>
- ORDER BY e_timestamp_ms, i_timestamp_ms
</select>
<select id="selectDetailsList" resultMap="jobUpdateDetailsMap">
@@ -494,6 +484,29 @@
ORDER BY e.timestamp_ms
</select>
+ <select id="selectUpdateEvents" resultMap="jobUpdateEventMap">
+ SELECT
+ e.id,
+ status,
+ timestamp_ms,
+ e.user,
+ message
+ FROM job_update_events as e
+ WHERE update_row_id = #{id}
+ ORDER BY e.timestamp_ms
+ </select>
+
+ <select id="selectInstanceEvents" resultMap="jobInstanceUpdateMap">
+ SELECT
+ e.id,
+ instance_id,
+ timestamp_ms,
+ action
+ FROM job_instance_update_events as e
+ WHERE update_row_id = #{id}
+ ORDER BY e.timestamp_ms
+ </select>
+
<delete id="truncate">
DELETE FROM job_updates;
</delete>