You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2015/06/22 18:51:59 UTC
aurora git commit: DbTaskStore perf: optimize queries scoped to a
task ID.
Repository: aurora
Updated Branches:
refs/heads/master d24092683 -> cbd8a1427
DbTaskStore perf: optimize queries scoped to a task ID.
Bugs closed: AURORA-1298
Reviewed at https://reviews.apache.org/r/35672/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/cbd8a142
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/cbd8a142
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/cbd8a142
Branch: refs/heads/master
Commit: cbd8a14272911285f5b08ae1f332957df8859e1e
Parents: d240926
Author: Bill Farner <wf...@apache.org>
Authored: Mon Jun 22 09:51:36 2015 -0700
Committer: Bill Farner <wf...@apache.org>
Committed: Mon Jun 22 09:51:36 2015 -0700
----------------------------------------------------------------------
.../aurora/scheduler/storage/TaskStore.java | 56 ++++++++++++++++++++
.../scheduler/storage/db/DbTaskStore.java | 32 +++++++----
.../aurora/scheduler/storage/db/TaskMapper.java | 11 ++++
.../scheduler/storage/mem/MemTaskStore.java | 56 ++++----------------
.../aurora/scheduler/storage/db/TaskMapper.xml | 15 ++++--
5 files changed, 111 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/cbd8a142/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
index cceac8a..142e4ac 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
@@ -16,13 +16,18 @@ package org.apache.aurora.scheduler.storage;
import java.util.Set;
import com.google.common.base.Function;
+import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
+import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
+import static com.google.common.base.CharMatcher.WHITESPACE;
+
/**
* Stores all tasks configured with the scheduler.
*/
@@ -106,4 +111,55 @@ public interface TaskStore {
*/
boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration);
}
+
+ final class Util {
+ private Util() {
+ // Utility class.
+ }
+
+ public static Predicate<IScheduledTask> queryFilter(final Query.Builder queryBuilder) {
+ return new Predicate<IScheduledTask>() {
+ @Override
+ public boolean apply(IScheduledTask task) {
+ TaskQuery query = queryBuilder.get();
+ ITaskConfig config = task.getAssignedTask().getTask();
+ // TODO(wfarner): Investigate why blank inputs are treated specially for the role field.
+ if (query.getRole() != null
+ && !WHITESPACE.matchesAllOf(query.getRole())
+ && !query.getRole().equals(config.getJob().getRole())) {
+ return false;
+ }
+ if (query.getEnvironment() != null
+ && !query.getEnvironment().equals(config.getEnvironment())) {
+ return false;
+ }
+ if (query.getJobName() != null && !query.getJobName().equals(config.getJobName())) {
+ return false;
+ }
+
+ if (query.getJobKeysSize() > 0
+ && !query.getJobKeys().contains(config.getJob().newBuilder())) {
+ return false;
+ }
+ if (query.getTaskIds() != null && !query.getTaskIds().contains(Tasks.id(task))) {
+ return false;
+ }
+
+ if (query.getStatusesSize() > 0 && !query.getStatuses().contains(task.getStatus())) {
+ return false;
+ }
+ if (query.getSlaveHostsSize() > 0
+ && !query.getSlaveHosts().contains(task.getAssignedTask().getSlaveHost())) {
+ return false;
+ }
+ if (query.getInstanceIdsSize() > 0
+ && !query.getInstanceIds().contains(task.getAssignedTask().getInstanceId())) {
+ return false;
+ }
+
+ return true;
+ }
+ };
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/cbd8a142/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java
index 9b30b01..30e3469 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java
@@ -23,6 +23,8 @@ import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -291,9 +293,24 @@ class DbTaskStore implements TaskStore.Mutable {
return Functions.compose(REPLACE_UNION_TYPES, linkPopulator);
}
- private FluentIterable<ScheduledTaskWrapper> fetchRows(Query.Builder query) {
+ private FluentIterable<IScheduledTask> matches(Query.Builder query) {
+ Iterable<ScheduledTaskWrapper> results;
+ Predicate<IScheduledTask> filter;
+ if (query.get().getTaskIdsSize() == 1) {
+ // Optimize queries that are scoped to a single task, as the dynamic SQL used for arbitrary
+ // queries comes with a performance penalty.
+ results = Optional.fromNullable(
+ taskMapper.selectById(Iterables.getOnlyElement(query.get().getTaskIds())))
+ .asSet();
+ filter = Util.queryFilter(query);
+ } else {
+ results = taskMapper.select(query.get());
+ // Additional filtering is not necessary in this case, since the query does it for us.
+ filter = Predicates.alwaysTrue();
+ }
+
final Function<TaskConfigRow, TaskConfig> configSaturator = getConfigSaturator();
- return FluentIterable.from(taskMapper.select(query.get()))
+ return FluentIterable.from(results)
.transform(populateAssignedPorts)
.transform(new Function<ScheduledTaskWrapper, ScheduledTaskWrapper>() {
@Override
@@ -304,7 +321,10 @@ class DbTaskStore implements TaskStore.Mutable {
task.getTask().getAssignedTask().getTask()));
return task;
}
- });
+ })
+ .transform(UNWRAP)
+ .transform(IScheduledTask.FROM_BUILDER)
+ .filter(filter);
}
private final Function<ScheduledTaskWrapper, ScheduledTaskWrapper> populateAssignedPorts =
@@ -320,12 +340,6 @@ class DbTaskStore implements TaskStore.Mutable {
}
};
- private FluentIterable<IScheduledTask> matches(Query.Builder query) {
- return fetchRows(query)
- .transform(UNWRAP)
- .transform(IScheduledTask.FROM_BUILDER);
- }
-
private static final Function<ScheduledTaskWrapper, ScheduledTask> UNWRAP =
new Function<ScheduledTaskWrapper, ScheduledTask>() {
@Override
http://git-wip-us.apache.org/repos/asf/aurora/blob/cbd8a142/src/main/java/org/apache/aurora/scheduler/storage/db/TaskMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskMapper.java
index 8270407..7d499af 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskMapper.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskMapper.java
@@ -16,6 +16,8 @@ package org.apache.aurora.scheduler.storage.db;
import java.util.List;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.aurora.gen.JobKey;
import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.storage.db.views.AssignedPort;
@@ -44,6 +46,15 @@ interface TaskMapper {
List<ScheduledTaskWrapper> select(TaskQuery query);
/**
+ * Gets a task by ID.
+ *
+ * @param taskId ID of the task to fetch.
+ * @return Task with the specified ID.
+ */
+ @Nullable
+ ScheduledTaskWrapper selectById(@Param("taskId") String taskId);
+
+ /**
* Gets job keys of all stored tasks.
*
* @return Job keys.
http://git-wip-us.apache.org/repos/asf/aurora/blob/cbd8a142/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
index 4b67f6b..b871575 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
@@ -48,7 +48,6 @@ import com.twitter.common.stats.StatsProvider;
import org.apache.aurora.gen.ScheduledTask;
import org.apache.aurora.gen.TaskConfig;
-import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.base.JobKeys;
import org.apache.aurora.scheduler.base.Query;
import org.apache.aurora.scheduler.base.Tasks;
@@ -59,8 +58,6 @@ import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import static java.util.Objects.requireNonNull;
-import static com.google.common.base.CharMatcher.WHITESPACE;
-
/**
* An in-memory task store.
*/
@@ -247,48 +244,15 @@ class MemTaskStore implements TaskStore.Mutable {
}
}
- private static Predicate<Task> queryFilter(final TaskQuery query) {
- return new Predicate<Task>() {
- @Override
- public boolean apply(Task canonicalTask) {
- IScheduledTask task = canonicalTask.storedTask;
- ITaskConfig config = task.getAssignedTask().getTask();
- if (query.getRole() != null
- && !WHITESPACE.matchesAllOf(query.getRole())
- && !query.getRole().equals(config.getJob().getRole())) {
- return false;
- }
- if (query.getEnvironment() != null
- && !query.getEnvironment().equals(config.getEnvironment())) {
- return false;
- }
- if (query.getJobName() != null && !query.getJobName().equals(config.getJobName())) {
- return false;
- }
-
- if (query.getJobKeysSize() > 0
- && !query.getJobKeys().contains(config.getJob().newBuilder())) {
- return false;
- }
- if (query.getTaskIds() != null && !query.getTaskIds().contains(Tasks.id(task))) {
- return false;
- }
-
- if (query.getStatusesSize() > 0 && !query.getStatuses().contains(task.getStatus())) {
- return false;
- }
- if (query.getSlaveHostsSize() > 0
- && !query.getSlaveHosts().contains(task.getAssignedTask().getSlaveHost())) {
- return false;
- }
- if (query.getInstanceIdsSize() > 0
- && !query.getInstanceIds().contains(task.getAssignedTask().getInstanceId())) {
- return false;
- }
-
- return true;
- }
- };
+ private static Predicate<Task> queryFilter(Query.Builder query) {
+ return Predicates.compose(
+ Util.queryFilter(query),
+ new Function<Task, IScheduledTask>() {
+ @Override
+ public IScheduledTask apply(Task canonicalTask) {
+ return canonicalTask.storedTask;
+ }
+ });
}
private Iterable<Task> fromIdIndex(Iterable<String> taskIds) {
@@ -323,7 +287,7 @@ class MemTaskStore implements TaskStore.Mutable {
}
}
- return FluentIterable.from(from.get()).filter(queryFilter(query.get()));
+ return FluentIterable.from(from.get()).filter(queryFilter(query));
}
private static final Function<Task, IScheduledTask> TO_SCHEDULED =
http://git-wip-us.apache.org/repos/asf/aurora/blob/cbd8a142/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml
index 7c27f37..e9660b7 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml
@@ -89,10 +89,7 @@
<association property="task" resultMap="scheduledTaskMap"/>
</resultMap>
- <!-- TODO(wfarner): Consider adding selectById and/or selectByIds methods. The dynamic SQL here
- seems to come with a ~3x performance hit.
- -->
- <select id="select" resultMap="taskWrapperMap">
+ <sql id="unscopedSelect">
SELECT
t.id AS row_id,
t.task_config_row_id AS task_config_row_id,
@@ -110,6 +107,16 @@
INNER JOIN task_configs as c ON c.id = t.task_config_row_id
INNER JOIN job_keys AS j ON j.id = c.job_key_id
LEFT OUTER JOIN host_attributes AS h ON h.id = t.slave_row_id
+ </sql>
+
+ <select id="selectById" resultMap="taskWrapperMap">
+ <include refid="unscopedSelect"/>
+ WHERE
+ t.task_id = #{taskId}
+ </select>
+
+ <select id="select" resultMap="taskWrapperMap">
+ <include refid="unscopedSelect"/>
<where>
<if test="role != null">
j.role = #{role}