You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2016/03/14 21:49:46 UTC
aurora git commit: Add scheduler support for running tasks using the
mesos Docker containerizer.
Repository: aurora
Updated Branches:
refs/heads/master de128a10a -> 3806e626a
Add scheduler support for running tasks using the mesos Docker containerizer.
This is currently labeled as experimental.
Only the most basic wiring is added here, and assumes that the provided image
includes an ENTRYPOINT. Unlike Docker support via the thermos executor, this
approach allows containers with an entrypoint, and does not impose environment
requirements on the image (e.g. python interpreter, libmesos dependencies).
Note that when using this, other familiar Aurora facilities that relate to the
thermos executor will not work. For example, browsing task logs is not
supported.
Support for exercising this from the client will come shortly.
Reviewed at https://reviews.apache.org/r/44685/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/3806e626
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/3806e626
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/3806e626
Branch: refs/heads/master
Commit: 3806e626a244a9338d9040d4e7132e02deb5065e
Parents: de128a1
Author: Bill Farner <wf...@apache.org>
Authored: Mon Mar 14 13:49:13 2016 -0700
Committer: Bill Farner <wf...@apache.org>
Committed: Mon Mar 14 13:49:13 2016 -0700
----------------------------------------------------------------------
NEWS | 2 +
.../aurora/benchmark/SnapshotBenchmarks.java | 4 +-
.../aurora/benchmark/TaskStoreBenchmarks.java | 2 +-
.../aurora/benchmark/ThriftApiBenchmarks.java | 9 +-
.../apache/aurora/scheduler/app/AppModule.java | 31 +++-
.../aurora/scheduler/base/TaskTestUtil.java | 7 +-
.../configuration/ConfigurationManager.java | 28 +++-
.../scheduler/mesos/MesosTaskFactory.java | 91 ++++++------
.../aurora/scheduler/storage/db/DbModule.java | 48 +++++--
.../aurora/scheduler/storage/db/DbUtil.java | 7 +-
.../aurora/scheduler/storage/db/schema.sql | 4 +-
.../app/local/FakeNonVolatileStorage.java | 2 +-
.../configuration/ConfigurationManagerTest.java | 62 ++++----
.../mesos/MesosTaskFactoryImplTest.java | 24 +++-
.../storage/db/DbCronJobStoreTest.java | 2 +-
.../storage/db/DbJobUpdateStoreTest.java | 2 +-
.../scheduler/storage/db/DbTaskStoreTest.java | 2 +-
.../storage/db/RowGarbageCollectorTest.java | 2 +-
.../storage/log/SnapshotStoreImplIT.java | 5 +-
.../storage/mem/InMemTaskStoreTest.java | 2 +-
.../storage/mem/MemCronJobStoreTest.java | 2 +-
.../aurora/scheduler/thrift/ThriftIT.java | 143 +++++++++++++------
.../aurora/scheduler/updater/JobUpdaterIT.java | 2 +-
23 files changed, 319 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/NEWS
----------------------------------------------------------------------
diff --git a/NEWS b/NEWS
index da3e4ce..26fc7f3 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,8 @@ Deprecations and removals:
- Task ID strings are no longer prefixed by a timestamp.
- Scheduler H2 in-memory database is now using MVStore: http://www.h2database.com/html/mvstore.html.
In addition, scheduler thrift snapshots are now supporting full DB dumps for faster restarts.
+- Added scheduler argument `-require_docker_use_executor` that indicates whether the scheduler
+ should accept tasks that use the Docker containerizer without an executor (experimental).
0.12.0
------
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
index 2c56b2e..ae59f3d 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
@@ -97,7 +97,9 @@ public class SnapshotBenchmarks {
.toInstance(true);
}
},
- DbModule.testModule(keyFactory, Optional.of(new DbModule.TaskStoreModule(keyFactory))));
+ DbModule.testModuleWithWorkQueue(
+ keyFactory,
+ Optional.of(new DbModule.TaskStoreModule(keyFactory))));
storage = injector.getInstance(Key.get(Storage.class, Storage.Volatile.class));
storage.prepare();
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
index 2ec6abd..f2f00b9 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
@@ -83,7 +83,7 @@ public class TaskStoreBenchmarks {
public void setUp() {
storage = Guice.createInjector(
Modules.combine(
- DbModule.testModule(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
+ DbModule.testModuleWithWorkQueue(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
new AbstractModule() {
@Override
protected void configure() {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/jmh/java/org/apache/aurora/benchmark/ThriftApiBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/ThriftApiBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/ThriftApiBenchmarks.java
index 6074638..7497fb9 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/ThriftApiBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/ThriftApiBenchmarks.java
@@ -17,7 +17,6 @@ import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.util.concurrent.TimeUnit;
-import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.gson.Gson;
import com.google.inject.AbstractModule;
@@ -28,12 +27,12 @@ import org.apache.aurora.benchmark.fakes.FakeStatsProvider;
import org.apache.aurora.common.inject.Bindings;
import org.apache.aurora.common.stats.StatsProvider;
import org.apache.aurora.common.util.Clock;
-import org.apache.aurora.gen.Container;
import org.apache.aurora.gen.ReadOnlyScheduler;
import org.apache.aurora.gen.Response;
import org.apache.aurora.gen.ScheduleStatus;
import org.apache.aurora.gen.TaskQuery;
import org.apache.aurora.scheduler.async.AsyncModule;
+import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.configuration.ConfigurationManager;
import org.apache.aurora.scheduler.cron.CronPredictor;
import org.apache.aurora.scheduler.quota.QuotaManager;
@@ -152,11 +151,7 @@ public class ThriftApiBenchmarks {
bind(QuotaManager.class).toInstance(createThrowingFake(QuotaManager.class));
bind(LockManager.class).toInstance(createThrowingFake(LockManager.class));
bind(StatsProvider.class).toInstance(new FakeStatsProvider());
- bind(ConfigurationManager.class).toInstance(
- new ConfigurationManager(
- ImmutableSet.of(Container._Fields.MESOS),
- /* allowDockerParameters */ false,
- /* defaultDockerParameters */ ImmutableMultimap.of()));
+ bind(ConfigurationManager.class).toInstance(TaskTestUtil.CONFIGURATION_MANAGER);
}
},
new AsyncModule(),
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
index a0d2a71..97d87ff 100644
--- a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
@@ -17,6 +17,7 @@ import java.util.Set;
import javax.inject.Singleton;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
@@ -54,6 +55,8 @@ import org.apache.aurora.scheduler.thrift.Thresholds;
import org.apache.aurora.scheduler.updater.UpdaterModule;
import org.apache.mesos.Scheduler;
+import static java.util.Objects.requireNonNull;
+
/**
* Binding module for the aurora scheduler application.
*/
@@ -64,8 +67,7 @@ public class AppModule extends AbstractModule {
@CmdLine(name = "max_tasks_per_job", help = "Maximum number of allowed tasks in a single job.")
public static final Arg<Integer> MAX_TASKS_PER_JOB = Arg.create(DEFAULT_MAX_TASKS_PER_JOB);
- private static final int DEFAULT_MAX_UPDATE_INSTANCE_FAILURES =
- DEFAULT_MAX_TASKS_PER_JOB * 5;
+ private static final int DEFAULT_MAX_UPDATE_INSTANCE_FAILURES = DEFAULT_MAX_TASKS_PER_JOB * 5;
@Positive
@CmdLine(name = "max_update_instance_failures", help = "Upper limit on the number of "
@@ -88,13 +90,28 @@ public class AppModule extends AbstractModule {
private static final Arg<Multimap<String, String>> DEFAULT_DOCKER_PARAMETERS =
Arg.create(ImmutableMultimap.of());
+ @CmdLine(name = "require_docker_use_executor",
+ help = "If false, Docker tasks may run without an executor (EXPERIMENTAL)")
+ private static final Arg<Boolean> REQUIRE_DOCKER_USE_EXECUTOR = Arg.create(true);
+
+ private final ConfigurationManager configurationManager;
+
+ @VisibleForTesting
+ public AppModule(ConfigurationManager configurationManager) {
+ this.configurationManager = requireNonNull(configurationManager);
+ }
+
+ public AppModule() {
+ this(new ConfigurationManager(
+ ImmutableSet.copyOf(ALLOWED_CONTAINER_TYPES.get()),
+ ENABLE_DOCKER_PARAMETERS.get(),
+ DEFAULT_DOCKER_PARAMETERS.get(),
+ REQUIRE_DOCKER_USE_EXECUTOR.get()));
+ }
+
@Override
protected void configure() {
- bind(ConfigurationManager.class).toInstance(
- new ConfigurationManager(
- ImmutableSet.copyOf(ALLOWED_CONTAINER_TYPES.get()),
- ENABLE_DOCKER_PARAMETERS.get(),
- DEFAULT_DOCKER_PARAMETERS.get()));
+ bind(ConfigurationManager.class).toInstance(configurationManager);
bind(Thresholds.class)
.toInstance(new Thresholds(MAX_TASKS_PER_JOB.get(), MAX_UPDATE_INSTANCE_FAILURES.get()));
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
index 1de6966..9cf8002 100644
--- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
+++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
@@ -53,8 +53,11 @@ public final class TaskTestUtil {
public static final TierInfo REVOCABLE_TIER = new TierInfo(true);
public static final TierConfig DEV_TIER_CONFIG =
new TierConfig(ImmutableMap.of("tier-dev", new TierInfo(false)));
- public static final ConfigurationManager CONFIGURATION_MANAGER =
- new ConfigurationManager(ImmutableSet.of(_Fields.MESOS), false, ImmutableMultimap.of());
+ public static final ConfigurationManager CONFIGURATION_MANAGER = new ConfigurationManager(
+ ImmutableSet.of(_Fields.MESOS),
+ false,
+ ImmutableMultimap.of(),
+ true);
private TaskTestUtil() {
// Utility class.
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
index e700fa3..b3b8ccf 100644
--- a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
@@ -18,6 +18,7 @@ import java.util.Objects;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
@@ -101,15 +102,18 @@ public class ConfigurationManager {
private final ImmutableSet<Container._Fields> allowedContainerTypes;
private final boolean allowDockerParameters;
private final Multimap<String, String> defaultDockerParameters;
+ private final boolean requireDockerUseExecutor;
public ConfigurationManager(
ImmutableSet<Container._Fields> allowedContainerTypes,
boolean allowDockerParameters,
- Multimap<String, String> defaultDockerParameters) {
+ Multimap<String, String> defaultDockerParameters,
+ boolean requireDockerUseExecutor) {
this.allowedContainerTypes = Objects.requireNonNull(allowedContainerTypes);
this.allowDockerParameters = allowDockerParameters;
this.defaultDockerParameters = Objects.requireNonNull(defaultDockerParameters);
+ this.requireDockerUseExecutor = requireDockerUseExecutor;
}
private static String getRole(IValueConstraint constraint) {
@@ -174,6 +178,14 @@ public class ConfigurationManager {
return IJobConfiguration.build(builder);
}
+ @VisibleForTesting
+ static final String NO_DOCKER_PARAMETERS =
+ "This scheduler is configured to disallow Docker parameters.";
+
+ @VisibleForTesting
+ static final String EXECUTOR_REQUIRED_WITH_DOCKER =
+ "This scheduler is configured to require an executor for Docker-based tasks.";
+
/**
* Check validity of and populates defaults in a task configuration. This will return a deep copy
* of the provided task configuration with default configuration values applied, and configuration
@@ -202,7 +214,10 @@ public class ConfigurationManager {
throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
}
- if (!builder.isSetExecutorConfig()) {
+ // A task must either have an executor configuration or specify a Docker container.
+ if (!builder.isSetExecutorConfig()
+ && !(builder.isSetContainer() && builder.getContainer().isSetDocker())) {
+
throw new TaskDescriptionException("Configuration may not be null");
}
@@ -246,9 +261,13 @@ public class ConfigurationManager {
}
} else {
if (!allowDockerParameters) {
- throw new TaskDescriptionException("Docker parameters not allowed.");
+ throw new TaskDescriptionException(NO_DOCKER_PARAMETERS);
}
}
+
+ if (requireDockerUseExecutor && !config.isSetExecutorConfig()) {
+ throw new TaskDescriptionException(EXECUTOR_REQUIRED_WITH_DOCKER);
+ }
}
} else {
// Default to mesos container type if unset.
@@ -259,7 +278,8 @@ public class ConfigurationManager {
}
if (!allowedContainerTypes.contains(containerType.get())) {
throw new TaskDescriptionException(
- "The container type " + containerType.get().toString() + " is not allowed");
+ "This scheduler is not configured to allow the container type "
+ + containerType.get().toString());
}
return ITaskConfig.build(builder);
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
index a34af4d..20cbd41 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
@@ -14,6 +14,7 @@
package org.apache.aurora.scheduler.mesos;
import java.util.List;
+import java.util.Set;
import javax.inject.Inject;
@@ -36,8 +37,10 @@ import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings;
import org.apache.aurora.scheduler.storage.entities.IAssignedTask;
import org.apache.aurora.scheduler.storage.entities.IDockerContainer;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
+import org.apache.aurora.scheduler.storage.entities.IMetadata;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import org.apache.mesos.Protos;
+import org.apache.mesos.Protos.CommandInfo;
import org.apache.mesos.Protos.ContainerInfo;
import org.apache.mesos.Protos.ExecutorID;
import org.apache.mesos.Protos.ExecutorInfo;
@@ -45,7 +48,6 @@ import org.apache.mesos.Protos.Label;
import org.apache.mesos.Protos.Labels;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.SlaveID;
import org.apache.mesos.Protos.TaskID;
import org.apache.mesos.Protos.TaskInfo;
import org.slf4j.Logger;
@@ -103,20 +105,19 @@ public interface MesosTaskFactory {
return String.format("%s.%s", getJobSourceName(task), instanceId);
}
- @Override
- public TaskInfo createFrom(IAssignedTask task, Offer offer) throws SchedulerException {
- requireNonNull(task);
- requireNonNull(offer);
-
- SlaveID slaveId = offer.getSlaveId();
-
- byte[] taskInBytes;
+ private static byte[] serializeTask(IAssignedTask task) throws SchedulerException {
try {
- taskInBytes = ThriftBinaryCodec.encode(task.newBuilder());
+ return ThriftBinaryCodec.encode(task.newBuilder());
} catch (ThriftBinaryCodec.CodingException e) {
LOG.error("Unable to serialize task.", e);
throw new SchedulerException("Internal error.", e);
}
+ }
+
+ @Override
+ public TaskInfo createFrom(IAssignedTask task, Offer offer) throws SchedulerException {
+ requireNonNull(task);
+ requireNonNull(offer);
ITaskConfig config = task.getTask();
AcceptedOffer acceptedOffer;
@@ -138,69 +139,69 @@ public interface MesosTaskFactory {
LOG.debug(
"Setting task resources to {}",
Iterables.transform(resources, Protobufs::toString));
- TaskInfo.Builder taskBuilder =
- TaskInfo.newBuilder()
- .setName(JobKeys.canonicalString(Tasks.getJob(task)))
- .setTaskId(TaskID.newBuilder().setValue(task.getTaskId()))
- .setSlaveId(slaveId)
- .addAllResources(resources)
- .setData(ByteString.copyFrom(taskInBytes));
- configureTaskLabels(config, taskBuilder);
+ TaskInfo.Builder taskBuilder = TaskInfo.newBuilder()
+ .setName(JobKeys.canonicalString(Tasks.getJob(task)))
+ .setTaskId(TaskID.newBuilder().setValue(task.getTaskId()))
+ .setSlaveId(offer.getSlaveId())
+ .addAllResources(resources);
+
+ configureTaskLabels(config.getMetadata(), taskBuilder);
if (config.getContainer().isSetMesos()) {
- configureTaskForNoContainer(task, config, taskBuilder, acceptedOffer);
+ configureTaskForNoContainer(task, taskBuilder, acceptedOffer);
} else if (config.getContainer().isSetDocker()) {
- configureTaskForDockerContainer(task, config, taskBuilder, acceptedOffer);
+ IDockerContainer dockerContainer = config.getContainer().getDocker();
+ if (config.isSetExecutorConfig()) {
+ ExecutorInfo.Builder execBuilder = configureTaskForExecutor(task, acceptedOffer)
+ .setContainer(getDockerContainerInfo(dockerContainer));
+ taskBuilder.setExecutor(execBuilder.build());
+ } else {
+ LOG.warn("Running Docker-based task without an executor.");
+ taskBuilder.setContainer(getDockerContainerInfo(dockerContainer))
+ .setCommand(CommandInfo.newBuilder().setShell(false));
+ }
} else {
throw new SchedulerException("Task had no supported container set.");
}
- return ResourceSlot.matchResourceTypes(taskBuilder.build());
+ if (taskBuilder.hasExecutor()) {
+ taskBuilder.setData(ByteString.copyFrom(serializeTask(task)));
+ return ResourceSlot.matchResourceTypes(taskBuilder.build());
+ } else {
+ return taskBuilder.build();
+ }
}
private void configureTaskForNoContainer(
IAssignedTask task,
- ITaskConfig config,
TaskInfo.Builder taskBuilder,
AcceptedOffer acceptedOffer) {
- taskBuilder.setExecutor(configureTaskForExecutor(task, config, acceptedOffer).build());
+ taskBuilder.setExecutor(configureTaskForExecutor(task, acceptedOffer).build());
}
- private void configureTaskForDockerContainer(
- IAssignedTask task,
- ITaskConfig taskConfig,
- TaskInfo.Builder taskBuilder,
- AcceptedOffer acceptedOffer) {
-
- IDockerContainer config = taskConfig.getContainer().getDocker();
+ private ContainerInfo getDockerContainerInfo(IDockerContainer config) {
Iterable<Protos.Parameter> parameters = Iterables.transform(config.getParameters(),
item -> Protos.Parameter.newBuilder().setKey(item.getName())
.setValue(item.getValue()).build());
ContainerInfo.DockerInfo.Builder dockerBuilder = ContainerInfo.DockerInfo.newBuilder()
.setImage(config.getImage()).addAllParameters(parameters);
- ContainerInfo.Builder containerBuilder = ContainerInfo.newBuilder()
+ return ContainerInfo.newBuilder()
.setType(ContainerInfo.Type.DOCKER)
- .setDocker(dockerBuilder.build());
-
- configureContainerVolumes(containerBuilder);
-
- ExecutorInfo.Builder execBuilder = configureTaskForExecutor(task, taskConfig, acceptedOffer)
- .setContainer(containerBuilder.build());
-
- taskBuilder.setExecutor(execBuilder.build());
+ .setDocker(dockerBuilder.build())
+ .addAllVolumes(executorSettings.getExecutorConfig().getVolumeMounts())
+ .build();
}
private ExecutorInfo.Builder configureTaskForExecutor(
IAssignedTask task,
- ITaskConfig config,
AcceptedOffer acceptedOffer) {
ExecutorInfo.Builder builder = executorSettings.getExecutorConfig().getExecutor().toBuilder()
.setExecutorId(getExecutorId(task.getTaskId()))
- .setSource(getInstanceSourceName(config, task.getInstanceId()));
+ .setSource(getInstanceSourceName(task.getTask(), task.getInstanceId()));
List<Resource> executorResources = acceptedOffer.getExecutorResources();
LOG.debug(
"Setting executor resources to {}",
@@ -209,12 +210,8 @@ public interface MesosTaskFactory {
return builder;
}
- private void configureContainerVolumes(ContainerInfo.Builder containerBuilder) {
- containerBuilder.addAllVolumes(executorSettings.getExecutorConfig().getVolumeMounts());
- }
-
- private void configureTaskLabels(ITaskConfig taskConfig, TaskInfo.Builder taskBuilder) {
- ImmutableSet<Label> labels = taskConfig.getMetadata().stream()
+ private void configureTaskLabels(Set<IMetadata> metadata, TaskInfo.Builder taskBuilder) {
+ ImmutableSet<Label> labels = metadata.stream()
.map(m -> Label.newBuilder()
.setKey(METADATA_LABEL_PREFIX + m.getKey())
.setValue(m.getValue())
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
index ff663fa..baf460e 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
@@ -144,6 +144,30 @@ public final class DbModule extends PrivateModule {
ImmutableMap.of("DB_CLOSE_DELAY", "-1"));
}
+ @VisibleForTesting
+ public static Module testModule(KeyFactory keyFactory, Optional<Module> taskStoreModule) {
+ return new DbModule(
+ keyFactory,
+ taskStoreModule.isPresent() ? taskStoreModule.get() : getTaskStoreModule(keyFactory),
+ "testdb-" + UUID.randomUUID().toString(),
+ // A non-zero close delay is used here to avoid eager database cleanup in tests that
+ // make use of multiple threads. Since all test databases are separately scoped by the
+ // included UUID, multiple DB instances will overlap in time but they should be distinct
+ // in content.
+ ImmutableMap.of("DB_CLOSE_DELAY", "5"));
+ }
+
+ /**
+ * Same as {@link #testModuleWithWorkQueue(KeyFactory, Optional)} but with default task store and
+ * key factory.
+ *
+ * @return A new database module for testing.
+ */
+ @VisibleForTesting
+ public static Module testModule() {
+ return testModule(KeyFactory.PLAIN, Optional.of(new TaskStoreModule(KeyFactory.PLAIN)));
+ }
+
/**
* Creates a module that will prepare a private in-memory database, using a specific task store
* implementation bound within the key factory and provided module.
@@ -153,7 +177,10 @@ public final class DbModule extends PrivateModule {
* @return A new database module for testing.
*/
@VisibleForTesting
- public static Module testModule(KeyFactory keyFactory, Optional<Module> taskStoreModule) {
+ public static Module testModuleWithWorkQueue(
+ KeyFactory keyFactory,
+ Optional<Module> taskStoreModule) {
+
return Modules.combine(
new AbstractModule() {
@Override
@@ -169,26 +196,21 @@ public final class DbModule extends PrivateModule {
});
}
},
- new DbModule(
- keyFactory,
- taskStoreModule.isPresent() ? taskStoreModule.get() : getTaskStoreModule(keyFactory),
- "testdb-" + UUID.randomUUID().toString(),
- // A non-zero close delay is used here to avoid eager database cleanup in tests that
- // make use of multiple threads. Since all test databases are separately scoped by the
- // included UUID, multiple DB instances will overlap in time but they should be distinct
- // in content.
- ImmutableMap.of("DB_CLOSE_DELAY", "5"))
+ testModule(keyFactory, taskStoreModule)
);
}
/**
- * Same as {@link #testModule(KeyFactory, Optional)} but with default task store and key factory.
+ * Same as {@link #testModuleWithWorkQueue(KeyFactory, Optional)} but with default task store and
+ * key factory.
*
* @return A new database module for testing.
*/
@VisibleForTesting
- public static Module testModule() {
- return testModule(KeyFactory.PLAIN, Optional.of(new TaskStoreModule(KeyFactory.PLAIN)));
+ public static Module testModuleWithWorkQueue() {
+ return testModuleWithWorkQueue(
+ KeyFactory.PLAIN,
+ Optional.of(new TaskStoreModule(KeyFactory.PLAIN)));
}
private static Module getTaskStoreModule(KeyFactory keyFactory) {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java
index 708be56..6f4e671 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java
@@ -26,7 +26,7 @@ import org.apache.aurora.scheduler.storage.Storage;
import org.apache.aurora.scheduler.testing.FakeStatsProvider;
import static org.apache.aurora.common.inject.Bindings.KeyFactory.PLAIN;
-import static org.apache.aurora.scheduler.storage.db.DbModule.testModule;
+import static org.apache.aurora.scheduler.storage.db.DbModule.testModuleWithWorkQueue;
/**
* Utility class for creating ad-hoc storage instances.
@@ -68,7 +68,7 @@ public final class DbUtil {
* @return A new storage instance.
*/
public static Storage createStorage() {
- return createStorageInjector(testModule()).getInstance(Storage.class);
+ return createStorageInjector(testModuleWithWorkQueue()).getInstance(Storage.class);
}
/**
@@ -77,6 +77,7 @@ public final class DbUtil {
* @return A new storage instance.
*/
public static Storage createFlaggedStorage() {
- return createStorageInjector(testModule(PLAIN, Optional.absent())).getInstance(Storage.class);
+ return createStorageInjector(testModuleWithWorkQueue(PLAIN, Optional.absent()))
+ .getInstance(Storage.class);
}
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
index c6c1f0a..be60c3b 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
@@ -95,8 +95,8 @@ CREATE TABLE task_configs(
max_task_failures INTEGER NOT NULL,
production BOOLEAN NOT NULL,
contact_email VARCHAR,
- executor_name VARCHAR NOT NULL,
- executor_data VARCHAR NOT NULL,
+ executor_name VARCHAR,
+ executor_data VARCHAR,
tier VARCHAR
);
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/app/local/FakeNonVolatileStorage.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/app/local/FakeNonVolatileStorage.java b/src/test/java/org/apache/aurora/scheduler/app/local/FakeNonVolatileStorage.java
index 3336f8c..8cf6871 100644
--- a/src/test/java/org/apache/aurora/scheduler/app/local/FakeNonVolatileStorage.java
+++ b/src/test/java/org/apache/aurora/scheduler/app/local/FakeNonVolatileStorage.java
@@ -22,7 +22,7 @@ import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage;
/**
* A storage system that implements non-volatile storage operations, but is actually volatile.
*/
-class FakeNonVolatileStorage implements NonVolatileStorage {
+public class FakeNonVolatileStorage implements NonVolatileStorage {
private final Storage delegate;
@Inject
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
index 11062e3..1a520b3 100644
--- a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
@@ -37,7 +37,6 @@ import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
import org.apache.aurora.scheduler.storage.entities.IDockerParameter;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
-import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -58,7 +57,7 @@ public class ConfigurationManagerTest {
public ExpectedException expectedException = ExpectedException.none();
private static final ImmutableSet<Container._Fields> ALL_CONTAINER_TYPES =
- ImmutableSet.of(Container._Fields.DOCKER, Container._Fields.MESOS);
+ ImmutableSet.copyOf(Container._Fields.values());
private static final JobKey JOB_KEY = new JobKey("owner-role", "devel", "email_stats");
private static final JobConfiguration UNSANITIZED_JOB_CONFIGURATION = new JobConfiguration()
@@ -102,17 +101,16 @@ public class ConfigurationManagerTest {
private static final ITaskConfig CONFIG_WITH_CONTAINER =
TaskTestUtil.makeConfig(JobKeys.from("role", "env", "job"));
- private ConfigurationManager configurationManager;
- private ConfigurationManager dockerConfigurationManager;
-
- @Before
- public void setUp() {
- configurationManager = new ConfigurationManager(
- ALL_CONTAINER_TYPES, false, ImmutableMultimap.of());
-
- dockerConfigurationManager = new ConfigurationManager(
- ALL_CONTAINER_TYPES, true, ImmutableMultimap.of("foo", "bar"));
- }
+ private static final ConfigurationManager CONFIGURATION_MANAGER = new ConfigurationManager(
+ ALL_CONTAINER_TYPES,
+ false,
+ ImmutableMultimap.of(),
+ true);
+ private static final ConfigurationManager DOCKER_CONFIGURATION_MANAGER = new ConfigurationManager(
+ ALL_CONTAINER_TYPES,
+ true,
+ ImmutableMultimap.of("foo", "bar"),
+ false);
@Test
public void testIsGoodIdentifier() {
@@ -130,7 +128,7 @@ public class ConfigurationManagerTest {
taskConfig.getContainer().getDocker().setImage(null);
expectTaskDescriptionException("A container must specify an image");
- configurationManager.validateAndPopulate(ITaskConfig.build(taskConfig));
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(taskConfig));
}
@Test
@@ -138,11 +136,26 @@ public class ConfigurationManagerTest {
TaskConfig taskConfig = CONFIG_WITH_CONTAINER.newBuilder();
taskConfig.getContainer().getDocker().addToParameters(new DockerParameter("foo", "bar"));
- ConfigurationManager noParamsManager = new ConfigurationManager(
- ALL_CONTAINER_TYPES, false, ImmutableMultimap.of());
+ expectTaskDescriptionException(ConfigurationManager.NO_DOCKER_PARAMETERS);
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(taskConfig));
+ }
+
+ @Test
+ public void testDisallowNoExecutorDockerTask() throws TaskDescriptionException {
+ TaskConfig builder = CONFIG_WITH_CONTAINER.newBuilder();
+ builder.getContainer().getDocker().unsetParameters();
+ builder.unsetExecutorConfig();
+
+ expectTaskDescriptionException(ConfigurationManager.EXECUTOR_REQUIRED_WITH_DOCKER);
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
+ }
+
+ @Test
+ public void testAllowNoExecutorDockerTask() throws TaskDescriptionException {
+ TaskConfig builder = CONFIG_WITH_CONTAINER.newBuilder();
+ builder.unsetExecutorConfig();
- expectTaskDescriptionException("Docker parameters not allowed");
- noParamsManager.validateAndPopulate(ITaskConfig.build(taskConfig));
+ DOCKER_CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
}
@Test
@@ -151,7 +164,7 @@ public class ConfigurationManagerTest {
.setTier("pr/d"));
expectTaskDescriptionException("Tier contains illegal characters");
- configurationManager.validateAndPopulate(config);
+ CONFIGURATION_MANAGER.validateAndPopulate(config);
}
@Test
@@ -159,7 +172,8 @@ public class ConfigurationManagerTest {
TaskConfig builder = CONFIG_WITH_CONTAINER.newBuilder();
builder.getContainer().getDocker().setParameters(ImmutableList.of());
- ITaskConfig result = dockerConfigurationManager.validateAndPopulate(ITaskConfig.build(builder));
+ ITaskConfig result =
+ DOCKER_CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
// The resulting task config should contain parameters supplied to the ConfigurationManager.
List<IDockerParameter> params = result.getContainer().getDocker().getParameters();
@@ -174,7 +188,7 @@ public class ConfigurationManagerTest {
taskConfig.getContainer().getDocker().getParameters().clear();
taskConfig.getContainer().getDocker().addToParameters(userParameter);
- ITaskConfig result = dockerConfigurationManager.validateAndPopulate(
+ ITaskConfig result = DOCKER_CONFIGURATION_MANAGER.validateAndPopulate(
ITaskConfig.build(taskConfig));
// The resulting task config should contain parameters supplied from user config.
@@ -190,7 +204,7 @@ public class ConfigurationManagerTest {
.setConstraint(TaskConstraint.value(
new ValueConstraint(false, ImmutableSet.of(JOB_KEY.getRole() + "/f"))))));
- configurationManager.validateAndPopulate(ITaskConfig.build(builder));
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
}
@Test
@@ -200,7 +214,7 @@ public class ConfigurationManagerTest {
.setName(DEDICATED_ATTRIBUTE)
.setConstraint(TaskConstraint.value(new ValueConstraint(false, ImmutableSet.of("*/f"))))));
- configurationManager.validateAndPopulate(ITaskConfig.build(builder));
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
}
@Test
@@ -211,7 +225,7 @@ public class ConfigurationManagerTest {
.setConstraint(TaskConstraint.value(new ValueConstraint(false, ImmutableSet.of("r/f"))))));
expectTaskDescriptionException("Only r may use hosts dedicated for that role.");
- configurationManager.validateAndPopulate(ITaskConfig.build(builder));
+ CONFIGURATION_MANAGER.validateAndPopulate(ITaskConfig.build(builder));
}
private void expectTaskDescriptionException(String message) {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
index 3db531b..5103bd0 100644
--- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
@@ -38,7 +38,9 @@ import org.apache.aurora.scheduler.mesos.MesosTaskFactory.MesosTaskFactoryImpl;
import org.apache.aurora.scheduler.storage.entities.IAssignedTask;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import org.apache.mesos.Protos;
+import org.apache.mesos.Protos.ContainerInfo;
import org.apache.mesos.Protos.ContainerInfo.DockerInfo;
+import org.apache.mesos.Protos.ContainerInfo.Type;
import org.apache.mesos.Protos.ExecutorInfo;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.Protos.Parameter;
@@ -60,6 +62,7 @@ import static org.apache.aurora.scheduler.mesos.TestExecutorSettings.THERMOS_CON
import static org.apache.aurora.scheduler.mesos.TestExecutorSettings.THERMOS_EXECUTOR;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class MesosTaskFactoryImplTest extends EasyMockTest {
@@ -279,10 +282,29 @@ public class MesosTaskFactoryImplTest extends EasyMockTest {
.map(m -> METADATA_LABEL_PREFIX + m.getKey() + m.getValue())
.collect(GuavaUtils.toImmutableSet());
- assertTrue(labels.size() > 0);
assertEquals(labels, metadata);
}
+ @Test
+ public void testDockerTaskWithoutExecutor() {
+ AssignedTask builder = TASK.newBuilder();
+ builder.getTask()
+ .setContainer(Container.docker(new DockerContainer()
+ .setImage("hello-world")))
+ .unsetExecutorConfig();
+
+ TaskInfo task = getDockerTaskInfo(IAssignedTask.build(builder));
+ assertTrue(task.hasCommand());
+ assertFalse(task.getCommand().getShell());
+ assertFalse(task.hasData());
+ ContainerInfo expectedContainer = ContainerInfo.newBuilder()
+ .setType(Type.DOCKER)
+ .setDocker(DockerInfo.newBuilder()
+ .setImage("hello-world"))
+ .build();
+ assertEquals(expectedContainer, task.getContainer());
+ }
+
private static ResourceSlot getTotalTaskResources(TaskInfo task) {
Resources taskResources = fromResourceList(task.getResourcesList());
Resources executorResources = fromResourceList(task.getExecutor().getResourcesList());
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/db/DbCronJobStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/db/DbCronJobStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/db/DbCronJobStoreTest.java
index fa0d0dc..b68298a 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/db/DbCronJobStoreTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/db/DbCronJobStoreTest.java
@@ -27,7 +27,7 @@ public class DbCronJobStoreTest extends AbstractCronJobStoreTest {
@Override
protected Module getStorageModule() {
return Modules.combine(
- DbModule.testModule(),
+ DbModule.testModuleWithWorkQueue(),
new AbstractModule() {
@Override
protected void configure() {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java
index 54defc2..0853039 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java
@@ -98,7 +98,7 @@ public class DbJobUpdateStoreTest {
@Before
public void setUp() throws Exception {
- Injector injector = DbUtil.createStorageInjector(DbModule.testModule());
+ Injector injector = DbUtil.createStorageInjector(DbModule.testModuleWithWorkQueue());
storage = injector.getInstance(Storage.class);
stats = injector.getInstance(FakeStatsProvider.class);
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/db/DbTaskStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/db/DbTaskStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/db/DbTaskStoreTest.java
index ecddc66..bcf5be4 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/db/DbTaskStoreTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/db/DbTaskStoreTest.java
@@ -27,7 +27,7 @@ public class DbTaskStoreTest extends AbstractTaskStoreTest {
@Override
protected Module getStorageModule() {
return Modules.combine(
- DbModule.testModule(),
+ DbModule.testModuleWithWorkQueue(),
new AbstractModule() {
@Override
protected void configure() {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollectorTest.java b/src/test/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollectorTest.java
index 58b4c93..3e5296e 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollectorTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollectorTest.java
@@ -51,7 +51,7 @@ public class RowGarbageCollectorTest {
@Before
public void setUp() {
Injector injector = Guice.createInjector(
- DbModule.testModule(),
+ DbModule.testModuleWithWorkQueue(),
new DbModule.GarbageCollectorModule(),
new AbstractModule() {
@Override
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
index 6a39d89..d93eed9 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
@@ -54,6 +54,7 @@ import org.apache.aurora.scheduler.base.JobKeys;
import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.storage.SnapshotStore;
import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.db.DbModule;
import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
@@ -69,7 +70,6 @@ import org.junit.Test;
import static org.apache.aurora.common.inject.Bindings.KeyFactory.PLAIN;
import static org.apache.aurora.common.util.testing.FakeBuildInfo.generateBuildInfo;
import static org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import static org.apache.aurora.scheduler.storage.db.DbModule.testModule;
import static org.apache.aurora.scheduler.storage.db.DbUtil.createStorage;
import static org.apache.aurora.scheduler.storage.db.DbUtil.createStorageInjector;
import static org.junit.Assert.assertEquals;
@@ -88,7 +88,8 @@ public class SnapshotStoreImplIT {
storage = dbTaskStore
? createStorage()
: createStorageInjector(
- testModule(PLAIN, Optional.of(new InMemStoresModule(PLAIN)))).getInstance(Storage.class);
+ DbModule.testModuleWithWorkQueue(PLAIN, Optional.of(new InMemStoresModule(PLAIN))))
+ .getInstance(Storage.class);
FakeClock clock = new FakeClock();
clock.setNowMillis(NOW);
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/mem/InMemTaskStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/mem/InMemTaskStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/mem/InMemTaskStoreTest.java
index d18ce20..2e560c0 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/mem/InMemTaskStoreTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/mem/InMemTaskStoreTest.java
@@ -39,7 +39,7 @@ public class InMemTaskStoreTest extends AbstractTaskStoreTest {
protected Module getStorageModule() {
statsProvider = new FakeStatsProvider();
return Modules.combine(
- DbModule.testModule(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
+ DbModule.testModuleWithWorkQueue(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
new AbstractModule() {
@Override
protected void configure() {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStoreTest.java
index d3a026c..79999e1 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStoreTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStoreTest.java
@@ -29,7 +29,7 @@ public class MemCronJobStoreTest extends AbstractCronJobStoreTest {
@Override
protected Module getStorageModule() {
return Modules.combine(
- DbModule.testModule(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
+ DbModule.testModuleWithWorkQueue(PLAIN, Optional.of(new InMemStoresModule(PLAIN))),
new AbstractModule() {
@Override
protected void configure() {
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java b/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java
index 860d960..b1c71a6 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java
@@ -15,37 +15,57 @@ package org.apache.aurora.scheduler.thrift;
import java.util.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
+import com.google.inject.Key;
import com.google.inject.Provides;
+import com.google.inject.TypeLiteral;
-import org.apache.aurora.common.application.ShutdownRegistry;
+import org.apache.aurora.common.application.ShutdownStage;
+import org.apache.aurora.common.base.Command;
+import org.apache.aurora.common.net.pool.DynamicHostSet;
import org.apache.aurora.common.testing.easymock.EasyMockTest;
+import org.apache.aurora.common.thrift.ServiceInstance;
import org.apache.aurora.gen.AuroraAdmin;
+import org.apache.aurora.gen.Container;
+import org.apache.aurora.gen.Container._Fields;
+import org.apache.aurora.gen.DockerContainer;
+import org.apache.aurora.gen.DockerParameter;
+import org.apache.aurora.gen.JobConfiguration;
import org.apache.aurora.gen.ResourceAggregate;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.gen.ScheduledTask;
import org.apache.aurora.gen.ServerInfo;
-import org.apache.aurora.scheduler.TaskIdGenerator;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.TaskQuery;
+import org.apache.aurora.scheduler.TierModule;
+import org.apache.aurora.scheduler.app.AppModule;
+import org.apache.aurora.scheduler.app.LifecycleModule;
+import org.apache.aurora.scheduler.app.local.FakeNonVolatileStorage;
import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.configuration.ConfigurationManager;
-import org.apache.aurora.scheduler.cron.CronJobManager;
-import org.apache.aurora.scheduler.cron.CronPredictor;
-import org.apache.aurora.scheduler.quota.QuotaManager;
-import org.apache.aurora.scheduler.state.LockManager;
-import org.apache.aurora.scheduler.state.MaintenanceController;
-import org.apache.aurora.scheduler.state.StateManager;
-import org.apache.aurora.scheduler.state.UUIDGenerator;
+import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings;
+import org.apache.aurora.scheduler.cron.quartz.CronModule;
+import org.apache.aurora.scheduler.mesos.DriverFactory;
+import org.apache.aurora.scheduler.mesos.DriverSettings;
+import org.apache.aurora.scheduler.mesos.TestExecutorSettings;
+import org.apache.aurora.scheduler.quota.QuotaModule;
+import org.apache.aurora.scheduler.stats.StatsModule;
import org.apache.aurora.scheduler.storage.Storage;
import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage;
import org.apache.aurora.scheduler.storage.backup.Recovery;
import org.apache.aurora.scheduler.storage.backup.StorageBackup;
+import org.apache.aurora.scheduler.storage.db.DbModule;
import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
import org.apache.aurora.scheduler.storage.entities.IServerInfo;
-import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
import org.apache.aurora.scheduler.thrift.aop.AnnotatedAuroraAdmin;
-import org.apache.aurora.scheduler.updater.JobUpdateController;
+import org.apache.mesos.Protos.FrameworkInfo;
import org.apache.shiro.subject.Subject;
-import org.junit.Before;
import org.junit.Test;
import static org.apache.aurora.gen.ResponseCode.OK;
@@ -56,18 +76,11 @@ public class ThriftIT extends EasyMockTest {
private static final String USER = "someuser";
private static final IResourceAggregate QUOTA =
IResourceAggregate.build(new ResourceAggregate(1, 1, 1));
+ private static final IServerInfo SERVER_INFO = IServerInfo.build(new ServerInfo());
private AuroraAdmin.Iface thrift;
- private StorageTestUtil storageTestUtil;
- private QuotaManager quotaManager;
- @Before
- public void setUp() {
- quotaManager = createMock(QuotaManager.class);
- createThrift();
- }
-
- private void createThrift() {
+ private void createThrift(ConfigurationManager configurationManager) {
Injector injector = Guice.createInjector(
new ThriftModule(),
new AbstractModule() {
@@ -79,24 +92,32 @@ public class ThriftIT extends EasyMockTest {
@Override
protected void configure() {
- bindMock(CronJobManager.class);
- bindMock(MaintenanceController.class);
+ install(new LifecycleModule());
+ install(new StatsModule());
+ install(DbModule.testModule());
+ install(new QuotaModule());
+ install(new CronModule());
+ install(new TierModule(TaskTestUtil.DEV_TIER_CONFIG));
+ bind(ExecutorSettings.class).toInstance(TestExecutorSettings.THERMOS_EXECUTOR);
+
+ install(new AppModule(configurationManager));
+
+ bind(NonVolatileStorage.class).to(FakeNonVolatileStorage.class);
+
+ DynamicHostSet<ServiceInstance> schedulers =
+ createMock(new Clazz<DynamicHostSet<ServiceInstance>>() { });
+ bind(new TypeLiteral<DynamicHostSet<ServiceInstance>>() { }).toInstance(schedulers);
+ bindMock(DriverFactory.class);
+ bind(DriverSettings.class).toInstance(new DriverSettings(
+ "fakemaster",
+ com.google.common.base.Optional.absent(),
+ FrameworkInfo.newBuilder()
+ .setUser("framework user")
+ .setName("test framework")
+ .build()));
bindMock(Recovery.class);
- bindMock(LockManager.class);
- bindMock(ShutdownRegistry.class);
- bindMock(StateManager.class);
- bindMock(TaskIdGenerator.class);
- bindMock(UUIDGenerator.class);
- bindMock(JobUpdateController.class);
- bind(ConfigurationManager.class).toInstance(TaskTestUtil.CONFIGURATION_MANAGER);
- bind(Thresholds.class).toInstance(new Thresholds(1000, 2000));
- storageTestUtil = new StorageTestUtil(ThriftIT.this);
- bind(Storage.class).toInstance(storageTestUtil.storage);
- bind(NonVolatileStorage.class).toInstance(storageTestUtil.storage);
bindMock(StorageBackup.class);
- bind(QuotaManager.class).toInstance(quotaManager);
- bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo()));
- bindMock(CronPredictor.class);
+ bind(IServerInfo.class).toInstance(SERVER_INFO);
}
@Provides
@@ -105,21 +126,59 @@ public class ThriftIT extends EasyMockTest {
}
}
);
+
+ Command shutdownCommand =
+ injector.getInstance(Key.get(Command.class, ShutdownStage.class));
+ addTearDown(shutdownCommand::execute);
+
thrift = injector.getInstance(AnnotatedAuroraAdmin.class);
+ Storage storage = injector.getInstance(Storage.class);
+ storage.prepare();
}
@Test
public void testSetQuota() throws Exception {
- storageTestUtil.expectOperations();
- quotaManager.saveQuota(
- USER,
- QUOTA,
- storageTestUtil.mutableStoreProvider);
+ createThrift(TaskTestUtil.CONFIGURATION_MANAGER);
control.replay();
assertEquals(
OK,
thrift.setQuota(USER, QUOTA.newBuilder()).getResponseCode());
+
+ assertEquals(
+ QUOTA.newBuilder(),
+ thrift.getQuota(USER).getResult().getGetQuotaResult().getQuota());
+ }
+
+ @Test
+ public void testSubmitNoExecutorDockerTask() throws Exception {
+ ConfigurationManager configurationManager = new ConfigurationManager(
+ ImmutableSet.of(_Fields.DOCKER),
+ true,
+ ImmutableMultimap.of(),
+ false);
+
+ createThrift(configurationManager);
+
+ control.replay();
+
+ TaskConfig task = TaskTestUtil.makeConfig(TaskTestUtil.JOB).newBuilder();
+ task.unsetExecutorConfig();
+ task.setProduction(false)
+ .setContainer(Container.docker(new DockerContainer()
+ .setImage("image")
+ .setParameters(
+ ImmutableList.of(new DockerParameter("a", "b"), new DockerParameter("c", "d")))));
+ JobConfiguration job = new JobConfiguration()
+ .setKey(task.getJob())
+ .setTaskConfig(task)
+ .setInstanceCount(1);
+
+ assertEquals(OK, thrift.createJob(job, null).getResponseCode());
+ ScheduledTask scheduledTask = Iterables.getOnlyElement(
+ thrift.getTasksStatus(new TaskQuery()).getResult().getScheduleStatusResult().getTasks());
+ assertEquals(ScheduleStatus.PENDING, scheduledTask.getStatus());
+ assertEquals(task, scheduledTask.getAssignedTask().getTask());
}
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/3806e626/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
index cc88915..e157c0d 100644
--- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
@@ -171,7 +171,7 @@ public class JobUpdaterIT extends EasyMockTest {
Injector injector = Guice.createInjector(
new UpdaterModule(executor),
- DbModule.testModule(),
+ DbModule.testModuleWithWorkQueue(),
new AbstractModule() {
@Override
protected void configure() {