You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2016/01/03 18:39:48 UTC
aurora git commit: Remove several scheduler command line arguments.
Repository: aurora
Updated Branches:
refs/heads/master 297ebc157 -> 6b4045397
Remove several scheduler command line arguments.
Reviewed at https://reviews.apache.org/r/41786/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6b404539
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6b404539
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6b404539
Branch: refs/heads/master
Commit: 6b4045397305040616affdf1989c73d0f78b667a
Parents: 297ebc1
Author: Bill Farner <wf...@apache.org>
Authored: Sun Jan 3 09:39:44 2016 -0800
Committer: Bill Farner <wf...@apache.org>
Committed: Sun Jan 3 09:39:44 2016 -0800
----------------------------------------------------------------------
NEWS | 5 +
docs/scheduler-storage.md | 47 --------
.../aurora/scheduler/http/api/ApiModule.java | 7 +-
.../aurora/scheduler/storage/log/Entries.java | 4 +-
.../scheduler/storage/log/LogManager.java | 16 ---
.../scheduler/storage/log/LogStorageModule.java | 12 --
.../storage/log/StreamManagerImpl.java | 23 +---
.../aurora/scheduler/thrift/aop/AopModule.java | 91 +---------------
.../thrift/aop/FeatureToggleInterceptor.java | 46 --------
.../scheduler/storage/log/LogManagerTest.java | 19 ++--
.../scheduler/storage/log/LogStorageTest.java | 23 ++--
.../storage/log/testing/LogOpMatcher.java | 14 ++-
.../scheduler/thrift/aop/AopModuleTest.java | 109 -------------------
.../aop/FeatureToggleInterceptorTest.java | 84 --------------
14 files changed, 43 insertions(+), 457 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/NEWS
----------------------------------------------------------------------
diff --git a/NEWS b/NEWS
index 394b31c..965fbf4 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,11 @@
- Upgraded Mesos to 0.25.0.
- Env variables can be passed through to task processes by passing `--preserve_env`
to thermos.
+- Removed scheduler command line arguments:
+ - `-enable_cors_support`. Enabling CORS is now implicit by setting the argument
+ `-enable_cors_for`.
+ - `-deduplicate_snapshots` and `-deflate_snapshots`. These features are good to always enable.
+ - `-enable_job_updates` and `-enable_job_creation`
0.11.0
------
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/docs/scheduler-storage.md
----------------------------------------------------------------------
diff --git a/docs/scheduler-storage.md b/docs/scheduler-storage.md
deleted file mode 100644
index 1cd02f8..0000000
--- a/docs/scheduler-storage.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# Snapshot Performance
-
-Periodically the scheduler writes a full snapshot of its state to the replicated log. To do this
-it needs to hold a global storage write lock while it writes out this data. In large clusters
-this has been observed to take up to 40 seconds. Long pauses can cause issues in the system,
-including delays in scheduling new tasks.
-
-The scheduler has two optimizations to reduce the size of snapshots and thus improve snapshot
-performance: compression and deduplication. Most users will want to enable both compression
-and deduplication.
-
-## Compression
-
-To reduce the size of the snapshot the DEFLATE algorithm can be applied to the serialized bytes
-of the snapshot as they are written to the stream. This reduces the total number of bytes that
-need to be written to the replicated log at the cost of CPU and generally reduces the amount
-of time a snapshot takes. Most users will want to enable both compression and deduplication.
-
-### Enabling Compression
-
-Snapshot compression is enabled via the `-deflate_snapshots` flag. This is the default since
-Aurora 0.5.0. All released versions of Aurora can read both compressed and uncompressed snapshots,
-so there are no backwards compatibility concerns associated with changing this flag.
-
-### Disabling compression
-
-Disable compression by passing `-deflate_snapshots=false`.
-
-## Deduplication
-
-In Aurora 0.6.0 a new snapshot format was introduced. Rather than write one configuration blob
-per Mesos task this format stores each configuration blob once, and each Mesos task with a
-pointer to its blob. This format is not backwards compatible with earlier versions of Aurora.
-
-### Enabling Deduplication
-
-After upgrading Aurora to 0.6.0, enable deduplication with the `-deduplicate_snapshots` flag.
-After the first snapshot the cluster will be using the deduplicated format to write to the
-replicated log. Snapshots are created periodically by the scheduler (according to
-the `-dlog_snapshot_interval` flag). An administrator can also force a snapshot operation with
-`aurora_admin snapshot`.
-
-### Disabling Deduplication
-
-To disable deduplication, for example to rollback to Aurora, restart all of the cluster's
-schedulers with `-deduplicate_snapshots=false` and either wait for a snapshot or force one
-using `aurora_admin snapshot`.
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
index da6894e..cd5adf9 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
@@ -35,16 +35,13 @@ import org.eclipse.jetty.util.resource.Resource;
public class ApiModule extends ServletModule {
public static final String API_PATH = "/api";
- @CmdLine(name = "enable_cors_support", help = "Enable CORS support for thrift end points.")
- private static final Arg<Boolean> ENABLE_CORS_SUPPORT = Arg.create(false);
-
/**
* Set the {@code Access-Control-Allow-Origin} header for API requests. See
* http://www.w3.org/TR/cors/
*/
@CmdLine(name = "enable_cors_for",
help = "List of domains for which CORS support should be enabled.")
- private static final Arg<String> ENABLE_CORS_FOR = Arg.create("*");
+ private static final Arg<String> ENABLE_CORS_FOR = Arg.create(null);
private static final String API_CLIENT_ROOT = Resource
.newClassPathResource("org/apache/aurora/scheduler/gen/client")
@@ -52,7 +49,7 @@ public class ApiModule extends ServletModule {
@Override
protected void configureServlets() {
- if (ENABLE_CORS_SUPPORT.get()) {
+ if (ENABLE_CORS_FOR.get() != null) {
filter(API_PATH).through(new CorsFilter(ENABLE_CORS_FOR.get()));
}
serve(API_PATH).with(TServlet.class);
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
index 548b5e7..413d59a 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
@@ -25,7 +25,7 @@ import org.apache.aurora.gen.storage.LogEntry._Fields;
/**
* Utility class for working with log entries.
*/
-final class Entries {
+public final class Entries {
private static final Logger LOG = Logger.getLogger(Entries.class.getName());
@@ -46,7 +46,7 @@ final class Entries {
* of the original entry.
* @throws CodingException If the value could not be encoded or deflated.
*/
- static LogEntry deflate(LogEntry entry) throws CodingException {
+ public static LogEntry deflate(LogEntry entry) throws CodingException {
return LogEntry.deflatedEntry(ThriftBinaryCodec.deflateNonNull(entry));
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
index 4099503..cfa9c56 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
@@ -39,22 +39,6 @@ public class LogManager {
public @interface MaxEntrySize { }
/**
- * When true, enable snapshot deflation.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ElementType.METHOD, ElementType.PARAMETER})
- @Qualifier
- public @interface DeflateSnapshots { }
-
- /**
- * When true, enable snapshot deduplication.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ElementType.METHOD, ElementType.PARAMETER})
- @Qualifier
- public @interface DeduplicateSnapshots { }
-
- /**
* Hash function used to verify log entries.
*/
@Retention(RetentionPolicy.RUNTIME)
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
index 561c70b..ed63a74 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
@@ -31,12 +31,10 @@ import org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage;
import org.apache.aurora.scheduler.storage.DistributedSnapshotStore;
import org.apache.aurora.scheduler.storage.Storage;
import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots;
import org.apache.aurora.scheduler.storage.log.LogManager.MaxEntrySize;
import org.apache.aurora.scheduler.storage.log.LogStorage.Settings;
import static org.apache.aurora.scheduler.storage.log.EntrySerializer.EntrySerializerImpl;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots;
import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction;
import static org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl;
@@ -64,14 +62,6 @@ public class LogStorageModule extends PrivateModule {
public static final Arg<Amount<Integer, Data>> MAX_LOG_ENTRY_SIZE =
Arg.create(Amount.of(512, Data.KB));
- @CmdLine(name = "deduplicate_snapshots",
- help = "Write snapshots in deduplicated format. For details and backwards compatibility "
- + "concerns see docs/scheduler-storage.md.")
- private static final Arg<Boolean> DEDUPLICATE_SNAPSHOTS = Arg.create(false);
-
- @CmdLine(name = "deflate_snapshots", help = "Whether snapshots should be deflate-compressed.")
- private static final Arg<Boolean> DEFLATE_SNAPSHOTS = Arg.create(true);
-
@Override
protected void configure() {
bind(Settings.class)
@@ -80,8 +70,6 @@ public class LogStorageModule extends PrivateModule {
bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class)
.toInstance(MAX_LOG_ENTRY_SIZE.get());
bind(LogManager.class).in(Singleton.class);
- bindConstant().annotatedWith(DeduplicateSnapshots.class).to(DEDUPLICATE_SNAPSHOTS.get());
- bindConstant().annotatedWith(DeflateSnapshots.class).to(DEFLATE_SNAPSHOTS.get());
bind(LogStorage.class).in(Singleton.class);
install(CallOrderEnforcingStorage.wrappingModule(LogStorage.class));
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
index b252468..7a63515 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
@@ -55,8 +55,6 @@ import static org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
import static org.apache.aurora.common.inject.TimedInterceptor.Timed;
import static org.apache.aurora.scheduler.log.Log.Stream.InvalidPositionException;
import static org.apache.aurora.scheduler.log.Log.Stream.StreamAccessException;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots;
import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction;
class StreamManagerImpl implements StreamManager {
@@ -79,26 +77,20 @@ class StreamManagerImpl implements StreamManager {
private final Object writeMutex = new Object();
private final Log.Stream stream;
private final EntrySerializer entrySerializer;
- private final boolean deflateSnapshots;
private final HashFunction hashFunction;
private final SnapshotDeduplicator snapshotDeduplicator;
- private final boolean deduplicateSnapshots;
@Inject
StreamManagerImpl(
@Assisted Stream stream,
EntrySerializer entrySerializer,
- @DeflateSnapshots boolean deflateSnapshots,
@LogEntryHashFunction HashFunction hashFunction,
- SnapshotDeduplicator snapshotDeduplicator,
- @DeduplicateSnapshots boolean deduplicateSnapshots) {
+ SnapshotDeduplicator snapshotDeduplicator) {
this.stream = requireNonNull(stream);
this.entrySerializer = requireNonNull(entrySerializer);
- this.deflateSnapshots = deflateSnapshots;
this.hashFunction = requireNonNull(hashFunction);
this.snapshotDeduplicator = requireNonNull(snapshotDeduplicator);
- this.deduplicateSnapshots = deduplicateSnapshots;
}
@Override
@@ -203,17 +195,8 @@ class StreamManagerImpl implements StreamManager {
public void snapshot(Snapshot snapshot)
throws CodingException, InvalidPositionException, StreamAccessException {
- LogEntry entry;
- if (deduplicateSnapshots) {
- entry = LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot));
- } else {
- entry = LogEntry.snapshot(snapshot);
- }
-
- if (deflateSnapshots) {
- entry = deflate(entry);
- }
-
+ LogEntry entry =
+ deflate(LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot)));
Log.Position position = appendAndGetPosition(entry);
vars.snapshots.incrementAndGet();
vars.unSnapshottedTransactions.set(0);
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
index 7f29b79..f59ee1a 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
@@ -13,33 +13,13 @@
*/
package org.apache.aurora.scheduler.thrift.aop;
-import java.lang.reflect.Method;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
import com.google.inject.AbstractModule;
import com.google.inject.Binder;
-import com.google.inject.Key;
-import com.google.inject.PrivateModule;
-import com.google.inject.TypeLiteral;
import com.google.inject.matcher.Matcher;
import com.google.inject.matcher.Matchers;
import org.aopalliance.intercept.MethodInterceptor;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.gen.AuroraSchedulerManager;
import org.apache.aurora.gen.Response;
import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
@@ -48,66 +28,20 @@ import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
*/
public class AopModule extends AbstractModule {
- @CmdLine(name = "enable_job_updates", help = "Whether new job updates should be accepted.")
- private static final Arg<Boolean> ENABLE_UPDATES = Arg.create(true);
-
- @CmdLine(name = "enable_job_creation",
- help = "Allow new jobs to be created, if false all job creation requests will be denied.")
- private static final Arg<Boolean> ENABLE_JOB_CREATION = Arg.create(true);
-
private static final Matcher<? super Class<?>> THRIFT_IFACE_MATCHER =
Matchers.subclassesOf(AnnotatedAuroraAdmin.class)
.and(Matchers.annotatedWith(DecoratedThrift.class));
- private final Map<String, Boolean> toggledMethods;
-
- public AopModule() {
- this(ImmutableMap.of(
- "createJob", ENABLE_JOB_CREATION.get(),
- "acquireLock", ENABLE_UPDATES.get()));
- }
-
- @VisibleForTesting
- AopModule(Map<String, Boolean> toggledMethods) {
- this.toggledMethods = ImmutableMap.copyOf(toggledMethods);
- }
-
- private static final Function<Method, String> GET_NAME = Method::getName;
-
@Override
protected void configure() {
// Layer ordering:
- // APIVersion -> Log -> FeatureToggle -> StatsExporter -> SchedulerThriftInterface
+ // APIVersion -> Log -> StatsExporter -> SchedulerThriftInterface
// It's important for this interceptor to be registered first to ensure it's at the 'top' of
// the stack and the standard message is always applied.
bindThriftDecorator(new ServerInfoInterceptor());
bindThriftDecorator(new LoggingInterceptor());
-
- install(new PrivateModule() {
- @Override
- protected void configure() {
- // Ensure that the provided methods exist on the decorated interface.
- List<Method> methods =
- ImmutableList.copyOf(AuroraSchedulerManager.Iface.class.getMethods());
- for (String toggledMethod : toggledMethods.keySet()) {
- Preconditions.checkArgument(
- Iterables.any(methods,
- Predicates.compose(Predicates.equalTo(toggledMethod), GET_NAME)),
- String.format("Method %s was not found in class %s",
- toggledMethod,
- AuroraSchedulerManager.Iface.class));
- }
-
- bind(new TypeLiteral<Map<String, Boolean>>() { }).toInstance(toggledMethods);
- bind(IsFeatureEnabled.class).in(Singleton.class);
- Key<Predicate<Method>> predicateKey = Key.get(new TypeLiteral<Predicate<Method>>() { });
- bind(predicateKey).to(IsFeatureEnabled.class);
- expose(predicateKey);
- }
- });
- bindThriftDecorator(new FeatureToggleInterceptor());
bindThriftDecorator(new ThriftStatsExporterInterceptor());
}
@@ -127,27 +61,4 @@ public class AopModule extends AbstractModule {
interceptor);
binder.requestInjection(interceptor);
}
-
- private static class IsFeatureEnabled implements Predicate<Method> {
- private final Predicate<String> methodEnabled;
-
- @Inject
- IsFeatureEnabled(Map<String, Boolean> toggleMethods) {
- Predicate<String> builder = Predicates.alwaysTrue();
- for (Map.Entry<String, Boolean> toggleMethod : toggleMethods.entrySet()) {
- Predicate<String> enableMethod = Predicates.or(
- toggleMethod.getValue()
- ? Predicates.alwaysTrue()
- : Predicates.alwaysFalse(),
- Predicates.not(Predicates.equalTo(toggleMethod.getKey())));
- builder = Predicates.and(builder, enableMethod);
- }
- methodEnabled = builder;
- }
-
- @Override
- public boolean apply(Method method) {
- return methodEnabled.apply(method.getName());
- }
- }
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
deleted file mode 100644
index 2492796..0000000
--- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Predicate;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.scheduler.thrift.Responses;
-
-/**
- * A method interceptor that blocks access to features based on a supplied predicate.
- */
-public class FeatureToggleInterceptor implements MethodInterceptor {
-
- @Inject private Predicate<Method> allowMethod;
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- Method method = invocation.getMethod();
- if (allowMethod.apply(method)) {
- return invocation.proceed();
- } else {
- return Responses.addMessage(
- Responses.empty(),
- ResponseCode.ERROR,
- "The " + method.getName() + " feature is currently disabled on this scheduler.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
index 692ace0..0443bb3 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
@@ -42,6 +42,7 @@ import org.apache.aurora.gen.HostAttributes;
import org.apache.aurora.gen.ScheduleStatus;
import org.apache.aurora.gen.ScheduledTask;
import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
import org.apache.aurora.gen.storage.Frame;
import org.apache.aurora.gen.storage.FrameChunk;
import org.apache.aurora.gen.storage.FrameHeader;
@@ -103,10 +104,8 @@ public class LogManagerTest extends EasyMockTest {
return new StreamManagerImpl(
stream,
new EntrySerializer.EntrySerializerImpl(maxEntrySize, Hashing.md5()),
- false,
Hashing.md5(),
- new SnapshotDeduplicatorImpl(),
- false);
+ new SnapshotDeduplicatorImpl());
}
@Test
@@ -261,7 +260,8 @@ public class LogManagerTest extends EasyMockTest {
@Test
public void testTransactionSnapshot() throws CodingException {
Snapshot snapshot = createSnapshot();
- expectAppend(position1, LogEntry.snapshot(snapshot));
+ DeduplicatedSnapshot deduplicated = new SnapshotDeduplicatorImpl().deduplicate(snapshot);
+ expectAppend(position1, Entries.deflate(LogEntry.deduplicatedSnapshot(deduplicated)));
stream.truncateBefore(position1);
control.replay();
@@ -393,10 +393,8 @@ public class LogManagerTest extends EasyMockTest {
final StreamManagerImpl streamManager = new StreamManagerImpl(
mockStream,
new EntrySerializer.EntrySerializerImpl(message1.chunkSize, Hashing.md5()),
- false,
Hashing.md5(),
- new SnapshotDeduplicatorImpl(),
- false);
+ new SnapshotDeduplicatorImpl());
StreamTransaction tr1 = streamManager.startTransaction();
tr1.add(op1);
@@ -485,7 +483,8 @@ public class LogManagerTest extends EasyMockTest {
public void testWriteAndReadDeflatedEntry() throws Exception {
Snapshot snapshot = createSnapshot();
LogEntry snapshotLogEntry = LogEntry.snapshot(snapshot);
- LogEntry deflatedSnapshotEntry = Entries.deflate(snapshotLogEntry);
+ LogEntry deflatedSnapshotEntry = Entries.deflate(
+ LogEntry.deduplicatedSnapshot(new SnapshotDeduplicatorImpl().deduplicate(snapshot)));
Entry snapshotEntry = createMock(Entry.class);
expect(stream.append(entryEq(deflatedSnapshotEntry))).andReturn(position1);
@@ -504,10 +503,8 @@ public class LogManagerTest extends EasyMockTest {
StreamManagerImpl streamManager = new StreamManagerImpl(
stream,
new EntrySerializer.EntrySerializerImpl(NO_FRAMES_EVER_SIZE, md5),
- true,
md5,
- new SnapshotDeduplicatorImpl(),
- false);
+ new SnapshotDeduplicatorImpl());
streamManager.snapshot(snapshot);
streamManager.readFromBeginning(reader);
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
index 216f92f..4305270 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
@@ -56,6 +56,7 @@ import org.apache.aurora.gen.ResourceAggregate;
import org.apache.aurora.gen.ScheduleStatus;
import org.apache.aurora.gen.ScheduledTask;
import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
import org.apache.aurora.gen.storage.LogEntry;
import org.apache.aurora.gen.storage.Op;
import org.apache.aurora.gen.storage.PruneJobUpdateHistory;
@@ -78,6 +79,7 @@ import org.apache.aurora.gen.storage.Transaction;
import org.apache.aurora.gen.storage.storageConstants;
import org.apache.aurora.scheduler.base.JobKeys;
import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.base.TaskTestUtil;
import org.apache.aurora.scheduler.base.Tasks;
import org.apache.aurora.scheduler.events.EventSink;
import org.apache.aurora.scheduler.events.PubsubEvent;
@@ -104,6 +106,7 @@ import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
import org.apache.aurora.scheduler.storage.log.LogStorage.SchedulingService;
+import org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl;
import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher;
import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher.StreamMatcher;
import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
@@ -131,6 +134,7 @@ public class LogStorageTest extends EasyMockTest {
private LogStorage logStorage;
private Log log;
+ private SnapshotDeduplicator deduplicator;
private Stream stream;
private Position position;
private StreamMatcher streamMatcher;
@@ -142,19 +146,15 @@ public class LogStorageTest extends EasyMockTest {
@Before
public void setUp() {
log = createMock(Log.class);
- SnapshotDeduplicator snapshotDeduplicator = createMock(SnapshotDeduplicator.class);
+ deduplicator = createMock(SnapshotDeduplicator.class);
StreamManagerFactory streamManagerFactory = logStream -> {
HashFunction md5 = Hashing.md5();
return new StreamManagerImpl(
logStream,
- new EntrySerializer.EntrySerializerImpl(
- Amount.of(1, Data.GB),
- md5),
- false,
+ new EntrySerializer.EntrySerializerImpl(Amount.of(1, Data.GB), md5),
md5,
- snapshotDeduplicator,
- false);
+ deduplicator);
};
LogManager logManager = new LogManager(log, streamManagerFactory);
@@ -227,11 +227,12 @@ public class LogStorageTest extends EasyMockTest {
Snapshot snapshotContents = new Snapshot()
.setTimestamp(NOW)
.setTasks(ImmutableSet.of(
- new ScheduledTask()
- .setStatus(ScheduleStatus.RUNNING)
- .setAssignedTask(new AssignedTask().setTaskId("task_id"))));
+ TaskTestUtil.makeTask("task_id", TaskTestUtil.JOB).newBuilder()));
expect(snapshotStore.createSnapshot()).andReturn(snapshotContents);
- streamMatcher.expectSnapshot(snapshotContents).andReturn(position);
+ DeduplicatedSnapshot deduplicated =
+ new SnapshotDeduplicatorImpl().deduplicate(snapshotContents);
+ expect(deduplicator.deduplicate(snapshotContents)).andReturn(deduplicated);
+ streamMatcher.expectSnapshot(deduplicated).andReturn(position);
stream.truncateBefore(position);
Capture<MutateWork<Void, RuntimeException>> snapshotWork = createCapture();
expect(storageUtil.storage.write(capture(snapshotWork))).andAnswer(
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
index 9abdbbb..5a2524d 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
@@ -15,17 +15,19 @@ package org.apache.aurora.scheduler.storage.log.testing;
import java.util.Objects;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import org.apache.aurora.codec.ThriftBinaryCodec;
import org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
import org.apache.aurora.gen.storage.LogEntry;
import org.apache.aurora.gen.storage.Op;
-import org.apache.aurora.gen.storage.Snapshot;
import org.apache.aurora.gen.storage.Transaction;
import org.apache.aurora.gen.storage.storageConstants;
import org.apache.aurora.scheduler.log.Log.Position;
import org.apache.aurora.scheduler.log.Log.Stream;
+import org.apache.aurora.scheduler.storage.log.Entries;
import org.easymock.EasyMock;
import org.easymock.IArgumentMatcher;
import org.easymock.IExpectationSetters;
@@ -92,9 +94,13 @@ public class LogOpMatcher implements IArgumentMatcher {
* @param snapshot Expected snapshot.
* @return An expectation setter.
*/
- public IExpectationSetters<Position> expectSnapshot(Snapshot snapshot) {
- LogEntry entry = LogEntry.snapshot(snapshot);
- return expect(stream.append(sameEntry(entry)));
+ public IExpectationSetters<Position> expectSnapshot(DeduplicatedSnapshot snapshot) {
+ try {
+ LogEntry entry = Entries.deflate(LogEntry.deduplicatedSnapshot(snapshot));
+ return expect(stream.append(sameEntry(entry)));
+ } catch (CodingException e) {
+ throw Throwables.propagate(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
deleted file mode 100644
index 19b7415..0000000
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.AbstractModule;
-import com.google.inject.CreationException;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.apache.aurora.gen.AuroraAdmin.Iface;
-import org.apache.aurora.gen.JobConfiguration;
-import org.apache.aurora.gen.Response;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.gen.ServerInfo;
-import org.apache.aurora.scheduler.storage.entities.IServerInfo;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-
-public class AopModuleTest extends EasyMockTest {
- private AnnotatedAuroraAdmin mockThrift;
-
- @Before
- public void setUp() throws Exception {
- mockThrift = createMock(AnnotatedAuroraAdmin.class);
- }
-
- private Iface getIface(Map<String, Boolean> toggledMethods) {
- Injector injector = Guice.createInjector(
- new AbstractModule() {
- @Override
- protected void configure() {
- bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo()));
- MockDecoratedThrift.bindForwardedMock(binder(), mockThrift);
- }
- },
- new AopModule(toggledMethods));
- return injector.getInstance(AnnotatedAuroraAdmin.class);
- }
-
- @Test
- public void testNonFlaggedMethod() throws Exception {
- assertCreateAllowed(ImmutableMap.of("acquireLock", false));
- }
-
- @Test
- public void testNoFlaggedMethods() throws Exception {
- assertCreateAllowed(ImmutableMap.of());
- }
-
- @Test
- public void testFlaggedMethodEnabled() throws Exception {
- assertCreateAllowed(ImmutableMap.of("createJob", true));
- }
-
- @Test
- public void testFlaggedMethodDisabled() throws Exception {
- JobConfiguration job = new JobConfiguration();
-
- control.replay();
-
- Iface thrift = getIface(ImmutableMap.of("createJob", false));
- assertEquals(ResponseCode.ERROR, thrift.createJob(job, null).getResponseCode());
- }
-
- @Test(expected = CreationException.class)
- public void testMissingMethod() {
- control.replay();
- getIface(ImmutableMap.of("notamethod", true));
- }
-
- private void assertCreateAllowed(Map<String, Boolean> toggledMethods) throws Exception {
- JobConfiguration job = new JobConfiguration();
- Response response = new Response();
- expect(mockThrift.createJob(job, null)).andReturn(response);
-
- control.replay();
-
- Iface thrift = getIface(toggledMethods);
- assertSame(response, thrift.createJob(job, null));
- }
-
- @Test
- public void assertToStringNotIntercepted() {
- control.replay();
-
- Iface thrift = getIface(ImmutableMap.of());
- assertNotNull(thrift.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
deleted file mode 100644
index 0edb315..0000000
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import com.google.common.base.Predicate;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.TypeLiteral;
-import com.google.inject.matcher.Matchers;
-
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.apache.aurora.gen.Response;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.gen.TaskQuery;
-import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertSame;
-
-public class FeatureToggleInterceptorTest extends EasyMockTest {
-
- private AnnotatedAuroraAdmin realThrift;
- private AnnotatedAuroraAdmin decoratedThrift;
- private Predicate<Method> predicate;
-
- @Before
- public void setUp() {
- realThrift = createMock(AnnotatedAuroraAdmin.class);
- predicate = createMock(new Clazz<Predicate<Method>>() { });
- Injector injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- MockDecoratedThrift.bindForwardedMock(binder(), realThrift);
- bind(new TypeLiteral<Predicate<Method>>() { }).toInstance(predicate);
- AopModule.bindThriftDecorator(
- binder(),
- Matchers.annotatedWith(DecoratedThrift.class),
- new FeatureToggleInterceptor());
- }
- });
- decoratedThrift = injector.getInstance(AnnotatedAuroraAdmin.class);
- }
-
- @Test
- public void testPredicatePass() throws Exception {
- TaskQuery query = new TaskQuery();
- Response response = new Response()
- .setResponseCode(ResponseCode.OK);
-
- expect(predicate.apply(EasyMock.anyObject())).andReturn(true);
- expect(realThrift.getTasksStatus(query)).andReturn(response);
-
- control.replay();
-
- assertSame(response, decoratedThrift.getTasksStatus(query));
- }
-
- @Test
- public void testPredicateDeny() throws Exception {
- TaskQuery query = new TaskQuery();
- expect(predicate.apply(EasyMock.anyObject())).andReturn(false);
-
- control.replay();
-
- assertSame(ResponseCode.ERROR, decoratedThrift.getTasksStatus(query).getResponseCode());
- }
-}