You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ke...@apache.org on 2013/12/05 20:58:13 UTC
[1/9] git commit: Fix the twitter.thermos sdist to properly depend
upon twitter.thermos.monitoring.
Updated Branches:
refs/heads/master c72c92cf3 -> d51fecab9
Fix the twitter.thermos sdist to properly depend upon twitter.thermos.monitoring.
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/13ddf83c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/13ddf83c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/13ddf83c
Branch: refs/heads/master
Commit: 13ddf83c8072b3f7d4126a28e3731b27e501721f
Parents: c72c92c
Author: Brian Wickman <wi...@twitter.com>
Authored: Tue Dec 3 15:44:05 2013 -0800
Committer: Brian Wickman <wi...@twitter.com>
Committed: Tue Dec 3 15:44:05 2013 -0800
----------------------------------------------------------------------
src/main/python/twitter/thermos/BUILD | 1 +
src/main/python/twitter/thermos/bin/thermos.py | 19 ++++++++-----------
2 files changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/13ddf83c/src/main/python/twitter/thermos/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/BUILD b/src/main/python/twitter/thermos/BUILD
index 8235d01..7c5d760 100644
--- a/src/main/python/twitter/thermos/BUILD
+++ b/src/main/python/twitter/thermos/BUILD
@@ -4,6 +4,7 @@ python_library(
name = 'thermos',
dependencies = [
pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos/monitoring'),
],
provides = setup_py(
name = 'twitter.thermos',
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/13ddf83c/src/main/python/twitter/thermos/bin/thermos.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/bin/thermos.py b/src/main/python/twitter/thermos/bin/thermos.py
index 75e54d7..39ab1f2 100644
--- a/src/main/python/twitter/thermos/bin/thermos.py
+++ b/src/main/python/twitter/thermos/bin/thermos.py
@@ -467,16 +467,6 @@ def gc(args, options):
print('Cancelling gc.')
-# TODO(wickman) Implement.
-@app.command
-def monitor(args, options):
- """Monitor task(s)
-
- Usage: thermos monitor task_id [task_id_2 ...]
- """
- pass
-
-
@app.command
@app.command_option("--verbosity", default=0, dest='verbose', type='int',
help="Display more verbosity")
@@ -637,6 +627,8 @@ def help(args, options):
print('unknown command: %s' % args[0], file=sys.stderr)
+
+
def generate_usage():
usage = """
thermos
@@ -652,4 +644,9 @@ commands:
LogOptions.set_disk_log_level('NONE')
LogOptions.set_stdout_log_level('INFO')
generate_usage()
-app.main()
+
+
+proxy_main = app.main
+
+
+proxy_main()
[8/9] git commit: Returning custom response code in case of lock
conflict
Posted by ke...@apache.org.
Returning custom response code in case of lock conflict
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/b4da3c3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/b4da3c3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/b4da3c3f
Branch: refs/heads/master
Commit: b4da3c3fc7ad5d3d0d9b7ccf1ce2c051420c85a9
Parents: 24c9906
Author: Maxim Khutornenko <mk...@twitter.com>
Authored: Thu Dec 5 11:23:43 2013 -0800
Committer: Maxim Khutornenko <mk...@twitter.com>
Committed: Thu Dec 5 11:23:43 2013 -0800
----------------------------------------------------------------------
.../thrift/SchedulerThriftInterface.java | 28 +++++++++++---------
.../thrift/com/twitter/aurora/gen/api.thrift | 3 ++-
.../thrift/SchedulerThriftInterfaceTest.java | 15 ++++++-----
3 files changed, 26 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/b4da3c3f/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
index b8ba3f2..503ac44 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -126,6 +126,7 @@ import static com.twitter.aurora.auth.SessionValidator.SessionContext;
import static com.twitter.aurora.gen.ResponseCode.AUTH_FAILED;
import static com.twitter.aurora.gen.ResponseCode.ERROR;
import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
+import static com.twitter.aurora.gen.ResponseCode.LOCK_ERROR;
import static com.twitter.aurora.gen.ResponseCode.OK;
import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
import static com.twitter.common.base.MorePreconditions.checkNotBlank;
@@ -246,7 +247,9 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
response.setResponseCode(OK)
.setMessage(String.format("%d new tasks pending for job %s",
sanitized.getJobConfig().getInstanceCount(), JobKeys.toPath(job)));
- } catch (LockException | TaskDescriptionException | ScheduleException e) {
+ } catch (LockException e) {
+ response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
}
@@ -286,10 +289,8 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
return response.setResponseCode(OK).setMessage("Replaced template for: " + jobKey);
} catch (LockException e) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- } catch (TaskDescriptionException e) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- } catch (ScheduleException e) {
+ return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
}
}
@@ -490,10 +491,9 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
validateLockForTasks(Optional.fromNullable(mutablelock).transform(ILock.FROM_BUILDER), tasks);
schedulerCore.killTasks(Query.arbitrary(query), context.get().getIdentity());
} catch (LockException e) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
+ return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
} catch (ScheduleException e) {
- response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- return response;
+ return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
}
// TODO(William Farner): Move this into the client.
@@ -549,7 +549,9 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
schedulerCore.restartShards(jobKey, shardIds, context.getIdentity());
response.setResponseCode(OK).setMessage("Shards are restarting.");
- } catch (LockException | ScheduleException e) {
+ } catch (LockException e) {
+ response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
+ } catch (ScheduleException e) {
response.setResponseCode(ResponseCode.INVALID_REQUEST).setMessage(e.getMessage());
}
@@ -894,7 +896,9 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
return resp.setResponseCode(OK).setMessage("Successfully added instances.");
} catch (AuthFailedException e) {
return resp.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- } catch (TaskDescriptionException | LockException | ScheduleException e) {
+ } catch (LockException e) {
+ return resp.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
return resp.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
}
}
@@ -930,7 +934,7 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
} catch (AuthFailedException e) {
return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
} catch (LockException e) {
- return response.setResponseCode(ResponseCode.INVALID_REQUEST).setMessage(e.getMessage());
+ return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
}
}
@@ -956,7 +960,7 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
} catch (AuthFailedException e) {
return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
} catch (LockException e) {
- return response.setResponseCode(ResponseCode.INVALID_REQUEST).setMessage(e.getMessage());
+ return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/b4da3c3f/src/main/thrift/com/twitter/aurora/gen/api.thrift
----------------------------------------------------------------------
diff --git a/src/main/thrift/com/twitter/aurora/gen/api.thrift b/src/main/thrift/com/twitter/aurora/gen/api.thrift
index 1ce4041..9b3680b 100644
--- a/src/main/thrift/com/twitter/aurora/gen/api.thrift
+++ b/src/main/thrift/com/twitter/aurora/gen/api.thrift
@@ -21,7 +21,8 @@ enum ResponseCode {
OK = 1,
ERROR = 2,
WARNING = 3,
- AUTH_FAILED = 4
+ AUTH_FAILED = 4,
+ LOCK_ERROR = 5 // Raised when a Lock-protected operation failed due to lock validation.
}
struct APIVersion {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/b4da3c3f/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java b/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
index 30d3947..1e1e7b5 100644
--- a/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
@@ -106,6 +106,7 @@ import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
import static com.twitter.aurora.gen.ResponseCode.AUTH_FAILED;
import static com.twitter.aurora.gen.ResponseCode.ERROR;
import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
+import static com.twitter.aurora.gen.ResponseCode.LOCK_ERROR;
import static com.twitter.aurora.gen.ResponseCode.OK;
import static com.twitter.aurora.gen.ResponseCode.WARNING;
import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
@@ -239,7 +240,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response response = thrift.createJob(job.newBuilder(), LOCK.newBuilder(), SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, response.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, response.getResponseCode());
}
@Test
@@ -364,7 +365,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response response = thrift.killTasks(query.get(), LOCK.newBuilder(), SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, response.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, response.getResponseCode());
}
@Test
@@ -520,7 +521,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response resp = thrift.restartShards(JOB_KEY.newBuilder(), shards, LOCK.newBuilder(), SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, resp.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, resp.getResponseCode());
}
@Test
@@ -664,7 +665,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
assertEquals(
- INVALID_REQUEST,
+ LOCK_ERROR,
thrift.replaceCronTemplate(CRON_JOB, LOCK.newBuilder(), SESSION).getResponseCode());
}
@@ -1146,7 +1147,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response response = thrift.addInstances(config, LOCK.newBuilder(), SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, response.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, response.getResponseCode());
}
@Test
@@ -1221,7 +1222,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response response = thrift.acquireLock(LOCK_KEY.newBuilder(), SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, response.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, response.getResponseCode());
}
@Test
@@ -1265,7 +1266,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
control.replay();
Response response = thrift.releaseLock(LOCK.newBuilder(), CHECKED, SESSION);
- assertEquals(ResponseCode.INVALID_REQUEST, response.getResponseCode());
+ assertEquals(ResponseCode.LOCK_ERROR, response.getResponseCode());
}
private static JobConfiguration makeJob() {
[4/9] git commit: Fixing the scope for hasProperty in publish
Posted by ke...@apache.org.
Fixing the scope for hasProperty in publish
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/ecda2b3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/ecda2b3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/ecda2b3b
Branch: refs/heads/master
Commit: ecda2b3b17dd60e2676599168a41e3443ab3cc34
Parents: 6436a93
Author: Maxim Khutornenko <mk...@twitter.com>
Authored: Wed Dec 4 15:04:34 2013 -0800
Committer: Maxim Khutornenko <mk...@twitter.com>
Committed: Wed Dec 4 15:04:34 2013 -0800
----------------------------------------------------------------------
build.gradle | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/ecda2b3b/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 366512d..98f7701 100644
--- a/build.gradle
+++ b/build.gradle
@@ -40,7 +40,7 @@ publishing {
}
}
}
- if (hasProperty('internalMavenUrl')) {
+ if (project.hasProperty('internalMavenUrl')) {
repositories {
maven {
credentials {
@@ -80,7 +80,7 @@ sourceSets {
}
jar {
- from sourceSets.generated.output
+ from sourceSets.generated.output
}
dependencies {
[7/9] git commit: Fixing error message for a held lock.
Posted by ke...@apache.org.
Fixing error message for a held lock.
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/24c99066
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/24c99066
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/24c99066
Branch: refs/heads/master
Commit: 24c99066791698274f8eb6f9669d58590fd2ac19
Parents: e7b3da9
Author: Maxim Khutornenko <mk...@twitter.com>
Authored: Thu Dec 5 10:51:16 2013 -0800
Committer: Maxim Khutornenko <mk...@twitter.com>
Committed: Thu Dec 5 10:51:16 2013 -0800
----------------------------------------------------------------------
.../aurora/scheduler/state/LockManagerImpl.java | 17 ++++++++++---
.../scheduler/state/LockManagerImplTest.java | 25 ++++++++++++++++----
2 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/24c99066/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java b/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
index 46aed98..4667f9f 100644
--- a/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
+++ b/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
@@ -22,6 +22,7 @@ import javax.inject.Inject;
import com.google.common.base.Optional;
import com.twitter.aurora.gen.Lock;
+import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.storage.LockStore;
import com.twitter.aurora.scheduler.storage.Storage;
import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
@@ -62,7 +63,7 @@ class LockManagerImpl implements LockManager {
if (existingLock.isPresent()) {
throw new LockException(String.format(
"Operation for: %s is already in progress. Started at: %s. Current owner: %s.",
- lockKey,
+ formatLockKey(lockKey),
new Date(existingLock.get().getTimestampMs()).toString(),
existingLock.get().getUser()));
}
@@ -110,10 +111,20 @@ class LockManagerImpl implements LockManager {
if (stored.isPresent()) {
throw new LockException(String.format(
"Unable to perform operation for: %s. Use override/cancel option.",
- context));
+ formatLockKey(context)));
} else if (heldLock.isPresent()) {
- throw new LockException(String.format("Invalid operation context: %s", context));
+ throw new LockException(
+ String.format("Invalid operation context: %s", formatLockKey(context)));
}
}
}
+
+ private static String formatLockKey(ILockKey lockKey) {
+ switch (lockKey.getSetField()) {
+ case JOB:
+ return JobKeys.toPath(lockKey.getJob());
+ default:
+ return "Unknown lock key type: " + lockKey.getSetField();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/24c99066/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
index cc6f3b7..ec0c8dd 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
@@ -21,7 +21,9 @@ import com.google.common.base.Optional;
import org.easymock.EasyMock;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import com.twitter.aurora.gen.Identity;
import com.twitter.aurora.gen.Lock;
@@ -52,6 +54,9 @@ public class LockManagerImplTest extends EasyMockTest {
private LockManager lockManager;
private long timestampMs;
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@Before
public void setUp() throws Exception {
FakeClock clock = new FakeClock();
@@ -77,8 +82,9 @@ public class LockManagerImplTest extends EasyMockTest {
assertEquals(expected, actual);
}
- @Test(expected = LockException.class)
+ @Test
public void testAcquireLockInProgress() throws Exception {
+ expectLockException(JOB_KEY);
lockManager.acquireLock(LOCK_KEY, USER);
lockManager.acquireLock(LOCK_KEY, USER);
}
@@ -103,23 +109,32 @@ public class LockManagerImplTest extends EasyMockTest {
lockManager.validateIfLocked(LOCK_KEY, Optional.<ILock>absent());
}
- @Test(expected = LockException.class)
+ @Test
public void testValidateLockStoredNotEqualHeld() throws Exception {
+ expectLockException(JOB_KEY);
ILock lock = lockManager.acquireLock(LOCK_KEY, USER);
lock = ILock.build(lock.newBuilder().setUser("bob"));
lockManager.validateIfLocked(LOCK_KEY, Optional.of(lock));
}
- @Test(expected = LockException.class)
+ @Test
public void testValidateLockStoredNotEqualHeldWithHeldNull() throws Exception {
+ expectLockException(JOB_KEY);
lockManager.acquireLock(LOCK_KEY, USER);
lockManager.validateIfLocked(LOCK_KEY, Optional.<ILock>absent());
}
- @Test(expected = LockException.class)
+ @Test
public void testValidateLockNotStoredHeld() throws Exception {
+ IJobKey jobKey = JobKeys.from("r", "e", "n");
+ expectLockException(jobKey);
ILock lock = lockManager.acquireLock(LOCK_KEY, USER);
- ILockKey key = ILockKey.build(LockKey.job(JobKeys.from("r", "e", "n").newBuilder()));
+ ILockKey key = ILockKey.build(LockKey.job(jobKey.newBuilder()));
lockManager.validateIfLocked(key, Optional.of(lock));
}
+
+ private void expectLockException(IJobKey key) {
+ expectedException.expect(LockException.class);
+ expectedException.expectMessage(JobKeys.toPath(key));
+ }
}
[2/9] git commit: Remove the deprecated instance ID field.
Posted by ke...@apache.org.
Remove the deprecated instance ID field.
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/87afc2c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/87afc2c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/87afc2c5
Branch: refs/heads/master
Commit: 87afc2c5e9c79225fd524287b2a1466ec0c27231
Parents: 13ddf83
Author: Bill Farner <bi...@twitter.com>
Authored: Wed Dec 4 10:16:42 2013 -0800
Committer: Bill Farner <bi...@twitter.com>
Committed: Wed Dec 4 10:16:42 2013 -0800
----------------------------------------------------------------------
.../aurora/scheduler/async/TaskGroups.java | 11 +++---
.../twitter/aurora/scheduler/base/Tasks.java | 7 ----
.../configuration/ConfigurationManager.java | 16 --------
.../scheduler/state/StateManagerImpl.java | 2 +-
.../scheduler/storage/StorageBackfill.java | 28 --------------
.../python/twitter/aurora/client/api/updater.py | 15 --------
.../thrift/com/twitter/aurora/gen/api.thrift | 6 ---
.../state/BaseSchedulerCoreImplTest.java | 8 +---
.../scheduler/state/StateManagerImplTest.java | 5 +--
.../scheduler/storage/StorageBackfillTest.java | 39 ++------------------
.../twitter/aurora/client/api/test_updater.py | 13 -------
11 files changed, 11 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
index ff51e93..9ea0229 100644
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
+++ b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
@@ -45,7 +45,6 @@ import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
@@ -340,15 +339,15 @@ public class TaskGroups implements EventSubscriber {
}
static class GroupKey {
- private final ITaskConfig scrubbedCanonicalTask;
+ private final ITaskConfig canonicalTask;
GroupKey(ITaskConfig task) {
- this.scrubbedCanonicalTask = ConfigurationManager.scrubNonUniqueTaskFields(task);
+ this.canonicalTask = task;
}
@Override
public int hashCode() {
- return Objects.hashCode(scrubbedCanonicalTask);
+ return Objects.hashCode(canonicalTask);
}
@Override
@@ -357,12 +356,12 @@ public class TaskGroups implements EventSubscriber {
return false;
}
GroupKey other = (GroupKey) o;
- return Objects.equal(scrubbedCanonicalTask, other.scrubbedCanonicalTask);
+ return Objects.equal(canonicalTask, other.canonicalTask);
}
@Override
public String toString() {
- return JobKeys.toPath(Tasks.INFO_TO_JOB_KEY.apply(scrubbedCanonicalTask));
+ return JobKeys.toPath(Tasks.INFO_TO_JOB_KEY.apply(canonicalTask));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java b/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
index f560fc8..557ac3a 100644
--- a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
+++ b/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
@@ -70,13 +70,6 @@ public final class Tasks {
public static final Function<IScheduledTask, String> SCHEDULED_TO_ID =
Functions.compose(ASSIGNED_TO_ID, SCHEDULED_TO_ASSIGNED);
- public static final Function<ITaskConfig, Integer> INFO_TO_INSTANCE_ID_DEPRECATED =
- new Function<ITaskConfig, Integer>() {
- @Override public Integer apply(ITaskConfig task) {
- return task.getInstanceIdDEPRECATED();
- }
- };
-
public static final Function<IAssignedTask, Integer> ASSIGNED_TO_INSTANCE_ID =
new Function<IAssignedTask, Integer>() {
@Override public Integer apply(IAssignedTask task) {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
index 8f0e055..6d40d11 100644
--- a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
+++ b/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
@@ -207,22 +207,6 @@ public final class ConfigurationManager {
}
/**
- * Resets fields within a task configuration that would make it unique from originally-equal
- * configurations.
- *
- * @param task Task to scrub.
- * @return Scrubbed task.
- */
- public static ITaskConfig scrubNonUniqueTaskFields(ITaskConfig task) {
- TaskConfig copy = task.newBuilder();
- // Unsetting only changes the isset bit vector. For equals() comparison, the value must also be
- // canonical.
- copy.setInstanceIdDEPRECATED(0);
- copy.unsetInstanceIdDEPRECATED();
- return ITaskConfig.build(copy);
- }
-
- /**
* Check validity of and populates defaults in a job configuration. This will return a deep copy
* of the provided job configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java b/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
index f2b8c88..37d13f4 100644
--- a/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
+++ b/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
@@ -122,7 +122,7 @@ public class StateManagerImpl implements StateManager {
AssignedTask assigned = new AssignedTask()
.setTaskId(taskIdGenerator.generate(task, entry.getKey()))
.setInstanceId(entry.getKey())
- .setTask(task.newBuilder().setInstanceIdDEPRECATED(entry.getKey()));
+ .setTask(task.newBuilder());
return IScheduledTask.build(new ScheduledTask()
.setStatus(INIT)
.setAssignedTask(assigned));
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java b/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
index 343cc7e..df5b603 100644
--- a/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
+++ b/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
@@ -23,7 +23,6 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
-import com.twitter.aurora.gen.AssignedTask;
import com.twitter.aurora.gen.JobConfiguration;
import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.gen.ScheduledTask;
@@ -109,32 +108,6 @@ public final class StorageBackfill {
Stats.exportLong("instance_ids_inconsistent");
/**
- * Ensures backwards-compatible data is present for both the new and deprecated instance ID
- * fields.
- *
- * @param task Task to possibly modify when ensuring backwards compatibility.
- */
- private static void dualWriteInstanceId(AssignedTask task) {
- boolean oldFieldSet = task.getTask().isSetInstanceIdDEPRECATED();
- boolean newFieldSet = task.isSetInstanceId();
- if (oldFieldSet && newFieldSet) {
- BOTH_FIELDS_SET.incrementAndGet();
- if (task.getInstanceId() != task.getTask().getInstanceIdDEPRECATED()) {
- FIELDS_INCONSISTENT.incrementAndGet();
- }
- } else if (oldFieldSet) {
- OLD_FIELD_SET.incrementAndGet();
- task.setInstanceId(task.getTask().getInstanceIdDEPRECATED());
- } else if (newFieldSet) {
- NEW_FIELD_SET.incrementAndGet();
- task.getTask().setInstanceIdDEPRECATED(task.getInstanceId());
- } else {
- throw new IllegalStateException(
- "Task " + task.getTaskId() + " does not have an instance id.");
- }
- }
-
- /**
* Ensures backwards-compatibility of the throttled state, which exists in this version but is
* not handled.
*
@@ -164,7 +137,6 @@ public final class StorageBackfill {
// TODO(ksweeney): Guarantee tasks pass current validation code here and quarantine if they
// don't.
guaranteeShardUniqueness(builder, storeProvider.getUnsafeTaskStore(), clock);
- dualWriteInstanceId(builder.getAssignedTask());
rewriteThrottledState(builder);
return IScheduledTask.build(builder);
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/python/twitter/aurora/client/api/updater.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/updater.py b/src/main/python/twitter/aurora/client/api/updater.py
index 9d930be..3e6e642 100644
--- a/src/main/python/twitter/aurora/client/api/updater.py
+++ b/src/main/python/twitter/aurora/client/api/updater.py
@@ -193,7 +193,6 @@ invoking cancel_update.
for instance_id in instance_ids:
from_config = operation_configs.from_config.get(instance_id)
to_config = operation_configs.to_config.get(instance_id)
- self._unset_deprecated_fields(from_config, to_config)
if from_config and to_config:
# Sort internal dicts before comparing to rule out differences due to hashing.
@@ -213,20 +212,6 @@ invoking cancel_update.
return to_kill, to_add
- def _unset_deprecated_fields(self, from_config, to_config):
- """Unsets deprecated fields in task configs to provide common base for diffing.
-
- Arguments:
- from_config - task config to update from.
- to_config - task config to update to.
- """
- if from_config:
- from_config.instanceIdDEPRECATED = None
-
- if to_config:
- to_config.instanceIdDEPRECATED = None
-
-
def _update_instances(self, instance_ids, operation_configs):
"""Applies kill/add actions for the specified batch instances.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/main/thrift/com/twitter/aurora/gen/api.thrift
----------------------------------------------------------------------
diff --git a/src/main/thrift/com/twitter/aurora/gen/api.thrift b/src/main/thrift/com/twitter/aurora/gen/api.thrift
index 3f8d61d..1ce4041 100644
--- a/src/main/thrift/com/twitter/aurora/gen/api.thrift
+++ b/src/main/thrift/com/twitter/aurora/gen/api.thrift
@@ -156,12 +156,6 @@ struct TaskConfig {
10: i64 diskMb
11: i32 priority
13: i32 maxTaskFailures
- 14: i32 instanceIdDEPRECATED // TODO(William Farner): Deprecated. Use
- // AssignedTask.instanceId instead.
- // The instance ID for this task.
- // Instance IDs must be unique and contiguous within a
- // job, and will be in the range [0, N-1] (inclusive)
- // for a job that has N instances.
18: optional bool production // Whether this is a production task, which can preempt
// non-production tasks.
20: set<Constraint> constraints
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/test/java/com/twitter/aurora/scheduler/state/BaseSchedulerCoreImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/BaseSchedulerCoreImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/BaseSchedulerCoreImplTest.java
index 0bfa3ab..3e4a502 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/BaseSchedulerCoreImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/BaseSchedulerCoreImplTest.java
@@ -224,19 +224,13 @@ public abstract class BaseSchedulerCoreImplTest extends EasyMockTest {
assertFalse(state.getAssignedTask().isSetSlaveId());
assertEquals(
validateAndPopulate(job.getJobConfig()).getTaskConfig(),
- ConfigurationManager.scrubNonUniqueTaskFields(state.getAssignedTask().getTask()));
+ state.getAssignedTask().getTask());
}
Set<Integer> expectedInstanceIds =
ContiguousSet.create(Range.closedOpen(0, numTasks), DiscreteDomain.integers());
assertEquals(
expectedInstanceIds,
FluentIterable.from(tasks).transform(Tasks.SCHEDULED_TO_INSTANCE_ID).toSet());
- assertEquals(
- expectedInstanceIds,
- FluentIterable.from(tasks)
- .transform(Tasks.SCHEDULED_TO_INFO)
- .transform(Tasks.INFO_TO_INSTANCE_ID_DEPRECATED)
- .toSet());
}
private static Constraint dedicatedConstraint(Set<String> values) {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
index 2e45de8..664ec64 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
@@ -42,14 +42,12 @@ import com.twitter.aurora.gen.TaskConfig;
import com.twitter.aurora.gen.TaskEvent;
import com.twitter.aurora.scheduler.Driver;
import com.twitter.aurora.scheduler.TaskIdGenerator;
-import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
import com.twitter.aurora.scheduler.events.PubsubEvent;
import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
import com.twitter.aurora.scheduler.storage.mem.MemStorage;
@@ -75,7 +73,6 @@ public class StateManagerImplTest extends EasyMockTest {
private static final String HOST_A = "host_a";
private static final Identity JIM = new Identity("jim", "jim-user");
private static final String MY_JOB = "myJob";
- private static final IJobKey JOB_KEY = JobKeys.from(JIM.getRole(), DEFAULT_ENVIRONMENT, MY_JOB);
private Driver driver;
private TaskIdGenerator taskIdGenerator;
@@ -180,7 +177,7 @@ public class StateManagerImplTest extends EasyMockTest {
.setAssignedTask(new AssignedTask()
.setInstanceId(3)
.setTaskId(taskId)
- .setTask(task.newBuilder().setInstanceIdDEPRECATED(3)));
+ .setTask(task.newBuilder()));
assertEquals(ImmutableSet.of(IScheduledTask.build(expected)),
Storage.Util.consistentFetchTasks(storage, Query.taskScoped(taskId)));
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/test/java/com/twitter/aurora/scheduler/storage/StorageBackfillTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/storage/StorageBackfillTest.java b/src/test/java/com/twitter/aurora/scheduler/storage/StorageBackfillTest.java
index 647f89e..56f96b5 100644
--- a/src/test/java/com/twitter/aurora/scheduler/storage/StorageBackfillTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/storage/StorageBackfillTest.java
@@ -1,6 +1,5 @@
package com.twitter.aurora.scheduler.storage;
-import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@@ -36,10 +35,7 @@ public class StorageBackfillTest {
clock = new FakeClock();
}
- private static IScheduledTask makeTask(
- String id,
- Optional<Integer> oldInstanceId,
- Optional<Integer> newInstanceId) {
+ private static IScheduledTask makeTask(String id, int instanceId) {
TaskConfig config = new TaskConfig()
.setOwner(new Identity("user", "role"))
@@ -54,43 +50,14 @@ public class StorageBackfillTest {
ScheduledTask task = new ScheduledTask().setAssignedTask(
new AssignedTask().setTask(config));
task.getAssignedTask().setTaskId(id);
-
- if (oldInstanceId.isPresent()) {
- task.getAssignedTask().getTask().setInstanceIdDEPRECATED(oldInstanceId.get());
- }
- if (newInstanceId.isPresent()) {
- task.getAssignedTask().setInstanceId(newInstanceId.get());
- }
+ task.getAssignedTask().setInstanceId(instanceId);
return IScheduledTask.build(task);
}
@Test
- public void testInstanceIdBackfill() {
- final String bothId = "both";
- final String oldId = "old";
- final String newId = "new";
- final IScheduledTask bothFields = makeTask(bothId, Optional.of(5), Optional.of(5));
- final IScheduledTask oldField = makeTask(oldId, Optional.of(6), Optional.<Integer>absent());
- final IScheduledTask newField = makeTask(newId, Optional.<Integer>absent(), Optional.of(7));
-
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- storeProvider.getUnsafeTaskStore().saveTasks(
- ImmutableSet.of(bothFields, oldField, newField));
- StorageBackfill.backfill(storeProvider, clock);
- }
- });
-
- assertEquals(makeTask(bothId, Optional.of(5), Optional.of(5)), getTask(bothId));
- assertEquals(makeTask(oldId, Optional.of(6), Optional.of(6)), getTask(oldId));
- assertEquals(makeTask(newId, Optional.of(7), Optional.of(7)), getTask(newId));
- }
-
- @Test
public void testRewriteThrottledState() {
final IScheduledTask savedTask =
- IScheduledTask.build(makeTask("id", Optional.<Integer>absent(), Optional.of(0)).newBuilder()
- .setStatus(ScheduleStatus.THROTTLED));
+ IScheduledTask.build(makeTask("id", 0).newBuilder().setStatus(ScheduleStatus.THROTTLED));
storage.write(new MutateWork.NoResult.Quiet() {
@Override protected void execute(MutableStoreProvider storeProvider) {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/87afc2c5/src/test/python/twitter/aurora/client/api/test_updater.py
----------------------------------------------------------------------
diff --git a/src/test/python/twitter/aurora/client/api/test_updater.py b/src/test/python/twitter/aurora/client/api/test_updater.py
index f98dc0c..3c148d3 100644
--- a/src/test/python/twitter/aurora/client/api/test_updater.py
+++ b/src/test/python/twitter/aurora/client/api/test_updater.py
@@ -403,19 +403,6 @@ class UpdaterTest(TestCase):
self.expect_finish()
self.replay_mocks()
- def test_noop_update_with_deprecated_fields(self):
- """Deprecated fields in task config do not affect diffing."""
- old_configs = self.make_task_configs(5)
- new_config = old_configs[0]
- old_configs[1].instanceIdDEPRECATED = 7
- job_config = self.make_job_config(new_config, 5)
- self._config.job_config = job_config
- self.expect_start()
- self.expect_get_tasks(old_configs)
- self.expect_populate(job_config)
- self.expect_finish()
- self.replay_mocks()
-
def test_update_rollback(self):
"""Update process failures exceed total allowable count and update is rolled back."""
update_config = self.UPDATE_CONFIG.copy()
[3/9] git commit: Bumping up twitter-aurora version to 0.0.11
Posted by ke...@apache.org.
Bumping up twitter-aurora version to 0.0.11
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/6436a932
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/6436a932
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/6436a932
Branch: refs/heads/master
Commit: 6436a9321caa69c5b79d8b521019b270fc8d3315
Parents: 87afc2c
Author: Maxim Khutornenko <mk...@twitter.com>
Authored: Wed Dec 4 11:23:36 2013 -0800
Committer: Maxim Khutornenko <mk...@twitter.com>
Committed: Wed Dec 4 11:23:36 2013 -0800
----------------------------------------------------------------------
build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/6436a932/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index b39d837..366512d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -31,7 +31,7 @@ publishing {
maven(MavenPublication) {
groupId 'com.twitter.aurora'
artifactId 'twitter-aurora'
- version '0.0.10'
+ version '0.0.11'
from components.java
[6/9] git commit: Upgrade Thrift to 0.9.1
Posted by ke...@apache.org.
Upgrade Thrift to 0.9.1
This upgrades the Java side of Aurora to libthrift 0.9.1 and the thrift compiler 0.9.1.
This change covers the following:
1. Upgrades the compiler and runtime.
2. Fixes the Aurora source code to use the new way of accessing Thrift
Constants.
3. com.twitter.common.ThriftServer was copied into the source tree and modified
to deal with the breakage caused by
https://issues.apache.org/jira/browse/THRIFT-745
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/e7b3da9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/e7b3da9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/e7b3da9c
Branch: refs/heads/master
Commit: e7b3da9c47390ed40aac8a5b17aef6b5a6404274
Parents: aa9f826
Author: Zameer Manji <zm...@twitter.com>
Authored: Thu Dec 5 10:13:23 2013 -0800
Committer: Zameer Manji <zm...@twitter.com>
Committed: Thu Dec 5 10:13:23 2013 -0800
----------------------------------------------------------------------
build-support/thrift/Makefile | 8 +-
build.gradle | 10 +-
.../twitter/aurora/scheduler/base/Tasks.java | 6 +-
.../configuration/ConfigurationManager.java | 4 +-
.../scheduler/cron/testing/AbstractCronIT.java | 2 +-
.../aurora/scheduler/http/SchedulerzJob.java | 4 +-
.../aurora/scheduler/http/SchedulerzRole.java | 6 +-
.../aurora/scheduler/storage/log/Entries.java | 2 +-
.../scheduler/storage/log/LogManager.java | 4 +-
.../storage/log/SnapshotStoreImpl.java | 2 +-
.../storage/log/testing/LogOpMatcher.java | 4 +-
.../scheduler/thrift/SchedulerAPIServlet.java | 2 +-
.../thrift/SchedulerThriftInterface.java | 2 +-
.../scheduler/thrift/SchedulerThriftServer.java | 28 -----
.../aurora/scheduler/thrift/ThriftModule.java | 3 +-
.../aurora/scheduler/thrift/ThriftServer.java | 107 +++++++++++++++++++
.../scheduler/thrift/ThriftServerLauncher.java | 14 +--
.../thrift/aop/APIVersionInterceptor.java | 2 +-
.../aurora/scheduler/app/SchedulerIT.java | 4 +-
.../configuration/ConfigurationManagerTest.java | 6 +-
.../scheduler/state/CronJobManagerTest.java | 2 +-
.../scheduler/state/LockManagerImplTest.java | 2 +-
.../scheduler/state/StateManagerImplTest.java | 2 +-
.../scheduler/storage/backup/RecoveryTest.java | 2 +-
.../scheduler/storage/log/LogManagerTest.java | 4 +-
.../scheduler/storage/log/LogStorageTest.java | 4 +-
.../storage/log/SnapshotStoreImplTest.java | 2 +-
.../thrift/SchedulerThriftInterfaceTest.java | 2 +-
28 files changed, 160 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/build-support/thrift/Makefile
----------------------------------------------------------------------
diff --git a/build-support/thrift/Makefile b/build-support/thrift/Makefile
index 10f201f..56e28fd 100644
--- a/build-support/thrift/Makefile
+++ b/build-support/thrift/Makefile
@@ -11,9 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-THRIFT_VERSION = 0.5.0
+THRIFT_VERSION = 0.9.1
THRIFT_DISTNAME = thrift-$(THRIFT_VERSION)
-THRIFT_URL = http://archive.apache.org/dist/incubator/thrift/$(THRIFT_VERSION)-incubating/$(THRIFT_DISTNAME).tar.gz
+THRIFT_URL = http://www.us.apache.org/dist/thrift/$(THRIFT_VERSION)/$(THRIFT_DISTNAME).tar.gz
+#TODO(zmanji): Verify the MD5 of the downloaded file.
THRIFT = ./thrift-$(THRIFT_VERSION)/compiler/cpp/thrift
@@ -21,7 +22,8 @@ all: $(THRIFT)
$(THRIFT):
curl $(THRIFT_URL) | tar zxv
- (cd $(THRIFT_DISTNAME) && ./configure --without-php --without-php_extension --without-ruby --without-haskell)
+ (cd $(THRIFT_DISTNAME) && ./configure --without-php --without-php_extension --without-ruby --without-haskell \
+ --without-d --without-go --without-perl --without-python --without-erlang --without-java --without-c_glib)
(cd $(THRIFT_DISTNAME) && make -j4)
clean:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 98f7701..12ff86f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -78,6 +78,11 @@ sourceSets {
runtimeClasspath += sourceSets.generated.output
}
}
+/* A note on libthrift: All of com.twitter.common depends on libthrift 0.5.x. We depend on
+ libthrift 0.9.x. There are binary incompatibilities between the two versions and resolving
+ them involved forking com.twitter.common classes into Aurora to use the new libthrift API. Be
+ very cautious when either upgrading libthrift or com.twitter.common dependencies!!!
+ */
jar {
from sourceSets.generated.output
@@ -102,7 +107,7 @@ dependencies {
compile 'log4j:log4j:1.2.17'
compile 'org.antlr:stringtemplate:3.2.1'
compile 'org.apache.mesos:mesos:0.15.0-rc3'
- def thriftLib = 'org.apache.thrift:libthrift:0.5.0-1'
+ def thriftLib = 'org.apache.thrift:libthrift:0.9.1'
compile thriftLib
compile 'org.apache.zookeeper:zookeeper:3.3.4'
def slf4jRev = '1.6.1'
@@ -158,10 +163,9 @@ checkstyle {
sourceSets = [ sourceSets.main ]
}
-def thriftBinary = 'build-support/thrift/thrift-0.5.0/compiler/cpp/thrift'
+def thriftBinary = 'build-support/thrift/thrift-0.9.1/compiler/cpp/thrift'
task bootstrapThrift {
- // build-support/thrift/thrift-0.5.0/compiler/cpp/thrift
inputs.file file(thriftBinary)
outputs.dir file(thriftBinary)
exec {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java b/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
index 557ac3a..d98da3f 100644
--- a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
+++ b/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
@@ -30,9 +30,9 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Ordering;
-import com.twitter.aurora.gen.Constants;
import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.apiConstants;
import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
import com.twitter.aurora.scheduler.storage.entities.IJobKey;
import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
@@ -97,13 +97,13 @@ public final class Tasks {
* Different states that an active task may be in.
*/
public static final EnumSet<ScheduleStatus> ACTIVE_STATES =
- EnumSet.copyOf(Constants.ACTIVE_STATES);
+ EnumSet.copyOf(apiConstants.ACTIVE_STATES);
/**
* Terminal states, which a task should not move from.
*/
public static final Set<ScheduleStatus> TERMINAL_STATES =
- EnumSet.copyOf(Constants.TERMINAL_STATES);
+ EnumSet.copyOf(apiConstants.TERMINAL_STATES);
public static final Predicate<ITaskConfig> IS_PRODUCTION =
new Predicate<ITaskConfig>() {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
index 6d40d11..4839d0f 100644
--- a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
+++ b/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
@@ -47,8 +47,8 @@ import com.twitter.aurora.scheduler.storage.entities.IValueConstraint;
import com.twitter.common.base.Closure;
import com.twitter.common.base.MorePreconditions;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
-import static com.twitter.aurora.gen.Constants.GOOD_IDENTIFIER_PATTERN_JVM;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
+import static com.twitter.aurora.gen.apiConstants.GOOD_IDENTIFIER_PATTERN_JVM;
/**
* Manages translation from a string-mapped configuration to a concrete configuration type, and
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java b/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
index 4aa279d..6bfc909 100644
--- a/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
+++ b/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
@@ -25,7 +25,7 @@ import com.twitter.common.testing.easymock.EasyMockTest;
import static org.junit.Assert.assertTrue;
-import static com.twitter.aurora.gen.test.Constants.VALID_CRON_SCHEDULES;
+import static com.twitter.aurora.gen.test.testConstants.VALID_CRON_SCHEDULES;
/**
* Abstract test to verify conformance with the {@link CronScheduler} interface.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
index 9420533..12b0bec 100644
--- a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
+++ b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
@@ -50,8 +50,8 @@ import com.google.common.collect.Ordering;
import org.antlr.stringtemplate.StringTemplate;
-import com.twitter.aurora.gen.Constants;
import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.apiConstants;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
@@ -187,7 +187,7 @@ public class SchedulerzJob extends JerseyTemplateServlet {
};
Map<String, String> links = ImmutableMap.of();
- if (Constants.LIVE_STATES.contains(scheduledTask.getStatus())) {
+ if (apiConstants.LIVE_STATES.contains(scheduledTask.getStatus())) {
links =
ImmutableMap.copyOf(Maps.transformValues(task.getTask().getTaskLinks(), expander));
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
index 1dd20d7..756c672 100644
--- a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
+++ b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
@@ -49,9 +49,9 @@ import com.google.common.collect.Sets;
import org.antlr.stringtemplate.StringTemplate;
-import com.twitter.aurora.gen.Constants;
import com.twitter.aurora.gen.CronCollisionPolicy;
import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.apiConstants;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
@@ -82,8 +82,8 @@ import static com.twitter.common.base.MorePreconditions.checkNotBlank;
public class SchedulerzRole extends JerseyTemplateServlet {
private static final List<ScheduleStatus> STATUSES = ImmutableList.<ScheduleStatus>builder()
- .addAll(Constants.TERMINAL_STATES)
- .addAll(Constants.ACTIVE_STATES)
+ .addAll(apiConstants.TERMINAL_STATES)
+ .addAll(apiConstants.ACTIVE_STATES)
.build();
// The freshest task is the latest active task
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
index 108f728..74e8c07 100644
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
+++ b/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
@@ -99,7 +99,7 @@ final class Entries {
Preconditions.checkArgument(entry.isSet(_Fields.DEFLATED_ENTRY));
ByteArrayOutputStream inflated = new ByteArrayOutputStream();
- ByteBuffer data = entry.getDeflatedEntry();
+ ByteBuffer data = entry.bufferForDeflatedEntry();
LOG.info("Inflating deflated log entry of size " + data.remaining());
InflaterInputStream inflater = new InflaterInputStream(
new ByteArrayInputStream(data.array(), data.position(), data.remaining()));
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
index 22a2e42..da29401 100644
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
+++ b/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
@@ -44,7 +44,6 @@ import com.google.inject.BindingAnnotation;
import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.storage.Constants;
import com.twitter.aurora.gen.storage.Frame;
import com.twitter.aurora.gen.storage.FrameChunk;
import com.twitter.aurora.gen.storage.FrameHeader;
@@ -56,6 +55,7 @@ import com.twitter.aurora.gen.storage.SaveHostAttributes;
import com.twitter.aurora.gen.storage.SaveTasks;
import com.twitter.aurora.gen.storage.Snapshot;
import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
import com.twitter.aurora.scheduler.log.Log;
import com.twitter.aurora.scheduler.log.Log.Entry;
import com.twitter.aurora.scheduler.log.Log.Position;
@@ -384,7 +384,7 @@ public final class LogManager {
*/
final class StreamTransaction {
private final Transaction transaction =
- new Transaction().setSchemaVersion(Constants.CURRENT_SCHEMA_VERSION);
+ new Transaction().setSchemaVersion(storageConstants.CURRENT_SCHEMA_VERSION);
private final AtomicBoolean committed = new AtomicBoolean(false);
private StreamTransaction() {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
index 1b449b6..df6b899 100644
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
+++ b/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
@@ -48,7 +48,7 @@ import com.twitter.common.util.Clock;
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.twitter.aurora.gen.Constants.CURRENT_API_VERSION;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
/**
* Snapshot store implementation that delegates to underlying snapshot stores by
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
index 66e9373..a4c0126 100644
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
+++ b/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
@@ -24,11 +24,11 @@ import org.easymock.IExpectationSetters;
import com.twitter.aurora.codec.ThriftBinaryCodec;
import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.Constants;
import com.twitter.aurora.gen.storage.LogEntry;
import com.twitter.aurora.gen.storage.Op;
import com.twitter.aurora.gen.storage.Snapshot;
import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
import com.twitter.aurora.scheduler.log.Log.Position;
import com.twitter.aurora.scheduler.log.Log.Stream;
@@ -84,7 +84,7 @@ public class LogOpMatcher implements IArgumentMatcher {
*/
public IExpectationSetters<Position> expectTransaction(Op...ops) {
LogEntry entry = LogEntry.transaction(
- new Transaction(ImmutableList.copyOf(ops), Constants.CURRENT_SCHEMA_VERSION));
+ new Transaction(ImmutableList.copyOf(ops), storageConstants.CURRENT_SCHEMA_VERSION));
return expect(stream.append(sameEntry(entry)));
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
index 83dbf26..2acf5c8 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
@@ -14,6 +14,6 @@ class SchedulerAPIServlet extends TServlet {
@Inject
SchedulerAPIServlet(AuroraAdmin.Iface schedulerThriftInterface) {
- super(new AuroraAdmin.Processor(schedulerThriftInterface), new TJSONProtocol.Factory());
+ super(new AuroraAdmin.Processor<>(schedulerThriftInterface), new TJSONProtocol.Factory());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
index 5237a6e..b8ba3f2 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -123,11 +123,11 @@ import com.twitter.common.util.BackoffHelper;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.twitter.aurora.auth.SessionValidator.SessionContext;
-import static com.twitter.aurora.gen.Constants.CURRENT_API_VERSION;
import static com.twitter.aurora.gen.ResponseCode.AUTH_FAILED;
import static com.twitter.aurora.gen.ResponseCode.ERROR;
import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
import static com.twitter.aurora.gen.ResponseCode.OK;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
import static com.twitter.common.base.MorePreconditions.checkNotBlank;
/**
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftServer.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftServer.java
deleted file mode 100644
index 2538254..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftServer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import com.twitter.common.thrift.ThriftServer;
-
-/**
- * Thin implementation of ThriftServer.
- */
-class SchedulerThriftServer extends ThriftServer {
-
- SchedulerThriftServer() {
- super("TwitterMesosScheduler", "1");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
index a1ec633..cca9053 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
@@ -23,7 +23,6 @@ import com.twitter.aurora.gen.AuroraAdmin;
import com.twitter.aurora.scheduler.thrift.aop.AopModule;
import com.twitter.common.application.http.Registration;
import com.twitter.common.application.modules.LifecycleModule;
-import com.twitter.common.thrift.ThriftServer;
/**
* Binding module to configure a thrift server.
@@ -33,7 +32,7 @@ public class ThriftModule extends AbstractModule {
@Override
protected void configure() {
bind(AuroraAdmin.Iface.class).to(SchedulerThriftInterface.class);
- bind(ThriftServer.class).to(SchedulerThriftServer.class).in(Singleton.class);
+ bind(ThriftServer.class).in(Singleton.class);
LifecycleModule.bindServiceRunner(binder(), ThriftServerLauncher.class);
Registration.registerServlet(binder(), "/api", SchedulerAPIServlet.class, true);
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
new file mode 100644
index 0000000..7b9abd1
--- /dev/null
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import java.net.ServerSocket;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+
+import com.twitter.thrift.Status;
+
+class ThriftServer {
+ private static final Logger LOG = Logger.getLogger(ThriftServer.class.getName());
+
+ private TServer server = null;
+
+ // Current health status of the server.
+ private Status status = Status.STARTING;
+
+ /**
+ * Starts the server.
+ * This may be called at any point except when the server is already alive. That is, it's
+ * allowable to start, stop, and re-start the server.
+ *
+ * @param socket The socket to use.
+ * @param processor The processor to handle requests.
+ */
+ public synchronized void start(ServerSocket socket, TProcessor processor) {
+ Preconditions.checkNotNull(socket);
+ Preconditions.checkNotNull(processor);
+ Preconditions.checkState(status != Status.ALIVE, "Server must only be started once.");
+ setStatus(Status.ALIVE);
+ TThreadPoolServer.Args args = new TThreadPoolServer.Args(new TServerSocket(socket))
+ .processor(processor)
+ .protocolFactory(new TBinaryProtocol.Factory(false, true));
+
+ final TServer starting = new TThreadPoolServer(args);
+ server = starting;
+ LOG.info("Starting thrift server on port " + socket.getLocalPort());
+
+ Thread listeningThread = new ThreadFactoryBuilder().setDaemon(false).build()
+ .newThread(new Runnable() {
+ @Override public void run() {
+ try {
+ starting.serve();
+ } catch (Throwable t) {
+ LOG.log(Level.WARNING,
+ "Uncaught exception while attempting to handle service requests: " + t, t);
+ setStatus(Status.DEAD);
+ }
+ }
+ });
+
+ listeningThread.start();
+ }
+
+ private synchronized void setStatus(Status status) {
+ LOG.info("Moving from status " + this.status + " to " + status);
+ this.status = status;
+ }
+
+ /**
+ * Attempts to shut down the server.
+ * The server may be shut down at any time, though the request will be ignored if the server is
+ * already stopped.
+ */
+ public synchronized void shutdown() {
+ if (status == Status.STOPPED) {
+ LOG.info("Server already stopped, shutdown request ignored.");
+ return;
+ }
+
+ LOG.info("Received shutdown request, stopping server.");
+ setStatus(Status.STOPPING);
+
+ // TODO(William Farner): Figure out what happens to queued / in-process requests when the server
+ // is stopped. Might want to allow a sleep period for the active requests to be completed.
+
+ if (server != null) {
+ server.stop();
+ }
+
+ server = null;
+ setStatus(Status.STOPPED);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
index 500508a..6743060 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
@@ -35,8 +35,6 @@ import com.twitter.aurora.gen.AuroraAdmin.Iface;
import com.twitter.common.application.modules.LifecycleModule.ServiceRunner;
import com.twitter.common.application.modules.LocalServiceRegistry.LocalService;
import com.twitter.common.base.Command;
-import com.twitter.common.thrift.ThriftServer;
-import com.twitter.common.thrift.ThriftServer.ServerSetup;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -69,12 +67,10 @@ class ThriftServerLauncher implements ServiceRunner {
@Override
public LocalService launch() {
- ServerSetup setup = new ServerSetup(
- 0, // TODO(John Sirois): unused, fix ServerSetup constructors
- new AuroraAdmin.Processor(schedulerThriftInterface),
- ThriftServer.BINARY_PROTOCOL.get());
- setup.setSocket(getServerSocket());
- schedulerThriftServer.start(setup);
+ ServerSocket socket = getServerSocket();
+ schedulerThriftServer.start(
+ socket,
+ new AuroraAdmin.Processor<>(schedulerThriftInterface));
Command shutdown = new Command() {
@Override public void execute() {
@@ -83,7 +79,7 @@ class ThriftServerLauncher implements ServiceRunner {
}
};
- return LocalService.primaryService(schedulerThriftServer.getListeningPort(), shutdown);
+ return LocalService.primaryService(socket.getLocalPort(), shutdown);
}
private ServerSocket getServerSocket() {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
index f0fdf47..d66a2b2 100644
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
+++ b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
@@ -5,7 +5,7 @@ import org.aopalliance.intercept.MethodInvocation;
import com.twitter.aurora.gen.Response;
-import static com.twitter.aurora.gen.Constants.CURRENT_API_VERSION;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
class APIVersionInterceptor implements MethodInterceptor {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/app/SchedulerIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/app/SchedulerIT.java b/src/test/java/com/twitter/aurora/scheduler/app/SchedulerIT.java
index 98085d5..4c381b9 100644
--- a/src/test/java/com/twitter/aurora/scheduler/app/SchedulerIT.java
+++ b/src/test/java/com/twitter/aurora/scheduler/app/SchedulerIT.java
@@ -63,13 +63,13 @@ import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.gen.ScheduledTask;
import com.twitter.aurora.gen.TaskConfig;
import com.twitter.aurora.gen.TaskEvent;
-import com.twitter.aurora.gen.storage.Constants;
import com.twitter.aurora.gen.storage.LogEntry;
import com.twitter.aurora.gen.storage.Op;
import com.twitter.aurora.gen.storage.SaveFrameworkId;
import com.twitter.aurora.gen.storage.SaveTasks;
import com.twitter.aurora.gen.storage.Snapshot;
import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
import com.twitter.aurora.scheduler.DriverFactory;
import com.twitter.aurora.scheduler.MesosTaskFactory.ExecutorConfig;
import com.twitter.aurora.scheduler.SchedulerLifecycle.ShutdownOnDriverExit;
@@ -316,7 +316,7 @@ public class SchedulerIT extends BaseZooKeeperTest {
LogEntry.snapshot(new Snapshot().setTasks(ImmutableSet.of(snapshotTask))),
LogEntry.transaction(new Transaction(
ImmutableList.of(Op.saveTasks(new SaveTasks(ImmutableSet.of(transactionTask)))),
- Constants.CURRENT_SCHEMA_VERSION)));
+ storageConstants.CURRENT_SCHEMA_VERSION)));
expect(log.open()).andReturn(logStream);
expect(logStream.readAll()).andReturn(recoveredEntries.iterator()).anyTimes();
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/configuration/ConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/configuration/ConfigurationManagerTest.java b/src/test/java/com/twitter/aurora/scheduler/configuration/ConfigurationManagerTest.java
index 688d106..ec9c21c 100644
--- a/src/test/java/com/twitter/aurora/scheduler/configuration/ConfigurationManagerTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/configuration/ConfigurationManagerTest.java
@@ -37,9 +37,9 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
-import static com.twitter.aurora.gen.test.Constants.INVALID_IDENTIFIERS;
-import static com.twitter.aurora.gen.test.Constants.VALID_IDENTIFIERS;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
+import static com.twitter.aurora.gen.test.testConstants.INVALID_IDENTIFIERS;
+import static com.twitter.aurora.gen.test.testConstants.VALID_IDENTIFIERS;
import static com.twitter.aurora.scheduler.configuration.ConfigurationManager.isGoodIdentifier;
// TODO(Sathya): Improve test coverage for this class.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/state/CronJobManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/CronJobManagerTest.java b/src/test/java/com/twitter/aurora/scheduler/state/CronJobManagerTest.java
index 4405cd9..b88ccff 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/CronJobManagerTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/CronJobManagerTest.java
@@ -62,8 +62,8 @@ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.junit.Assert.fail;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
import static com.twitter.aurora.gen.CronCollisionPolicy.RUN_OVERLAP;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
public class CronJobManagerTest extends EasyMockTest {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
index 64d9b19..cc6f3b7 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/LockManagerImplTest.java
@@ -39,7 +39,7 @@ import com.twitter.common.util.testing.FakeClock;
import static org.junit.Assert.assertEquals;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
public class LockManagerImplTest extends EasyMockTest {
private static final String USER = "jim-user";
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
index 664ec64..7de377c 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/StateManagerImplTest.java
@@ -60,13 +60,13 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
import static com.twitter.aurora.gen.ScheduleStatus.ASSIGNED;
import static com.twitter.aurora.gen.ScheduleStatus.INIT;
import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
import static com.twitter.aurora.gen.ScheduleStatus.UNKNOWN;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
public class StateManagerImplTest extends EasyMockTest {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/storage/backup/RecoveryTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/storage/backup/RecoveryTest.java b/src/test/java/com/twitter/aurora/scheduler/storage/backup/RecoveryTest.java
index e1c913d..ded3a3f 100644
--- a/src/test/java/com/twitter/aurora/scheduler/storage/backup/RecoveryTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/storage/backup/RecoveryTest.java
@@ -58,7 +58,7 @@ import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
-import static com.twitter.aurora.gen.Constants.CURRENT_API_VERSION;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
public class RecoveryTest extends EasyMockTest {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/storage/log/LogManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/storage/log/LogManagerTest.java b/src/test/java/com/twitter/aurora/scheduler/storage/log/LogManagerTest.java
index c1ce79b..d14fb4b 100644
--- a/src/test/java/com/twitter/aurora/scheduler/storage/log/LogManagerTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/storage/log/LogManagerTest.java
@@ -44,7 +44,6 @@ import com.twitter.aurora.gen.HostAttributes;
import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.gen.ScheduledTask;
import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.storage.Constants;
import com.twitter.aurora.gen.storage.Frame;
import com.twitter.aurora.gen.storage.FrameChunk;
import com.twitter.aurora.gen.storage.FrameHeader;
@@ -56,6 +55,7 @@ import com.twitter.aurora.gen.storage.SaveFrameworkId;
import com.twitter.aurora.gen.storage.SaveTasks;
import com.twitter.aurora.gen.storage.Snapshot;
import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.log.Log;
import com.twitter.aurora.scheduler.log.Log.Entry;
@@ -537,7 +537,7 @@ public class LogManagerTest extends EasyMockTest {
private LogEntry createLogEntry(Op... ops) {
return LogEntry.transaction(
- new Transaction(ImmutableList.copyOf(ops), Constants.CURRENT_SCHEMA_VERSION));
+ new Transaction(ImmutableList.copyOf(ops), storageConstants.CURRENT_SCHEMA_VERSION));
}
private void expectAppend(Position position, LogEntry logEntry) throws CodingException {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/storage/log/LogStorageTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/com/twitter/aurora/scheduler/storage/log/LogStorageTest.java
index caedab4..e2d1836 100644
--- a/src/test/java/com/twitter/aurora/scheduler/storage/log/LogStorageTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/storage/log/LogStorageTest.java
@@ -46,7 +46,6 @@ import com.twitter.aurora.gen.Quota;
import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.gen.ScheduledTask;
import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.storage.Constants;
import com.twitter.aurora.gen.storage.LogEntry;
import com.twitter.aurora.gen.storage.Op;
import com.twitter.aurora.gen.storage.RemoveJob;
@@ -62,6 +61,7 @@ import com.twitter.aurora.gen.storage.SaveQuota;
import com.twitter.aurora.gen.storage.SaveTasks;
import com.twitter.aurora.gen.storage.Snapshot;
import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
@@ -676,7 +676,7 @@ public class LogStorageTest extends EasyMockTest {
private LogEntry createTransaction(Op... ops) {
return LogEntry.transaction(
- new Transaction(ImmutableList.copyOf(ops), Constants.CURRENT_SCHEMA_VERSION));
+ new Transaction(ImmutableList.copyOf(ops), storageConstants.CURRENT_SCHEMA_VERSION));
}
private static IScheduledTask task(String id, ScheduleStatus status) {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImplTest.java b/src/test/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImplTest.java
index 85358f1..5062795 100644
--- a/src/test/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImplTest.java
@@ -49,7 +49,7 @@ import com.twitter.common.util.testing.FakeClock;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
-import static com.twitter.aurora.gen.Constants.CURRENT_API_VERSION;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
public class SnapshotStoreImplTest extends EasyMockTest {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/e7b3da9c/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java b/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
index 1f3cd6e..30d3947 100644
--- a/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
@@ -99,7 +99,6 @@ import static org.junit.Assert.assertTrue;
import static com.twitter.aurora.auth.CapabilityValidator.Capability.ROOT;
import static com.twitter.aurora.auth.SessionValidator.SessionContext;
-import static com.twitter.aurora.gen.Constants.DEFAULT_ENVIRONMENT;
import static com.twitter.aurora.gen.LockValidation.CHECKED;
import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
import static com.twitter.aurora.gen.MaintenanceMode.NONE;
@@ -109,6 +108,7 @@ import static com.twitter.aurora.gen.ResponseCode.ERROR;
import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
import static com.twitter.aurora.gen.ResponseCode.OK;
import static com.twitter.aurora.gen.ResponseCode.WARNING;
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
import static com.twitter.aurora.scheduler.configuration.ConfigurationManager.DEDICATED_ATTRIBUTE;
import static com.twitter.aurora.scheduler.thrift.SchedulerThriftInterface.transitionMessage;
[9/9] git commit: Document ASF review process and tooling.
Posted by ke...@apache.org.
Document ASF review process and tooling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/d51fecab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/d51fecab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/d51fecab
Branch: refs/heads/master
Commit: d51fecab9eb75a92e1b774fff31f746b55111c82
Parents: b4da3c3
Author: Kevin Sweeney <ke...@apache.org>
Authored: Thu Dec 5 11:54:13 2013 -0800
Committer: Kevin Sweeney <ke...@apache.org>
Committed: Thu Dec 5 11:54:13 2013 -0800
----------------------------------------------------------------------
.gitignore | 3 ++-
.reviewboardrc | 3 +++
build-support/virtualenv | 25 ++++++++++++++++++++++++
docs/contributing.md | 45 +++++++++++++++++++++++++++++++++++++++++++
pants | 27 +++-----------------------
rbt | 12 ++++++++++++
6 files changed, 90 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 3731eeb..bf8c4e5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,9 +6,10 @@
.coverage
.gradle/
.pants.*
-.reviewboardrc
.vagrant/
build/
+build-support/rbtools/
+build-support/virtualenv-*
dist/
gradle-app.setting
out/
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/.reviewboardrc
----------------------------------------------------------------------
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 0000000..a66e6fd
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,3 @@
+REPOSITORY = "aurora"
+REVIEWBOARD_URL = "https://reviews.apache.org"
+TARGET_GROUPS = "Aurora"
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/build-support/virtualenv
----------------------------------------------------------------------
diff --git a/build-support/virtualenv b/build-support/virtualenv
new file mode 100755
index 0000000..b44991a
--- /dev/null
+++ b/build-support/virtualenv
@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+# Wrapper for self-bootstrapping virtualenv
+VIRTUALENV_VERSION=1.10.1
+
+if which python2.7 >/dev/null; then
+ PY=`which python2.7`
+elif which python2.6 >/dev/null; then
+ PY=`which python2.6`
+else
+ echo 'No python interpreter found on the path. Python will not work!' 1>&2
+ exit 1
+fi
+
+echo "Using $PY" >&2
+
+if ! [ -f "build-support/virtualenv-$VIRTUALENV_VERSION/BOOTSTRAPPED" ]; then
+ pushd build-support
+ curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-$VIRTUALENV_VERSION.tar.gz
+ tar zxvf virtualenv-$VIRTUALENV_VERSION.tar.gz
+ # TODO(ksweeney): Checksum
+ touch virtualenv-$VIRTUALENV_VERSION/BOOTSTRAPPED # 2PC
+ popd
+fi
+
+exec "$PY" build-support/virtualenv-$VIRTUALENV_VERSION/virtualenv.py "$@"
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/docs/contributing.md
----------------------------------------------------------------------
diff --git a/docs/contributing.md b/docs/contributing.md
new file mode 100644
index 0000000..cac4429
--- /dev/null
+++ b/docs/contributing.md
@@ -0,0 +1,45 @@
+Getting your ReviewBoard Account
+--------------------------------
+Go to https://reviews.apache.org and create an account.
+
+Setting up your ReviewBoard Environment
+---------------------------------------
+Run `./rbt status`. The first time this runs it will bootstrap and you will be asked to login.
+Subsequent runs will cache your login credentials.
+
+Submitting a Patch for Review
+-----------------------------
+Post a review with `rbt`, fill out the fields in your browser and hit Publish.
+
+ ./rbt post -o -g
+
+Updating an Existing Review
+---------------------------
+Incorporate review feedback, make some more commits, update your existing review, fill out the
+fields in your browser and hit Publish.
+
+ ./rbt post -o -r <RB_ID>
+
+Merging Your Own Review (Committers)
+------------------------------------
+Once you have shipits from the right committers, merge your changes in a single squash commit
+and mark the review as submitted. The typical workflow is
+
+ git checkout master
+ git pull origin master
+ git merge --squash my_feature_branch
+ git commit
+ git show master # Verify everything looks sane
+ git push origin master
+ ./rbt close <RB_ID>
+ git branch -d my_feature_branch
+
+Merging Someone Else's Review
+-----------------------------
+Sometimes you'll need to merge someone else's RB. The typical workflow for this is
+
+ git checkout master
+ git pull origin master
+ ./rbt patch -c <RB_ID>
+ git show master # Verify everything looks sane, author is correct
+ git push origin master
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/pants
----------------------------------------------------------------------
diff --git a/pants b/pants
index c0d0cab..2772c64 100755
--- a/pants
+++ b/pants
@@ -2,7 +2,6 @@
PANTS_ROOT=$(dirname "$0")
PANTS_VERSION=$(cat "$PANTS_ROOT/.pantsversion")
-VIRTUALENV_VERSION=1.10.1
if [[ -e "$PANTS_ROOT/pants.pex" ]]; then
cat <<EOF | PANTS_ROOT="$PANTS_ROOT" PANTS_VERSION=$PANTS_VERSION python
@@ -31,34 +30,14 @@ fi
PANTS_TEMP=$(mktemp -d /tmp/pants.XXXXXX)
-if which python2.7 >/dev/null; then
- PY=$(which python2.7)
-elif which python2.6 >/dev/null; then
- PY=$(which python2.6)
-elif which python3 >/dev/null; then
- PY=$(which python3)
-else
- echo 'No python interpreter found on the path. Python will not work!' 1>&2
- exit 1
-fi
-
-echo 'Using '$PY 1>&2
-
+./build-support/virtualenv "$PANTS_TEMP/tcp"
pushd "$PANTS_TEMP"
-curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-$VIRTUALENV_VERSION.tar.gz
-tar -zxf virtualenv-$VIRTUALENV_VERSION.tar.gz
-$PY virtualenv-$VIRTUALENV_VERSION/virtualenv.py tcp
source tcp/bin/activate
pip install twitter.common.python==0.2.0
-cat <<EOF | \
- PYTHON_VERBOSE=1 \
- PEX_VERBOSE=1 \
- PANTS_VERSION=$PANTS_VERSION python
-import os
-
+PYTHON_VERBOSE=1 PEX_VERBOSE=1 python <<EOF
from twitter.common.python.fetcher import Fetcher, PyPIFetcher
from twitter.common.python.http import Crawler
from twitter.common.python.obtainer import Obtainer
@@ -68,7 +47,7 @@ from twitter.common.python.translator import Translator
pb = PEXBuilder()
resolver = Resolver(crawler=Crawler(), fetchers=[PyPIFetcher()])
-req = 'twitter.pants==' + os.environ['PANTS_VERSION']
+req = 'twitter.pants==$PANTS_VERSION'
pb.add_requirement(req)
for dist in resolver.resolve(req):
pb.add_distribution(dist)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d51fecab/rbt
----------------------------------------------------------------------
diff --git a/rbt b/rbt
new file mode 100755
index 0000000..8f01ff9
--- /dev/null
+++ b/rbt
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+# Wrapper script for self-bootstrapping rbt.
+# TODO(ksweeney): Use ./pants py here instead of virtualenv.
+if ! [ -f build-support/rbtools/BOOTSTRAPPED ]; then
+ echo Bootstrapping rbtools
+ ./build-support/virtualenv build-support/rbtools
+ source build-support/rbtools/bin/activate
+ pip install RBTools==0.5
+ touch build-support/rbtools/BOOTSTRAPPED
+fi
+source build-support/rbtools/bin/activate
+exec rbt "$@"
[5/9] git commit: Adding random jitter on pending task rescheduling
after startup
Posted by ke...@apache.org.
Adding random jitter on pending task rescheduling after startup
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/aa9f826f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/aa9f826f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/aa9f826f
Branch: refs/heads/master
Commit: aa9f826f297e1aa7f92abb8c3972d3d6c0d0e5ea
Parents: ecda2b3
Author: Maxim Khutornenko <mk...@twitter.com>
Authored: Thu Dec 5 08:09:41 2013 -0800
Committer: Maxim Khutornenko <mk...@twitter.com>
Committed: Thu Dec 5 08:09:41 2013 -0800
----------------------------------------------------------------------
.../aurora/scheduler/async/AsyncModule.java | 23 ++-
.../scheduler/async/RescheduleCalculator.java | 188 +++++++++++++++++++
.../aurora/scheduler/async/TaskGroups.java | 162 +++-------------
.../scheduler/async/TaskSchedulerTest.java | 11 +-
4 files changed, 239 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/aa9f826f/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java b/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
index 37913a8..db07841 100644
--- a/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
+++ b/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
@@ -30,7 +30,9 @@ import com.google.inject.TypeLiteral;
import com.twitter.aurora.scheduler.async.OfferQueue.OfferQueueImpl;
import com.twitter.aurora.scheduler.async.OfferQueue.OfferReturnDelay;
+import com.twitter.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl;
import com.twitter.aurora.scheduler.async.TaskGroups.SchedulingAction;
+import com.twitter.aurora.scheduler.async.TaskGroups.TaskGroupsSettings;
import com.twitter.aurora.scheduler.events.PubsubEventModule;
import com.twitter.common.args.Arg;
import com.twitter.common.args.CmdLine;
@@ -45,8 +47,6 @@ import com.twitter.common.util.TruncatedBinaryBackoff;
import static com.twitter.aurora.scheduler.async.HistoryPruner.PruneThreshold;
import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl;
import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl.PreemptionDelay;
-import static com.twitter.aurora.scheduler.async.TaskGroups.FlappingTaskSettings;
-import static com.twitter.aurora.scheduler.async.TaskGroups.SchedulingSettings;
/**
* Binding module for async task management.
@@ -103,6 +103,11 @@ public class AsyncModule extends AbstractModule {
private static final Arg<Amount<Long, Time>> MAX_FLAPPING_DELAY =
Arg.create(Amount.of(5L, Time.MINUTES));
+ @CmdLine(name = "max_reschedule_task_delay_on_startup",
+ help = "Upper bound of random delay for pending task rescheduling on scheduler startup.")
+ private static final Arg<Amount<Integer, Time>> MAX_RESCHEDULING_DELAY =
+ Arg.create(Amount.of(30, Time.SECONDS));
+
@CmdLine(name = "preemption_delay",
help = "Time interval after which a pending task becomes eligible to preempt other tasks")
private static final Arg<Amount<Long, Time>> PREEMPTION_DELAY =
@@ -148,13 +153,17 @@ public class AsyncModule extends AbstractModule {
binder().install(new PrivateModule() {
@Override protected void configure() {
- bind(SchedulingSettings.class).toInstance(new SchedulingSettings(
+ bind(TaskGroupsSettings.class).toInstance(new TaskGroupsSettings(
new TruncatedBinaryBackoff(INITIAL_SCHEDULE_DELAY.get(), MAX_SCHEDULE_DELAY.get()),
RateLimiter.create(MAX_SCHEDULE_ATTEMPTS_PER_SEC.get())));
- bind(FlappingTaskSettings.class).toInstance(new FlappingTaskSettings(
- new TruncatedBinaryBackoff(INITIAL_FLAPPING_DELAY.get(), MAX_FLAPPING_DELAY.get()),
- FLAPPING_THRESHOLD.get()
- ));
+
+ bind(RescheduleCalculatorImpl.RescheduleCalculatorSettings.class)
+ .toInstance(new RescheduleCalculatorImpl.RescheduleCalculatorSettings(
+ new TruncatedBinaryBackoff(INITIAL_FLAPPING_DELAY.get(), MAX_FLAPPING_DELAY.get()),
+ FLAPPING_THRESHOLD.get(),
+ MAX_RESCHEDULING_DELAY.get()));
+
+ bind(RescheduleCalculator.class).to(RescheduleCalculatorImpl.class).in(Singleton.class);
bind(SchedulingAction.class).to(TaskScheduler.class);
bind(TaskScheduler.class).in(Singleton.class);
if (ENABLE_PREEMPTOR.get()) {
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/aa9f826f/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java b/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
new file mode 100644
index 0000000..eefc03a
--- /dev/null
+++ b/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.BackoffStrategy;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.Random;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
+import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
+
+/**
+ * Calculates scheduling delays for tasks.
+ */
+interface RescheduleCalculator {
+ /**
+ * Gets a timestamp for the task to become eligible for (re)scheduling at scheduler startup.
+ *
+ * @param task Task to calculate timestamp for.
+ * @return Timestamp in msec.
+ */
+ long getStartupReadyTimeMs(IScheduledTask task);
+
+ /**
+ * Gets a timestamp for the task to become eligible for (re)scheduling.
+ *
+ * @param task Task to calculate timestamp for.
+ * @return Timestamp in msec.
+ */
+ long getReadyTimeMs(IScheduledTask task);
+
+ class RescheduleCalculatorImpl implements RescheduleCalculator {
+
+ private static final Logger LOG = Logger.getLogger(TaskGroups.class.getName());
+
+ private final Storage storage;
+ private final RescheduleCalculatorSettings settings;
+ private final Clock clock;
+ private final Random random = new Random.SystemRandom(new java.util.Random());
+
+ private static final Predicate<ScheduleStatus> IS_ACTIVE_STATUS =
+ Predicates.in(Tasks.ACTIVE_STATES);
+
+ private static final Function<ITaskEvent, ScheduleStatus> TO_STATUS =
+ new Function<ITaskEvent, ScheduleStatus>() {
+ @Override public ScheduleStatus apply(ITaskEvent input) {
+ return input.getStatus();
+ }
+ };
+
+ private static final Set<ScheduleStatus> INTERRUPTED_TASK_STATES =
+ EnumSet.of(RESTARTING, KILLING);
+
+ private final Predicate<IScheduledTask> flapped = new Predicate<IScheduledTask>() {
+ @Override public boolean apply(IScheduledTask task) {
+ if (!task.isSetTaskEvents()) {
+ return false;
+ }
+
+ List<ITaskEvent> events = Lists.reverse(task.getTaskEvents());
+
+ // Avoid penalizing tasks that were interrupted by outside action, such as a user
+ // restarting them.
+ if (Iterables.any(Iterables.transform(events, TO_STATUS),
+ Predicates.in(INTERRUPTED_TASK_STATES))) {
+ return false;
+ }
+
+ ITaskEvent terminalEvent = Iterables.get(events, 0);
+ ScheduleStatus terminalState = terminalEvent.getStatus();
+ Preconditions.checkState(Tasks.isTerminated(terminalState));
+
+ ITaskEvent activeEvent =
+ Iterables.find(events, Predicates.compose(IS_ACTIVE_STATUS, TO_STATUS));
+
+ long thresholdMs = settings.flappingTaskThreashold.as(Time.MILLISECONDS);
+
+ return (terminalEvent.getTimestamp() - activeEvent.getTimestamp()) < thresholdMs;
+ }
+ };
+
+ static class RescheduleCalculatorSettings {
+ private final BackoffStrategy flappingTaskBackoff;
+ private final Amount<Long, Time> flappingTaskThreashold;
+ private final Amount<Integer, Time> maxStartupRescheduleDelay;
+
+ RescheduleCalculatorSettings(
+ BackoffStrategy flappingTaskBackoff,
+ Amount<Long, Time> flappingTaskThreashold,
+ Amount<Integer, Time> maxStartupRescheduleDelay) {
+
+ this.flappingTaskBackoff = checkNotNull(flappingTaskBackoff);
+ this.flappingTaskThreashold = checkNotNull(flappingTaskThreashold);
+ this.maxStartupRescheduleDelay = checkNotNull(maxStartupRescheduleDelay);
+ }
+ }
+
+ @Inject
+ RescheduleCalculatorImpl(
+ Storage storage,
+ RescheduleCalculatorSettings settings,
+ Clock clock) {
+
+ this.storage = checkNotNull(storage);
+ this.settings = checkNotNull(settings);
+ this.clock = checkNotNull(clock);
+ }
+
+ @Override
+ public long getStartupReadyTimeMs(IScheduledTask task) {
+ return random.nextInt(settings.maxStartupRescheduleDelay.as(Time.MILLISECONDS))
+ + getTaskReadyTimestamp(task);
+ }
+
+ @Override
+ public long getReadyTimeMs(IScheduledTask task) {
+ return getTaskReadyTimestamp(task);
+ }
+
+ private Optional<IScheduledTask> getTaskAncestor(IScheduledTask task) {
+ if (!task.isSetAncestorId()) {
+ return Optional.absent();
+ }
+
+ ImmutableSet<IScheduledTask> res =
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.taskScoped(task.getAncestorId()));
+
+ return Optional.fromNullable(Iterables.getOnlyElement(res, null));
+ }
+
+ private long getTaskReadyTimestamp(IScheduledTask task) {
+ Optional<IScheduledTask> curTask = getTaskAncestor(task);
+ long penaltyMs = 0;
+ while (curTask.isPresent() && flapped.apply(curTask.get())) {
+ LOG.info(
+ String.format("Ancestor of %s flapped: %s", Tasks.id(task), Tasks.id(curTask.get())));
+ long newPenalty = settings.flappingTaskBackoff.calculateBackoffMs(penaltyMs);
+ // If the backoff strategy is truncated then there is no need for us to continue.
+ if (newPenalty == penaltyMs) {
+ break;
+ }
+ penaltyMs = newPenalty;
+ curTask = getTaskAncestor(curTask.get());
+ }
+
+ return penaltyMs + clock.nowMillis();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/aa9f826f/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
index 9ea0229..f95f719 100644
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
+++ b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
@@ -15,9 +15,6 @@
*/
package com.twitter.aurora.scheduler.async;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -25,23 +22,16 @@ import java.util.logging.Logger;
import javax.inject.Inject;
-import com.google.common.base.Function;
import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
import com.google.common.eventbus.Subscribe;
import com.google.common.util.concurrent.RateLimiter;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.twitter.aurora.gen.ScheduleStatus;
import com.twitter.aurora.scheduler.base.JobKeys;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
@@ -53,7 +43,6 @@ import com.twitter.aurora.scheduler.storage.Storage;
import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
import com.twitter.common.application.ShutdownRegistry;
import com.twitter.common.base.Command;
import com.twitter.common.quantity.Amount;
@@ -65,9 +54,7 @@ import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
import static com.twitter.aurora.scheduler.async.TaskGroup.GroupState;
/**
@@ -85,51 +72,58 @@ public class TaskGroups implements EventSubscriber {
private final Storage storage;
private final LoadingCache<GroupKey, TaskGroup> groups;
- private final Amount<Long, Time> flappingThreshold;
- private final BackoffStrategy flappingBackoffStrategy;
private final Clock clock;
+ private final RescheduleCalculator rescheduleCalculator;
private final Preemptor preemptor;
+ static class TaskGroupsSettings {
+ private final BackoffStrategy taskGroupBackoff;
+ private final RateLimiter rateLimiter;
+
+ TaskGroupsSettings(BackoffStrategy taskGroupBackoff, RateLimiter rateLimiter) {
+ this.taskGroupBackoff = checkNotNull(taskGroupBackoff);
+ this.rateLimiter = checkNotNull(rateLimiter);
+ }
+ }
+
@Inject
TaskGroups(
ShutdownRegistry shutdownRegistry,
Storage storage,
- SchedulingSettings schedulingSettings,
+ TaskGroupsSettings settings,
SchedulingAction schedulingAction,
- FlappingTaskSettings flappingTaskSettings,
Clock clock,
+ RescheduleCalculator rescheduleCalculator,
Preemptor preemptor) {
this(
createThreadPool(shutdownRegistry),
storage,
- schedulingSettings.getBackoff(),
- schedulingSettings.getRateLimit(),
+ settings.taskGroupBackoff,
+ settings.rateLimiter,
schedulingAction,
- flappingTaskSettings.getFlappingThreashold(),
clock,
- flappingTaskSettings.getBackoff(),
+ rescheduleCalculator,
preemptor);
}
TaskGroups(
final ScheduledExecutorService executor,
final Storage storage,
- final BackoffStrategy backoffStrategy,
+ final BackoffStrategy taskGroupBackoffStrategy,
final RateLimiter rateLimiter,
final SchedulingAction schedulingAction,
- final Amount<Long, Time> flappingThreshold,
final Clock clock,
- final BackoffStrategy flappingBackoffStrategy,
+ final RescheduleCalculator rescheduleCalculator,
final Preemptor preemptor) {
this.storage = checkNotNull(storage);
checkNotNull(executor);
- checkNotNull(backoffStrategy);
+ checkNotNull(taskGroupBackoffStrategy);
+ checkNotNull(rateLimiter);
checkNotNull(schedulingAction);
- this.flappingThreshold = checkNotNull(flappingThreshold);
this.clock = checkNotNull(clock);
- this.flappingBackoffStrategy = checkNotNull(flappingBackoffStrategy);
+ this.rescheduleCalculator = checkNotNull(rescheduleCalculator);
this.preemptor = checkNotNull(preemptor);
final SchedulingAction rateLimitedAction = new SchedulingAction() {
@@ -141,7 +135,7 @@ public class TaskGroups implements EventSubscriber {
groups = CacheBuilder.newBuilder().build(new CacheLoader<GroupKey, TaskGroup>() {
@Override public TaskGroup load(GroupKey key) {
- TaskGroup group = new TaskGroup(key, backoffStrategy);
+ TaskGroup group = new TaskGroup(key, taskGroupBackoffStrategy);
LOG.info("Evaluating group " + key + " in " + group.getPenaltyMs() + " ms");
startGroup(group, executor, rateLimitedAction);
return group;
@@ -216,76 +210,6 @@ public class TaskGroups implements EventSubscriber {
groups.getUnchecked(new GroupKey(task.getTask())).push(task.getTaskId(), readyTimestamp);
}
- private Optional<IScheduledTask> getTaskAncestor(IScheduledTask task) {
- if (!task.isSetAncestorId()) {
- return Optional.absent();
- }
-
- ImmutableSet<IScheduledTask> res =
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.taskScoped(task.getAncestorId()));
-
- return Optional.fromNullable(Iterables.getOnlyElement(res, null));
- }
-
- private static final Predicate<ScheduleStatus> IS_ACTIVE_STATUS =
- Predicates.in(Tasks.ACTIVE_STATES);
-
- private static final Function<ITaskEvent, ScheduleStatus> TO_STATUS =
- new Function<ITaskEvent, ScheduleStatus>() {
- @Override public ScheduleStatus apply(ITaskEvent input) {
- return input.getStatus();
- }
- };
-
- private static final Set<ScheduleStatus> INTERRUPTED_TASK_STATES =
- EnumSet.of(RESTARTING, KILLING);
-
- private final Predicate<IScheduledTask> flapped = new Predicate<IScheduledTask>() {
- @Override public boolean apply(IScheduledTask task) {
- if (!task.isSetTaskEvents()) {
- return false;
- }
-
- List<ITaskEvent> events = Lists.reverse(task.getTaskEvents());
-
- // Avoid penalizing tasks that were interrupted by outside action, such as a user
- // restarting them.
- if (Iterables.any(Iterables.transform(events, TO_STATUS),
- Predicates.in(INTERRUPTED_TASK_STATES))) {
- return false;
- }
-
- ITaskEvent terminalEvent = Iterables.get(events, 0);
- ScheduleStatus terminalState = terminalEvent.getStatus();
- Preconditions.checkState(Tasks.isTerminated(terminalState));
-
- ITaskEvent activeEvent =
- Iterables.find(events, Predicates.compose(IS_ACTIVE_STATUS, TO_STATUS));
-
- long thresholdMs = flappingThreshold.as(Time.MILLISECONDS);
-
- return (terminalEvent.getTimestamp() - activeEvent.getTimestamp()) < thresholdMs;
- }
- };
-
- private long getTaskReadyTimestamp(IScheduledTask task) {
- Optional<IScheduledTask> curTask = getTaskAncestor(task);
- long penaltyMs = 0;
- while (curTask.isPresent() && flapped.apply(curTask.get())) {
- LOG.info(
- String.format("Ancestor of %s flapped: %s", Tasks.id(task), Tasks.id(curTask.get())));
- long newPenalty = flappingBackoffStrategy.calculateBackoffMs(penaltyMs);
- // If the backoff strategy is truncated then there is no need for us to continue.
- if (newPenalty == penaltyMs) {
- break;
- }
- penaltyMs = newPenalty;
- curTask = getTaskAncestor(curTask.get());
- }
-
- return penaltyMs + clock.nowMillis();
- }
-
/**
* Informs the task groups of a task state change.
* <p>
@@ -297,7 +221,9 @@ public class TaskGroups implements EventSubscriber {
@Subscribe
public synchronized void taskChangedState(TaskStateChange stateChange) {
if (stateChange.getNewState() == PENDING) {
- add(stateChange.getTask().getAssignedTask(), getTaskReadyTimestamp(stateChange.getTask()));
+ add(
+ stateChange.getTask().getAssignedTask(),
+ rescheduleCalculator.getReadyTimeMs(stateChange.getTask()));
}
}
@@ -314,7 +240,7 @@ public class TaskGroups implements EventSubscriber {
for (IScheduledTask task
: Storage.Util.consistentFetchTasks(storage, Query.unscoped().byStatus(PENDING))) {
- add(task.getAssignedTask(), getTaskReadyTimestamp(task));
+ add(task.getAssignedTask(), rescheduleCalculator.getStartupReadyTimeMs(task));
}
}
@@ -374,40 +300,4 @@ public class TaskGroups implements EventSubscriber {
*/
boolean schedule(String taskId);
}
-
- static class SchedulingSettings {
- private final BackoffStrategy backoff;
- private final RateLimiter rateLimit;
-
- SchedulingSettings(BackoffStrategy backoff, RateLimiter rateLimit) {
- this.backoff = checkNotNull(backoff);
- this.rateLimit = checkNotNull(rateLimit);
- }
-
- BackoffStrategy getBackoff() {
- return backoff;
- }
-
- RateLimiter getRateLimit() {
- return rateLimit;
- }
- }
-
- static class FlappingTaskSettings {
- private final BackoffStrategy backoff;
- private final Amount<Long, Time> flappingThreashold;
-
- FlappingTaskSettings(BackoffStrategy backoff, Amount<Long, Time> flappingThreashold) {
- this.backoff = checkNotNull(backoff);
- this.flappingThreashold = checkNotNull(flappingThreashold);
- }
-
- BackoffStrategy getBackoff() {
- return backoff;
- }
-
- Amount<Long, Time> getFlappingThreashold() {
- return flappingThreashold;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/aa9f826f/src/test/java/com/twitter/aurora/scheduler/async/TaskSchedulerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/async/TaskSchedulerTest.java b/src/test/java/com/twitter/aurora/scheduler/async/TaskSchedulerTest.java
index e7a2f21..a747f2b 100644
--- a/src/test/java/com/twitter/aurora/scheduler/async/TaskSchedulerTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/async/TaskSchedulerTest.java
@@ -49,6 +49,8 @@ import com.twitter.aurora.scheduler.Driver;
import com.twitter.aurora.scheduler.async.OfferQueue.OfferQueueImpl;
import com.twitter.aurora.scheduler.async.OfferQueue.OfferReturnDelay;
import com.twitter.aurora.scheduler.async.TaskGroups.SchedulingAction;
+import com.twitter.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl;
+import com.twitter.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl.RescheduleCalculatorSettings;
import com.twitter.aurora.scheduler.base.Query;
import com.twitter.aurora.scheduler.base.Tasks;
import com.twitter.aurora.scheduler.events.PubsubEvent.HostMaintenanceStateChange;
@@ -143,9 +145,14 @@ public class TaskSchedulerTest extends EasyMockTest {
retryStrategy,
rateLimiter,
scheduler,
- flappingThreshold,
clock,
- flappingStrategy,
+ new RescheduleCalculatorImpl(
+ storage,
+ new RescheduleCalculatorSettings(
+ flappingStrategy,
+ flappingThreshold,
+ Amount.of(5, Time.SECONDS)),
+ clock),
preemptor);
}