You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wi...@apache.org on 2014/06/05 18:49:36 UTC
[1/4] Make style consistent with build-support/python/checkstyle-check
Repository: incubator-aurora
Updated Branches:
refs/heads/master ca4adf93f -> 467f84285
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_failure_limit.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_failure_limit.py b/src/test/python/apache/thermos/core/test_failure_limit.py
index a37f44f..ae6503d 100644
--- a/src/test/python/apache/thermos/core/test_failure_limit.py
+++ b/src/test/python/apache/thermos/core/test_failure_limit.py
@@ -22,15 +22,15 @@ class TestFailureLimit(RunnerTestBase):
@classmethod
def task(cls):
task = Task(
- name = "failing_task",
- resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
- max_failures = 2,
- processes = [
- Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
- Process(name = "b", max_failures=1, min_duration=1, cmdline="exit 1"),
- Process(name = "c", max_failures=1, min_duration=1, cmdline="echo hello world")
+ name="failing_task",
+ resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
+ max_failures=2,
+ processes=[
+ Process(name="a", max_failures=1, min_duration=1, cmdline="echo hello world"),
+ Process(name="b", max_failures=1, min_duration=1, cmdline="exit 1"),
+ Process(name="c", max_failures=1, min_duration=1, cmdline="echo hello world")
],
- constraints = [{'order': ['a', 'b', 'c']}]
+ constraints=[{'order': ['a', 'b', 'c']}]
)
return task.interpolate()[0]
@@ -53,11 +53,12 @@ class TestTaskSucceedsIfMaxFailures0(RunnerTestBase):
ex = base(cmdline="exit 1")
hw = base(cmdline="echo hello world")
task = Task(
- name = "failing_task",
- resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
- max_failures = 0,
- processes = [ex(name='f1'), ex(name='f2'), ex(name='f3'),
- hw(name='s1'), hw(name='s2'), hw(name='s3')])
+ name="failing_task",
+ resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
+ max_failures=0,
+ processes=[
+ ex(name='f1'), ex(name='f2'), ex(name='f3'),
+ hw(name='s1'), hw(name='s2'), hw(name='s3')])
return task.interpolate()[0]
def test_runner_state_failure(self):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_finalization.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_finalization.py b/src/test/python/apache/thermos/core/test_finalization.py
index 972785f..2b0ee05 100644
--- a/src/test/python/apache/thermos/core/test_finalization.py
+++ b/src/test/python/apache/thermos/core/test_finalization.py
@@ -23,7 +23,7 @@ class TestRegularFinalizingTask(RunnerTestBase):
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
- task = Task(name="task_with_finalizer", processes = [main, finalizer])
+ task = Task(name="task_with_finalizer", processes=[main, finalizer])
return task.interpolate()[0]
def test_runner_state(self):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_helper.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_helper.py b/src/test/python/apache/thermos/core/test_helper.py
index 26288fa..5809611 100644
--- a/src/test/python/apache/thermos/core/test_helper.py
+++ b/src/test/python/apache/thermos/core/test_helper.py
@@ -82,7 +82,10 @@ def test_scan_process():
with mock.patch(PSUTIL_PATH) as p_mock:
class WrappedNoSuchProcess(psutil.NoSuchProcess):
# psutil.NoSuchProcess exception requires an argument, but mock doesn't support that.
- def __init__(self): pass
+
+ def __init__(self):
+ pass
+
p_mock.side_effect = WrappedNoSuchProcess
assert TRH.scan_process(
make_runner_state(cpid=None), PROCESS_NAME) == (None, None, set())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_process.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_process.py b/src/test/python/apache/thermos/core/test_process.py
index ed66f73..223393d 100644
--- a/src/test/python/apache/thermos/core/test_process.py
+++ b/src/test/python/apache/thermos/core/test_process.py
@@ -16,7 +16,6 @@ import grp
import os
import pwd
import random
-import threading
import time
import mock
@@ -35,19 +34,21 @@ class TestProcess(Process):
def execute(self):
super(TestProcess, self).execute()
os._exit(0)
+
def finish(self):
pass
def wait_for_rc(checkpoint, timeout=5.0):
start = time.time()
- trr = ThriftRecordReader(open(checkpoint), RunnerCkpt)
- while time.time() < start + timeout:
- record = trr.read()
- if record and record.process_status and record.process_status.return_code is not None:
- return record.process_status.return_code
- else:
- time.sleep(0.1)
+ with open(checkpoint) as fp:
+ trr = ThriftRecordReader(fp, RunnerCkpt)
+ while time.time() < start + timeout:
+ record = trr.read()
+ if record and record.process_status and record.process_status.return_code is not None:
+ return record.process_status.return_code
+ else:
+ time.sleep(0.1)
def get_other_nonroot_user():
@@ -94,7 +95,7 @@ def test_simple_process_other_user(*args):
p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox, user=some_user.pw_name)
p.start()
- rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
+ wait_for_rc(taskpath.getpath('process_checkpoint'))
# since we're not actually root, the best we can do is check the right things were attempted
assert os.setgroups.calledwith([g.gr_gid for g in grp.getgrall() if some_user.pw_name in g])
@@ -108,7 +109,7 @@ def test_other_user_fails_nonroot():
sandbox = setup_sandbox(td, taskpath)
with pytest.raises(Process.PermissionError):
- p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox,
+ TestProcess('process', 'echo hello world', 0, taskpath, sandbox,
user=get_other_nonroot_user().pw_name)
@@ -119,7 +120,7 @@ def test_log_permissions():
p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox)
p.start()
- rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
+ wait_for_rc(taskpath.getpath('process_checkpoint'))
stdout = taskpath.with_filename('stdout').getpath('process_logdir')
stderr = taskpath.with_filename('stderr').getpath('process_logdir')
@@ -142,7 +143,7 @@ def test_log_permissions_other_user(*mocks):
p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox, user=some_user.pw_name)
p.start()
- rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
+ wait_for_rc(taskpath.getpath('process_checkpoint'))
# since we're not actually root, the best we can do is check the right things were attempted
stdout = taskpath.with_filename('stdout').getpath('process_logdir')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_runner_integration.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_runner_integration.py b/src/test/python/apache/thermos/core/test_runner_integration.py
index fa84de7..55dbb12 100644
--- a/src/test/python/apache/thermos/core/test_runner_integration.py
+++ b/src/test/python/apache/thermos/core/test_runner_integration.py
@@ -15,36 +15,35 @@
import os
from textwrap import dedent
-import pytest
-
-from apache.thermos.config.schema import order, Process, Resources, SequentialTask, Task, Tasks
+from apache.thermos.config.schema import Process, Resources, SequentialTask, Task, Tasks
from apache.thermos.testing.runner import RunnerTestBase
from gen.apache.thermos.ttypes import ProcessState, TaskState
class TestRunnerBasic(RunnerTestBase):
- portmap = {'named_port': 8123}
+ portmap = {'named_port': 8123} # noqa
@classmethod
def task(cls):
- hello_template = Process(cmdline = "echo 1")
- t1 = hello_template(name = "t1", cmdline = "echo 1 port {{thermos.ports[named_port]}}")
- t2 = hello_template(name = "t2")
- t3 = hello_template(name = "t3")
- t4 = hello_template(name = "t4")
- t5 = hello_template(name = "t5")
- t6 = hello_template(name = "t6")
- tsk = Task(name = "complex", processes = [t1, t2, t3, t4, t5, t6])
+ hello_template = Process(cmdline="echo 1")
+ t1 = hello_template(name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}")
+ t2 = hello_template(name="t2")
+ t3 = hello_template(name="t3")
+ t4 = hello_template(name="t4")
+ t5 = hello_template(name="t5")
+ t6 = hello_template(name="t6")
+ tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6])
# three ways of tasks: t1 t2, t3 t4, t5 t6
- tsk = tsk(constraints = [{'order': ['t1', 't3']},
- {'order': ['t1', 't4']},
- {'order': ['t2', 't3']},
- {'order': ['t2', 't4']},
- {'order': ['t3', 't5']},
- {'order': ['t3', 't6']},
- {'order': ['t4', 't5']},
- {'order': ['t4', 't6']}])
+ tsk = tsk(constraints=[
+ {'order': ['t1', 't3']},
+ {'order': ['t1', 't4']},
+ {'order': ['t2', 't3']},
+ {'order': ['t2', 't4']},
+ {'order': ['t3', 't5']},
+ {'order': ['t3', 't6']},
+ {'order': ['t4', 't5']},
+ {'order': ['t4', 't6']}])
return tsk
def test_runner_state_success(self):
@@ -66,7 +65,7 @@ class TestRunnerBasic(RunnerTestBase):
def test_runner_has_expected_processes(self):
processes = self.state.processes
- process_names = set(['t%d'%k for k in range(1,7)])
+ process_names = set(['t%d' % k for k in range(1, 7)])
actual_process_names = set(processes.keys())
assert process_names == actual_process_names, "runner didn't run expected set of processes!"
for process in processes:
@@ -77,7 +76,7 @@ class TestRunnerBasic(RunnerTestBase):
history = self.state.processes[process]
assert history[-1].state == ProcessState.SUCCESS
if len(history) > 1:
- for run in range(len(history)-1):
+ for run in range(len(history) - 1):
assert history[run].state != ProcessState.SUCCESS, (
"nonterminal processes must not be in SUCCESS state!")
@@ -91,14 +90,14 @@ class TestRunnerBasic(RunnerTestBase):
class TestConcurrencyBasic(RunnerTestBase):
@classmethod
def task(cls):
- hello_template = Process(cmdline = "sleep 1")
+ hello_template = Process(cmdline="sleep 1")
tsk = Task(
- name = "complex",
- processes = [hello_template(name = "process1"),
- hello_template(name = "process2"),
- hello_template(name = "process3")],
- resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
- max_concurrency = 1)
+ name="complex",
+ processes=[hello_template(name="process1"),
+ hello_template(name="process2"),
+ hello_template(name="process3")],
+ resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
+ max_concurrency=1)
return tsk
def test_runner_state_success(self):
@@ -119,45 +118,44 @@ class TestConcurrencyBasic(RunnerTestBase):
class TestRunnerEnvironment(RunnerTestBase):
@classmethod
def task(cls):
- setup_bashrc = Process(name = "setup_bashrc", cmdline = dedent(
- """
- mkdir -p .profile.d
- cat <<EOF > .thermos_profile
- for i in .profile.d/*.sh ; do
- if [ -r "\\$i" ]; then
- . \\$i
- fi
- done
- EOF
- """))
-
- setup_foo = Process(name = "setup_foo", cmdline = dedent(
- """
- cat <<EOF > .profile.d/setup_foo.sh
- export FOO=1
- EOF
- """))
-
- setup_bar = Process(name = "setup_bar", cmdline = dedent(
- """
- cat <<EOF > .profile.d/setup_bar.sh
- export BAR=2
- EOF
- """))
-
- foo_recipe = SequentialTask(processes = [setup_bashrc, setup_foo])
- bar_recipe = SequentialTask(processes = [setup_bashrc, setup_bar])
+ setup_bashrc = Process(name="setup_bashrc", cmdline=dedent(
+ """
+ mkdir -p .profile.d
+ cat <<EOF > .thermos_profile
+ for i in .profile.d/*.sh ; do
+ if [ -r "\\$i" ]; then
+ . \\$i
+ fi
+ done
+ EOF
+ """))
+
+ setup_foo = Process(name="setup_foo", cmdline=dedent(
+ """
+ cat <<EOF > .profile.d/setup_foo.sh
+ export FOO=1
+ EOF
+ """))
+
+ setup_bar = Process(name="setup_bar", cmdline=dedent(
+ """
+ cat <<EOF > .profile.d/setup_bar.sh
+ export BAR=2
+ EOF
+ """))
+
+ foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
+ bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
all_recipes = Tasks.combine(foo_recipe, bar_recipe)
- run = Process(name = "run", cmdline = dedent(
- """
- echo $FOO $BAR > expected_output.txt
- """
- ))
+ run = Process(name="run", cmdline=dedent(
+ """
+ echo $FOO $BAR > expected_output.txt
+ """))
- my_task = Task(processes = [run],
- resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024))
- return Tasks.concat(all_recipes, my_task, name = "my_task")
+ my_task = Task(processes=[run],
+ resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024))
+ return Tasks.concat(all_recipes, my_task, name="my_task")
def test_runner_state_success(self):
assert self.state.statuses[-1].state == TaskState.SUCCESS
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_staged_kill.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_staged_kill.py b/src/test/python/apache/thermos/core/test_staged_kill.py
index e0a3d14..faa23ae 100644
--- a/src/test/python/apache/thermos/core/test_staged_kill.py
+++ b/src/test/python/apache/thermos/core/test_staged_kill.py
@@ -22,17 +22,14 @@ import pytest
from twitter.common.process import ProcessProviderFactory
from twitter.common.quantity import Amount, Time
-from apache.thermos.config.schema import Process, Resources, Task
+from apache.thermos.config.schema import Process, Task
from apache.thermos.core.runner import TaskRunner
from apache.thermos.monitoring.monitor import TaskMonitor
from apache.thermos.testing.runner import Runner
from gen.apache.thermos.ttypes import ProcessState, TaskState
-sleepy_process = Process(
- name = "sleepy",
- cmdline = "sleep 3",
- min_duration = 1)
+sleepy_process = Process(name="sleepy", cmdline="sleep 3", min_duration=1)
ignore_script = [
"import time, signal",
@@ -107,7 +104,7 @@ class ProcessPidTestCase(object):
class TestRunnerKill(RunnerBase, ProcessPidTestCase):
@classmethod
def task(cls):
- task = Task(name = "task", processes = [sleepy_process(name="process")])
+ task = Task(name="task", processes=[sleepy_process(name="process")])
return task.interpolate()[0]
def test_coordinator_kill(self):
@@ -136,9 +133,9 @@ class TestRunnerKill(RunnerBase, ProcessPidTestCase):
class TestRunnerKillProcessTrappingSIGTERM(RunnerBase):
@classmethod
def task(cls):
- task = Task(name = "task",
- finalization_wait = 3,
- processes = [ignorant_process(name="ignorant_process")])
+ task = Task(name="task",
+ finalization_wait=3,
+ processes=[ignorant_process(name="ignorant_process")])
return task.interpolate()[0]
def test_coordinator_kill(self):
@@ -224,7 +221,6 @@ class TestRunnerKillProcessTrappingSIGTERM(RunnerBase):
assert preempter.state.processes['ignorant_process'][-1].state == ProcessState.KILLED
-
SIMPLEFORK_SCRIPT = """
cat <<EOF | %(INTERPRETER)s -
from __future__ import print_function
@@ -244,12 +240,13 @@ else:
while not os.path.exists('exit.txt'):
time.sleep(0.1)
EOF
-""" % { 'INTERPRETER': sys.executable }
+""" % {'INTERPRETER': sys.executable}
+
class TestRunnerKillProcessGroup(RunnerBase):
@classmethod
def task(cls):
- task = Task(name = "task", processes = [Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
+ task = Task(name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
return task.interpolate()[0]
def test_pg_is_killed(self):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/monitoring/test_disk.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/monitoring/test_disk.py b/src/test/python/apache/thermos/monitoring/test_disk.py
index cc3c20a..913f87d 100644
--- a/src/test/python/apache/thermos/monitoring/test_disk.py
+++ b/src/test/python/apache/thermos/monitoring/test_disk.py
@@ -12,9 +12,7 @@
# limitations under the License.
#
-import atexit
import os
-import sys
import time
from tempfile import mkstemp
@@ -47,7 +45,7 @@ def _run_collector_tests(collector, target, wait):
wait()
assert collector.value >= TEST_AMOUNT_1.as_(Data.BYTES)
- f2 = make_file(TEST_AMOUNT_2, dir=target)
+ make_file(TEST_AMOUNT_2, dir=target)
wait()
assert collector.value >= TEST_AMOUNT_SUM.as_(Data.BYTES)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/sh/org/apache/aurora/e2e/http_example.py
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http_example.py b/src/test/sh/org/apache/aurora/e2e/http_example.py
index 04f7fa5..675ece8 100644
--- a/src/test/sh/org/apache/aurora/e2e/http_example.py
+++ b/src/test/sh/org/apache/aurora/e2e/http_example.py
@@ -12,8 +12,10 @@
# limitations under the License.
#
from __future__ import print_function
-from sys import argv
+
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from sys import argv
+
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
[2/4] Make style consistent with build-support/python/checkstyle-check
Posted by wi...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_kill.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_kill.py b/src/test/python/apache/aurora/client/cli/test_kill.py
index 666ec3a..e9e1679 100644
--- a/src/test/python/apache/aurora/client/cli/test_kill.py
+++ b/src/test/python/apache/aurora/client/cli/test_kill.py
@@ -22,18 +22,9 @@ from apache.aurora.client.cli import EXIT_TIMEOUT
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.options import parse_instances
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.common.aurora_job_key import AuroraJobKey
-from gen.apache.aurora.api.ttypes import (
- AssignedTask,
- Identity,
- ScheduledTask,
- ScheduleStatus,
- ScheduleStatusResult,
- TaskEvent,
- TaskQuery
-)
+from gen.apache.aurora.api.ttypes import Identity, ScheduleStatus, ScheduleStatusResult, TaskQuery
class TestInstancesParser(unittest.TestCase):
@@ -67,7 +58,6 @@ class TestClientKillCommand(AuroraClientCommandTest):
return TaskQuery(taskIds=None, jobName=cls.TEST_JOB, environment=cls.TEST_ENV,
instanceIds=instance_ids, owner=Identity(role=cls.TEST_ROLE, user=None))
-
def test_killall_job(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
@@ -87,7 +77,8 @@ class TestClientKillCommand(AuroraClientCommandTest):
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
- cmd.execute(['job', 'killall', '--no-batching', '--config=%s' % fp.name, 'west/bozo/test/hello'])
+ cmd.execute(['job', 'killall', '--no-batching', '--config=%s' % fp.name,
+ 'west/bozo/test/hello'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 1
@@ -123,7 +114,7 @@ class TestClientKillCommand(AuroraClientCommandTest):
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), None)
self.assert_scheduler_called(api, self.get_expected_task_query(), 8)
- def test_killall_job(self):
+ def test_killall_job_something_else(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
mock_scheduler_proxy = Mock()
@@ -166,7 +157,8 @@ class TestClientKillCommand(AuroraClientCommandTest):
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
- cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--no-batching', 'west/bozo/test/hello/0,2,4-6'])
+ cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--no-batching',
+ 'west/bozo/test/hello/0,2,4-6'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 1
@@ -196,7 +188,6 @@ class TestClientKillCommand(AuroraClientCommandTest):
# Now check that the right API calls got made.
assert api.kill_job.call_count == 0
-
def test_kill_job_with_invalid_instances_nonstrict(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
@@ -222,7 +213,6 @@ class TestClientKillCommand(AuroraClientCommandTest):
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), instances)
self.assert_scheduler_called(api, self.get_expected_task_query(instances), 2)
-
def test_kill_job_with_instances_batched(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
@@ -292,7 +282,8 @@ class TestClientKillCommand(AuroraClientCommandTest):
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
- cmd.execute(['job', 'kill', '--max-total-failures=1', '--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'])
+ cmd.execute(['job', 'kill', '--max-total-failures=1', '--config=%s' % fp.name,
+ 'west/bozo/test/hello/0,2,4-13'])
# Now check that the right API calls got made. We should have aborted after the second batch.
assert api.kill_job.call_count == 2
@@ -322,7 +313,6 @@ class TestClientKillCommand(AuroraClientCommandTest):
# Now check that the right API calls got made.
assert api.kill_job.call_count == 0
-
def test_kill_job_with_instances_deep_api(self):
"""Test kill client-side API logic."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_open.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_open.py b/src/test/python/apache/aurora/client/cli/test_open.py
index 5557c35..c20649f 100644
--- a/src/test/python/apache/aurora/client/cli/test_open.py
+++ b/src/test/python/apache/aurora/client/cli/test_open.py
@@ -12,11 +12,9 @@
# limitations under the License.
#
-import contextlib
+from mock import patch
-from mock import Mock, patch
-
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, EXIT_OK
+from apache.aurora.client.cli import EXIT_OK
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_plugins.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_plugins.py b/src/test/python/apache/aurora/client/cli/test_plugins.py
index 00b4583..ce85318 100644
--- a/src/test/python/apache/aurora/client/cli/test_plugins.py
+++ b/src/test/python/apache/aurora/client/cli/test_plugins.py
@@ -12,21 +12,13 @@
# limitations under the License.
#
-import contextlib
-
from mock import Mock, patch
from twitter.common.contextutil import temporary_file
-from apache.aurora.client.cli import (
- ConfigurationPlugin,
- EXIT_COMMAND_FAILURE,
- EXIT_INVALID_CONFIGURATION,
- EXIT_OK
-)
+from apache.aurora.client.cli import ConfigurationPlugin, EXIT_OK
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.options import CommandOption
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.config import AuroraConfig
from gen.apache.aurora.api.ttypes import (
@@ -53,10 +45,9 @@ class BogusPlugin(ConfigurationPlugin):
def before_dispatch(self, args):
if args[0] == '--bogus_bogus':
- args = args[1:]
+ args = args[1:]
return args
-
def before_execution(self, context):
context.bogosity = context.options.bogosity
@@ -64,9 +55,11 @@ class BogusPlugin(ConfigurationPlugin):
context.after = True
raise self.Error("Oops")
+
class EmptyPlugin(ConfigurationPlugin):
pass
+
class TestPlugins(AuroraClientCommandTest):
@classmethod
@@ -150,7 +143,7 @@ class TestPlugins(AuroraClientCommandTest):
self.assert_scheduler_called(api, mock_query, 1)
# Check that the plugin did its job.
assert mock_context.bogosity == "maximum"
- assert mock_context.after == True
+ assert mock_context.after
def test_empty_plugins_in_create_job(self):
"""Installs a plugin that doesn't implement any of the plugin methods.
@@ -185,7 +178,6 @@ class TestPlugins(AuroraClientCommandTest):
for str in str.split('\n'):
self.err_transcript.append(str)
-
def test_plugin_options_in_help(self):
cmd = AuroraCommandLine()
cmd.register_plugin(BogusPlugin())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_restart.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_restart.py b/src/test/python/apache/aurora/client/cli/test_restart.py
index 50acc09..407eb41 100644
--- a/src/test/python/apache/aurora/client/cli/test_restart.py
+++ b/src/test/python/apache/aurora/client/cli/test_restart.py
@@ -22,14 +22,7 @@ from apache.aurora.client.cli import EXIT_API_ERROR, EXIT_INVALID_PARAMETER
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.util import AuroraClientCommandTest
-from gen.apache.aurora.api.ttypes import (
- AssignedTask,
- JobKey,
- PopulateJobResult,
- ScheduledTask,
- ScheduleStatusResult,
- TaskConfig
-)
+from gen.apache.aurora.api.ttypes import JobKey, PopulateJobResult, TaskConfig
class TestRestartCommand(AuroraClientCommandTest):
@@ -51,7 +44,6 @@ class TestRestartCommand(AuroraClientCommandTest):
populate.result.populateJobResult.populated = set(configs)
return populate
-
@classmethod
def setup_health_checks(cls, mock_api):
mock_health_check = Mock(spec=StatusHealthCheck)
@@ -111,7 +103,6 @@ class TestRestartCommand(AuroraClientCommandTest):
assert mock_scheduler_proxy.getTasksStatus.call_count == 0
assert mock_scheduler_proxy.restartShards.call_count == 0
-
def test_restart_failed_status(self):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
@@ -139,7 +130,6 @@ class TestRestartCommand(AuroraClientCommandTest):
self.setup_mock_scheduler_for_simple_restart(mock_api)
# Make getTasksStatus return an error, which is what happens when a job is not found.
mock_scheduler_proxy.getTasksStatus.return_value = self.create_error_response()
- mock_logger = Mock()
with contextlib.nested(
patch('apache.aurora.client.cli.print_aurora_log'),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
@@ -152,7 +142,8 @@ class TestRestartCommand(AuroraClientCommandTest):
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
- result = cmd.execute(['job', 'restart', '--batch-size=5', 'west/bozo/test/hello/1-3', fp.name])
+ result = cmd.execute(['job', 'restart', '--batch-size=5', 'west/bozo/test/hello/1-3',
+ fp.name])
# We need to check tat getTasksStatus was called, but that restartShards wasn't.
# In older versions of the client, if shards were specified, but the job didn't
# exist, the error wouldn't be detected unti0 restartShards was called, which generated
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_status.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_status.py b/src/test/python/apache/aurora/client/cli/test_status.py
index 4cc3f9d..dc1c730 100644
--- a/src/test/python/apache/aurora/client/cli/test_status.py
+++ b/src/test/python/apache/aurora/client/cli/test_status.py
@@ -14,7 +14,7 @@
import contextlib
-from mock import call, Mock, patch
+from mock import Mock, patch
from apache.aurora.client.cli import EXIT_INVALID_PARAMETER
from apache.aurora.client.cli.client import AuroraCommandLine
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_task_run.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_task_run.py b/src/test/python/apache/aurora/client/cli/test_task_run.py
index 2663398..167f490 100644
--- a/src/test/python/apache/aurora/client/cli/test_task_run.py
+++ b/src/test/python/apache/aurora/client/cli/test_task_run.py
@@ -16,10 +16,8 @@ import contextlib
from mock import Mock, patch
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER
from apache.aurora.client.cli.client import AuroraCommandLine
-from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
-from apache.aurora.common.aurora_job_key import AuroraJobKey
+from apache.aurora.client.cli.util import AuroraClientCommandTest
from gen.apache.aurora.api.ttypes import (
AssignedTask,
@@ -108,7 +106,8 @@ class TestRunCommand(AuroraClientCommandTest):
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.cli.task.CLUSTERS', new=self.TEST_CLUSTERS),
- patch('apache.aurora.client.api.command_runner.InstanceDistributedCommandRunner.sandbox_args',
+ patch('apache.aurora.client.api.command_runner.'
+ 'InstanceDistributedCommandRunner.sandbox_args',
return_value=sandbox_args),
patch('subprocess.Popen', return_value=self.create_mock_process())) as (
mock_scheduler_proxy_class,
@@ -134,6 +133,7 @@ class TestRunCommand(AuroraClientCommandTest):
'slaverun/sandbox;ls'],
stderr=-2, stdout=-1)
+
class TestSshCommand(AuroraClientCommandTest):
@classmethod
def create_mock_scheduled_tasks(cls):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_update.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_update.py b/src/test/python/apache/aurora/client/cli/test_update.py
index a2abc5e..08be49e 100644
--- a/src/test/python/apache/aurora/client/cli/test_update.py
+++ b/src/test/python/apache/aurora/client/cli/test_update.py
@@ -180,8 +180,8 @@ class TestUpdateCommand(AuroraClientCommandTest):
@classmethod
def setup_quota_check(cls):
mock_quota_check = Mock(spec=QuotaCheck)
- mock_quota_check.validate_quota_from_requested.return_value = \
- cls.create_simple_success_response()
+ mock_quota_check.validate_quota_from_requested.return_value = (
+ cls.create_simple_success_response())
return mock_quota_check
@classmethod
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/util.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/util.py b/src/test/python/apache/aurora/client/cli/util.py
index dac4928..070718c 100644
--- a/src/test/python/apache/aurora/client/cli/util.py
+++ b/src/test/python/apache/aurora/client/cli/util.py
@@ -91,6 +91,8 @@ class FakeAuroraCommandContext(AuroraCommandContext):
class AuroraClientCommandTest(unittest.TestCase):
+ FAKE_TIME = 42131
+
@classmethod
def create_blank_response(cls, code, msg):
response = Mock(spec=Response)
@@ -163,9 +165,6 @@ class AuroraClientCommandTest(unittest.TestCase):
status_response = cls.create_status_call_result()
scheduler.getTasksStatus.return_value = status_response
-
- FAKE_TIME = 42131
-
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_admin_sla.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_admin_sla.py b/src/test/python/apache/aurora/client/commands/test_admin_sla.py
index 84a91d5..da8015f 100644
--- a/src/test/python/apache/aurora/client/commands/test_admin_sla.py
+++ b/src/test/python/apache/aurora/client/commands/test_admin_sla.py
@@ -19,7 +19,7 @@ from mock import Mock, patch
from twitter.common.contextutil import temporary_file
from apache.aurora.client.api import AuroraClientAPI
-from apache.aurora.client.api.sla import DomainUpTimeSlaVector
+from apache.aurora.client.api.sla import DomainUpTimeSlaVector, JobUpTimeDetails, JobUpTimeLimit
from apache.aurora.client.base import DEFAULT_GROUPING
from apache.aurora.client.commands.admin import sla_list_safe_domain, sla_probe_hosts
from apache.aurora.client.commands.util import AuroraClientCommandTest
@@ -55,7 +55,7 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
for i in range(num_hosts):
host_name = 'h%s' % i
job = AuroraJobKey.from_path('west/role/env/job%s' % i)
- hosts[host_name].append(DomainUpTimeSlaVector.JobUpTimeLimit(job, percentage, duration))
+ hosts[host_name].append(JobUpTimeLimit(job, percentage, duration))
return [hosts]
@classmethod
@@ -214,9 +214,8 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
sla_list_safe_domain(['west', '50', '100s'])
job_key = AuroraJobKey.from_path('west/role/env/job1')
- override = {job_key: DomainUpTimeSlaVector.JobUpTimeLimit(job_key, 30, 200)}
- mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0,
- override, DEFAULT_GROUPING)
+ override = {job_key: JobUpTimeLimit(job_key, 30, 200)}
+ mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, override, DEFAULT_GROUPING)
mock_print_results.assert_called_once_with(['h0', 'h1', 'h2'])
def test_safe_domain_list_jobs(self):
@@ -247,8 +246,7 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
def test_safe_domain_invalid_percentage(self):
"""Tests execution of the sla_list_safe_domain command with invalid percentage"""
mock_options = self.setup_mock_options()
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
-
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_list_safe_domain(['west', '0', '100s'])
except SystemExit:
@@ -262,7 +260,7 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
fp.write('30 200s')
fp.flush()
mock_options = self.setup_mock_options(override=fp.name)
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_list_safe_domain(['west', '50', '100s'])
@@ -274,7 +272,7 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
def test_safe_domain_hosts_error(self):
"""Tests execution of the sla_list_safe_domain command with both include file and list"""
mock_options = self.setup_mock_options(include='file', include_list='list')
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_list_safe_domain(['west', '50', '100s'])
@@ -286,7 +284,7 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
def test_safe_domain_grouping_error(self):
"""Tests execution of the sla_list_safe_domain command invalid grouping"""
mock_options = self.setup_mock_options(grouping='foo')
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_list_safe_domain(['west', '50', '100s'])
@@ -319,7 +317,7 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
for i in range(num_hosts):
host_name = 'h%s' % i
job = AuroraJobKey.from_path('west/role/env/job%s' % i)
- hosts[host_name].append(DomainUpTimeSlaVector.JobUpTimeDetails(job, predicted, safe, safe_in))
+ hosts[host_name].append(JobUpTimeDetails(job, predicted, safe, safe_in))
return [hosts]
def test_probe_hosts_with_list(self):
@@ -385,7 +383,7 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
fp.write('h0')
fp.flush()
mock_options = self.setup_mock_options(hosts='h0', filename=fp.name)
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_probe_hosts(['west', '50', '100s'])
@@ -397,7 +395,7 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
def test_probe_grouping_error(self):
"""Tests execution of the sla_probe_hosts command with invalid grouping."""
mock_options = self.setup_mock_options(hosts='h0', grouping='foo')
- with patch('twitter.common.app.get_options', return_value=mock_options) as (_):
+ with patch('twitter.common.app.get_options', return_value=mock_options):
try:
sla_probe_hosts(['west', '50', '100s'])
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_cancel_update.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_cancel_update.py b/src/test/python/apache/aurora/client/commands/test_cancel_update.py
index f8df6ae..5f05ef7 100644
--- a/src/test/python/apache/aurora/client/commands/test_cancel_update.py
+++ b/src/test/python/apache/aurora/client/commands/test_cancel_update.py
@@ -13,17 +13,13 @@
#
import contextlib
-import unittest
from mock import Mock, patch
from twitter.common.contextutil import temporary_file
from apache.aurora.client.commands.core import cancel_update
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.common.aurora_job_key import AuroraJobKey
-from apache.aurora.common.cluster import Cluster
-from apache.aurora.common.clusters import Clusters
from gen.apache.aurora.api.ttypes import (
Identity,
@@ -84,7 +80,6 @@ class TestClientCancelUpdateCommand(AuroraClientCommandTest):
AuroraJobKey(cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB),
config=None)
-
def test_simple_successful_cancel_update(self):
"""Run a test of the "kill" command against a mocked-out API:
Verifies that the kill command sends the right API RPCs, and performs the correct
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_create.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_create.py b/src/test/python/apache/aurora/client/commands/test_create.py
index 75f0682..4cd1343 100644
--- a/src/test/python/apache/aurora/client/commands/test_create.py
+++ b/src/test/python/apache/aurora/client/commands/test_create.py
@@ -13,24 +13,18 @@
#
import contextlib
-import unittest
from mock import Mock, patch
from pystachio.config import Config
-from twitter.common import app
from twitter.common.contextutil import temporary_file
from apache.aurora.client.commands.core import create
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.config import AuroraConfig
from gen.apache.aurora.api.ttypes import (
AssignedTask,
Identity,
- Response,
- ResponseCode,
- Result,
ScheduledTask,
ScheduleStatus,
ScheduleStatusResult,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_hooks.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_hooks.py b/src/test/python/apache/aurora/client/commands/test_hooks.py
index 7648bf4..0c76aea 100644
--- a/src/test/python/apache/aurora/client/commands/test_hooks.py
+++ b/src/test/python/apache/aurora/client/commands/test_hooks.py
@@ -13,11 +13,8 @@
#
import contextlib
-import unittest
from mock import Mock, patch
-from pystachio.config import Config
-from twitter.common import app
from twitter.common.contextutil import temporary_file
from apache.aurora.client.commands.core import create
@@ -28,9 +25,6 @@ from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from gen.apache.aurora.api.ttypes import (
AssignedTask,
Identity,
- Response,
- ResponseCode,
- Result,
ScheduledTask,
ScheduleStatus,
ScheduleStatusResult,
@@ -135,11 +129,10 @@ class TestClientCreateCommand(AuroraClientCommandTest):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
- patch('apache.aurora.client.api.SchedulerProxy', return_value = mock_scheduler_proxy),
+ patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('twitter.common.app.get_options', return_value=mock_options)):
- mock_query = self.create_mock_query()
- mock_scheduler_proxy.createJob.return_value=self.get_createjob_response()
+ mock_scheduler_proxy.createJob.return_value = self.get_createjob_response()
mock_scheduler_proxy.getTasksStatus.side_effect = [
self.create_mock_status_query_result(ScheduleStatus.INIT),
@@ -175,11 +168,10 @@ class TestClientCreateCommand(AuroraClientCommandTest):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
- patch('apache.aurora.client.api.SchedulerProxy', return_value = mock_scheduler_proxy),
+ patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('twitter.common.app.get_options', return_value=mock_options)):
- mock_query = self.create_mock_query()
- mock_scheduler_proxy.createJob.return_value=self.get_createjob_response()
+ mock_scheduler_proxy.createJob.return_value = self.get_createjob_response()
mock_scheduler_proxy.getTasksStatus.side_effect = [
self.create_mock_status_query_result(ScheduleStatus.INIT),
@@ -200,7 +192,6 @@ class TestClientCreateCommand(AuroraClientCommandTest):
assert mock_scheduler_proxy.createJob.call_count == 0
assert len(hook.created_jobs) == 1
-
def test_block_hooks(self):
"""Run a test of the "create" command against a mocked API;
verifies that a required hook runs, even though the config doesn't mention it.
@@ -213,19 +204,16 @@ class TestClientCreateCommand(AuroraClientCommandTest):
GlobalHookRegistry.register_global_hook(hook)
mock_options.disable_all_hooks_reason = "Because I said so."
-
# create first calls get_job_config, which calls get_config. As long as we've got the options
# set up correctly, this should work.
-
# Next, create gets an API object via make_client. We need to replace that with a mock API.
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
- patch('apache.aurora.client.api.SchedulerProxy', return_value = mock_scheduler_proxy),
+ patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('twitter.common.app.get_options', return_value=mock_options)):
- mock_query = self.create_mock_query()
- mock_scheduler_proxy.createJob.return_value=self.get_createjob_response()
+ mock_scheduler_proxy.createJob.return_value = self.get_createjob_response()
mock_scheduler_proxy.getTasksStatus.side_effect = [
self.create_mock_status_query_result(ScheduleStatus.INIT),
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_kill.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_kill.py b/src/test/python/apache/aurora/client/commands/test_kill.py
index 3e2ac1f..94bbe1b 100644
--- a/src/test/python/apache/aurora/client/commands/test_kill.py
+++ b/src/test/python/apache/aurora/client/commands/test_kill.py
@@ -13,17 +13,13 @@
#
import contextlib
-import unittest
from mock import Mock, patch
from twitter.common.contextutil import temporary_file
from apache.aurora.client.commands.core import kill, killall
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.common.aurora_job_key import AuroraJobKey
-from apache.aurora.common.cluster import Cluster
-from apache.aurora.common.clusters import Clusters
from gen.apache.aurora.api.ttypes import (
AssignedTask,
@@ -61,20 +57,6 @@ class TestClientKillCommand(AuroraClientCommandTest):
return mock_api_factory
@classmethod
- def create_mock_status_query_result(cls, scheduleStatus):
- mock_query_result = Mock(spec=Response)
- mock_query_result.result = Mock(spec=Result)
- mock_query_result.result.scheduleStatusResult = Mock(spec=ScheduleStatusResult)
- if scheduleStatus == ScheduleStatus.INIT:
- # status query result for before job is launched.
- mock_query_result.result.scheduleStatusResult.tasks = []
- else:
- mock_task_one = cls.create_mock_task('hello', 0, 1000, scheduleStatus)
- mock_task_two = cls.create_mock_task('hello', 1, 1004, scheduleStatus)
- mock_query_result.result.scheduleStatusResult.tasks = [mock_task_one, mock_task_two]
- return mock_query_result
-
- @classmethod
def get_kill_job_response(cls):
return cls.create_simple_success_response()
@@ -82,19 +64,11 @@ class TestClientKillCommand(AuroraClientCommandTest):
def get_kill_job_error_response(cls):
return cls.create_error_response()
-
@classmethod
def assert_kill_job_called(cls, mock_api):
assert mock_api.kill_job.call_count == 1
@classmethod
- def get_expected_task_query(cls, instances=None):
- """Helper to create the query that will be a parameter to job kill."""
- instance_ids = frozenset(instances) if instances is not None else None
- return TaskQuery(taskIds=None, jobName=cls.TEST_JOB, environment=cls.TEST_ENV,
- instanceIds=instance_ids, owner=Identity(role=cls.TEST_ROLE, user=None))
-
- @classmethod
def create_mock_task(cls, task_id, instance_id, initial_time, status):
mock_task = Mock(spec=ScheduledTask)
mock_task.assignedTask = Mock(spec=AssignedTask)
@@ -144,13 +118,9 @@ class TestClientKillCommand(AuroraClientCommandTest):
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
- patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)) as (
- mock_sleep,
- mock_api_patch,
- mock_scheduler_proxy_class,
- mock_clusters,
- options,
- mock_get_job_config):
+ patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)
+ ) as (mock_sleep, mock_api_patch, mock_scheduler_proxy_class, mock_clusters, options,
+ mock_get_job_config):
with temporary_file() as fp:
fp.write(self.get_valid_config())
@@ -408,17 +378,17 @@ class TestClientKillCommand(AuroraClientCommandTest):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_api.check_status.return_value = self.create_status_call_result()
mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_call_result()
- mock_api.kill_job.side_effect = [self.get_kill_job_error_response(), self.get_kill_job_response()]
+ mock_api.kill_job.side_effect = [
+ self.get_kill_job_error_response(), self.get_kill_job_response()]
with contextlib.nested(
patch('apache.aurora.client.factory.make_client', return_value=mock_api),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
- patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)) as (
- mock_api_factory_patch,
- mock_scheduler_proxy_class,
- mock_clusters,
- options, mock_get_job_config):
+ patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)
+ ) as (mock_api_factory_patch, mock_scheduler_proxy_class, mock_clusters, options,
+ mock_get_job_config):
+
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_listjobs.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_listjobs.py b/src/test/python/apache/aurora/client/commands/test_listjobs.py
index 69dd6b4..f2a325e 100644
--- a/src/test/python/apache/aurora/client/commands/test_listjobs.py
+++ b/src/test/python/apache/aurora/client/commands/test_listjobs.py
@@ -13,14 +13,11 @@
#
import contextlib
-import unittest
from mock import Mock, patch
from apache.aurora.client.commands.core import list_jobs
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.common.cluster import Cluster
-from apache.aurora.common.clusters import Clusters
from gen.apache.aurora.api.ttypes import GetJobsResult, JobKey
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_restart.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_restart.py b/src/test/python/apache/aurora/client/commands/test_restart.py
index 6e0159f..0c6d5a0 100644
--- a/src/test/python/apache/aurora/client/commands/test_restart.py
+++ b/src/test/python/apache/aurora/client/commands/test_restart.py
@@ -37,9 +37,10 @@ class TestRestartCommand(AuroraClientCommandTest):
@classmethod
def setup_mock_options(cls):
"""set up to get a mock options object."""
- mock_options = Mock(spec=['json', 'bindings', 'open_browser', 'shards', 'cluster',
- 'health_check_interval_seconds', 'batch_size', 'max_per_shard_failures',
- 'max_total_failures', 'restart_threshold', 'watch_secs'])
+ mock_options = Mock(
+ spec=['json', 'bindings', 'open_browser', 'shards', 'cluster',
+ 'health_check_interval_seconds', 'batch_size', 'max_per_shard_failures',
+ 'max_total_failures', 'restart_threshold', 'watch_secs'])
mock_options.json = False
mock_options.bindings = {}
mock_options.open_browser = False
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_status.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_status.py b/src/test/python/apache/aurora/client/commands/test_status.py
index cb053d1..a5c0ef0 100644
--- a/src/test/python/apache/aurora/client/commands/test_status.py
+++ b/src/test/python/apache/aurora/client/commands/test_status.py
@@ -13,14 +13,11 @@
#
import contextlib
-import unittest
from mock import Mock, patch
from apache.aurora.client.commands.core import status
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.common.cluster import Cluster
-from apache.aurora.common.clusters import Clusters
from gen.apache.aurora.api.ttypes import (
AssignedTask,
@@ -146,10 +143,8 @@ class TestListJobs(AuroraClientCommandTest):
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
- patch('twitter.common.app.get_options', return_value=mock_options)) as (
- mock_scheduler_proxy_class,
- mock_clusters,
- options):
+ patch('twitter.common.app.get_options', return_value=mock_options)
+ ) as (mock_scheduler_proxy_class, mock_clusters, options):
status(['west/mchucarroll/test/hello'], mock_options)
mock_scheduler_proxy.getTasksStatus.assert_called_with(TaskQuery(jobName='hello',
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/commands/test_update.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_update.py b/src/test/python/apache/aurora/client/commands/test_update.py
index c5afbd3..ea8e092 100644
--- a/src/test/python/apache/aurora/client/commands/test_update.py
+++ b/src/test/python/apache/aurora/client/commands/test_update.py
@@ -14,7 +14,6 @@
import contextlib
import functools
-import unittest
from mock import Mock, patch
from twitter.common.contextutil import temporary_file
@@ -25,9 +24,6 @@ from apache.aurora.client.api.quota_check import QuotaCheck
from apache.aurora.client.api.updater import Updater
from apache.aurora.client.commands.core import update
from apache.aurora.client.commands.util import AuroraClientCommandTest
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
-from apache.aurora.common.cluster import Cluster
-from apache.aurora.common.clusters import Clusters
from apache.aurora.config import AuroraConfig
from gen.apache.aurora.api.constants import ACTIVE_STATES
@@ -204,7 +200,8 @@ class TestUpdateCommand(AuroraClientCommandTest):
@classmethod
def setup_quota_check(cls):
mock_quota_check = Mock(spec=QuotaCheck)
- mock_quota_check.validate_quota_from_requested.return_value = cls.create_simple_success_response()
+ mock_quota_check.validate_quota_from_requested.return_value = (
+ cls.create_simple_success_response())
return mock_quota_check
@classmethod
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/test_binding_helper.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/test_binding_helper.py b/src/test/python/apache/aurora/client/test_binding_helper.py
index cc9397b..ee00300 100644
--- a/src/test/python/apache/aurora/client/test_binding_helper.py
+++ b/src/test/python/apache/aurora/client/test_binding_helper.py
@@ -12,8 +12,6 @@
# limitations under the License.
#
-import textwrap
-
from pystachio import Ref
from pystachio.matcher import Any, Matcher
from twitter.common.contextutil import temporary_file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/test_config.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/test_config.py b/src/test/python/apache/aurora/client/test_config.py
index 4b3e149..901c337 100644
--- a/src/test/python/apache/aurora/client/test_config.py
+++ b/src/test/python/apache/aurora/client/test_config.py
@@ -19,9 +19,16 @@ from twitter.common.contextutil import temporary_dir, temporary_file
from apache.aurora.client import config
from apache.aurora.config import AuroraConfig
-from apache.aurora.config.schema.base import HealthCheckConfig, UpdateConfig
from apache.aurora.config.loader import AuroraConfigLoader
-from apache.aurora.config.schema.base import Announcer, Job, MB, Resources, Task
+from apache.aurora.config.schema.base import (
+ Announcer,
+ HealthCheckConfig,
+ Job,
+ MB,
+ Resources,
+ Task,
+ UpdateConfig
+)
from gen.apache.aurora.api.constants import DEFAULT_ENVIRONMENT
@@ -172,20 +179,22 @@ def test_update_config_passes_with_default_values():
config._validate_update_config(AuroraConfig(base_job))
+
def test_update_config_passes_with_min_requirement_values():
base_job = Job(
name='hello_world', role='john_doe', cluster='test-cluster',
- update_config = UpdateConfig(watch_secs=26),
- health_check_config = HealthCheckConfig(max_consecutive_failures=1),
+ update_config=UpdateConfig(watch_secs=26),
+ health_check_config=HealthCheckConfig(max_consecutive_failures=1),
task=Task(name='main', processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
config._validate_update_config(AuroraConfig(base_job))
+
def test_update_config_fails_insufficient_watch_secs_less_than_target():
base_job = Job(
name='hello_world', role='john_doe', cluster='test-cluster',
- update_config = UpdateConfig(watch_secs=10),
+ update_config=UpdateConfig(watch_secs=10),
task=Task(name='main', processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
@@ -196,8 +205,8 @@ def test_update_config_fails_insufficient_watch_secs_less_than_target():
def test_update_config_fails_insufficient_watch_secs_equal_to_target():
base_job = Job(
name='hello_world', role='john_doe', cluster='test-cluster',
- update_config = UpdateConfig(watch_secs=25),
- health_check_config = HealthCheckConfig(max_consecutive_failures=1),
+ update_config=UpdateConfig(watch_secs=25),
+ health_check_config=HealthCheckConfig(max_consecutive_failures=1),
task=Task(name='main', processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/common/test_cluster.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_cluster.py b/src/test/python/apache/aurora/common/test_cluster.py
index 8559dfc..1d9422e 100644
--- a/src/test/python/apache/aurora/common/test_cluster.py
+++ b/src/test/python/apache/aurora/common/test_cluster.py
@@ -20,15 +20,15 @@ from apache.aurora.common.cluster import Cluster
def test_simple():
class AudubonTrait(Cluster.Trait):
- master_role = String
- slave_role = Default(String, 'slave')
- version = Required(Integer)
+ master_role = String # noqa
+ slave_role = Default(String, 'slave') # noqa
+ version = Required(Integer) # noqa
- west = Cluster(name = 'west',
- master_role = 'west.master',
- slave_role = 'west.slave',
- version = 10)
- east = Cluster(name = 'east', version = 11)
+ west = Cluster(name='west',
+ master_role='west.master',
+ slave_role='west.slave',
+ version=10)
+ east = Cluster(name='east', version=11)
assert east.name == 'east'
with pytest.raises(AttributeError):
@@ -39,4 +39,4 @@ def test_simple():
with pytest.raises(TypeError):
# requires version at least
- Cluster(name = 'east').with_traits(AudubonTrait)
+ Cluster(name='east').with_traits(AudubonTrait)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/common/test_cluster_option.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_cluster_option.py b/src/test/python/apache/aurora/common/test_cluster_option.py
index af671ee..2213d8c 100644
--- a/src/test/python/apache/aurora/common/test_cluster_option.py
+++ b/src/test/python/apache/aurora/common/test_cluster_option.py
@@ -20,8 +20,8 @@ from apache.aurora.common.cluster_option import ClusterOption
from apache.aurora.common.clusters import Clusters
CLUSTER_LIST = Clusters((
- Cluster(name = 'smf1'),
- Cluster(name = 'smf1-test'),
+ Cluster(name='smf1'),
+ Cluster(name='smf1-test'),
))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/common/test_clusters.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_clusters.py b/src/test/python/apache/aurora/common/test_clusters.py
index 2f5632f..45250e6 100644
--- a/src/test/python/apache/aurora/common/test_clusters.py
+++ b/src/test/python/apache/aurora/common/test_clusters.py
@@ -51,14 +51,13 @@ def validate_loaded_clusters(clusters):
cluster = clusters[cluster_name]
assert cluster.name == cluster_name
assert cluster.dc == cluster_name
- assert cluster.force_notunnel == False
+ assert cluster.force_notunnel is False
assert cluster.slave_root == '/var/lib/mesos'
assert cluster.slave_run_directory == 'latest'
assert cluster.auth_mechanism == 'UNAUTHENTICATED'
assert clusters['cluster1'].zk == 'zookeeper.cluster1.example.com'
-
def test_load_json():
with temporary_dir() as td:
clusters_json = os.path.join(td, 'clusters.json')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/common/test_http_signaler.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_http_signaler.py b/src/test/python/apache/aurora/common/test_http_signaler.py
index 6bac80a..f5f8419 100644
--- a/src/test/python/apache/aurora/common/test_http_signaler.py
+++ b/src/test/python/apache/aurora/common/test_http_signaler.py
@@ -69,7 +69,7 @@ class TestHttpSignaler(unittest.TestCase):
self._mox.StubOutWithMock(urllib_request, 'urlopen')
urllib_request.urlopen(
'http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndRaise(
- SocketTimeout('Timed out'))
+ SocketTimeout('Timed out'))
self._mox.ReplayAll()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/common/test_shellify.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_shellify.py b/src/test/python/apache/aurora/common/test_shellify.py
index 6bdecb7..e569b63 100644
--- a/src/test/python/apache/aurora/common/test_shellify.py
+++ b/src/test/python/apache/aurora/common/test_shellify.py
@@ -32,8 +32,7 @@ def test_shellify():
"hi": [0],
},
]
- }
- , prefix="TEST_"))
+ }, prefix="TEST_"))
assert set(dump) == set([
"TEST_NUM=123",
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/config/test_base.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_base.py b/src/test/python/apache/aurora/config/test_base.py
index 72a711a..66480db 100644
--- a/src/test/python/apache/aurora/config/test_base.py
+++ b/src/test/python/apache/aurora/config/test_base.py
@@ -16,17 +16,7 @@ import pytest
from twitter.common.contextutil import temporary_file
from apache.aurora.config import AuroraConfig, PortResolver
-from apache.aurora.config.schema.base import (
- Announcer,
- Empty,
- Integer,
- Job,
- Process,
- Resources,
- Task
-)
-
-from gen.apache.aurora.api.ttypes import Identity
+from apache.aurora.config.schema.base import Announcer, Empty, Job, Process, Resources, Task
resolve = PortResolver.resolve
@@ -56,7 +46,6 @@ def test_cycle():
resolve(portmap)
-
MESOS_CONFIG = """
HELLO_WORLD = Job(
name = 'hello_world',
@@ -72,6 +61,7 @@ HELLO_WORLD = Job(
jobs = [HELLO_WORLD]
"""
+
LIMITED_MESOS_CONFIG = """
HELLO_WORLD = Job(
name = 'hello_world',
@@ -96,33 +86,31 @@ jobs = [HELLO_WORLD]
REIFIED_CONFIG = Job(
- name = 'hello_world',
- role = 'john_doe',
- environment = 'staging42',
- cluster = 'smf1-test',
- task = Task(
- name = 'main',
- processes = [Process(name = 'hello_world', cmdline = 'echo {{mesos.instance}}')],
- resources = Resources(cpu = 0.1, ram = 64 * 1048576, disk = 64 * 1048576),
+ name='hello_world',
+ role='john_doe',
+ environment='staging42',
+ cluster='smf1-test',
+ task=Task(
+ name='main',
+ processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')],
+ resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576),
)
)
REIFIED_LIMITED_CONFIG = Job(
- name = 'hello_world',
- role = 'john_doe',
- environment = 'staging42',
- cluster = 'smf1-test',
- task = Task(
- name = 'main',
- processes = [Process(name = 'hello_world_fails_0', cmdline = 'echo hello world',
- max_failures = 0),
- Process(name = 'hello_world_fails_50', cmdline = 'echo hello world',
- max_failures = 50),
- Process(name = 'hello_world_fails_100', cmdline = 'echo hello world',
- max_failures = 100),
- Process(name = 'hello_world_fails_200', cmdline = 'echo hello world',
- max_failures = 200)],
- resources = Resources(cpu = 0.1, ram = 64 * 1048576, disk = 64 * 1048576),
+ name='hello_world',
+ role='john_doe',
+ environment='staging42',
+ cluster='smf1-test',
+ task=Task(
+ name='main',
+ processes=[
+ Process(name='hello_world_fails_0', cmdline='echo hello world', max_failures=0),
+ Process(name='hello_world_fails_50', cmdline='echo hello world', max_failures=50),
+ Process(name='hello_world_fails_100', cmdline='echo hello world', max_failures=100),
+ Process(name='hello_world_fails_200', cmdline='echo hello world', max_failures=200)
+ ],
+ resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576),
)
)
@@ -165,27 +153,33 @@ def test_simple_config():
def make_config(announce, *ports):
- process = Process(name = 'hello',
- cmdline = ' '.join('{{thermos.ports[%s]}}' % port for port in ports))
+ process = Process(
+ name='hello',
+ cmdline=' '.join('{{thermos.ports[%s]}}' % port for port in ports))
return AuroraConfig(Job(
- name = 'hello_world', environment = 'staging42', role = 'john_doe', cluster = 'smf1-test',
- announce = announce,
- task = Task(name = 'main', processes = [process],
- resources = Resources(cpu = 0.1, ram = 64 * 1048576, disk = 64 * 1048576))))
+ name='hello_world',
+ environment='staging42',
+ role='john_doe',
+ cluster='smf1-test',
+ announce=announce,
+ task=Task(
+ name='main',
+ processes=[process],
+ resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576))))
def test_ports():
- announce = Announcer(portmap = {'http': 80})
+ announce = Announcer(portmap={'http': 80})
assert make_config(announce).ports() == set()
assert make_config(announce, 'http').ports() == set()
assert make_config(announce, 'http', 'thrift').ports() == set(['thrift'])
- announce = Announcer(portmap = {'http': 'aurora'})
+ announce = Announcer(portmap={'http': 'aurora'})
assert make_config(announce).ports() == set(['aurora'])
assert make_config(announce, 'http').ports() == set(['aurora'])
assert make_config(announce, 'http', 'thrift').ports() == set(['thrift', 'aurora'])
- announce = Announcer(portmap = {'aurora': 'http'})
+ announce = Announcer(portmap={'aurora': 'http'})
assert make_config(announce).ports() == set(['http'])
assert make_config(announce, 'http').ports() == set(['http'])
assert make_config(announce, 'http', 'thrift').ports() == set(['http', 'thrift'])
@@ -196,8 +190,8 @@ def test_ports():
def test_static_port_aliasing():
- announce = Announcer(primary_port = 'thrift',
- portmap = {'thrift': 8081, 'health': 8300, 'aurora': 'health'})
+ announce = Announcer(primary_port='thrift',
+ portmap={'thrift': 8081, 'health': 8300, 'aurora': 'health'})
assert make_config(announce).ports() == set()
assert make_config(announce).job().taskConfig.requestedPorts == set()
assert make_config(announce, 'thrift').ports() == set()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/config/test_constraint_parsing.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_constraint_parsing.py b/src/test/python/apache/aurora/config/test_constraint_parsing.py
index 385f6a3..fac8436 100644
--- a/src/test/python/apache/aurora/config/test_constraint_parsing.py
+++ b/src/test/python/apache/aurora/config/test_constraint_parsing.py
@@ -12,8 +12,6 @@
# limitations under the License.
#
-import unittest
-
import pytest
from apache.aurora.config.thrift import constraints_to_thrift
@@ -25,7 +23,7 @@ def test_parsing_constraints():
}
c = constraints_to_thrift(constraints_dict).pop()
assert c.name == 'int'
- assert c.constraint.value.negated == False
+ assert c.constraint.value.negated is False
assert c.constraint.value.values == set(['2'])
# Test negated value
@@ -34,7 +32,7 @@ def test_parsing_constraints():
}
c = constraints_to_thrift(constraints_dict).pop()
assert c.name == '!str'
- assert c.constraint.value.negated == True
+ assert c.constraint.value.negated
assert c.constraint.value.values == set(['foo'])
# Test list
@@ -43,7 +41,7 @@ def test_parsing_constraints():
}
c = constraints_to_thrift(constraints_dict).pop()
assert c.name == 'set'
- assert c.constraint.value.negated == False
+ assert c.constraint.value.negated is False
assert c.constraint.value.values == set(['1', '2', '3', 'a', 'b', 'c'])
constraints_dict = {
@@ -51,7 +49,7 @@ def test_parsing_constraints():
}
c = constraints_to_thrift(constraints_dict).pop()
assert c.name == '!set'
- assert c.constraint.value.negated == True
+ assert c.constraint.value.negated
assert c.constraint.value.values == set(['1', '2', '3', 'a', 'b', 'c'])
# Test limit
@@ -66,4 +64,4 @@ def test_parsing_constraints():
'limit': 'limit:a',
}
with pytest.raises(ValueError):
- constraints = constraints_to_thrift(constraints_dict)
+ constraints_to_thrift(constraints_dict)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/config/test_loader.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_loader.py b/src/test/python/apache/aurora/config/test_loader.py
index adde41b..00b6eab 100644
--- a/src/test/python/apache/aurora/config/test_loader.py
+++ b/src/test/python/apache/aurora/config/test_loader.py
@@ -16,18 +16,17 @@ import json
import tempfile
import pytest
-from pystachio import Environment
from twitter.common.contextutil import temporary_file
from apache.aurora.config import AuroraConfig
from apache.aurora.config.loader import AuroraConfigLoader
-from apache.thermos.config.loader import ThermosTaskWrapper
BAD_MESOS_CONFIG = """
3 2 1 3 2 4 2 3
"""
+
MESOS_CONFIG = """
HELLO_WORLD = MesosJob(
name = 'hello_world',
@@ -42,6 +41,7 @@ HELLO_WORLD = MesosJob(
jobs = [HELLO_WORLD]
"""
+
def test_enoent():
nonexistent_file = tempfile.mktemp()
with pytest.raises(AuroraConfigLoader.NotFound):
@@ -97,6 +97,6 @@ def test_pick():
hello_world = env['jobs'][0]
assert AuroraConfig.pick(env, 'hello_world', None) == hello_world
- env['jobs'][0] = env['jobs'][0](name = 'something_{{else}}')
+ env['jobs'][0] = env['jobs'][0](name='something_{{else}}')
assert str(AuroraConfig.pick(env, 'something_else', [{'else': 'else'}]).name()) == (
'something_else')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/config/test_thrift.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py
index 9fea33d..fd28313 100644
--- a/src/test/python/apache/aurora/config/test_thrift.py
+++ b/src/test/python/apache/aurora/config/test_thrift.py
@@ -30,14 +30,14 @@ from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey
from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS
HELLO_WORLD = Job(
- name = 'hello_world',
- role = 'john_doe',
- environment = 'staging66',
- cluster = 'smf1-test',
- task = Task(
- name = 'main',
- processes = [Process(name = 'hello_world', cmdline = 'echo {{mesos.instance}}')],
- resources = Resources(cpu = 0.1, ram = 64 * 1048576, disk = 64 * 1048576),
+ name='hello_world',
+ role='john_doe',
+ environment='staging66',
+ cluster='smf1-test',
+ task=Task(
+ name='main',
+ processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')],
+ resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576),
)
)
@@ -51,14 +51,14 @@ def test_simple_config():
environment=HELLO_WORLD.environment().get(),
name=HELLO_WORLD.name().get())
assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser())
- assert job.cronSchedule == None
+ assert job.cronSchedule is None
assert tti.jobName == 'hello_world'
- assert tti.isService == False
+ assert tti.isService is False
assert tti.numCpus == 0.1
assert tti.ramMb == 64
assert tti.diskMb == 64
assert tti.requestedPorts == set()
- assert tti.production == False
+ assert tti.production is False
assert tti.priority == 0
assert tti.maxTaskFailures == 1
assert tti.constraints == set()
@@ -68,23 +68,23 @@ def test_simple_config():
def test_config_with_options():
hwc = HELLO_WORLD(
- production = True,
- priority = 200,
- service = True,
- cron_policy = 'RUN_OVERLAP',
- constraints = {
- 'dedicated': 'your_mom',
+ production=True,
+ priority=200,
+ service=True,
+ cron_policy='RUN_OVERLAP',
+ constraints={
+ 'dedicated': 'root',
'cpu': 'x86_64'
},
- environment = 'prod'
+ environment='prod'
)
job = convert_pystachio_to_thrift(hwc)
assert job.instanceCount == 1
tti = job.taskConfig
- assert tti.production == True
+ assert tti.production
assert tti.priority == 200
- assert tti.isService == True
+ assert tti.isService
assert job.cronCollisionPolicy == CronCollisionPolicy.RUN_OVERLAP
assert len(tti.constraints) == 2
assert tti.environment == 'prod'
@@ -93,10 +93,10 @@ def test_config_with_options():
def test_config_with_ports():
hwc = HELLO_WORLD(
- task = HELLO_WORLD.task()(
- processes = [
- Process(name = 'hello_world',
- cmdline = 'echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
+ task=HELLO_WORLD.task()(
+ processes=[
+ Process(name='hello_world',
+ cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]
)
)
@@ -112,23 +112,23 @@ def test_config_with_bad_resources():
convert_pystachio_to_thrift(HELLO_WORLD)
good_resources = [
- Resources(cpu = 1.0, ram = 1 * MB, disk = 1 * MB)
+ Resources(cpu=1.0, ram=1 * MB, disk=1 * MB)
]
bad_resources = [
- Resources(cpu = 0, ram = 1 * MB, disk = 1 * MB),
- Resources(cpu = 1, ram = 0 * MB, disk = 1 * MB),
- Resources(cpu = 1, ram = 1 * MB, disk = 0 * MB),
- Resources(cpu = 1, ram = 1 * MB - 1, disk = 1 * MB),
- Resources(cpu = 1, ram = 1 * MB, disk = 1 * MB - 1)
+ Resources(cpu=0, ram=1 * MB, disk=1 * MB),
+ Resources(cpu=1, ram=0 * MB, disk=1 * MB),
+ Resources(cpu=1, ram=1 * MB, disk=0 * MB),
+ Resources(cpu=1, ram=1 * MB - 1, disk=1 * MB),
+ Resources(cpu=1, ram=1 * MB, disk=1 * MB - 1)
]
for resource in good_resources:
- convert_pystachio_to_thrift(HELLO_WORLD(task = hwtask(resources = resource)))
+ convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
for resource in bad_resources:
with pytest.raises(ValueError):
- convert_pystachio_to_thrift(HELLO_WORLD(task = hwtask(resources = resource)))
+ convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
def test_config_with_task_links():
@@ -141,7 +141,7 @@ def test_config_with_task_links():
'foo': 'http://%host%:%port:foo%',
'bar': 'http://%host%:%port:bar%/%shard_id%'
}
- aurora_config = AuroraConfig(HELLO_WORLD(task_links = tl(unresolved_tl)))
+ aurora_config = AuroraConfig(HELLO_WORLD(task_links=tl(unresolved_tl)))
assert aurora_config.task_links() == tl(resolved_tl)
assert aurora_config.job().taskConfig.taskLinks == frozendict(resolved_tl)
@@ -154,12 +154,12 @@ def test_config_with_task_links():
def test_unbound_references():
def job_command(cmdline):
- return AuroraConfig(HELLO_WORLD(task = SimpleTask('hello_world', cmdline))).raw()
+ return AuroraConfig(HELLO_WORLD(task=SimpleTask('hello_world', cmdline))).raw()
# bindingless and bad => good bindings should work
convert_pystachio_to_thrift(job_command('echo hello world'))
convert_pystachio_to_thrift(job_command('echo {{mesos.user}}')
- .bind(mesos = {'user': '{{mesos.role}}'}))
+ .bind(mesos={'user': '{{mesos.role}}'}))
# unbound
with pytest.raises(InvalidConfig):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/common/test_directory_sandbox.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_directory_sandbox.py b/src/test/python/apache/aurora/executor/common/test_directory_sandbox.py
index 4d62cf9..a80c6ef 100644
--- a/src/test/python/apache/aurora/executor/common/test_directory_sandbox.py
+++ b/src/test/python/apache/aurora/executor/common/test_directory_sandbox.py
@@ -18,7 +18,7 @@ import mock
import pytest
from twitter.common.contextutil import temporary_dir
-from apache.aurora.executor.common.sandbox import DirectorySandbox, SandboxInterface
+from apache.aurora.executor.common.sandbox import DirectorySandbox
def test_directory_sandbox():
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/common/test_health_checker.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_health_checker.py b/src/test/python/apache/aurora/executor/common/test_health_checker.py
index 4071b40..baeaba2 100644
--- a/src/test/python/apache/aurora/executor/common/test_health_checker.py
+++ b/src/test/python/apache/aurora/executor/common/test_health_checker.py
@@ -24,6 +24,7 @@ from apache.aurora.executor.common.health_checker import HealthCheckerThread
def thread_yield():
time.sleep(0.1)
+
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/common/test_task_info.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_task_info.py b/src/test/python/apache/aurora/executor/common/test_task_info.py
index 394ee71..344fd67 100644
--- a/src/test/python/apache/aurora/executor/common/test_task_info.py
+++ b/src/test/python/apache/aurora/executor/common/test_task_info.py
@@ -26,25 +26,26 @@ from apache.aurora.executor.common.task_info import mesos_task_instance_from_ass
from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, TaskConfig
-BASE_MTI = MesosTaskInstance(instance = 0, role = getpass.getuser())
-BASE_TASK = Task(resources = Resources(cpu=1.0, ram=16*MB, disk=32*MB))
+BASE_MTI = MesosTaskInstance(instance=0, role=getpass.getuser())
+BASE_TASK = Task(resources=Resources(cpu=1.0, ram=16 * MB, disk=32 * MB))
HELLO_WORLD_TASK_ID = 'hello_world-001'
HELLO_WORLD = BASE_TASK(
- name = 'hello_world',
- processes = [Process(name = 'hello_world_{{thermos.task_id}}', cmdline = 'echo hello world')])
+ name='hello_world',
+ processes=[Process(name='hello_world_{{thermos.task_id}}', cmdline='echo hello world')])
HELLO_WORLD_MTI = BASE_MTI(task=HELLO_WORLD)
-SLEEP60 = BASE_TASK(processes = [Process(name = 'sleep60', cmdline = 'sleep 60')])
-SLEEP2 = BASE_TASK(processes = [Process(name = 'sleep2', cmdline = 'sleep 2')])
+SLEEP60 = BASE_TASK(processes=[Process(name='sleep60', cmdline='sleep 60')])
+SLEEP2 = BASE_TASK(processes=[Process(name='sleep2', cmdline='sleep 2')])
SLEEP60_MTI = BASE_MTI(task=SLEEP60)
MESOS_JOB = MesosJob(
- name = 'does_not_matter',
- instances = 1,
- role = getpass.getuser(),
+ name='does_not_matter',
+ instances=1,
+ role=getpass.getuser(),
)
+
def test_deserialize_thermos_task():
task_config = TaskConfig(
executorConfig=ExecutorConfig(name='thermos', data=MESOS_JOB(task=HELLO_WORLD).json_dumps()))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/test_executor_vars.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/test_executor_vars.py b/src/test/python/apache/aurora/executor/test_executor_vars.py
index b35387f..af1c7b7 100644
--- a/src/test/python/apache/aurora/executor/test_executor_vars.py
+++ b/src/test/python/apache/aurora/executor/test_executor_vars.py
@@ -43,7 +43,7 @@ def test_extract_pexinfo():
def test_init():
self = ExecutorVars()
- assert self._orphan == False
+ assert self._orphan is False
samples = self.metrics.sample()
assert samples['version'] == 'UNKNOWN'
assert samples['orphan'] == 0
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/test_gc_executor.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/test_gc_executor.py b/src/test/python/apache/aurora/executor/test_gc_executor.py
index e98bffa..0b2278a 100644
--- a/src/test/python/apache/aurora/executor/test_gc_executor.py
+++ b/src/test/python/apache/aurora/executor/test_gc_executor.py
@@ -105,10 +105,10 @@ class ProxyDriver(object):
def stop(self):
self.stopped.set()
- def sendStatusUpdate(self, update):
+ def sendStatusUpdate(self, update): # noqa
self.updates.append(StatusUpdate(update.state, update.task_id.value))
- def sendFrameworkMessage(self, message):
+ def sendFrameworkMessage(self, message): # noqa
self.messages.append(thrift_deserialize(SchedulerMessage(), message))
@@ -261,6 +261,7 @@ def test_state_reconciliation_terminal_active():
assert tgc.len_results == (0, 0, 0, 0)
assert llen(lgc, rgc, updates) == (0, 0, 1)
+
def test_state_reconciliation_corrupt_tasks():
for st0, st1 in product(THERMOS_TERMINALS, LIVE_STATES):
tgc, driver = make_pair({}, {'foo': st0}, corrupt_tasks=['foo'])
@@ -268,6 +269,7 @@ def test_state_reconciliation_corrupt_tasks():
assert tgc.len_results == (0, 0, 0, 0)
assert llen(lgc, rgc, updates) == (1, 0, 0)
+
def test_state_reconciliation_terminal_nexist():
for st0, st1 in product(THERMOS_TERMINALS, LIVE_STATES):
tgc, driver = make_pair({}, {'foo': st0})
@@ -329,8 +331,8 @@ def run_gc_with(active_executors, retained_tasks, lose=False):
assert len(executor._gc_task_queue) == 0
assert not executor._task_id
assert len(proxy_driver.updates) >= 1
- if not lose: # if the task is lost it will be cleaned out of band (by clean_orphans),
- # so we don't care when the GC task actually finishes
+ if not lose: # if the task is lost it will be cleaned out of band (by clean_orphans),
+ # so we don't care when the GC task actually finishes
assert proxy_driver.updates[-1][0] == mesos.TASK_FINISHED
assert proxy_driver.updates[-1][1] == TASK_ID
return executor, proxy_driver
@@ -563,8 +565,9 @@ class TestRealGC(unittest.TestCase):
"""
Test functions against the actual garbage_collect() functionality of the GC executor
"""
+
def setUp(self):
- self.HELLO_WORLD= SimpleTask(name="foo", command="echo hello world")
+ self.HELLO_WORLD = SimpleTask(name="foo", command="echo hello world")
def setup_task(self, task, root, finished=False, corrupt=False):
"""Set up the checkpoint stream for the given task in the given checkpoint root, optionally
@@ -590,15 +593,28 @@ class TestRealGC(unittest.TestCase):
def run_gc(self, root, task_id, retain=False):
"""Run the garbage collection process against the given task_id in the given checkpoint root"""
class FakeTaskKiller(object):
- def __init__(self, task_id, checkpoint_root): pass
- def kill(self): pass
- def lose(self): pass
+ def __init__(self, task_id, checkpoint_root):
+ pass
+
+ def kill(self):
+ pass
+
+ def lose(self):
+ pass
+
class FakeTaskGarbageCollector(object):
- def __init__(self, root): pass
- def erase_logs(self, task_id): pass
- def erase_metadata(self, task_id): pass
+ def __init__(self, root):
+ pass
+
+ def erase_logs(self, task_id):
+ pass
+
+ def erase_metadata(self, task_id):
+ pass
+
class FastThermosGCExecutor(ThermosGCExecutor):
POLL_WAIT = Amount(1, Time.MICROSECONDS)
+
detector = functools.partial(FakeExecutorDetector, task_id) if retain else FakeExecutorDetector
executor = FastThermosGCExecutor(
checkpoint_root=root,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/executor/test_thermos_executor.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/test_thermos_executor.py b/src/test/python/apache/aurora/executor/test_thermos_executor.py
index b28fc32..55747a1 100644
--- a/src/test/python/apache/aurora/executor/test_thermos_executor.py
+++ b/src/test/python/apache/aurora/executor/test_thermos_executor.py
@@ -147,23 +147,23 @@ def make_task(thermos_config, assigned_ports={}, **kw):
return td
-BASE_MTI = MesosTaskInstance(instance = 0, role = getpass.getuser())
-BASE_TASK = Task(resources = Resources(cpu=1.0, ram=16*MB, disk=32*MB))
+BASE_MTI = MesosTaskInstance(instance=0, role=getpass.getuser())
+BASE_TASK = Task(resources=Resources(cpu=1.0, ram=16 * MB, disk=32 * MB))
HELLO_WORLD_TASK_ID = 'hello_world-001'
HELLO_WORLD = BASE_TASK(
- name = 'hello_world',
- processes = [Process(name = 'hello_world_{{thermos.task_id}}', cmdline = 'echo hello world')])
+ name='hello_world',
+ processes=[Process(name='hello_world_{{thermos.task_id}}', cmdline='echo hello world')])
HELLO_WORLD_MTI = BASE_MTI(task=HELLO_WORLD)
-SLEEP60 = BASE_TASK(processes = [Process(name = 'sleep60', cmdline = 'sleep 60')])
-SLEEP2 = BASE_TASK(processes = [Process(name = 'sleep2', cmdline = 'sleep 2')])
+SLEEP60 = BASE_TASK(processes=[Process(name='sleep60', cmdline='sleep 60')])
+SLEEP2 = BASE_TASK(processes=[Process(name='sleep2', cmdline='sleep 2')])
SLEEP60_MTI = BASE_MTI(task=SLEEP60)
MESOS_JOB = MesosJob(
- name = 'does_not_matter',
- instances = 1,
- role = getpass.getuser(),
+ name='does_not_matter',
+ instances=1,
+ role=getpass.getuser(),
)
@@ -244,12 +244,15 @@ class SignalServer(ExceptionalThread):
super(SignalServer, self).__init__()
self.daemon = True
self._stop = threading.Event()
+
def run(self):
while not self._stop.is_set():
self._server.handle_request()
+
def __enter__(self):
self.start()
return self._server.server_port
+
def __exit__(self, exc_type, exc_val, traceback):
self._stop.set()
@@ -350,7 +353,7 @@ class TestThermosExecutor(object):
assert len(updates) == 3
assert updates[-1][0][0].state == mesos_pb.TASK_KILLED
- def test_killTask(self):
+ def test_killTask(self): # noqa
proxy_driver = ProxyDriver()
with temporary_dir() as checkpoint_root:
@@ -393,12 +396,13 @@ class TestThermosExecutor(object):
with SignalServer(UnhealthyHandler) as port:
with temporary_dir() as checkpoint_root:
health_check_config = HealthCheckConfig(initial_interval_secs=0.1, interval_secs=0.1)
- _, executor = make_executor(proxy_driver,
- checkpoint_root,
- MESOS_JOB(task=SLEEP60, health_check_config=health_check_config),
- ports={'health': port},
- fast_status=True,
- status_providers=(HealthCheckerProvider(),))
+ _, executor = make_executor(
+ proxy_driver,
+ checkpoint_root,
+ MESOS_JOB(task=SLEEP60, health_check_config=health_check_config),
+ ports={'health': port},
+ fast_status=True,
+ status_providers=(HealthCheckerProvider(),))
executor.terminated.wait()
updates = proxy_driver.method_calls['sendStatusUpdate']
@@ -468,7 +472,7 @@ class TestThermosExecutor(object):
te._sandbox._init_start.set()
- def test_killTask_during_runner_initialize(self):
+ def test_killTask_during_runner_initialize(self): # noqa
proxy_driver = ProxyDriver()
task = make_task(HELLO_WORLD_MTI)
@@ -500,7 +504,7 @@ class TestThermosExecutor(object):
assert len(updates) == 2
assert updates[-1][0][0].state == mesos_pb.TASK_KILLED
- def test_launchTask_deserialization_fail(self):
+ def test_launchTask_deserialization_fail(self): # noqa
proxy_driver = ProxyDriver()
role = getpass.getuser()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/bin/test_thermos.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/bin/test_thermos.py b/src/test/python/apache/thermos/bin/test_thermos.py
index 67fa90b..2d9d333 100644
--- a/src/test/python/apache/thermos/bin/test_thermos.py
+++ b/src/test/python/apache/thermos/bin/test_thermos.py
@@ -12,4 +12,4 @@
# limitations under the License.
#
-from apache.thermos.bin import thermos
+from apache.thermos.bin import thermos # noqa
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/common/test_planner.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/common/test_planner.py b/src/test/python/apache/thermos/common/test_planner.py
index f8cf49b..b58b5bf 100644
--- a/src/test/python/apache/thermos/common/test_planner.py
+++ b/src/test/python/apache/thermos/common/test_planner.py
@@ -20,11 +20,14 @@ from apache.thermos.common.planner import Planner
def details(planner):
return planner.runnable, planner.running, planner.finished
-def _(*processes):
+
+def _(*processes): # noqa
return set(processes)
+
empty = set()
+
def test_planner_empty():
p = Planner(set(), {})
assert details(p) == (empty, empty, empty)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/common/test_task_planner.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/common/test_task_planner.py b/src/test/python/apache/thermos/common/test_task_planner.py
index ffdc30b..132c1ec 100644
--- a/src/test/python/apache/thermos/common/test_task_planner.py
+++ b/src/test/python/apache/thermos/common/test_task_planner.py
@@ -13,23 +13,26 @@
#
import pytest
-from twitter.common.testing.clock import ThreadedClock
from apache.thermos.common.planner import TaskPlanner
-from apache.thermos.config.schema import *
+from apache.thermos.config.schema import Constraint, Process, Task
-p1 = Process(name = "p1", cmdline = "")
-p2 = Process(name = "p2", cmdline = "")
-p3 = Process(name = "p3", cmdline = "")
+p1 = Process(name="p1", cmdline="")
+p2 = Process(name="p2", cmdline="")
+p3 = Process(name="p3", cmdline="")
-unordered_task = Task(name = "unordered", processes = [p1, p2, p3])
-ordered_task = unordered_task(constraints = [{'order': ['p1', 'p2', 'p3']}])
-empty_task = Task(name = "empty", processes = [])
+unordered_task = Task(name="unordered", processes=[p1, p2, p3])
+ordered_task = unordered_task(constraints=[{'order': ['p1', 'p2', 'p3']}])
+empty_task = Task(name="empty", processes=[])
-def _(*processes):
+
+def _(*processes): # noqa
return set(processes)
+
+
empty = set()
+
def approx_equal(a, b):
return abs(a - b) < 0.001
@@ -203,7 +206,7 @@ def test_task_waits():
def test_task_fails():
dt = p1(max_failures=1, min_duration=1)
- p = TaskPlanner(empty_task(processes = [dt(name='d1'), dt(name='d2')]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1'), dt(name='d2')]))
assert p.runnable_at(timestamp=0) == _('d1', 'd2')
p.set_running('d1')
p.set_running('d2')
@@ -213,7 +216,7 @@ def test_task_fails():
assert p.runnable_at(timestamp=0) == empty
assert p.min_wait(timestamp=0) == TaskPlanner.INFINITY
- p = TaskPlanner(empty_task(processes = [dt(name='d1'), dt(name='d2')]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1'), dt(name='d2')]))
assert p.runnable_at(timestamp=0) == _('d1', 'd2')
p.set_running('d1')
p.set_failed('d1')
@@ -225,7 +228,7 @@ def test_task_fails():
assert p.min_wait(timestamp=0) == TaskPlanner.INFINITY
# test max_failures=0 && daemon==True ==> retries forever
- p = TaskPlanner(empty_task(processes = [dt(name='d1', max_failures=0, daemon=True)]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1', max_failures=0, daemon=True)]))
for k in range(10):
p.set_running('d1')
assert 'd1' in p.running
@@ -241,7 +244,7 @@ def test_task_fails():
assert 'd1' not in p.failed
assert 'd1' not in p.finished
- p = TaskPlanner(empty_task(processes = [dt(name='d1', max_failures=0)]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1', max_failures=0)]))
p.set_running('d1')
assert 'd1' in p.running
assert 'd1' not in p.failed
@@ -261,14 +264,14 @@ def test_task_lost():
dt = p1(max_failures=2, min_duration=1)
# regular success behavior
- p = TaskPlanner(empty_task(processes = [dt(name='d1')]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1')]))
assert p.runnable_at(timestamp=0) == _('d1')
p.set_running('d1')
p.add_success('d1', timestamp=0)
assert p.min_wait(timestamp=0) == TaskPlanner.INFINITY
# regular failure behavior
- p = TaskPlanner(empty_task(processes = [dt(name='d1')]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1')]))
assert p.runnable_at(timestamp=0) == _('d1')
p.set_running('d1')
p.add_failure('d1', timestamp=1)
@@ -278,7 +281,7 @@ def test_task_lost():
assert p.min_wait(timestamp=3) == TaskPlanner.INFINITY
# lost behavior
- p = TaskPlanner(empty_task(processes = [dt(name='d1')]))
+ p = TaskPlanner(empty_task(processes=[dt(name='d1')]))
assert p.runnable_at(timestamp=0) == _('d1')
p.set_running('d1')
p.add_failure('d1', timestamp=1)
@@ -293,18 +296,21 @@ def test_task_lost():
def test_task_filters():
t = p1
- task = empty_task(processes = [t(name='p1'), t(name='p2'), t(name='p3'),
- t(name='f1'), t(name='f2'), t(name='f3')],
- constraints = [Constraint(order=['p1','p2','p3']),
- Constraint(order=['f1','f2','f3'])])
+ task = empty_task(
+ processes=[
+ t(name='p1'), t(name='p2'), t(name='p3'),
+ t(name='f1'), t(name='f2'), t(name='f3')],
+ constraints=[
+ Constraint(order=['p1', 'p2', 'p3']),
+ Constraint(order=['f1', 'f2', 'f3'])])
assert TaskPlanner(task, process_filter=lambda proc: proc.name().get().startswith('p'))
assert TaskPlanner(task, process_filter=lambda proc: proc.name().get().startswith('f'))
with pytest.raises(TaskPlanner.InvalidSchedule):
- TaskPlanner(task(constraints=[Constraint(order=['p1','f1'])]),
+ TaskPlanner(task(constraints=[Constraint(order=['p1', 'f1'])]),
process_filter=lambda proc: proc.name().get().startswith('p'))
with pytest.raises(TaskPlanner.InvalidSchedule):
- TaskPlanner(task(constraints=[Constraint(order=['p1','f1'])]),
+ TaskPlanner(task(constraints=[Constraint(order=['p1', 'f1'])]),
process_filter=lambda proc: proc.name().get().startswith('f'))
@@ -313,7 +319,7 @@ def test_task_max_runs():
TOTAL_RUN_LIMIT = 2
dt = p1(daemon=True, max_failures=0)
- p = CappedTaskPlanner(empty_task(processes = [dt(name = 'd1', max_failures=100, daemon=False)]))
+ p = CappedTaskPlanner(empty_task(processes=[dt(name='d1', max_failures=100, daemon=False)]))
p.set_running('d1')
p.add_failure('d1', timestamp=1)
assert 'd1' in p.runnable
@@ -321,7 +327,7 @@ def test_task_max_runs():
p.add_failure('d1', timestamp=2)
assert 'd1' not in p.runnable
- p = CappedTaskPlanner(empty_task(processes = [dt(name = 'd1', max_failures=100)]))
+ p = CappedTaskPlanner(empty_task(processes=[dt(name='d1', max_failures=100)]))
p.set_running('d1')
p.add_failure('d1', timestamp=1)
assert 'd1' in p.runnable
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/config/test_schema.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/config/test_schema.py b/src/test/python/apache/thermos/config/test_schema.py
index f48f6c6..85b04c0 100644
--- a/src/test/python/apache/thermos/config/test_schema.py
+++ b/src/test/python/apache/thermos/config/test_schema.py
@@ -20,7 +20,6 @@ from apache.thermos.config.schema import (
concat_tasks,
Constraint,
java_options,
- List,
order,
Process,
python_options,
@@ -50,6 +49,7 @@ def test_order():
with pytest.raises(ValueError):
order(None)
+
def test_add_resources():
assert Units.resources_sum(Resources(), Resources()) == Resources(cpu=0, ram=0, disk=0)
@@ -120,8 +120,8 @@ def test_tasklets():
recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19])
recipe_php = SequentialTask(processes=[install_thermosrc, setup_php])
all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php)
- my_task = Task(processes = [Process(name='my_process')])
- my_new_task = Tasks.concat(all_recipes, my_task)(name = 'my_task')
+ my_task = Task(processes=[Process(name='my_process')])
+ my_new_task = Tasks.concat(all_recipes, my_task)(name='my_task')
# TODO(wickman) Probably should have Tasks.combine/concat do constraint
# minimization since many constraints are redundant.
@@ -129,7 +129,6 @@ def test_tasklets():
assert p in my_new_task.processes()
-
def test_render_options():
def eq(o1, o2):
return set(o1.split()) == set(o2.split())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_angry.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_angry.py b/src/test/python/apache/thermos/core/test_angry.py
index 02280ed..126ff79 100644
--- a/src/test/python/apache/thermos/core/test_angry.py
+++ b/src/test/python/apache/thermos/core/test_angry.py
@@ -12,8 +12,6 @@
# limitations under the License.
#
-import random
-
from apache.thermos.config.schema import Process, Task
from apache.thermos.testing.runner import Runner
@@ -22,14 +20,14 @@ from gen.apache.thermos.ttypes import ProcessState, TaskState
def flaky_task():
task = Task(
- name = "failing_task",
- max_failures = 2,
- processes = [
- Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
- Process(name = "b", max_failures=2, min_duration=1, cmdline="exit 1"),
- Process(name = "c", max_failures=1, min_duration=1, final=True, cmdline="echo hello world")
+ name="failing_task",
+ max_failures=2,
+ processes=[
+ Process(name="a", max_failures=1, min_duration=1, cmdline="echo hello world"),
+ Process(name="b", max_failures=2, min_duration=1, cmdline="exit 1"),
+ Process(name="c", max_failures=1, min_duration=1, final=True, cmdline="echo hello world")
],
- constraints = [{'order': ['a', 'b']}]
+ constraints=[{'order': ['a', 'b']}]
)
return task.interpolate()[0]
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_ephemerals.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_ephemerals.py b/src/test/python/apache/thermos/core/test_ephemerals.py
index 34b29aa..c81d0bb 100644
--- a/src/test/python/apache/thermos/core/test_ephemerals.py
+++ b/src/test/python/apache/thermos/core/test_ephemerals.py
@@ -12,7 +12,7 @@
# limitations under the License.
#
-from apache.thermos.config.schema import Process, Resources, Task
+from apache.thermos.config.schema import Process, Task
from apache.thermos.testing.runner import RunnerTestBase
from gen.apache.thermos.ttypes import ProcessState, TaskState
@@ -22,11 +22,11 @@ class TestEphemeralTask(RunnerTestBase):
@classmethod
def task(cls):
task = Task(
- name = "task_with_ephemeral",
- processes = [
- Process(name = "ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"),
- Process(name = "sleepy", cmdline="sleep 1")
- ])
+ name="task_with_ephemeral",
+ processes=[
+ Process(name="ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"),
+ Process(name="sleepy", cmdline="sleep 1")
+ ])
return task.interpolate()[0]
def test_runner_state(self):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/thermos/core/test_failing_runner.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_failing_runner.py b/src/test/python/apache/thermos/core/test_failing_runner.py
index 0c4e812..36a9fc0 100644
--- a/src/test/python/apache/thermos/core/test_failing_runner.py
+++ b/src/test/python/apache/thermos/core/test_failing_runner.py
@@ -25,21 +25,21 @@ class TestFailingRunner(RunnerTestBase):
name="{{process_name}}",
min_duration=1,
max_failures=5,
- cmdline = "echo {{process_name}} pinging; "
- "echo ping >> {{process_name}}; "
- "echo current count $(cat {{process_name}} | wc -l); "
- "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
- " exit 0; "
- "else "
- " exit 1; "
- "fi ")
+ cmdline="echo {{process_name}} pinging; "
+ "echo ping >> {{process_name}}; "
+ "echo current count $(cat {{process_name}} | wc -l); "
+ "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
+ " exit 0; "
+ "else "
+ " exit 1; "
+ "fi ")
tsk = Task(
- name = "pingping",
- resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
- processes = [
- ping_template.bind(process_name = "p1", num_runs = 1),
- ping_template.bind(process_name = "p2", num_runs = 2),
- ping_template.bind(process_name = "p3", num_runs = 3),
+ name="pingping",
+ resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
+ processes=[
+ ping_template.bind(process_name="p1", num_runs=1),
+ ping_template.bind(process_name="p2", num_runs=2),
+ ping_template.bind(process_name="p3", num_runs=3),
]
)
return tsk.interpolate()[0]
@@ -49,10 +49,10 @@ class TestFailingRunner(RunnerTestBase):
def test_runner_processes_have_expected_runs(self):
processes = self.state.processes
- for k in range(1,4):
+ for k in range(1, 4):
process_name = 'p%d' % k
assert process_name in processes
assert len(processes[process_name]) == k
- for j in range(k-1):
+ for j in range(k - 1):
assert processes[process_name][j].state == ProcessState.FAILED
- assert processes[process_name][k-1].state == ProcessState.SUCCESS
+ assert processes[process_name][k - 1].state == ProcessState.SUCCESS
[4/4] git commit: Make style consistent with
build-support/python/checkstyle-check
Posted by wi...@apache.org.
Make style consistent with build-support/python/checkstyle-check
Reviewed at https://reviews.apache.org/r/22168/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/467f8428
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/467f8428
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/467f8428
Branch: refs/heads/master
Commit: 467f84285d7fd0c9f861b839f25449b194e391e4
Parents: ca4adf9
Author: Brian Wickman <wi...@apache.org>
Authored: Thu Jun 5 09:48:43 2014 -0700
Committer: Brian Wickman <wi...@apache.org>
Committed: Thu Jun 5 09:48:43 2014 -0700
----------------------------------------------------------------------
.../apache/aurora/admin/host_maintenance.py | 8 +-
.../python/apache/aurora/client/api/__init__.py | 17 +--
.../apache/aurora/client/api/command_runner.py | 21 ++-
.../apache/aurora/client/api/health_check.py | 6 +-
.../aurora/client/api/scheduler_client.py | 29 +++--
src/main/python/apache/aurora/client/api/sla.py | 23 ++--
.../python/apache/aurora/client/api/updater.py | 37 +++---
.../apache/aurora/client/api/updater_util.py | 2 +-
src/main/python/apache/aurora/client/base.py | 36 ++++--
.../apache/aurora/client/bin/aurora_admin.py | 5 +-
.../apache/aurora/client/bin/aurora_client.py | 5 +-
.../apache/aurora/client/binding_helper.py | 10 +-
.../python/apache/aurora/client/cli/__init__.py | 21 +--
.../python/apache/aurora/client/cli/bridge.py | 1 -
.../python/apache/aurora/client/cli/client.py | 8 +-
.../apache/aurora/client/cli/command_hooks.py | 6 +-
.../python/apache/aurora/client/cli/config.py | 12 +-
.../python/apache/aurora/client/cli/context.py | 4 +-
.../python/apache/aurora/client/cli/cron.py | 4 +-
.../python/apache/aurora/client/cli/jobs.py | 15 +--
.../python/apache/aurora/client/cli/logsetup.py | 9 +-
.../python/apache/aurora/client/cli/options.py | 7 +-
.../python/apache/aurora/client/cli/quota.py | 11 +-
.../aurora/client/cli/standalone_client.py | 3 +-
.../python/apache/aurora/client/cli/task.py | 45 ++-----
.../apache/aurora/client/commands/admin.py | 10 +-
.../apache/aurora/client/commands/core.py | 7 +-
.../apache/aurora/client/commands/help.py | 4 +-
.../aurora/client/commands/maintenance.py | 1 -
src/main/python/apache/aurora/client/config.py | 7 +-
src/main/python/apache/aurora/client/factory.py | 1 -
.../apache/aurora/client/hooks/hooked_api.py | 4 +-
.../apache/aurora/common/auth/__init__.py | 12 +-
.../apache/aurora/common/auth/auth_module.py | 2 -
src/main/python/apache/aurora/common/cluster.py | 2 +-
.../apache/aurora/common/cluster_option.py | 2 +-
.../python/apache/aurora/common/clusters.py | 4 +-
.../apache/aurora/common/http_signaler.py | 1 -
src/main/python/apache/aurora/config/loader.py | 1 -
.../python/apache/aurora/config/schema/base.py | 3 +
src/main/python/apache/aurora/config/thrift.py | 6 +-
.../apache/aurora/executor/aurora_executor.py | 2 +-
.../aurora/executor/bin/thermos_runner_main.py | 2 +-
.../aurora/executor/common/health_checker.py | 1 +
.../aurora/executor/common/kill_manager.py | 1 +
.../apache/aurora/executor/common/sandbox.py | 1 +
.../apache/aurora/executor/gc_executor.py | 26 ++--
.../apache/aurora/executor/thermos_runner.py | 2 +-
.../aurora/tools/java/organize_imports.py | 128 ++++++++++---------
.../aurora/tools/java/thrift_wrapper_codegen.py | 8 +-
src/main/python/apache/thermos/bin/thermos.py | 43 +++----
.../python/apache/thermos/bin/thermos_ckpt.py | 47 ++++---
src/main/python/apache/thermos/common/ckpt.py | 6 +-
.../python/apache/thermos/common/options.py | 2 +
src/main/python/apache/thermos/common/path.py | 20 +--
.../python/apache/thermos/common/planner.py | 33 ++---
.../apache/thermos/config/bin/config_load.py | 29 ++---
.../apache/thermos/config/bin/config_repl.py | 2 +-
src/main/python/apache/thermos/config/loader.py | 30 ++---
src/main/python/apache/thermos/config/schema.py | 4 +-
.../python/apache/thermos/config/schema_base.py | 2 +
.../apache/thermos/config/schema_helpers.py | 5 +
.../python/apache/thermos/core/inspector.py | 4 +-
src/main/python/apache/thermos/core/muxer.py | 23 ++--
src/main/python/apache/thermos/core/process.py | 4 +-
src/main/python/apache/thermos/core/runner.py | 44 +++----
src/main/python/apache/thermos/monitoring/BUILD | 2 +-
.../apache/thermos/monitoring/detector.py | 6 +-
.../python/apache/thermos/monitoring/disk.py | 7 +-
.../python/apache/thermos/monitoring/garbage.py | 6 +-
.../python/apache/thermos/monitoring/process.py | 35 ++---
.../monitoring/process_collector_psutil.py | 3 +-
.../apache/thermos/monitoring/resource.py | 11 +-
.../thermos/observer/bin/thermos_observer.py | 2 -
.../thermos/observer/http/file_browser.py | 2 -
.../thermos/observer/http/http_observer.py | 25 ++--
.../thermos/observer/http/static_assets.py | 1 +
.../python/apache/thermos/testing/runner.py | 18 +--
.../aurora/admin/test_host_maintenance.py | 1 -
.../aurora/client/api/test_disambiguator.py | 4 +-
.../aurora/client/api/test_health_check.py | 1 -
.../aurora/client/api/test_instance_watcher.py | 22 +++-
.../aurora/client/api/test_job_monitor.py | 3 +-
.../aurora/client/api/test_quota_check.py | 1 -
.../apache/aurora/client/api/test_restarter.py | 28 ++--
.../aurora/client/api/test_scheduler_client.py | 37 ++++--
.../python/apache/aurora/client/api/test_sla.py | 16 +--
.../apache/aurora/client/api/test_updater.py | 15 +--
.../aurora/client/cli/test_command_hooks.py | 12 +-
.../aurora/client/cli/test_config_noun.py | 3 +-
.../apache/aurora/client/cli/test_create.py | 2 -
.../apache/aurora/client/cli/test_cron.py | 5 +-
.../apache/aurora/client/cli/test_kill.py | 26 ++--
.../apache/aurora/client/cli/test_open.py | 6 +-
.../apache/aurora/client/cli/test_plugins.py | 18 +--
.../apache/aurora/client/cli/test_restart.py | 15 +--
.../apache/aurora/client/cli/test_status.py | 2 +-
.../apache/aurora/client/cli/test_task_run.py | 8 +-
.../apache/aurora/client/cli/test_update.py | 4 +-
.../python/apache/aurora/client/cli/util.py | 5 +-
.../aurora/client/commands/test_admin_sla.py | 24 ++--
.../client/commands/test_cancel_update.py | 5 -
.../aurora/client/commands/test_create.py | 6 -
.../apache/aurora/client/commands/test_hooks.py | 24 +---
.../apache/aurora/client/commands/test_kill.py | 48 ++-----
.../aurora/client/commands/test_listjobs.py | 3 -
.../aurora/client/commands/test_restart.py | 7 +-
.../aurora/client/commands/test_status.py | 9 +-
.../aurora/client/commands/test_update.py | 7 +-
.../apache/aurora/client/test_binding_helper.py | 2 -
.../python/apache/aurora/client/test_config.py | 23 +++-
.../python/apache/aurora/common/test_cluster.py | 18 +--
.../apache/aurora/common/test_cluster_option.py | 4 +-
.../apache/aurora/common/test_clusters.py | 3 +-
.../apache/aurora/common/test_http_signaler.py | 2 +-
.../apache/aurora/common/test_shellify.py | 3 +-
.../python/apache/aurora/config/test_base.py | 86 ++++++-------
.../aurora/config/test_constraint_parsing.py | 12 +-
.../python/apache/aurora/config/test_loader.py | 6 +-
.../python/apache/aurora/config/test_thrift.py | 70 +++++-----
.../executor/common/test_directory_sandbox.py | 2 +-
.../executor/common/test_health_checker.py | 1 +
.../aurora/executor/common/test_task_info.py | 19 +--
.../aurora/executor/test_executor_vars.py | 2 +-
.../apache/aurora/executor/test_gc_executor.py | 38 ++++--
.../aurora/executor/test_thermos_executor.py | 40 +++---
.../python/apache/thermos/bin/test_thermos.py | 2 +-
.../apache/thermos/common/test_planner.py | 5 +-
.../apache/thermos/common/test_task_planner.py | 54 ++++----
.../python/apache/thermos/config/test_schema.py | 7 +-
.../python/apache/thermos/core/test_angry.py | 16 +--
.../apache/thermos/core/test_ephemerals.py | 12 +-
.../apache/thermos/core/test_failing_runner.py | 34 ++---
.../apache/thermos/core/test_failure_limit.py | 27 ++--
.../apache/thermos/core/test_finalization.py | 2 +-
.../python/apache/thermos/core/test_helper.py | 5 +-
.../python/apache/thermos/core/test_process.py | 25 ++--
.../thermos/core/test_runner_integration.py | 128 +++++++++----------
.../apache/thermos/core/test_staged_kill.py | 21 ++-
.../apache/thermos/monitoring/test_disk.py | 4 +-
.../sh/org/apache/aurora/e2e/http_example.py | 4 +-
141 files changed, 1000 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/admin/host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/host_maintenance.py b/src/main/python/apache/aurora/admin/host_maintenance.py
index ca26de1..f223c5e 100644
--- a/src/main/python/apache/aurora/admin/host_maintenance.py
+++ b/src/main/python/apache/aurora/admin/host_maintenance.py
@@ -13,18 +13,12 @@
#
import time
-from collections import defaultdict
from twitter.common import log
from twitter.common.quantity import Amount, Time
from apache.aurora.client.api import AuroraClientAPI
-from apache.aurora.client.base import (
- check_and_log_response,
- DEFAULT_GROUPING,
- group_hosts,
- GROUPING_FUNCTIONS
-)
+from apache.aurora.client.base import check_and_log_response, DEFAULT_GROUPING, group_hosts
from gen.apache.aurora.api.ttypes import Hosts, MaintenanceMode
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/__init__.py b/src/main/python/apache/aurora/client/api/__init__.py
index 1af0f8f..a73f23f 100644
--- a/src/main/python/apache/aurora/client/api/__init__.py
+++ b/src/main/python/apache/aurora/client/api/__init__.py
@@ -24,13 +24,7 @@ from .sla import Sla
from .updater import Updater
from gen.apache.aurora.api.constants import LIVE_STATES
-from gen.apache.aurora.api.ttypes import (
- Identity,
- ResourceAggregate,
- Response,
- ResponseCode,
- TaskQuery
-)
+from gen.apache.aurora.api.ttypes import Identity, ResourceAggregate, ResponseCode, TaskQuery
class AuroraClientAPI(object):
@@ -139,9 +133,12 @@ class AuroraClientAPI(object):
return resp
def restart(self, job_key, instances, updater_config, health_check_interval_seconds):
- """Perform a rolling restart of the job. If instances is None or [], restart all instances. Returns
- the scheduler response for the last restarted batch of instances (which allows the client to
- show the job URL), or the status check response if no tasks were active.
+ """Perform a rolling restart of the job.
+
+ If instances is None or [], restart all instances. Returns the
+ scheduler response for the last restarted batch of instances (which
+ allows the client to show the job URL), or the status check response
+ if no tasks were active.
"""
self._assert_valid_job_key(job_key)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/command_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/command_runner.py b/src/main/python/apache/aurora/client/api/command_runner.py
index 8b29f35..a291fd9 100644
--- a/src/main/python/apache/aurora/client/api/command_runner.py
+++ b/src/main/python/apache/aurora/client/api/command_runner.py
@@ -12,6 +12,8 @@
# limitations under the License.
#
+from __future__ import print_function
+
import posixpath
import subprocess
from multiprocessing.pool import ThreadPool
@@ -29,8 +31,8 @@ from gen.apache.aurora.api.ttypes import Identity, ResponseCode, TaskQuery
class CommandRunnerTrait(Cluster.Trait):
- slave_root = Required(String)
- slave_run_directory = Required(String)
+ slave_root = Required(String) # noqa
+ slave_run_directory = Required(String) # noqa
class DistributedCommandRunner(object):
@@ -121,20 +123,25 @@ class DistributedCommandRunner(object):
def process_arguments(self, command, **kw):
for task in self.resolve():
host = task.assignedTask.slaveHost
- role = task.assignedTask.task.owner.role
yield (host, self._ssh_user, self.substitute(command, task, self._cluster, **kw))
def run(self, command, parallelism=1, **kw):
threadpool = ThreadPool(processes=parallelism)
for result in threadpool.imap_unordered(self.execute, self.process_arguments(command, **kw)):
- print result
+ print(result)
+
class InstanceDistributedCommandRunner(DistributedCommandRunner):
"""A distributed command runner that only runs on specified instances of a job."""
@classmethod
def query_from(cls, role, env, job, instances=None):
- return TaskQuery(statuses=LIVE_STATES, owner=Identity(role), jobName=job, environment=env, instanceIds=instances)
+ return TaskQuery(
+ statuses=LIVE_STATES,
+ owner=Identity(role),
+ jobName=job,
+ environment=env,
+ instanceIds=instances)
def __init__(self, cluster, role, env, job, ssh_user=None, instances=None):
super(InstanceDistributedCommandRunner, self).__init__(cluster, role, env, [job], ssh_user)
@@ -148,6 +155,6 @@ class InstanceDistributedCommandRunner(DistributedCommandRunner):
for task in resp.result.scheduleStatusResult.tasks:
yield task
else:
- print_aurora_log(logging.ERROR,
- "Error: could not retrieve task information for run command: %s" % resp.messageDEPRECATED)
+ # This should be fixed by AURORA-503.
+ log.error('could not retrieve task information for run command: %s' % resp.messageDEPRECATED)
raise ValueError("Could not retrieve task information: %s" % resp.messageDEPRECATED)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/health_check.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/health_check.py b/src/main/python/apache/aurora/client/api/health_check.py
index d6ef596..0503c0b 100644
--- a/src/main/python/apache/aurora/client/api/health_check.py
+++ b/src/main/python/apache/aurora/client/api/health_check.py
@@ -58,6 +58,7 @@ class StatusHealthCheck(HealthCheck):
1. A task is in state RUNNING
2. A task that satisfies (1) and is already known has the same task id.
"""
+
def __init__(self):
self._task_ids = {}
@@ -68,7 +69,10 @@ class StatusHealthCheck(HealthCheck):
if status == ScheduleStatus.RUNNING:
if instance_id in self._task_ids:
- return Retriable.alive() if task_id == self._task_ids.get(instance_id) else NotRetriable.dead()
+ if task_id == self._task_ids.get(instance_id):
+ return Retriable.alive()
+ else:
+ return NotRetriable.dead()
else:
log.info('Detected RUNNING instance %s' % instance_id)
self._task_ids[instance_id] = task_id
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/scheduler_client.py b/src/main/python/apache/aurora/client/api/scheduler_client.py
index 7be974e..86ccdd9 100644
--- a/src/main/python/apache/aurora/client/api/scheduler_client.py
+++ b/src/main/python/apache/aurora/client/api/scheduler_client.py
@@ -17,7 +17,7 @@ import threading
import time
import traceback
-from pystachio import Boolean, Default, Integer, String
+from pystachio import Default, Integer, String
from thrift.protocol import TJSONProtocol
from thrift.transport import THttpClient, TTransport
from twitter.common import log
@@ -33,19 +33,20 @@ from gen.apache.aurora.api.constants import CURRENT_API_VERSION
class SchedulerClientTrait(Cluster.Trait):
- zk = String
- zk_port = Default(Integer, 2181)
- scheduler_zk_path = String
- scheduler_uri = String
- proxy_url = String
- auth_mechanism = Default(String, 'UNAUTHENTICATED')
+ zk = String # noqa
+ zk_port = Default(Integer, 2181) # noqa
+ scheduler_zk_path = String # noqa
+ scheduler_uri = String # noqa
+ proxy_url = String # noqa
+ auth_mechanism = Default(String, 'UNAUTHENTICATED') # noqa
class SchedulerClient(object):
THRIFT_RETRIES = 5
RETRY_TIMEOUT = Amount(1, Time.SECONDS)
- class CouldNotConnect(Exception): pass
+ class Error(Exception): pass
+ class CouldNotConnect(Error): pass
# TODO(wickman) Refactor per MESOS-3005 into two separate classes with separate traits:
# ZookeeperClientTrait
@@ -81,23 +82,23 @@ class SchedulerClient(object):
def _connect(self):
return None
- @staticmethod
- def _connect_scheduler(host, port, clock=time):
+ @classmethod
+ def _connect_scheduler(cls, host, port, clock=time):
transport = THttpClient.THttpClient('http://%s:%s/api' % (host, port))
protocol = TJSONProtocol.TJSONProtocol(transport)
schedulerClient = AuroraAdmin.Client(protocol)
- for _ in range(SchedulerClient.THRIFT_RETRIES):
+ for _ in range(cls.THRIFT_RETRIES):
try:
transport.open()
return schedulerClient
except TTransport.TTransportException:
- clock.sleep(SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
+ clock.sleep(cls.RETRY_TIMEOUT.as_(Time.SECONDS))
continue
except Exception as e:
# Monkey-patched proxies, like socks, can generate a proxy error here.
# without adding a dependency, we can't catch those in a more specific way.
- raise SchedulerClient.CouldNotConnect('Connection to scheduler failed: %s' % e)
- raise SchedulerClient.CouldNotConnect('Could not connect to %s:%s' % (host, port))
+ raise cls.CouldNotConnect('Connection to scheduler failed: %s' % e)
+ raise cls.CouldNotConnect('Could not connect to %s:%s' % (host, port))
class ZookeeperSchedulerClient(SchedulerClient):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/sla.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/sla.py b/src/main/python/apache/aurora/client/api/sla.py
index d15491a..a7b8760 100644
--- a/src/main/python/apache/aurora/client/api/sla.py
+++ b/src/main/python/apache/aurora/client/api/sla.py
@@ -15,13 +15,12 @@
import math
import time
from collections import defaultdict, namedtuple
-from copy import deepcopy
from apache.aurora.client.base import DEFAULT_GROUPING, group_hosts, log_response
from apache.aurora.common.aurora_job_key import AuroraJobKey
from gen.apache.aurora.api.constants import LIVE_STATES
-from gen.apache.aurora.api.ttypes import Identity, Response, ResponseCode, ScheduleStatus, TaskQuery
+from gen.apache.aurora.api.ttypes import Identity, ResponseCode, ScheduleStatus, TaskQuery
def job_key_from_scheduled(task, cluster):
@@ -99,7 +98,6 @@ class JobUpTimeSlaVector(object):
else:
return duration - sorted(self._uptime_map.values())[index]
-
def get_task_up_count(self, duration, total_tasks=None):
"""Returns the percentage of job tasks that stayed up longer than duration.
@@ -130,11 +128,19 @@ class JobUpTimeSlaVector(object):
for task in self._tasks:
for event in task.taskEvents:
if event.status == ScheduleStatus.RUNNING:
- instance_map[task.assignedTask.instanceId] = math.floor(self._now - event.timestamp / 1000)
+ instance_map[task.assignedTask.instanceId] = math.floor(
+ self._now - event.timestamp / 1000)
break
return instance_map
+JobUpTimeLimit = namedtuple('JobUpTimeLimit', ['job', 'percentage', 'duration_secs'])
+
+
+JobUpTimeDetails = namedtuple('JobUpTimeDetails',
+ ['job', 'predicted_percentage', 'safe', 'safe_in_secs'])
+
+
class DomainUpTimeSlaVector(object):
"""A grouping of all active tasks in the cluster by:
- job: Map of job_key -> task. Provides logical mapping between jobs and their active tasks.
@@ -143,10 +149,6 @@ class DomainUpTimeSlaVector(object):
"""
DEFAULT_MIN_INSTANCE_COUNT = 2
- JobUpTimeLimit = namedtuple('JobUpTimeLimit', ['job', 'percentage', 'duration_secs'])
- JobUpTimeDetails = namedtuple('JobUpTimeDetails',
- ['job', 'predicted_percentage', 'safe', 'safe_in_secs'])
-
def __init__(self, cluster, tasks, min_instance_count=DEFAULT_MIN_INSTANCE_COUNT, hosts=None):
self._cluster = cluster
self._tasks = tasks
@@ -188,7 +190,7 @@ class DomainUpTimeSlaVector(object):
break
for host in job_hosts:
- safe_hosts[host].append(self.JobUpTimeLimit(job_key, filtered_percentage, job_duration))
+ safe_hosts[host].append(JobUpTimeLimit(job_key, filtered_percentage, job_duration))
else:
safe_groups.append(safe_hosts)
@@ -226,7 +228,7 @@ class DomainUpTimeSlaVector(object):
for host in job_hosts:
probed_hosts[host].append(
- self.JobUpTimeDetails(job_key, filtered_percentage, safe, wait_to_sla))
+ JobUpTimeDetails(job_key, filtered_percentage, safe, wait_to_sla))
if probed_hosts:
probed_groups.append(probed_hosts)
@@ -248,7 +250,6 @@ class DomainUpTimeSlaVector(object):
for host in group.keys():
result[host].append(uptime_details)
-
def _simulate_hosts_down(self, job_key, hosts, duration):
unfiltered_tasks = self._tasks_by_job[job_key]
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/updater.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater.py b/src/main/python/apache/aurora/client/api/updater.py
index ea7285a..de7912c 100644
--- a/src/main/python/apache/aurora/client/api/updater.py
+++ b/src/main/python/apache/aurora/client/api/updater.py
@@ -39,19 +39,23 @@ from gen.apache.aurora.api.ttypes import (
TaskQuery
)
+InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
+
+
+OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
+
+
+InstanceConfigs = namedtuple(
+ 'InstanceConfigs',
+ ['remote_config_map', 'local_config_map', 'instances_to_process']
+)
+
class Updater(object):
"""Update the instances of a job in batches."""
class Error(Exception): pass
- InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
- OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
- InstanceConfigs = namedtuple(
- 'InstanceConfigs',
- ['remote_config_map', 'local_config_map', 'instances_to_process']
- )
-
def __init__(self,
config,
health_check_interval_seconds,
@@ -116,19 +120,19 @@ class Updater(object):
self._update_config.max_total_failures
)
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.remote_config_map,
to_config=instance_configs.local_config_map
)
remaining_instances = [
- self.InstanceState(instance_id, is_updated=False)
+ InstanceState(instance_id, is_updated=False)
for instance_id in instance_configs.instances_to_process
]
log.info('Starting job update.')
while remaining_instances and not failure_threshold.is_failed_update():
- batch_instances = remaining_instances[0 : self._update_config.batch_size]
+ batch_instances = remaining_instances[0:self._update_config.batch_size]
remaining_instances = list(set(remaining_instances) - set(batch_instances))
instances_to_restart = [s.instance_id for s in batch_instances if s.is_updated]
instances_to_update = [s.instance_id for s in batch_instances if not s.is_updated]
@@ -152,7 +156,7 @@ class Updater(object):
(unretryable_instances, self._update_config.max_per_instance_failures))
retryable_instances = list(set(failed_instances) - set(unretryable_instances))
remaining_instances += [
- self.InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
+ InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
]
remaining_instances.sort(key=lambda tup: tup.instance_id)
@@ -177,14 +181,14 @@ class Updater(object):
return
log.info('Reverting update for %s' % instances_to_rollback)
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.local_config_map,
to_config=instance_configs.remote_config_map
)
instances_to_rollback.sort(reverse=True)
failed_instances = []
while instances_to_rollback:
- batch_instances = instances_to_rollback[0 : self._update_config.batch_size]
+ batch_instances = instances_to_rollback[0:self._update_config.batch_size]
instances_to_rollback = list(set(instances_to_rollback) - set(batch_instances))
instances_to_rollback.sort(reverse=True)
instances_to_watch = self._update_instances(batch_instances, instance_operation)
@@ -236,7 +240,8 @@ class Updater(object):
if from_config and to_config:
diff_output = self._diff_configs(from_config, to_config)
if diff_output:
- log.debug('Task configuration changed for instance [%s]:\n%s' % (instance_id, diff_output))
+ log.debug('Task configuration changed for instance [%s]:\n%s' % (
+ instance_id, diff_output))
to_kill.append(instance_id)
to_add.append(instance_id)
elif from_config and not to_config:
@@ -318,7 +323,7 @@ class Updater(object):
Returns Response.OK if quota check was successful.
"""
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.remote_config_map,
to_config=instance_configs.local_config_map
)
@@ -379,7 +384,7 @@ class Updater(object):
# Populate local config map
local_config_map = dict.fromkeys(job_config_instances, local_task_config)
- return self.InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
+ return InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
def _get_existing_tasks(self):
"""Loads all existing tasks from the scheduler.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/updater_util.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater_util.py b/src/main/python/apache/aurora/client/api/updater_util.py
index 04105de..6b689c1 100644
--- a/src/main/python/apache/aurora/client/api/updater_util.py
+++ b/src/main/python/apache/aurora/client/api/updater_util.py
@@ -30,6 +30,7 @@ class UpdaterConfig(object):
An instance is also expected to remain healthy for at least watch_secs. If these conditions are
not satisfied, the instance is deemed unhealthy.
"""
+
def __init__(self,
batch_size,
restart_threshold,
@@ -58,7 +59,6 @@ class FailureThreshold(object):
self._max_total_failures = max_total_failures
self._failures_by_instance = collections.defaultdict(int)
-
def update_failure_counts(self, failed_instances):
"""Update the failure counts metrics based upon a batch of failed instances.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/base.py b/src/main/python/apache/aurora/client/base.py
index ef0855d..6716ba6 100644
--- a/src/main/python/apache/aurora/client/base.py
+++ b/src/main/python/apache/aurora/client/base.py
@@ -34,14 +34,17 @@ After checking on the above, you may release the update lock on the job by
invoking cancel_update.
"""
+
def die(msg):
log.fatal(msg)
sys.exit(1)
+
def log_response(resp):
log.info('Response from scheduler: %s (message: %s)'
% (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.messageDEPRECATED))
+
def check_and_log_response(resp):
log_response(resp)
if resp.responseCode != ResponseCode.OK:
@@ -49,6 +52,12 @@ def check_and_log_response(resp):
log.info(LOCKED_WARNING)
sys.exit(1)
+
+def check_and_log_locked_response(resp):
+ if resp.responseCode == ResponseCode.LOCK_ERROR:
+ log.info(LOCKED_WARNING)
+
+
def deprecation_warning(text):
log.warning('')
log.warning('*' * 80)
@@ -59,9 +68,9 @@ def deprecation_warning(text):
log.warning('')
-class requires(object):
- @staticmethod
- def wrap_function(fn, fnargs, comparator):
+class requires(object): # noqa
+ @classmethod
+ def wrap_function(cls, fn, fnargs, comparator):
@functools.wraps(fn)
def wrapped_function(args):
if not comparator(args, fnargs):
@@ -72,20 +81,20 @@ class requires(object):
return fn(*args)
return wrapped_function
- @staticmethod
- def exactly(*args):
+ @classmethod
+ def exactly(cls, *args):
def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
+ return cls.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
return wrap
- @staticmethod
- def at_least(*args):
+ @classmethod
+ def at_least(cls, *args):
def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
+ return cls.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
return wrap
- @staticmethod
- def nothing(fn):
+ @classmethod
+ def nothing(cls, fn):
@functools.wraps(fn)
def real_fn(line):
return fn(*line)
@@ -109,17 +118,21 @@ HOSTS_OPTION = optparse.Option(
def group_by_host(hostname):
return hostname
+
DEFAULT_GROUPING = 'by_host'
GROUPING_FUNCTIONS = {
'by_host': group_by_host,
}
+
def add_grouping(name, function):
GROUPING_FUNCTIONS[name] = function
+
def remove_grouping(name):
GROUPING_FUNCTIONS.pop(name)
+
def get_grouping_or_die(grouping_function):
try:
return GROUPING_FUNCTIONS[grouping_function]
@@ -127,6 +140,7 @@ def get_grouping_or_die(grouping_function):
die('Unknown grouping function %s. Must be one of: %s'
% (grouping_function, GROUPING_FUNCTIONS.keys()))
+
def group_hosts(hostnames, grouping_function=DEFAULT_GROUPING):
grouping_function = get_grouping_or_die(grouping_function)
groups = defaultdict(set)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/bin/aurora_admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_admin.py b/src/main/python/apache/aurora/client/bin/aurora_admin.py
index d1247e6..136cf60 100644
--- a/src/main/python/apache/aurora/client/bin/aurora_admin.py
+++ b/src/main/python/apache/aurora/client/bin/aurora_admin.py
@@ -16,10 +16,11 @@ from twitter.common import app
from twitter.common.log.options import LogOptions
from apache.aurora.client.base import generate_terse_usage
-from apache.aurora.client.commands import admin, help, maintenance
+from apache.aurora.client.commands import help as help_commands
+from apache.aurora.client.commands import admin, maintenance
from apache.aurora.client.options import add_verbosity_options
-app.register_commands_from(admin, help, maintenance)
+app.register_commands_from(admin, help_commands, maintenance)
add_verbosity_options()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/bin/aurora_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_client.py b/src/main/python/apache/aurora/client/bin/aurora_client.py
index 1317fae..4999265 100644
--- a/src/main/python/apache/aurora/client/bin/aurora_client.py
+++ b/src/main/python/apache/aurora/client/bin/aurora_client.py
@@ -16,7 +16,8 @@ from twitter.common import app
from twitter.common.log.options import LogOptions
from apache.aurora.client.base import generate_terse_usage
-from apache.aurora.client.commands import core, help, run, ssh
+from apache.aurora.client.commands import help as help_commands
+from apache.aurora.client.commands import core, run, ssh
from apache.aurora.client.options import add_verbosity_options
# These are are side-effecting imports in that they register commands via
@@ -25,7 +26,7 @@ from apache.aurora.client.options import add_verbosity_options
# composition.
app.register_commands_from(core, run, ssh)
-app.register_commands_from(help)
+app.register_commands_from(help_commands)
add_verbosity_options()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/binding_helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/binding_helper.py b/src/main/python/apache/aurora/client/binding_helper.py
index d17d57e..6d6a067 100644
--- a/src/main/python/apache/aurora/client/binding_helper.py
+++ b/src/main/python/apache/aurora/client/binding_helper.py
@@ -12,9 +12,6 @@
# limitations under the License.
#
-import inspect
-import os
-import sys
from abc import abstractmethod, abstractproperty
from twitter.common.lang import Interface
@@ -63,10 +60,10 @@ class BindingHelper(Interface):
(because a human reader wants to know the version of the package, not the meaningless
HDFS URL.
"""
- @staticmethod
- def register(helper):
- _BINDING_HELPERS.append(helper)
+ @classmethod
+ def register(cls, helper):
+ _BINDING_HELPERS.append(helper)
def apply(self, config, env=None, binding_dict=None):
for match in self.matcher.match(config.raw()):
@@ -88,6 +85,7 @@ class BindingHelper(Interface):
class CachingBindingHelper(BindingHelper):
"""A binding helper implementation that caches binding results"""
+
def __init__(self):
self.cache = {}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/__init__.py b/src/main/python/apache/aurora/client/cli/__init__.py
index fd6f96e..fc63dbf 100644
--- a/src/main/python/apache/aurora/client/cli/__init__.py
+++ b/src/main/python/apache/aurora/client/cli/__init__.py
@@ -41,6 +41,7 @@ from twitter.common.python.pex import PexInfo
from .command_hooks import GlobalCommandHookRegistry
from .logsetup import TRANSCRIPT
+from .options import CommandOption
# Constants for standard return codes.
EXIT_OK = 0
@@ -78,6 +79,7 @@ def print_aurora_log(sev, msg, *args, **kwargs):
kwargs["extra"] = extra
logger.log(sev, msg, *args, **kwargs)
+
def get_client_version():
try:
pexpath = sys.argv[0]
@@ -90,12 +92,11 @@ def get_client_version():
class Context(object):
class Error(Exception): pass
-
class ArgumentException(Error): pass
class CommandError(Error):
def __init__(self, code, msg):
- super(Context.CommandError, self).__init__(msg)
+ super(Context.CommandError, self).__init__(msg) # noqa:T800
self.msg = msg
self.code = code
self.options = None
@@ -151,7 +152,7 @@ class ConfigurationPlugin(object):
class Error(Exception):
def __init__(self, msg, code=0):
- super(ConfigurationPlugin.Error, self).__init__(msg)
+ super(ConfigurationPlugin.Error, self).__init__(msg) # noqa:T800
self.code = code
self.msg = msg
@@ -295,7 +296,6 @@ class CommandLine(object):
result.append("\nRun 'help noun' or 'help noun verb' for help about a specific command")
return "\n".join(result)
-
def register_nouns(self):
"""This method should overridden by applications to register the collection of nouns
that they can manipulate.
@@ -309,7 +309,6 @@ class CommandLine(object):
etc. You wouldn't want to clutter the help output with AWS commands for users
that weren't using AWS. So you could have the command-line check the cluster.json
file, and only register the AWS noun if there was an AWS cluster.
-
"""
@property
@@ -320,7 +319,9 @@ class CommandLine(object):
def _setup(self, args):
GlobalCommandHookRegistry.setup(GLOBAL_HOOK_SKIP_RULES_URL)
- nouns = self.registered_nouns
+ # Accessing registered_nouns has the side-effect of registering them, but since
+ # nouns is unused, we must disable checkstyle.
+ nouns = self.registered_nouns # noqa
for plugin in self.plugins:
args = plugin.before_dispatch(args)
return args
@@ -346,8 +347,8 @@ class CommandLine(object):
for plugin in self.plugins:
plugin.before_execution(context)
except ConfigurationPlugin.Error as e:
- print("Error in configuration plugin before execution: %s" % c.msg, file=sys.stderr)
- return c.code
+ print("Error in configuration plugin before execution: %s" % e.msg, file=sys.stderr)
+ return e.code
plugin_result = GlobalCommandHookRegistry.run_pre_hooks(context, context.options.noun,
context.options.verb)
if plugin_result != EXIT_OK:
@@ -383,8 +384,8 @@ class CommandLine(object):
result = noun.execute(context)
if result == EXIT_OK:
print_aurora_log(logging.INFO, "Command terminated successfully")
- GlobalCommandHookRegistry.run_post_hooks(context, context.options.noun, context.options.verb,
- result)
+ GlobalCommandHookRegistry.run_post_hooks(context, context.options.noun,
+ context.options.verb, result)
else:
print_aurora_log(logging.INFO, "Command terminated with error code %s", result)
self._run_post_plugins(context, result)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/bridge.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/bridge.py b/src/main/python/apache/aurora/client/cli/bridge.py
index d5eec8a..6528d90 100644
--- a/src/main/python/apache/aurora/client/cli/bridge.py
+++ b/src/main/python/apache/aurora/client/cli/bridge.py
@@ -74,7 +74,6 @@ class Bridge(object):
if self.default is not None:
return self.default.execute(args)
-
def execute(self, args):
"""Dispatch a command line to the appropriate CommandProcessor"""
if len(args) == 1:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/client.py b/src/main/python/apache/aurora/client/cli/client.py
index fa21576..e4409e5 100644
--- a/src/main/python/apache/aurora/client/cli/client.py
+++ b/src/main/python/apache/aurora/client/cli/client.py
@@ -11,20 +11,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import logging
import sys
-from apache.aurora.client.cli import CommandLine
from apache.aurora.client.cli.bridge import Bridge, CommandProcessor
from apache.aurora.client.cli.logsetup import setup_default_log_handlers
from apache.aurora.client.cli.standalone_client import AuroraCommandLine
-import logging
# TODO(mchucarroll): the entire bridged executable mechanism here is
# intended to be deprecated once clientv2 has proven to be stable
# and adopted by most users. Once we reach that point, this should
# be replaced by the standalone executable defined in standalone_client.py.
-
class AuroraClientV2CommandProcessor(CommandProcessor):
def __init__(self):
self.commandline = AuroraCommandLine()
@@ -50,17 +48,19 @@ class AuroraClientV1CommandProcessor(CommandProcessor):
def get_commands(self):
return ["cancel_update", "create", "diff", "get_quota", "inspect", "kill", "list_jobs",
- "open", "restart", "run", "ssh", "start_cron", "status", "update", "version" ]
+ "open", "restart", "run", "ssh", "start_cron", "status", "update", "version"]
def execute(self, args):
from apache.aurora.client.bin.aurora_client import proxy_main as clientone_proxy_main
return clientone_proxy_main()
+
def proxy_main():
v2 = AuroraClientV2CommandProcessor()
v1 = AuroraClientV1CommandProcessor()
bridge = Bridge([v2, v1], default=v1)
bridge.execute(sys.argv)
+
if __name__ == '__main__':
proxy_main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/command_hooks.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/command_hooks.py b/src/main/python/apache/aurora/client/cli/command_hooks.py
index c349824..aa850bf 100644
--- a/src/main/python/apache/aurora/client/cli/command_hooks.py
+++ b/src/main/python/apache/aurora/client/cli/command_hooks.py
@@ -137,7 +137,6 @@ class GlobalCommandHookRegistry(object):
except ValueError as e:
logging.error("Client could not decode hook skip rules: %s" % e)
-
@classmethod
def load_hooks_file(cls, path):
"""Load a file containing hooks. If there are any errors compiling or executing the file,
@@ -177,7 +176,7 @@ class GlobalCommandHookRegistry(object):
# it's an HG root if it contains a directory named ".hg"
return any(os.path.isdir(os.path.join(dir, rootname)) for rootname in [".git", ".hg"])
- filepath = os.path.join(dir, cls.HOOKS_FILE_NAME)
+ filepath = os.path.join(dir, cls.HOOKS_FILE_NAME)
if os.path.exists(filepath):
return filepath
elif is_repos_root(dir):
@@ -282,11 +281,12 @@ class GlobalCommandHookRegistry(object):
context.print_err("Error executing command hook %s: %s; aborting" % hook.name, c.msg)
return c.code
+
class CommandHook(object):
"""A hook which contains code that should be run before certain commands."""
class Error(Exception):
def __init__(self, code, msg):
- super(CommandHook.Error, self).__init__(msg)
+ super(CommandHook.Error, self).__init__(msg) # noqa:T800
self.code = code
self.msg = msg
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/config.py b/src/main/python/apache/aurora/client/cli/config.py
index 034c68e..637e7ac 100644
--- a/src/main/python/apache/aurora/client/cli/config.py
+++ b/src/main/python/apache/aurora/client/cli/config.py
@@ -19,17 +19,9 @@ operate in on configuration files.
from __future__ import print_function
-from apache.aurora.client.cli import (
- EXIT_COMMAND_FAILURE,
- EXIT_OK,
- Noun,
- Verb,
-)
+from apache.aurora.client.cli import EXIT_COMMAND_FAILURE, EXIT_OK, Noun, Verb
from apache.aurora.client.cli.context import AuroraCommandContext
-from apache.aurora.client.cli.options import (
- BIND_OPTION,
- CONFIG_ARGUMENT,
-)
+from apache.aurora.client.cli.options import BIND_OPTION, CONFIG_ARGUMENT
from apache.aurora.client.config import AuroraConfig
from apache.aurora.config.loader import AuroraConfigLoader
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/context.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/context.py b/src/main/python/apache/aurora/client/cli/context.py
index d1f1f3f..669add7 100644
--- a/src/main/python/apache/aurora/client/cli/context.py
+++ b/src/main/python/apache/aurora/client/cli/context.py
@@ -15,16 +15,14 @@
from __future__ import print_function
import logging
-import sys
from collections import namedtuple
from fnmatch import fnmatch
-from twitter.common import log
-
from apache.aurora.client.base import synthesize_url
from apache.aurora.client.cli import (
Context,
EXIT_API_ERROR,
+ EXIT_COMMAND_FAILURE,
EXIT_INVALID_CONFIGURATION,
EXIT_INVALID_PARAMETER
)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/cron.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/cron.py b/src/main/python/apache/aurora/client/cli/cron.py
index c30a0a6..cefeca4 100644
--- a/src/main/python/apache/aurora/client/cli/cron.py
+++ b/src/main/python/apache/aurora/client/cli/cron.py
@@ -21,7 +21,7 @@ from apache.aurora.client.cli import (
EXIT_INVALID_PARAMETER,
EXIT_OK,
Noun,
- Verb,
+ Verb
)
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import (
@@ -30,7 +30,7 @@ from apache.aurora.client.cli.options import (
CONFIG_ARGUMENT,
CONFIG_OPTION,
JOBSPEC_ARGUMENT,
- JSON_READ_OPTION,
+ JSON_READ_OPTION
)
from gen.apache.aurora.api.ttypes import ResponseCode
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/jobs.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/jobs.py b/src/main/python/apache/aurora/client/cli/jobs.py
index 8020c35..cd5070e 100644
--- a/src/main/python/apache/aurora/client/cli/jobs.py
+++ b/src/main/python/apache/aurora/client/cli/jobs.py
@@ -19,12 +19,10 @@ import logging
import os
import pprint
import subprocess
-import sys
import time
from datetime import datetime
from tempfile import NamedTemporaryFile
-from pystachio.config import Config
from thrift.protocol import TJSONProtocol
from thrift.TSerialization import serialize
@@ -403,8 +401,10 @@ class ListJobsCommand(Verb):
context.print_out('%s/%s/%s/%s' % (j.cluster, j.role, j.env, j.name))
return EXIT_OK
+
class OpenCommand(Verb):
# TODO(mchucarroll): this is awkward in the noun/verb framework: where does it actually belong?
+
@property
def name(self):
return 'open'
@@ -424,7 +424,6 @@ class OpenCommand(Verb):
while len(key_parts) < 4:
key_parts.append(None)
(cluster, role, env, name) = key_parts
- api = context.get_api(cluster)
context.open_scheduler_page(cluster, role, env, name)
return EXIT_OK
@@ -438,11 +437,11 @@ class RestartCommand(Verb):
return [BATCH_OPTION, BIND_OPTION, BROWSER_OPTION, FORCE_OPTION, HEALTHCHECK_OPTION,
JSON_READ_OPTION, WATCH_OPTION,
CommandOption('--max-per-instance-failures', type=int, default=0,
- help='Maximum number of restarts per instance during restart. Increments total failure '
- 'count when this limit is exceeded.'),
+ help='Maximum number of restarts per instance during restart. Increments total '
+ 'failure count when this limit is exceeded.'),
CommandOption('--restart-threshold', type=int, default=60,
help='Maximum number of seconds before a shard must move into the RUNNING state '
- 'before considered a failure.'),
+ 'before considered a failure.'),
MAX_TOTAL_FAILURES_OPTION,
STRICT_OPTION,
CommandOption('--rollback-on-failure', default=True, action='store_false',
@@ -467,7 +466,7 @@ Restarts are fully controlled client-side, so aborting halts the restart."""
job = context.options.instance_spec.jobkey
instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else
context.options.instance_spec.instance)
- if instances != None and context.options.strict:
+ if instances is not None and context.options.strict:
context.verify_shards_option_validity(job, instances)
api = context.get_api(job.cluster)
config = (context.get_job_config(job, context.options.config_file)
@@ -628,7 +627,7 @@ to preview what changes will take effect.
job = context.options.instance_spec.jobkey
instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else
context.options.instance_spec.instance)
- if instances != None and context.options.strict:
+ if instances is not None and context.options.strict:
context.verify_shards_option_validity(job, instances)
config = context.get_job_config(job, context.options.config_file)
api = context.get_api(config.cluster())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/logsetup.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/logsetup.py b/src/main/python/apache/aurora/client/cli/logsetup.py
index 15fb306..55d99c4 100644
--- a/src/main/python/apache/aurora/client/cli/logsetup.py
+++ b/src/main/python/apache/aurora/client/cli/logsetup.py
@@ -21,6 +21,7 @@ import sys
TRANSCRIPT = logging.DEBUG + 1
logging.addLevelName(TRANSCRIPT, "TRANSCRIPT")
+
class PlainFormatter(logging.Formatter):
"""
Format a log in a simple style:
@@ -31,8 +32,8 @@ class PlainFormatter(logging.Formatter):
LEVEL_MAP = {
logging.FATAL: "FATAL",
logging.ERROR: "ERROR",
- logging.WARN: "WARN",
- logging.INFO: "info",
+ logging.WARN: "WARN",
+ logging.INFO: "info",
TRANSCRIPT: "transcript",
logging.DEBUG: "debug"
}
@@ -46,8 +47,8 @@ class PlainFormatter(logging.Formatter):
except TypeError:
record_message = record.msg
try:
- level = PlainFormatter.LEVEL_MAP[record.levelno]
- except:
+ level = self.LEVEL_MAP[record.levelno]
+ except Exception:
level = "?????"
record_message = "log(%s): %s" % (level, record_message)
record.getMessage = lambda: record_message
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/options.py b/src/main/python/apache/aurora/client/cli/options.py
index 0d49bac..f10ef4c 100644
--- a/src/main/python/apache/aurora/client/cli/options.py
+++ b/src/main/python/apache/aurora/client/cli/options.py
@@ -39,11 +39,9 @@ class CommandOption(object):
self.type = kwargs.get('type')
self.help = kwargs.get('help', '')
-
def is_mandatory(self):
return self.kwargs.get('required', not self.name.startswith('--'))
-
def get_displayname(self):
"""Get a display name for a the expected format of a parameter value"""
if 'metavar' in self.kwargs:
@@ -58,7 +56,6 @@ class CommandOption(object):
displayname = "value"
return displayname
-
def render_usage(self):
"""Create a usage string for this option"""
if not self.name.startswith('--'):
@@ -73,7 +70,6 @@ class CommandOption(object):
else:
return "[%s=%s]" % (self.name, self.get_displayname())
-
def render_help(self):
"""Render a full help message for this option"""
result = ""
@@ -153,7 +149,7 @@ def parse_percentiles(percentiles):
return sorted(map(parse_percentile, percentiles.split(','))) if percentiles else None
-TaskInstanceKey = namedtuple('TaskInstanceKey', [ 'jobkey', 'instance' ])
+TaskInstanceKey = namedtuple('TaskInstanceKey', ['jobkey', 'instance'])
def parse_task_instance_key(key):
@@ -266,7 +262,6 @@ STRICT_OPTION = CommandOption('--strict', default=False, action='store_true',
"that are larger than the actual set of instances in the job"))
-
TASK_INSTANCE_ARGUMENT = CommandOption('task_instance', type=parse_task_instance_key,
help='A task instance specifier, in the form CLUSTER/ROLE/ENV/NAME/INSTANCE')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/quota.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/quota.py b/src/main/python/apache/aurora/client/cli/quota.py
index af07d83..5811825 100644
--- a/src/main/python/apache/aurora/client/cli/quota.py
+++ b/src/main/python/apache/aurora/client/cli/quota.py
@@ -17,7 +17,13 @@ from __future__ import print_function
from thrift.protocol import TJSONProtocol
from thrift.TSerialization import serialize
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, EXIT_OK, Noun, Verb
+from apache.aurora.client.cli import (
+ EXIT_COMMAND_FAILURE,
+ EXIT_INVALID_PARAMETER,
+ EXIT_OK,
+ Noun,
+ Verb
+)
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import JSON_WRITE_OPTION, ROLE_ARGUMENT
@@ -75,7 +81,8 @@ class GetQuotaCmd(Verb):
elif resp.responseCode == ResponseCode.INVALID_REQUEST:
raise context.CommandError(EXIT_COMMAND_FAILURE, 'Error retrieving quota for role %s' % role)
elif resp.responseCode == ResponseCode.AUTH_FAILED:
- raise context.CommandError(EXIT_COMMAND_FAILURE, 'Invalid authorization to retrieve quota for role %s' % role)
+ raise context.CommandError(EXIT_COMMAND_FAILURE,
+ 'Invalid authorization to retrieve quota for role %s' % role)
context.print_out(self.render_quota(context.options.write_json, resp))
return EXIT_OK
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/standalone_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/standalone_client.py b/src/main/python/apache/aurora/client/cli/standalone_client.py
index 1e63ead..5d1a4a5 100644
--- a/src/main/python/apache/aurora/client/cli/standalone_client.py
+++ b/src/main/python/apache/aurora/client/cli/standalone_client.py
@@ -11,11 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import logging
import sys
from apache.aurora.client.cli import CommandLine
from apache.aurora.client.cli.logsetup import setup_default_log_handlers
-import logging
class AuroraCommandLine(CommandLine):
@@ -50,4 +50,3 @@ def proxy_main():
if __name__ == '__main__':
proxy_main()
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/task.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/task.py b/src/main/python/apache/aurora/client/cli/task.py
index fe11f38..7fb3f55 100644
--- a/src/main/python/apache/aurora/client/cli/task.py
+++ b/src/main/python/apache/aurora/client/cli/task.py
@@ -17,54 +17,24 @@
from __future__ import print_function
-import json
-import os
-import pprint
import subprocess
-import sys
-import time
-from datetime import datetime
-
-from pystachio.config import Config
-from thrift.protocol import TJSONProtocol
-from thrift.TSerialization import serialize
from apache.aurora.client.api.command_runner import (
DistributedCommandRunner,
InstanceDistributedCommandRunner
)
-from apache.aurora.client.api.updater_util import UpdaterConfig
-from apache.aurora.client.cli import (
- EXIT_COMMAND_FAILURE,
- EXIT_INVALID_CONFIGURATION,
- EXIT_INVALID_PARAMETER,
- EXIT_OK,
- Noun,
- Verb
-)
+from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, Noun, Verb
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import (
- BATCH_OPTION,
- BIND_OPTION,
- BROWSER_OPTION,
CommandOption,
- CONFIG_ARGUMENT,
EXECUTOR_SANDBOX_OPTION,
- FORCE_OPTION,
- HEALTHCHECK_OPTION,
INSTANCES_SPEC_ARGUMENT,
- JOBSPEC_ARGUMENT,
- JSON_READ_OPTION,
- JSON_WRITE_OPTION,
SSH_USER_OPTION,
- TASK_INSTANCE_ARGUMENT,
- WATCH_OPTION
+ TASK_INSTANCE_ARGUMENT
)
-from apache.aurora.common.aurora_job_key import AuroraJobKey
from apache.aurora.common.clusters import CLUSTERS
-from gen.apache.aurora.api.constants import ACTIVE_STATES, AURORA_EXECUTOR_NAME
-from gen.apache.aurora.api.ttypes import ExecutorConfig, ResponseCode, ScheduleStatus
+from gen.apache.aurora.api.ttypes import ResponseCode
class RunCommand(Verb):
@@ -98,7 +68,8 @@ class RunCommand(Verb):
# TODO(mchucarroll): add options to specify which instances to run on (AURORA-198)
(cluster_name, role, env, name), instances = context.options.instance_spec
cluster = CLUSTERS[cluster_name]
- dcr = InstanceDistributedCommandRunner(cluster, role, env, name, context.options.ssh_user, instances)
+ dcr = InstanceDistributedCommandRunner(
+ cluster, role, env, name, context.options.ssh_user, instances)
dcr.run(context.options.cmd, parallelism=context.options.num_threads,
executor_sandbox=context.options.executor_sandbox)
@@ -150,9 +121,11 @@ class SshCommand(Verb):
port, name = tunnel.split(':')
port = int(port)
except ValueError:
- die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
+ raise context.CommandError(EXIT_INVALID_PARAMETER,
+ 'Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in first_task.assignedTask.assignedPorts:
- die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
+ raise context.CommandError(EXIT_INVALID_PARAMETER,
+ 'Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/admin.py b/src/main/python/apache/aurora/client/commands/admin.py
index 919eea9..a3b5584 100644
--- a/src/main/python/apache/aurora/client/commands/admin.py
+++ b/src/main/python/apache/aurora/client/commands/admin.py
@@ -16,9 +16,6 @@ from __future__ import print_function
import json
import optparse
-import os
-import pipes
-import subprocess
import sys
from twitter.common import app, log
@@ -26,7 +23,7 @@ from twitter.common.quantity import Amount, Data, Time
from twitter.common.quantity.parse_simple import parse_data, parse_time
from apache.aurora.client.api import AuroraClientAPI
-from apache.aurora.client.api.sla import DomainUpTimeSlaVector
+from apache.aurora.client.api.sla import JobUpTimeLimit
from apache.aurora.client.base import (
check_and_log_response,
die,
@@ -395,7 +392,7 @@ def sla_list_safe_domain(cluster, percentage, duration):
if len(tokens) != 3:
die('Invalid line in %s:%s' % (filename, line))
job_key = AuroraJobKey.from_path(tokens[0])
- result[job_key] = DomainUpTimeSlaVector.JobUpTimeLimit(
+ result[job_key] = JobUpTimeLimit(
job=job_key,
percentage=parse_sla_percentage(tokens[1]),
duration_secs=parse_time(tokens[2]).as_(Time.SECONDS)
@@ -488,6 +485,7 @@ def sla_probe_hosts(cluster, percentage, duration):
print_results(results)
+
@app.command
@app.command_option('--sh', default=False, action="store_true",
help="Emit a shell script instead of JSON.")
@@ -506,5 +504,5 @@ def get_cluster_config(cluster):
if not options.sh:
json.dump(cluster, sys.stdout)
else:
- for line in shellify(cluster, options.export, prefix = "AURORA_CLUSTER_"):
+ for line in shellify(cluster, options.export, prefix="AURORA_CLUSTER_"):
print(line)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/core.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/core.py b/src/main/python/apache/aurora/client/commands/core.py
index 29e70a9..0939f05 100644
--- a/src/main/python/apache/aurora/client/commands/core.py
+++ b/src/main/python/apache/aurora/client/commands/core.py
@@ -17,7 +17,6 @@
from __future__ import print_function
-import collections
import json
import os
import pprint
@@ -385,7 +384,7 @@ def list_jobs(cluster_and_role):
# Take the cluster_and_role parameter, and split it into its two components.
if cluster_and_role.count('/') != 1:
die('list_jobs parameter must be in cluster/role format')
- (cluster,role) = cluster_and_role.split('/')
+ cluster, role = cluster_and_role.split('/')
api = make_client(cluster)
resp = api.get_jobs(role)
check_and_log_response(resp)
@@ -423,7 +422,6 @@ def kill(args, options):
wait_kill_tasks(api.scheduler_proxy, job_key, options.shards)
-
def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
""" Common behavior shared by kill and killAll for killing instances in
a sequence of batches.
@@ -437,7 +435,6 @@ def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
result.append(batch)
return result
-
resp = api.check_status(job_key)
if resp.responseCode is not ResponseCode.OK:
log.error("Job %s could not be found" % job_key)
@@ -465,7 +462,6 @@ def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
return 1
-
@app.command
@app.command_option(CLUSTER_INVOKE_OPTION)
@app.command_option(OPEN_BROWSER_OPTION)
@@ -589,7 +585,6 @@ def update(job_spec, config_file):
def warn_if_dangerous_change(api, job_spec, config):
# Get the current job status, so that we can check if there's anything
# dangerous about this update.
- job_key = AuroraJobKey(config.cluster(), config.role(), config.environment(), config.name())
resp = api.query(api.build_query(config.role(), config.name(),
statuses=ACTIVE_STATES, env=config.environment()))
if resp.responseCode != ResponseCode.OK:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/help.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/help.py b/src/main/python/apache/aurora/client/commands/help.py
index d59b299..f79e971 100644
--- a/src/main/python/apache/aurora/client/commands/help.py
+++ b/src/main/python/apache/aurora/client/commands/help.py
@@ -47,8 +47,8 @@ def generate_full_usage():
return 'Available commands:\n\n' + '\n'.join(usage)
-@app.command
-def help(args):
+@app.command(name='help')
+def help_command(args):
"""usage: help [subcommand]
Prints help for using the aurora client, or one of its specific subcommands.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/maintenance.py b/src/main/python/apache/aurora/client/commands/maintenance.py
index f6ebe3b..e4d60d4 100644
--- a/src/main/python/apache/aurora/client/commands/maintenance.py
+++ b/src/main/python/apache/aurora/client/commands/maintenance.py
@@ -12,7 +12,6 @@
# limitations under the License.
#
-import optparse
import os
import subprocess
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py
index 3b01792..e440f58 100644
--- a/src/main/python/apache/aurora/client/config.py
+++ b/src/main/python/apache/aurora/client/config.py
@@ -19,17 +19,14 @@ from __future__ import print_function
import functools
import math
-import posixpath
import re
import sys
-from pystachio import Empty, Ref
-from twitter.common import app, log
+from pystachio import Empty
from apache.aurora.client import binding_helper
from apache.aurora.client.base import deprecation_warning, die
from apache.aurora.config import AuroraConfig
-from apache.thermos.config.schema_helpers import Tasks
from gen.apache.aurora.api.constants import DEFAULT_ENVIRONMENT
@@ -125,6 +122,7 @@ Based on your job size (%s) you should use max_total_failures >= %s.
See http://go/auroraconfig for details.
'''
+
WATCH_SECS_INSUFFICIENT_ERROR_FORMAT = '''
You have specified an insufficiently short watch period (%d seconds) in your update configuration.
Your update will always succeed. In order for the updater to detect health check failures,
@@ -132,6 +130,7 @@ UpdateConfig.watch_secs must be greater than %d seconds to account for an initia
health check interval (%d seconds) plus %d consecutive failures at a check interval of %d seconds.
'''
+
def _validate_update_config(config):
job_size = config.instances()
update_config = config.update_config()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/factory.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/factory.py b/src/main/python/apache/aurora/client/factory.py
index 22805f0..4b31e85 100644
--- a/src/main/python/apache/aurora/client/factory.py
+++ b/src/main/python/apache/aurora/client/factory.py
@@ -26,7 +26,6 @@ from .base import die
# TODO(wickman) Kill make_client and make_client_factory as part of MESOS-3801.
# These are currently necessary indirections for the LiveJobDisambiguator among
# other things but can go away once those are scrubbed.
-
def make_client_factory():
verbose = getattr(app.get_options(), 'verbosity', 'normal') == 'verbose'
class TwitterAuroraClientAPI(HookedAuroraClientAPI):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/hooks/hooked_api.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/hooks/hooked_api.py b/src/main/python/apache/aurora/client/hooks/hooked_api.py
index a205777..91efe52 100644
--- a/src/main/python/apache/aurora/client/hooks/hooked_api.py
+++ b/src/main/python/apache/aurora/client/hooks/hooked_api.py
@@ -19,7 +19,6 @@ from twitter.common import log
from apache.aurora.client.api import AuroraClientAPI
from apache.aurora.client.config import GlobalHookRegistry
-from apache.aurora.common.aurora_job_key import AuroraJobKey
from gen.apache.aurora.api.ttypes import ResponseCode
@@ -181,4 +180,5 @@ class HookedAuroraClientAPI(NonHookedAuroraClientAPI):
def update_job(self, config, health_check_interval_seconds=3, instances=None):
return self._hooked_call(config, None,
_partial(super(HookedAuroraClientAPI, self).update_job,
- config, health_check_interval_seconds=health_check_interval_seconds, instances=instances))
+ config, health_check_interval_seconds=health_check_interval_seconds,
+ instances=instances))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/auth/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/__init__.py b/src/main/python/apache/aurora/common/auth/__init__.py
index 95e56ad..3119fd6 100644
--- a/src/main/python/apache/aurora/common/auth/__init__.py
+++ b/src/main/python/apache/aurora/common/auth/__init__.py
@@ -12,5 +12,13 @@
# limitations under the License.
#
-from .auth_module import AuthModule, InsecureAuthModule
-from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError
+from .auth_module import AuthModule, InsecureAuthModule # noqa
+from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError # noqa
+
+__all__ = (
+ 'AuthModule',
+ 'InsecureAuthModule',
+ 'make_session_key',
+ 'register_auth_module',
+ 'SessionKeyError',
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/auth/auth_module.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/auth_module.py b/src/main/python/apache/aurora/common/auth/auth_module.py
index aacb1b9..5f4116e 100644
--- a/src/main/python/apache/aurora/common/auth/auth_module.py
+++ b/src/main/python/apache/aurora/common/auth/auth_module.py
@@ -12,8 +12,6 @@
# limitations under the License.
#
-import getpass
-import time
from abc import abstractmethod, abstractproperty
from twitter.common.lang import Interface
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/cluster.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster.py b/src/main/python/apache/aurora/common/cluster.py
index f04718f..285b890 100644
--- a/src/main/python/apache/aurora/common/cluster.py
+++ b/src/main/python/apache/aurora/common/cluster.py
@@ -43,7 +43,7 @@ class Cluster(dict):
# Will use the default if none is provided on Cluster.
cluster.with_trait(ResolverTrait).scheduler_zk_path
"""
- Trait = Struct
+ Trait = Struct # noqa
def __init__(self, **kwargs):
self._traits = ()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/cluster_option.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster_option.py b/src/main/python/apache/aurora/common/cluster_option.py
index 9ddd868..77202c2 100644
--- a/src/main/python/apache/aurora/common/cluster_option.py
+++ b/src/main/python/apache/aurora/common/cluster_option.py
@@ -13,7 +13,7 @@
#
from copy import copy
-from optparse import NO_DEFAULT, Option, OptionValueError
+from optparse import Option, OptionValueError
def _check_mesos_cluster(option, opt, value):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/clusters.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/clusters.py b/src/main/python/apache/aurora/common/clusters.py
index 389b6f9..e55aa77 100644
--- a/src/main/python/apache/aurora/common/clusters.py
+++ b/src/main/python/apache/aurora/common/clusters.py
@@ -17,7 +17,6 @@ from __future__ import print_function
import itertools
import json
import os
-import sys
from collections import Mapping, namedtuple
from contextlib import contextmanager
@@ -40,7 +39,7 @@ __all__ = (
class NameTrait(Cluster.Trait):
- name = Required(String)
+ name = Required(String) # noqa
Parser = namedtuple('Parser', 'loader exception')
@@ -127,7 +126,6 @@ class Clusters(Mapping):
name, ', '.join(self._clusters.keys())))
-
DEFAULT_SEARCH_PATHS = (
os.environ.get('AURORA_CONFIG_ROOT') or '/etc/aurora',
os.path.expanduser('~/.aurora')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/http_signaler.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/http_signaler.py b/src/main/python/apache/aurora/common/http_signaler.py
index 4e5d7b4..e3e819d 100644
--- a/src/main/python/apache/aurora/common/http_signaler.py
+++ b/src/main/python/apache/aurora/common/http_signaler.py
@@ -14,7 +14,6 @@
import contextlib
import os
-import sys
from socket import timeout as SocketTimeout
from twitter.common import log
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/loader.py b/src/main/python/apache/aurora/config/loader.py
index 942f149..c8b045e 100644
--- a/src/main/python/apache/aurora/config/loader.py
+++ b/src/main/python/apache/aurora/config/loader.py
@@ -14,7 +14,6 @@
import json
import pkgutil
-import textwrap
from pystachio.config import Config as PystachioConfig
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
index 43ae5cf..3b90ccb 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -12,6 +12,9 @@
# limitations under the License.
#
+# Disable checkstyle for this entire file as it is a pystachio schema.
+# checkstyle: noqa
+
from apache.thermos.config.schema import *
from gen.apache.aurora.api.constants import DEFAULT_ENVIRONMENT
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
index 419625d..288fb40 100644
--- a/src/main/python/apache/aurora/config/thrift.py
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -51,8 +51,8 @@ def constraints_to_thrift(constraints):
result = set()
for attribute, constraint_value in constraints.items():
assert isinstance(attribute, Compatibility.string) and (
- isinstance(constraint_value, Compatibility.string)), (
- "Both attribute name and value in constraints must be string")
+ isinstance(constraint_value, Compatibility.string)), (
+ "Both attribute name and value in constraints must be string")
constraint = Constraint()
constraint.name = attribute
task_constraint = TaskConstraint()
@@ -230,7 +230,7 @@ def convert(job, metadata=frozenset(), ports=frozenset()):
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
- underlying_checked = underlying.bind(mesos = {'instance': 31337})
+ underlying_checked = underlying.bind(mesos={'instance': 31337})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/aurora_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/aurora_executor.py b/src/main/python/apache/aurora/executor/aurora_executor.py
index a71f1eb..8985aeb 100644
--- a/src/main/python/apache/aurora/executor/aurora_executor.py
+++ b/src/main/python/apache/aurora/executor/aurora_executor.py
@@ -233,7 +233,7 @@ class AuroraExecutor(ExecutorBase, Observable):
try:
assigned_task = assigned_task_from_mesos_task(task)
return assigned_task
- except Exception as e:
+ except Exception:
log.fatal('Could not deserialize AssignedTask')
log.fatal(traceback.format_exc())
return None
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
index 38a4c1d..f61caf0 100644
--- a/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
+++ b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
@@ -21,6 +21,6 @@ LogOptions.set_simple(True)
def proxy_main():
- main = runner_proxy_main
+ main = runner_proxy_main # noqa
app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/health_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/health_checker.py b/src/main/python/apache/aurora/executor/common/health_checker.py
index 05a89ee..671f497 100644
--- a/src/main/python/apache/aurora/executor/common/health_checker.py
+++ b/src/main/python/apache/aurora/executor/common/health_checker.py
@@ -31,6 +31,7 @@ class HealthCheckerThread(StatusChecker, ExceptionalThread):
respectively the health of the service and the reason for its failure (or None if the service is
still healthy).
"""
+
def __init__(self,
health_checker,
interval_secs=10,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/kill_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/kill_manager.py b/src/main/python/apache/aurora/executor/common/kill_manager.py
index 2bfb5dc..6e482cf 100644
--- a/src/main/python/apache/aurora/executor/common/kill_manager.py
+++ b/src/main/python/apache/aurora/executor/common/kill_manager.py
@@ -19,6 +19,7 @@ class KillManager(StatusChecker):
"""
A health interface that provides a kill-switch for a task monitored by the status manager.
"""
+
def __init__(self):
self._killed = False
self._reason = None
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/sandbox.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/sandbox.py b/src/main/python/apache/aurora/executor/common/sandbox.py
index b4125f4..f47a32b 100644
--- a/src/main/python/apache/aurora/executor/common/sandbox.py
+++ b/src/main/python/apache/aurora/executor/common/sandbox.py
@@ -57,6 +57,7 @@ class SandboxProvider(Interface):
class DirectorySandbox(SandboxInterface):
""" Basic sandbox implementation using a directory on the filesystem """
+
def __init__(self, root, user=getpass.getuser()):
self._root = root
self._user = user
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/gc_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/gc_executor.py b/src/main/python/apache/aurora/executor/gc_executor.py
index 1800ee7..c45d5eb 100644
--- a/src/main/python/apache/aurora/executor/gc_executor.py
+++ b/src/main/python/apache/aurora/executor/gc_executor.py
@@ -41,7 +41,7 @@ from apache.thermos.core.inspector import CheckpointInspector
from apache.thermos.monitoring.detector import TaskDetector
from apache.thermos.monitoring.garbage import TaskGarbageCollector
-from .common.sandbox import DirectorySandbox, SandboxInterface
+from .common.sandbox import DirectorySandbox
from .executor_base import ExecutorBase
from .executor_detector import ExecutorDetector
@@ -109,15 +109,17 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
ExceptionalThread.__init__(self)
self.daemon = True
self._stop_event = threading.Event()
- self._gc_task_queue = OrderedDict() # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in
- # the order in which they were received via a launchTask.
- self._driver = None # cache the ExecutorDriver provided by the slave, so we can use it
- # out of band from slave-initiated callbacks. This should be supplied by
- # ExecutorBase.registered() when the executor first registers with
- # the slave.
- self._slave_id = None # cache the slave ID provided by the slave
+ # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in the order in
+ # which they were received via a launchTask.
+ self._gc_task_queue = OrderedDict()
+ # cache the ExecutorDriver provided by the slave, so we can use it out
+ # of band from slave-initiated callbacks. This should be supplied by
+ # ExecutorBase.registered() when the executor first registers with the
+ # slave.
+ self._driver = None
+ self._slave_id = None # cache the slave ID provided by the slave
self._task_id = None # the task_id currently being executed by the ThermosGCExecutor, if any
- self._start_time = None # the start time of a task currently being executed, if any
+ self._start_time = None # the start time of a task currently being executed, if any
self._detector = executor_detector()
self._collector = task_garbage_collector(root=checkpoint_root)
self._clock = clock
@@ -465,7 +467,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
_, (task, retain_tasks, retain_start) = self._gc_task_queue.popitem(0)
last_gc_run = retain_start
self._run_gc(task, retain_tasks, retain_start)
- except KeyError: # no enqueued GC tasks
+ except KeyError: # no enqueued GC tasks
pass
if self._driver is not None:
self.clean_orphans(self._driver)
@@ -475,7 +477,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
if self._driver is not None:
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks
+ except KeyError: # no enqueued GC tasks
pass
else:
self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
@@ -509,7 +511,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
return
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks - reset counter
+ except KeyError: # no enqueued GC tasks - reset counter
self._dropped_tasks.write(0)
else:
self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/thermos_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_runner.py b/src/main/python/apache/aurora/executor/thermos_runner.py
index aef45d7..396753e 100644
--- a/src/main/python/apache/aurora/executor/thermos_runner.py
+++ b/src/main/python/apache/aurora/executor/thermos_runner.py
@@ -80,7 +80,7 @@ app.add_option(
callback=add_port_to('prebound_ports'),
dest='prebound_ports',
default={},
- metavar = "NAME:PORT",
+ metavar="NAME:PORT",
help="bind a numbered port PORT to name NAME")
[3/4] Make style consistent with build-support/python/checkstyle-check
Posted by wi...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/tools/java/organize_imports.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/tools/java/organize_imports.py b/src/main/python/apache/aurora/tools/java/organize_imports.py
index f5472ec..99b8ec4 100644
--- a/src/main/python/apache/aurora/tools/java/organize_imports.py
+++ b/src/main/python/apache/aurora/tools/java/organize_imports.py
@@ -12,10 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
'''
-Organizes a java source file's import statements in a way that pleases Apache Aurora's checkstyle
-configuration. This expects exactly one argument: the name of the file to modify with preferred import
-ordering.
+Organizes a java source file's import statements in a way that pleases
+Apache Aurora's checkstyle configuration. This expects exactly one
+argument: the name of the file to modify with preferred import ordering.
'''
from __future__ import print_function
@@ -25,6 +26,8 @@ import sys
from collections import defaultdict
IMPORT_RE = re.compile('import(?: static)? (.*);')
+
+
def get_group(import_statement):
matcher = IMPORT_RE.match(import_statement)
assert matcher, 'Could not parse import statement: %s' % import_statement
@@ -42,6 +45,8 @@ def index_by_group(import_statements):
IMPORT_CLASS_RE = re.compile(
'import(?: static)? (?P<outer>[^A-Z]*[A-Z]\w+)(?:\.(?P<inners>[\w][^;]*))?')
+
+
def get_all_group_lines(import_groups):
if not import_groups:
return []
@@ -75,64 +80,69 @@ def get_all_group_lines(import_groups):
all_lines += get_group_lines(group)
return all_lines
-
-if len(sys.argv) != 2:
- print('usage: %s FILE' % sys.argv[0])
- sys.exit(1)
-
BEFORE_IMPORTS = 'before_imports'
IMPORTS = 'imports'
STATIC_IMPORTS = 'static_imports'
AFTER_IMPORTS = 'after_imports'
-print('Organizing imports in %s' % sys.argv[1])
-lines_before_imports = []
-import_lines = []
-static_import_lines = []
-lines_after_imports = []
-with open(sys.argv[1], 'r') as f:
- position = BEFORE_IMPORTS
- for line in f:
- line = line.rstrip()
- if position == BEFORE_IMPORTS:
- if line.startswith('import'):
- position = IMPORTS
- else:
- lines_before_imports.append(line)
- if position == IMPORTS:
- if line.startswith('import static'):
- position = STATIC_IMPORTS
- elif line.startswith('import'):
- import_lines.append(line)
- elif line.strip():
- position = AFTER_IMPORTS
- if position == STATIC_IMPORTS:
- if line.startswith('import static'):
- static_import_lines.append(line)
- elif line.strip():
- position = AFTER_IMPORTS
- if position == AFTER_IMPORTS:
- lines_after_imports.append(line)
-
-import_groups = index_by_group(import_lines)
-static_import_groups = index_by_group(static_import_lines)
-
-def ensure_line_padding(lines):
- if lines and lines[-1] != '':
- lines.append('')
- return lines
-
-file_lines = lines_before_imports
-if import_groups:
- ensure_line_padding(file_lines)
- file_lines += get_all_group_lines(import_groups)
-if static_import_groups:
- ensure_line_padding(file_lines)
- file_lines += get_all_group_lines(static_import_groups)
-if lines_after_imports:
- ensure_line_padding(file_lines)
- file_lines += lines_after_imports
-
-with open(sys.argv[1], 'w') as f:
- for line in file_lines:
- print(line, file=f)
+
+def main(argv):
+ if len(argv) != 2:
+ print('usage: %s FILE' % argv[0])
+ sys.exit(1)
+
+ print('Organizing imports in %s' % argv[1])
+ lines_before_imports = []
+ import_lines = []
+ static_import_lines = []
+ lines_after_imports = []
+ with open(argv[1], 'r') as f:
+ position = BEFORE_IMPORTS
+ for line in f:
+ line = line.rstrip()
+ if position == BEFORE_IMPORTS:
+ if line.startswith('import'):
+ position = IMPORTS
+ else:
+ lines_before_imports.append(line)
+ if position == IMPORTS:
+ if line.startswith('import static'):
+ position = STATIC_IMPORTS
+ elif line.startswith('import'):
+ import_lines.append(line)
+ elif line.strip():
+ position = AFTER_IMPORTS
+ if position == STATIC_IMPORTS:
+ if line.startswith('import static'):
+ static_import_lines.append(line)
+ elif line.strip():
+ position = AFTER_IMPORTS
+ if position == AFTER_IMPORTS:
+ lines_after_imports.append(line)
+
+ import_groups = index_by_group(import_lines)
+ static_import_groups = index_by_group(static_import_lines)
+
+ def ensure_line_padding(lines):
+ if lines and lines[-1] != '':
+ lines.append('')
+ return lines
+
+ file_lines = lines_before_imports
+ if import_groups:
+ ensure_line_padding(file_lines)
+ file_lines += get_all_group_lines(import_groups)
+ if static_import_groups:
+ ensure_line_padding(file_lines)
+ file_lines += get_all_group_lines(static_import_groups)
+ if lines_after_imports:
+ ensure_line_padding(file_lines)
+ file_lines += lines_after_imports
+
+ with open(argv[1], 'w') as f:
+ for line in file_lines:
+ print(line, file=f)
+
+
+if __name__ == '__main__':
+ main(sys.argv)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
index ea9a482..9dc92fd 100644
--- a/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
+++ b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
@@ -110,7 +110,8 @@ STRUCT_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
: %(type)s.buildNoCopy(wrapped.%(fn_name)s());'''
-IMMUTABLE_COLLECTION_DECLARATION = '''private final Immutable%(collection)s<%(params)s> %(field)s;'''
+IMMUTABLE_COLLECTION_DECLARATION = (
+ '''private final Immutable%(collection)s<%(params)s> %(field)s;''')
IMMUTABLE_COLLECTION_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
? Immutable%(collection)s.<%(params)s>of()
: Immutable%(collection)s.copyOf(wrapped.%(fn_name)s());'''
@@ -273,7 +274,7 @@ class GeneratedCode(object):
'wrapped': self._wrapped_type,
'imports': '\n\n'.join(import_groups),
'accessors': '\n\n'.join(self._accessors),
- 'fields': (' ' + '\n '.join(self._fields) + '\n') if self._fields else '',
+ 'fields': (' ' + '\n '.join(self._fields) + '\n') if self._fields else '',
'assignments': ('\n ' + '\n '.join(self._assignments)) if self._assignments else '',
}, file=f)
@@ -344,7 +345,8 @@ def generate_java(struct):
# Accessor for each field.
for field in struct.fields:
- if not (isinstance(field.ttype, StructType) and (field.ttype.kind == 'enum' or struct.kind == 'union')):
+ if not (isinstance(field.ttype, StructType) and (
+ field.ttype.kind == 'enum' or struct.kind == 'union')):
code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
% {'type': 'boolean',
'fn_name': field.isset_method()})
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/bin/thermos.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/thermos.py b/src/main/python/apache/thermos/bin/thermos.py
index 196dfae..ebf5488 100644
--- a/src/main/python/apache/thermos/bin/thermos.py
+++ b/src/main/python/apache/thermos/bin/thermos.py
@@ -23,15 +23,12 @@ import pwd
import re
import sys
import time
-from collections import namedtuple
-from pystachio import Ref
from pystachio.naming import frozendict
from twitter.common import app, log
-from twitter.common.dirutil import du, tail_f
+from twitter.common.dirutil import tail_f
from twitter.common.dirutil.tail import tail as tail_closed
from twitter.common.log.options import LogOptions
-from twitter.common.quantity import Amount, Data, Time
from twitter.common.quantity.parse_simple import parse_data, parse_time
from twitter.common.recordio import RecordIO, ThriftRecordReader
@@ -101,9 +98,9 @@ def daemonize():
daemon_fork()
os.setsid()
daemon_fork()
- sys.stdin, sys.stdout, sys.stderr = (open('/dev/null', 'r'),
- open('/dev/null', 'a+'),
- open('/dev/null', 'a+', 0))
+ sys.stdin, sys.stdout, sys.stderr = (open('/dev/null', 'r'), # noqa
+ open('/dev/null', 'a+'), # noqa
+ open('/dev/null', 'a+', 0)) # noqa
def tasks_from_re(expressions, root, state=None):
@@ -274,16 +271,16 @@ def simplerun(args, options):
"""
try:
cutoff = args.index('--')
- cmdline = ' '.join(args[cutoff+1:])
+ cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(Task(
- name = options.name,
- resources = Resources(cpu = 1.0, ram = 256 * 1024 * 1024, disk = 0),
- processes = [Process(name = options.name, cmdline = cmdline)]))
+ name=options.name,
+ resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
+ processes=[Process(name=options.name, cmdline=cmdline)]))
_really_run(thermos_task,
options.root,
@@ -339,7 +336,7 @@ def read(args, options):
print('Recovered Task States:')
for task_status in state.statuses:
print(' %s [pid: %d] => %s' % (
- time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
+ time.asctime(time.localtime(task_status.timestamp_ms / 1000.0)),
task_status.runner_pid,
TaskState._VALUES_TO_NAMES[task_status.state]))
print('Recovered Processes:')
@@ -431,10 +428,11 @@ def gc(args, options):
gc_options['max_space'] = parse_data(options.max_space)
if options.max_tasks is not None:
gc_options['max_tasks'] = int(options.max_tasks)
- gc_options.update(include_data = not options.keep_data,
- include_metadata = not options.keep_metadata,
- include_logs = not options.keep_logs,
- verbose = True, logger = print)
+ gc_options.update(include_data=not options.keep_data,
+ include_metadata=not options.keep_metadata,
+ include_logs=not options.keep_logs,
+ verbose=True,
+ logger=print)
tgc = TaskGarbageCollector(root=options.root)
if args:
@@ -496,7 +494,7 @@ def status(args, options):
checkpoint_stat = os.stat(checkpoint_filename)
try:
checkpoint_owner = pwd.getpwuid(checkpoint_stat.st_uid).pw_name
- except:
+ except KeyError:
checkpoint_owner = 'uid:%s' % checkpoint_stat.st_uid
print(' %-20s [owner: %8s]' % (task_id, checkpoint_owner), end='')
if options.verbose == 0:
@@ -508,7 +506,7 @@ def status(args, options):
return
print(' state: %8s' % TaskState._VALUES_TO_NAMES.get(state.statuses[-1].state, 'Unknown'),
end='')
- print(' start: %25s' % time.asctime(time.localtime(state.header.launch_time_ms/1000.0)))
+ print(' start: %25s' % time.asctime(time.localtime(state.header.launch_time_ms / 1000.0)))
if options.verbose > 1:
print(' user: %s' % state.header.user, end='')
if state.header.ports:
@@ -557,7 +555,6 @@ def status(args, options):
sys.exit(1)
-
@app.command
@app.command_option("--stderr", default=False, dest='use_stderr', action='store_true',
help="Tail stderr instead of stdout")
@@ -595,7 +592,7 @@ def tail(args, options):
run=run, log_dir=log_dir).getpath('process_logdir')
logfile = os.path.join(logdir, 'stderr' if options.use_stderr else 'stdout')
- monitor = TaskMonitor(TaskPath(root = options.root), args[0])
+ monitor = TaskMonitor(TaskPath(root=options.root), args[0])
def log_is_active():
active_processes = monitor.get_active_processes()
for process_status, process_run in active_processes:
@@ -621,8 +618,8 @@ def tail(args, options):
next_check = time.time() + 5.0
-@app.command
-def help(args, options):
+@app.command(name='help')
+def help_command(args, options):
"""Get help about a specific command.
"""
if len(args) == 0:
@@ -635,8 +632,6 @@ def help(args, options):
print('unknown command: %s' % args[0], file=sys.stderr)
-
-
def generate_usage():
usage = """
thermos
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/bin/thermos_ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/thermos_ckpt.py b/src/main/python/apache/thermos/bin/thermos_ckpt.py
index a8e42bc..8ed69e0 100644
--- a/src/main/python/apache/thermos/bin/thermos_ckpt.py
+++ b/src/main/python/apache/thermos/bin/thermos_ckpt.py
@@ -12,7 +12,8 @@
# limitations under the License.
#
-import os
+from __future__ import print_function
+
import pprint
import sys
import time
@@ -24,47 +25,59 @@ from apache.thermos.common.ckpt import CheckpointDispatcher
from gen.apache.thermos.ttypes import RunnerCkpt, RunnerState, TaskState
-app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT",
- help = "read checkpoint from CKPT")
-app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True,
- help = "read checkpoint from CKPT")
+app.add_option(
+ "--checkpoint",
+ dest="ckpt",
+ metavar="CKPT",
+ help="read checkpoint from CKPT")
+
+app.add_option(
+ "--assemble",
+ dest="assemble",
+ metavar="CKPT",
+ default=True,
+ help="whether or not to replay the checkpoint records.")
+
def main(args):
values = app.get_options()
if len(args) > 0:
- print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args))
+ print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
app.help()
sys.exit(1)
if not values.ckpt:
- print >> sys.stderr, "ERROR: must supply --checkpoint"
+ print("ERROR: must supply --checkpoint", file=sys.stderr)
app.help()
sys.exit(1)
fp = file(values.ckpt, "r")
rr = ThriftRecordReader(fp, RunnerCkpt)
- wrs = RunnerState(processes = {})
+ wrs = RunnerState(processes={})
dispatcher = CheckpointDispatcher()
try:
for wts in rr:
- print 'Recovering: ', wts
+ print('Recovering: %s' % wts)
if values.assemble is True:
- dispatcher.dispatch(wrs, wts)
+ dispatcher.dispatch(wrs, wts)
except RecordIO.Error as err:
- print 'Error recovering checkpoint stream: %s' % err
+ print('Error recovering checkpoint stream: %s' % err, file=sys.stderr)
return
- print '\n\n\n'
+ print('\n\n\n')
if values.assemble:
- print 'Recovered Task Header'
+ print('Recovered Task Header')
pprint.pprint(wrs.header, indent=4)
- print '\nRecovered Task States'
+ print('\nRecovered Task States')
for task_status in wrs.statuses:
- print ' %s [pid: %d] => %s' % (time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
- task_status.runner_pid, TaskState._VALUES_TO_NAMES[task_status.state])
+ print(' %s [pid: %d] => %s' % (
+ time.asctime(time.localtime(task_status.timestamp_ms / 1000.0)),
+ task_status.runner_pid,
+ TaskState._VALUES_TO_NAMES[task_status.state]))
- print '\nRecovered Processes'
+ print('\nRecovered Processes')
pprint.pprint(wrs.processes, indent=4)
+
app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/common/ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/ckpt.py b/src/main/python/apache/thermos/common/ckpt.py
index af971f5..7df179b 100644
--- a/src/main/python/apache/thermos/common/ckpt.py
+++ b/src/main/python/apache/thermos/common/ckpt.py
@@ -22,8 +22,6 @@ Task state machines.
"""
-import os
-
from twitter.common import log
from twitter.common.recordio import RecordIO, ThriftRecordReader
@@ -41,6 +39,7 @@ class UniversalStateHandler(object):
Generic interface for a handler to be called on any process/state transition, and at task
initialization
"""
+
def on_process_transition(self, state, process_update):
pass
@@ -65,6 +64,7 @@ class ProcessStateHandler(object):
v | `---> [SUCCESS]
[LOST] <------'
"""
+
def on_waiting(self, process_update):
pass
@@ -301,7 +301,7 @@ class CheckpointDispatcher(object):
if process_update is None:
return False
process = process_update.process
- if process not in state.processes: # never seen before
+ if process not in state.processes: # never seen before
return True
else:
# if this sequence number is ahead of the current high water mark, it would
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/common/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/options.py b/src/main/python/apache/thermos/common/options.py
index f3debc3..d7abd8e 100644
--- a/src/main/python/apache/thermos/common/options.py
+++ b/src/main/python/apache/thermos/common/options.py
@@ -18,6 +18,7 @@ from pystachio import Ref
class ParseError(Exception):
pass
+
def add_port_to(option_name):
def add_port_callback(option, opt, value, parser):
if not getattr(parser.values, option_name, None):
@@ -34,6 +35,7 @@ def add_port_to(option_name):
getattr(parser.values, option_name)[name] = port
return add_port_callback
+
def add_binding_to(option_name):
def add_binding_callback(option, opt, value, parser):
if not getattr(parser.values, option_name, None):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/common/path.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/path.py b/src/main/python/apache/thermos/common/path.py
index 0278161..9e61705 100644
--- a/src/main/python/apache/thermos/common/path.py
+++ b/src/main/python/apache/thermos/common/path.py
@@ -52,22 +52,22 @@ class TaskPath(object):
class UnderspecifiedPath(Exception): pass
DEFAULT_CHECKPOINT_ROOT = "/var/run/thermos"
- KNOWN_KEYS = [ 'root', 'task_id', 'state', 'process', 'run', 'log_dir' ]
+ KNOWN_KEYS = ['root', 'task_id', 'state', 'process', 'run', 'log_dir']
LEGACY_KNOWN_KEYS = KNOWN_KEYS[:-1]
DIR_TEMPLATE = {
- 'task_path': ['%(root)s', 'tasks', '%(state)s', '%(task_id)s'],
+ 'task_path': ['%(root)s', 'tasks', '%(state)s', '%(task_id)s'],
'checkpoint_path': ['%(root)s', 'checkpoints', '%(task_id)s'],
- 'runner_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'runner'],
- 'process_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'coordinator.%(process)s'],
+ 'runner_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'runner'],
+ 'process_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'coordinator.%(process)s'],
'process_logbase': ['%(log_dir)s'],
- 'process_logdir': ['%(log_dir)s', '%(process)s', '%(run)s']
+ 'process_logdir': ['%(log_dir)s', '%(process)s', '%(run)s']
}
LEGACY_DIR_TEMPLATE = DIR_TEMPLATE.copy()
LEGACY_DIR_TEMPLATE.update(
- process_logbase = ['%(root)s', 'logs', '%(task_id)s'],
- process_logdir = ['%(root)s', 'logs', '%(task_id)s', '%(process)s', '%(run)s']
+ process_logbase=['%(root)s', 'logs', '%(task_id)s'],
+ process_logdir=['%(root)s', 'logs', '%(task_id)s', '%(process)s', '%(run)s']
)
def __init__(self, **kw):
@@ -85,7 +85,7 @@ class TaskPath(object):
def given(self, **kw):
""" Perform further interpolation of the templates given the kwargs """
- eval_dict = dict(self._data) # copy
+ eval_dict = dict(self._data)
eval_dict.update(kw)
tp = TaskPath(**eval_dict)
tp._filename = self._filename
@@ -107,9 +107,9 @@ class TaskPath(object):
path = os.path.join(*path)
interpolated_path = path % self._data
try:
- _ = interpolated_path % {}
+ interpolated_path % {}
except KeyError:
- raise TaskPath.UnderspecifiedPath(
+ raise self.UnderspecifiedPath(
"Tried to interpolate path with insufficient variables: %s as %s" % (
pathname, interpolated_path))
return interpolated_path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/common/planner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/planner.py b/src/main/python/apache/thermos/common/planner.py
index 58bd68d..da5120f 100644
--- a/src/main/python/apache/thermos/common/planner.py
+++ b/src/main/python/apache/thermos/common/planner.py
@@ -49,8 +49,8 @@ class Planner(object):
process_set -= given
return dependencies
- @staticmethod
- def satisfiable(processes, dependencies):
+ @classmethod
+ def satisfiable(cls, processes, dependencies):
"""
Given a set of processes and a dependency map, determine if this is a consistent
schedule without cycles.
@@ -61,27 +61,27 @@ class Planner(object):
scheduling = True
while scheduling:
scheduling = False
- runnables = Planner.filter_runnable(processes, dependencies)
+ runnables = cls.filter_runnable(processes, dependencies)
if runnables:
scheduling = True
processes -= runnables
- dependencies = Planner.filter_dependencies(dependencies, given=runnables)
+ dependencies = cls.filter_dependencies(dependencies, given=runnables)
return len(processes) == 0
def __init__(self, processes, dependencies):
self._processes = set(processes)
self._dependencies = dict((process, set(dependencies.get(process, [])))
for process in self._processes)
- if not Planner.satisfiable(self._processes, self._dependencies):
- raise Planner.InvalidSchedule("Cycles detected in the task schedule!")
+ if not self.satisfiable(self._processes, self._dependencies):
+ raise self.InvalidSchedule("Cycles detected in the task schedule!")
self._running = set()
self._finished = set()
self._failed = set()
@property
def runnable(self):
- return Planner.filter_runnable(self._processes - self._running - self._finished - self._failed,
- Planner.filter_dependencies(self._dependencies, given=self._finished))
+ return self.filter_runnable(self._processes - self._running - self._finished - self._failed,
+ self.filter_dependencies(self._dependencies, given=self._finished))
@property
def processes(self):
@@ -129,6 +129,7 @@ class Planner(object):
TaskAttributes = namedtuple('TaskAttributes', 'min_duration is_daemon max_failures is_ephemeral')
+
class TaskPlanner(object):
"""
A planner for the processes part of a Thermos task, taking into account ephemeral and daemon
@@ -155,12 +156,12 @@ class TaskPlanner(object):
| failed |
`--------'
"""
- InvalidSchedule = Planner.InvalidSchedule
+ InvalidSchedule = Planner.InvalidSchedule # noqa
INFINITY = sys.float_info.max
TOTAL_RUN_LIMIT = sys.maxsize
- @staticmethod
- def extract_dependencies(task, process_filter=None):
+ @classmethod
+ def extract_dependencies(cls, task, process_filter=None):
"""
Construct a set of processes and the process dependencies from a Thermos Task.
"""
@@ -175,16 +176,16 @@ class TaskPlanner(object):
process_name_set = set(process_names)
# either all process_names must be in processes or none should be
if process_name_set.issubset(processes) == process_name_set.isdisjoint(processes):
- raise TaskPlanner.InvalidSchedule('Invalid process dependencies!')
+ raise cls.InvalidSchedule('Invalid process dependencies!')
if not process_name_set.issubset(processes):
continue
for k in range(1, len(process_names)):
- pnk, pnk1 = process_names[k], process_names[k-1]
+ pnk, pnk1 = process_names[k], process_names[k - 1]
if process_map[pnk1].daemon().get():
- raise TaskPlanner.InvalidSchedule(
+ raise cls.InvalidSchedule(
'Process %s may not depend upon daemon process %s' % (pnk, pnk1))
if not process_map[pnk].ephemeral().get() and process_map[pnk1].ephemeral().get():
- raise TaskPlanner.InvalidSchedule(
+ raise cls.InvalidSchedule(
'Non-ephemeral process %s may not depend upon ephemeral process %s' % (pnk, pnk1))
dependencies[pnk].add(pnk1)
return (processes, dependencies)
@@ -195,7 +196,7 @@ class TaskPlanner(object):
'TaskPlanner must be given callable process filter.')
self._planner = Planner(*self.extract_dependencies(task, self._filter))
self._clock = clock
- self._last_terminal = {} # process => timestamp of last terminal state
+ self._last_terminal = {} # process => timestamp of last terminal state
self._failures = defaultdict(int)
self._successes = defaultdict(int)
self._attributes = {}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/bin/config_load.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/bin/config_load.py b/src/main/python/apache/thermos/config/bin/config_load.py
index a0b4800..d6e1f82 100644
--- a/src/main/python/apache/thermos/config/bin/config_load.py
+++ b/src/main/python/apache/thermos/config/bin/config_load.py
@@ -12,10 +12,10 @@
# limitations under the License.
#
-import copy
+from __future__ import print_function
+
import json
import pprint
-import sys
from twitter.common import app
@@ -23,41 +23,40 @@ from apache.thermos.config.loader import ThermosConfigLoader
def main(args):
- """
- Given .thermos configs, loads them and prints out information about them.
- """
+ """Given .thermos configs, loads them and prints out information about them."""
if len(args) == 0:
app.help()
for arg in args:
- print '\nparsing %s\n' % arg
+ print('\nparsing %s\n' % arg)
tc = ThermosConfigLoader.load(arg)
for task_wrapper in tc.tasks():
task = task_wrapper.task
if not task.has_name():
- print 'Found unnamed task! Skipping...'
+ print('Found unnamed task! Skipping...')
continue
- print 'Task: %s [check: %s]' % (task.name(), task.check())
+ print('Task: %s [check: %s]' % (task.name(), task.check()))
if not task.processes():
- print ' No processes.'
+ print(' No processes.')
else:
- print ' Processes:'
+ print(' Processes:')
for proc in task.processes():
- print ' %s' % proc
+ print(' %s' % proc)
ports = task_wrapper.ports()
if not ports:
- print ' No unbound ports.'
+ print(' No unbound ports.')
else:
- print ' Ports:'
+ print(' Ports:')
for port in ports:
- print ' %s' % port
+ print(' %s' % port)
- print 'raw:'
+ print('raw:')
pprint.pprint(json.loads(task_wrapper.to_json()))
+
app.set_usage("%s config1 config2 ..." % app.name())
app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/bin/config_repl.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/bin/config_repl.py b/src/main/python/apache/thermos/config/bin/config_repl.py
index 8776f9c..ae9ca3b 100644
--- a/src/main/python/apache/thermos/config/bin/config_repl.py
+++ b/src/main/python/apache/thermos/config/bin/config_repl.py
@@ -14,6 +14,6 @@
from code import interact
-from apache.thermos.config.schema import *
+from apache.thermos.config.schema import * # noqa
interact('Thermos Config REPL', local=locals())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/loader.py b/src/main/python/apache/thermos/config/loader.py
index 160027a..d77ab9a 100644
--- a/src/main/python/apache/thermos/config/loader.py
+++ b/src/main/python/apache/thermos/config/loader.py
@@ -12,7 +12,6 @@
# limitations under the License.
#
-import copy
import json
import os
import re
@@ -21,7 +20,6 @@ import textwrap
from pystachio import Ref
from pystachio.config import Config
from twitter.common.dirutil import safe_open
-from twitter.common.lang import Compatibility
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.config.schema import Task
@@ -30,8 +28,8 @@ from apache.thermos.config.schema import Task
class PortExtractor(object):
class InvalidPorts(Exception): pass
- @staticmethod
- def extract(obj):
+ @classmethod
+ def extract(cls, obj):
port_scope = Ref.from_address('thermos.ports')
_, uninterp = obj.interpolate()
ports = []
@@ -39,7 +37,7 @@ class PortExtractor(object):
subscope = port_scope.scoped_to(ref)
if subscope is not None:
if not subscope.is_index():
- raise PortExtractor.InvalidPorts(
+ raise cls.InvalidPorts(
'Bad port specification "%s" (should be of form "thermos.ports[name]"' % ref.address())
ports.append(subscope.action().value)
return ports
@@ -59,10 +57,10 @@ class ThermosProcessWrapper(object):
except PortExtractor.InvalidPorts:
raise self.InvalidProcess('Process has invalid ports scoping!')
- @staticmethod
- def assert_valid_process_name(name):
- if not ThermosProcessWrapper.VALID_PROCESS_NAME_RE.match(name):
- raise ThermosProcessWrapper.InvalidProcess('Invalid process name: %s' % name)
+ @classmethod
+ def assert_valid_process_name(cls, name):
+ if not cls.VALID_PROCESS_NAME_RE.match(name):
+ raise cls.InvalidProcess('Invalid process name: %s' % name)
class ThermosTaskWrapper(object):
@@ -72,7 +70,7 @@ class ThermosTaskWrapper(object):
if bindings:
task = task.bind(*bindings)
if not task.check().ok() and strict:
- raise ThermosTaskWrapper.InvalidTask(task.check().message())
+ raise self.InvalidTask(task.check().message())
self._task = task
@property
@@ -98,13 +96,13 @@ class ThermosTaskWrapper(object):
with safe_open(filename, 'w') as fp:
json.dump(ti.get(), fp)
- @staticmethod
- def from_file(filename, **kw):
+ @classmethod
+ def from_file(cls, filename, **kw):
try:
with safe_open(filename) as fp:
task = Task.json_load(fp)
- return ThermosTaskWrapper(task, **kw)
- except Exception as e:
+ return cls(task, **kw)
+ except Exception:
return None
@@ -121,8 +119,8 @@ class ThermosTaskValidator(object):
@classmethod
def assert_valid_plan(cls, task):
try:
- TaskPlanner(task, process_filter=lambda proc: proc.final().get() == False)
- TaskPlanner(task, process_filter=lambda proc: proc.final().get() == True)
+ TaskPlanner(task, process_filter=lambda proc: proc.final().get() is False)
+ TaskPlanner(task, process_filter=lambda proc: proc.final().get() is True)
except TaskPlanner.InvalidSchedule as e:
raise cls.InvalidTaskError('Task has invalid plan: %s' % e)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/schema.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema.py b/src/main/python/apache/thermos/config/schema.py
index ca16c57..9d858be 100644
--- a/src/main/python/apache/thermos/config/schema.py
+++ b/src/main/python/apache/thermos/config/schema.py
@@ -12,5 +12,5 @@
# limitations under the License.
#
-from .schema_base import *
-from .schema_helpers import *
+from .schema_base import * # noqa
+from .schema_helpers import * # noqa
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/schema_base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema_base.py b/src/main/python/apache/thermos/config/schema_base.py
index 5fc77a8..f9143cc 100644
--- a/src/main/python/apache/thermos/config/schema_base.py
+++ b/src/main/python/apache/thermos/config/schema_base.py
@@ -12,6 +12,8 @@
# limitations under the License.
#
+# checkstyle: noqa
+
from pystachio import Boolean, Default, Empty, Float, Integer, List, Map, Required, String, Struct
# Define constants for resources
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/config/schema_helpers.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema_helpers.py b/src/main/python/apache/thermos/config/schema_helpers.py
index 38620f7..3feef02 100644
--- a/src/main/python/apache/thermos/config/schema_helpers.py
+++ b/src/main/python/apache/thermos/config/schema_helpers.py
@@ -13,6 +13,9 @@
#
"""Helpers for composing Thermos workflows."""
+
+# checkstyle: noqa
+
import itertools
from pystachio import Empty, List
@@ -242,10 +245,12 @@ def SimpleTask(name, command):
"""A simple command-line Task with default resources"""
return Tasks.simple(name, command)
+
def SequentialTask(*args, **kw):
"""A Task whose processes are always sequential."""
return Tasks.sequential(Task(*args, **kw))
+
python_options = Options.python
java_options = Options.java
combine_tasks = Tasks.combine
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/core/inspector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/inspector.py b/src/main/python/apache/thermos/core/inspector.py
index f85b5f9..4fe8aa3 100644
--- a/src/main/python/apache/thermos/core/inspector.py
+++ b/src/main/python/apache/thermos/core/inspector.py
@@ -40,7 +40,7 @@ class CheckpointInspector(object):
@staticmethod
def get_timestamp(process_record):
- if process_record :
+ if process_record:
for timestamp in ('fork_time', 'start_time', 'stop_time'):
stamp = getattr(process_record, timestamp, None)
if stamp:
@@ -52,7 +52,7 @@ class CheckpointInspector(object):
Reconstructs the checkpoint stream and returns a CheckpointInspection.
"""
dispatcher = CheckpointDispatcher()
- state = RunnerState(processes = {})
+ state = RunnerState(processes={})
muxer = ProcessMuxer(self._path.given(task_id=task_id))
runner_processes = []
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/core/muxer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/muxer.py b/src/main/python/apache/thermos/core/muxer.py
index ea2665c..47e77f7 100644
--- a/src/main/python/apache/thermos/core/muxer.py
+++ b/src/main/python/apache/thermos/core/muxer.py
@@ -22,13 +22,14 @@ from gen.apache.thermos.ttypes import RunnerCkpt
class ProcessMuxer(object):
- class ProcessExists(Exception): pass
- class ProcessNotFound(Exception): pass
- class CorruptCheckpoint(Exception): pass
+ class Error(Exception): pass
+ class ProcessExists(Error): pass
+ class ProcessNotFound(Error): pass
+ class CorruptCheckpoint(Error): pass
def __init__(self, pathspec):
- self._processes = {} # process_name => fp
- self._watermarks = {} # process_name => sequence high watermark
+ self._processes = {} # process_name => fp
+ self._watermarks = {} # process_name => sequence high watermark
self._pathspec = pathspec
def __del__(self):
@@ -38,7 +39,7 @@ class ProcessMuxer(object):
def register(self, process_name, watermark=0):
log.debug('registering %s' % process_name)
if process_name in self._processes:
- raise ProcessMuxer.ProcessExists("Process %s is already registered" % process_name)
+ raise self.ProcessExists("Process %s is already registered" % process_name)
self._processes[process_name] = None
self._watermarks[process_name] = watermark
@@ -48,7 +49,7 @@ class ProcessMuxer(object):
process_ckpt = self._pathspec.given(process=process_name).getpath('process_checkpoint')
log.debug('ProcessMuxer binding %s => %s' % (process_name, process_ckpt))
try:
- self._processes[process_name] = open(process_ckpt, 'r')
+ self._processes[process_name] = open(process_ckpt, 'r') # noqa
except IOError as e:
if e.errno == errno.ENOENT:
log.debug(' => bind failed, checkpoint not available yet.')
@@ -92,7 +93,7 @@ class ProcessMuxer(object):
def unregister(self, process_name):
log.debug('unregistering %s' % process_name)
if process_name not in self._processes:
- raise ProcessMuxer.ProcessNotFound("No trace of process: %s" % process_name)
+ raise self.ProcessNotFound("No trace of process: %s" % process_name)
else:
self._watermarks.pop(process_name)
fp = self._processes.pop(process_name)
@@ -111,8 +112,8 @@ class ProcessMuxer(object):
rr = ThriftRecordReader(fp, RunnerCkpt)
old_pos = fp.tell()
try:
- expected_new_pos = os.fstat(fp.fileno()).st_size
- except OSError as e:
+ os.fstat(fp.fileno()).st_size
+ except OSError:
log.debug('ProcessMuxer could not fstat for process %s' % process)
return False
update = rr.try_read()
@@ -137,7 +138,7 @@ class ProcessMuxer(object):
for handle in filter(None, self._processes.values()):
try:
fstat = os.fstat(handle.fileno())
- except OSError as e:
+ except OSError:
log.error('Unable to fstat %s!' % handle.name)
continue
if handle.tell() > fstat.st_size:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/core/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/process.py b/src/main/python/apache/thermos/core/process.py
index aef1ec6..4889e63 100644
--- a/src/main/python/apache/thermos/core/process.py
+++ b/src/main/python/apache/thermos/core/process.py
@@ -20,7 +20,6 @@ commandline in a subprocess of its own.
"""
-import getpass
import grp
import os
import pwd
@@ -41,6 +40,7 @@ from gen.apache.thermos.ttypes import ProcessState, ProcessStatus, RunnerCkpt
class Platform(Interface):
"""Abstract representation of a platform encapsulating system-level functions"""
+
@abstractmethod
def clock(self):
pass
@@ -94,7 +94,7 @@ class ProcessBase(object):
self._stderr = None
self._user = user
if self._user:
- user, current_user = self._getpwuid() # may raise self.UnknownUserError
+ user, current_user = self._getpwuid() # may raise self.UnknownUserError
if user != current_user and os.geteuid() != 0:
raise self.PermissionError('Must be root to run processes as other users!')
self._ckpt = None
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/core/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/runner.py b/src/main/python/apache/thermos/core/runner.py
index b04c555..ec4cdb7 100644
--- a/src/main/python/apache/thermos/core/runner.py
+++ b/src/main/python/apache/thermos/core/runner.py
@@ -39,14 +39,12 @@ terminal state.
"""
-import errno
import os
import socket
import sys
import time
import traceback
from contextlib import contextmanager
-from functools import partial
from pystachio import Environment
from twitter.common import log
@@ -64,7 +62,6 @@ from apache.thermos.common.path import TaskPath
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.config.loader import (
ThermosConfigLoader,
- ThermosProcessWrapper,
ThermosTaskValidator,
ThermosTaskWrapper
)
@@ -279,7 +276,7 @@ class TaskRunnerStage(object):
raise NotImplementedError
-class TaskRunnerStage_ACTIVE(TaskRunnerStage):
+class TaskRunnerStage_ACTIVE(TaskRunnerStage): # noqa
"""
Run the regular plan (i.e. normal, non-finalizing processes.)
"""
@@ -315,7 +312,7 @@ class TaskRunnerStage_ACTIVE(TaskRunnerStage):
return TaskState.CLEANING
-class TaskRunnerStage_CLEANING(TaskRunnerStage):
+class TaskRunnerStage_CLEANING(TaskRunnerStage): # noqa
"""
Start the cleanup of the regular plan (e.g. if it failed.) On ACTIVE -> CLEANING,
we send SIGTERMs to all still-running processes. We wait at most finalization_wait
@@ -323,6 +320,7 @@ class TaskRunnerStage_CLEANING(TaskRunnerStage):
prior to that point in time, we transition to FINALIZING, which kicks into gear
the finalization schedule (if any.)
"""
+
def run(self):
log.debug('TaskRunnerStage[CLEANING]: Finalization remaining: %s' %
self.runner._finalization_remaining())
@@ -336,7 +334,7 @@ class TaskRunnerStage_CLEANING(TaskRunnerStage):
return TaskState.FINALIZING
-class TaskRunnerStage_FINALIZING(TaskRunnerStage):
+class TaskRunnerStage_FINALIZING(TaskRunnerStage): # noqa
"""
Run the finalizing plan, specifically the plan of tasks with the 'final'
bit marked (e.g. log savers, checkpointers and the like.) Anything in this
@@ -446,7 +444,7 @@ class TaskRunner(object):
raise TypeError('planner_class must be a TaskPlanner.')
self._clock = clock
launch_time = self._clock.time()
- launch_time_ms = '%06d' % int((launch_time - int(launch_time)) * 10**6)
+ launch_time_ms = '%06d' % int((launch_time - int(launch_time)) * (10 ** 6))
if not task_id:
self._task_id = '%s-%s.%s' % (task.name(),
time.strftime('%Y%m%d-%H%M%S', time.localtime(launch_time)),
@@ -481,11 +479,11 @@ class TaskRunner(object):
ThermosTaskValidator.assert_same_task(self._pathspec, self._task)
except ThermosTaskValidator.InvalidTaskError as e:
raise self.InvalidTask('Invalid task: %s' % e)
- self._plan = None # plan currently being executed (updated by Handlers)
+ self._plan = None # plan currently being executed (updated by Handlers)
self._regular_plan = planner_class(self._task, clock=clock,
- process_filter=lambda proc: proc.final().get() == False)
+ process_filter=lambda proc: proc.final().get() is False)
self._finalizing_plan = planner_class(self._task, clock=clock,
- process_filter=lambda proc: proc.final().get() == True)
+ process_filter=lambda proc: proc.final().get() is True)
self._chroot = chroot
self._sandbox = sandbox
self._terminal_state = None
@@ -496,7 +494,7 @@ class TaskRunner(object):
self._finalization_start = None
self._preemption_deadline = None
self._watcher = ProcessMuxer(self._pathspec)
- self._state = RunnerState(processes = {})
+ self._state = RunnerState(processes={})
# create runner state
universal_handler = universal_handler or TaskRunnerUniversalHandler
@@ -542,14 +540,14 @@ class TaskRunner(object):
file lock on the checkpoint stream.
"""
if self.is_terminal():
- raise TaskRunner.StateError('Cannot take control of a task in terminal state.')
+ raise self.StateError('Cannot take control of a task in terminal state.')
if self._sandbox:
safe_mkdir(self._sandbox)
ckpt_file = self._pathspec.getpath('runner_checkpoint')
try:
self._ckpt = TaskRunnerHelper.open_checkpoint(ckpt_file, force=force, state=self._state)
except TaskRunnerHelper.PermissionError:
- raise TaskRunner.PermissionError('Unable to open checkpoint %s' % ckpt_file)
+ raise self.PermissionError('Unable to open checkpoint %s' % ckpt_file)
log.debug('Flipping recovery mode off.')
self._recovery = False
self._set_task_status(self.task_state())
@@ -589,12 +587,11 @@ class TaskRunner(object):
"""
ckpt_file = self._pathspec.getpath('runner_checkpoint')
if os.path.exists(ckpt_file):
- fp = open(ckpt_file, "r")
- ckpt_recover = ThriftRecordReader(fp, RunnerCkpt)
- for record in ckpt_recover:
- log.debug('Replaying runner checkpoint record: %s' % record)
- self._dispatcher.dispatch(self._state, record, recovery=True)
- ckpt_recover.close()
+ with open(ckpt_file, 'r') as fp:
+ ckpt_recover = ThriftRecordReader(fp, RunnerCkpt)
+ for record in ckpt_recover:
+ log.debug('Replaying runner checkpoint record: %s' % record)
+ self._dispatcher.dispatch(self._state, record, recovery=True)
def _replay_process_ckpts(self):
"""
@@ -618,7 +615,7 @@ class TaskRunner(object):
if self._state.header is None:
header = RunnerHeader(
task_id=self._task_id,
- launch_time_ms=int(self._launch_time*1000),
+ launch_time_ms=int(self._launch_time * 1000),
sandbox=self._sandbox,
log_dir=self._log_dir,
hostname=socket.gethostname(),
@@ -643,8 +640,8 @@ class TaskRunner(object):
if self._finalization_start is None:
return sys.float_info.max
else:
- waited = max(0, self._clock.time() - self._finalization_start)
- return max(0, finalization_allocation - waited)
+ waited = max(0, self._clock.time() - self._finalization_start)
+ return max(0, finalization_allocation - waited)
def _set_process_status(self, process_name, process_state, **kw):
if 'sequence_number' in kw:
@@ -725,7 +722,7 @@ class TaskRunner(object):
def forked_but_never_came_up():
return current_run.state == ProcessState.FORKED and (
- self._clock.time() - current_run.fork_time > TaskRunner.LOST_TIMEOUT.as_(Time.SECONDS))
+ self._clock.time() - current_run.fork_time > self.LOST_TIMEOUT.as_(Time.SECONDS))
def running_but_coordinator_died():
if current_run.state != ProcessState.RUNNING:
@@ -846,7 +843,6 @@ class TaskRunner(object):
self._run()
def _run(self):
- iteration_time = self.MAX_ITERATION_TIME.as_(Time.SECONDS)
while not self.is_terminal():
start = self._clock.time()
# step 1: execute stage corresponding to the state we're currently in
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/BUILD b/src/main/python/apache/thermos/monitoring/BUILD
index 1ae5f52..79da0d5 100644
--- a/src/main/python/apache/thermos/monitoring/BUILD
+++ b/src/main/python/apache/thermos/monitoring/BUILD
@@ -18,7 +18,7 @@ python_library(
name = 'detector',
sources = ['detector.py'],
dependencies = [
- pants('src/main/python/apache/thermos/common:path')
+ pants('src/main/python/apache/thermos/common:path'),
]
)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/detector.py b/src/main/python/apache/thermos/monitoring/detector.py
index fa50fce..117aef5 100644
--- a/src/main/python/apache/thermos/monitoring/detector.py
+++ b/src/main/python/apache/thermos/monitoring/detector.py
@@ -48,7 +48,7 @@ class TaskDetector(object):
for path in paths:
try:
task_state, task_id = path_re.match(path).groups()
- except:
+ except Exception:
continue
if state is None or task_state == state:
yield (task_state, task_id)
@@ -69,7 +69,7 @@ class TaskDetector(object):
for path in paths:
try:
process, run = path_re.match(path).groups()
- except:
+ except Exception:
continue
yield process, int(run)
@@ -100,6 +100,6 @@ class TaskDetector(object):
for path in matching_paths:
try:
process, = path_re.match(path).groups()
- except:
+ except Exception:
continue
yield path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/disk.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/disk.py b/src/main/python/apache/thermos/monitoring/disk.py
index 23c1a93..175ed3a 100644
--- a/src/main/python/apache/thermos/monitoring/disk.py
+++ b/src/main/python/apache/thermos/monitoring/disk.py
@@ -28,7 +28,7 @@ Currently, there are two threads available:
import os
import threading
import time
-from Queue import Empty, Queue
+from Queue import Queue
from twitter.common import log
from twitter.common.dirutil import du, safe_bsize
@@ -47,6 +47,7 @@ from watchdog.observers import Observer as WatchdogObserver
class DiskCollectorThread(ExceptionalThread):
""" Thread to calculate aggregate disk usage under a given path using a simple algorithm """
+
def __init__(self, path):
self.path = path
self.value = None
@@ -66,6 +67,7 @@ class DiskCollectorThread(ExceptionalThread):
class DiskCollector(Lockable):
""" Spawn a background thread to sample disk usage """
+
def __init__(self, root):
self._root = root
self._thread = None
@@ -169,11 +171,12 @@ class InotifyDiskCollectorThread(ExceptionalThread, FileSystemEventHandler):
@property
def value(self):
- return sum(self._files.itervalues())
+ return sum(self._files.values())
class InotifyDiskCollector(object):
""" Spawn a background thread to sample disk usage """
+
def __init__(self, root):
self._root = root
self._thread = InotifyDiskCollectorThread(self._root)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/garbage.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/garbage.py b/src/main/python/apache/thermos/monitoring/garbage.py
index 11b8c76..69bf8e4 100644
--- a/src/main/python/apache/thermos/monitoring/garbage.py
+++ b/src/main/python/apache/thermos/monitoring/garbage.py
@@ -136,9 +136,9 @@ class DefaultCollector(TaskGarbageCollectionPolicy):
verbose: boolean (whether or not to log) [default: False]
logger: callable (function to call with log messages) [default: sys.stdout.write]
"""
- self._max_age = kw.get('max_age', Amount(10**10, Time.DAYS))
- self._max_space = kw.get('max_space', Amount(10**10, Data.TB))
- self._max_tasks = kw.get('max_tasks', 10**10)
+ self._max_age = kw.get('max_age', Amount(10 ** 10, Time.DAYS))
+ self._max_space = kw.get('max_space', Amount(10 ** 10, Data.TB))
+ self._max_tasks = kw.get('max_tasks', 10 ** 10)
self._include_metadata = kw.get('include_metadata', True)
self._include_logs = kw.get('include_logs', True)
self._verbose = kw.get('verbose', False)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/process.py b/src/main/python/apache/thermos/monitoring/process.py
index 99f7a3d..9529a00 100644
--- a/src/main/python/apache/thermos/monitoring/process.py
+++ b/src/main/python/apache/thermos/monitoring/process.py
@@ -40,24 +40,25 @@ class ProcessSample(namedtuple('ProcessSample', 'rate user system rss vms nice s
else:
status = other.status
return ProcessSample(
- rate = self.rate + other.rate,
- user = self.user + other.user,
- system = self.system + other.system,
- rss = self.rss + other.rss,
- vms = self.vms + other.vms,
- nice = nice,
- status = status,
- threads = self.threads + other.threads)
+ rate=self.rate + other.rate,
+ user=self.user + other.user,
+ system=self.system + other.system,
+ rss=self.rss + other.rss,
+ vms=self.vms + other.vms,
+ nice=nice,
+ status=status,
+ threads=self.threads + other.threads,
+ )
def to_dict(self):
return dict(
- cpu = self.rate,
- ram = self.rss,
- user = self.user,
- system = self.system,
- rss = self.rss,
- vms = self.vms,
- nice = self.nice,
- status = str(self.status),
- threads = self.threads
+ cpu=self.rate,
+ ram=self.rss,
+ user=self.user,
+ system=self.system,
+ rss=self.rss,
+ vms=self.vms,
+ nice=self.nice,
+ status=str(self.status),
+ threads=self.threads,
)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/process_collector_psutil.py b/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
index f972c83..298459b 100644
--- a/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
+++ b/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
@@ -43,11 +43,12 @@ def process_to_sample(process):
class ProcessTreeCollector(object):
""" Collect resource consumption statistics for a process and its children """
+
def __init__(self, pid):
""" Given a pid """
self._pid = pid
self._process = None # psutil.Process
- self._sampled_tree = {} # pid => ProcessSample
+ self._sampled_tree = {} # pid => ProcessSample
self._sample = ProcessSample.empty()
self._stamp = None
self._rate = 0.0
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/monitoring/resource.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/resource.py b/src/main/python/apache/thermos/monitoring/resource.py
index 9f8e24e..7b5bbe0 100644
--- a/src/main/python/apache/thermos/monitoring/resource.py
+++ b/src/main/python/apache/thermos/monitoring/resource.py
@@ -27,7 +27,6 @@ disk consumption and retaining a limited (FIFO) in-memory history of this data.
"""
-import platform
import threading
import time
from abc import abstractmethod
@@ -42,7 +41,6 @@ from twitter.common.lang import Interface
from twitter.common.quantity import Amount, Time
from .disk import DiskCollector
-from .monitor import TaskMonitor
from .process import ProcessSample
from .process_collector_psutil import ProcessTreeCollector
@@ -81,6 +79,7 @@ class ResourceHistory(object):
"""Simple class to contain a RingBuffer (fixed-length FIFO) history of resource samples, with the
mapping: timestamp => (number_of_procs, ProcessSample, disk_usage_in_bytes)
"""
+
def __init__(self, maxlen, initialize=True):
if not maxlen >= 1:
raise ValueError("maxlen must be greater than 0")
@@ -117,7 +116,7 @@ class TaskResourceMonitor(ResourceMonitorBase, threading.Thread):
history of previous sample results.
"""
- MAX_HISTORY = 10000 # magic number
+ MAX_HISTORY = 10000 # magic number
def __init__(self, task_monitor, sandbox,
process_collector=ProcessTreeCollector, disk_collector=DiskCollector,
@@ -128,10 +127,10 @@ class TaskResourceMonitor(ResourceMonitorBase, threading.Thread):
task_monitor: TaskMonitor object specifying the task whose resources should be monitored
sandbox: Directory for which to monitor disk utilisation
"""
- self._task_monitor = task_monitor # exposes PIDs, sandbox
+ self._task_monitor = task_monitor # exposes PIDs, sandbox
self._task_id = task_monitor._task_id
log.debug('Initialising resource collection for task %s' % self._task_id)
- self._process_collectors = dict() # ProcessStatus => ProcessTreeCollector
+ self._process_collectors = dict() # ProcessStatus => ProcessTreeCollector
# TODO(jon): sandbox is also available through task_monitor, but typically the first checkpoint
# isn't written (and hence the header is not available) by the time we initialise here
self._sandbox = sandbox
@@ -203,7 +202,7 @@ class TaskResourceMonitor(ResourceMonitorBase, threading.Thread):
log.debug('Adding process "%s" (pid %s) to resource monitoring' %
(process.process, process.pid))
self._process_collectors[process] = self._process_collector_factory(process.pid)
- for process, collector in self._process_collectors.iteritems():
+ for process, collector in self._process_collectors.items():
log.debug('Collecting sample for process "%s" (pid %s) and children' %
(process.process, process.pid))
collector.sample()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/observer/bin/thermos_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/bin/thermos_observer.py b/src/main/python/apache/thermos/observer/bin/thermos_observer.py
index c205f14..53c331c 100644
--- a/src/main/python/apache/thermos/observer/bin/thermos_observer.py
+++ b/src/main/python/apache/thermos/observer/bin/thermos_observer.py
@@ -14,9 +14,7 @@
from __future__ import print_function
-import socket
import sys
-import time
from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/observer/http/file_browser.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/file_browser.py b/src/main/python/apache/thermos/observer/http/file_browser.py
index 6b53c86..87ef9c8 100644
--- a/src/main/python/apache/thermos/observer/http/file_browser.py
+++ b/src/main/python/apache/thermos/observer/http/file_browser.py
@@ -13,11 +13,9 @@
#
import os
-import pprint
from xml.sax.saxutils import escape
import bottle
-from mako.template import Template
from twitter.common import log
from twitter.common.http import HttpServer
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/observer/http/http_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/http_observer.py b/src/main/python/apache/thermos/observer/http/http_observer.py
index 2bd079c..5bfc4f2 100644
--- a/src/main/python/apache/thermos/observer/http/http_observer.py
+++ b/src/main/python/apache/thermos/observer/http/http_observer.py
@@ -19,7 +19,6 @@ system. To do this, it relies heavily on the Thermos TaskObserver.
"""
-import os
import socket
from twitter.common import log
@@ -79,15 +78,15 @@ class BottleObserver(HttpServer, StaticAssets, TaskObserverFileBrowser, TaskObse
state = self._observer.state(task_id)
return dict(
- task_id = task_id,
- task = task,
- statuses = self._observer.task_statuses(task_id),
- user = task['user'],
- ports = task['ports'],
- processes = processes,
- chroot = state.get('sandbox', ''),
- launch_time = state.get('launch_time', 0),
- hostname = state.get('hostname', 'localhost'),
+ task_id=task_id,
+ task=task,
+ statuses=self._observer.task_statuses(task_id),
+ user=task['user'],
+ ports=task['ports'],
+ processes=processes,
+ chroot=state.get('sandbox', ''),
+ launch_time=state.get('launch_time', 0),
+ hostname=state.get('hostname', 'localhost'),
)
def get_task(self, task_id):
@@ -102,9 +101,9 @@ class BottleObserver(HttpServer, StaticAssets, TaskObserverFileBrowser, TaskObse
task = self.get_task(task_id)
state = self._observer.state(task_id)
return dict(
- hostname = state.get('hostname', 'localhost'),
- task_id = task_id,
- task_struct = task['task_struct']
+ hostname=state.get('hostname', 'localhost'),
+ task_id=task_id,
+ task_struct=task['task_struct']
)
@HttpServer.route("/process/:task_id/:process_id")
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/observer/http/static_assets.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/static_assets.py b/src/main/python/apache/thermos/observer/http/static_assets.py
index be881bf..83adeb3 100644
--- a/src/main/python/apache/thermos/observer/http/static_assets.py
+++ b/src/main/python/apache/thermos/observer/http/static_assets.py
@@ -25,6 +25,7 @@ class StaticAssets(object):
"""
Serve the /assets directory.
"""
+
def __init__(self):
self._assets = {}
self._detect_assets()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/thermos/testing/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/testing/runner.py b/src/main/python/apache/thermos/testing/runner.py
index 551311f..8b6ba73 100644
--- a/src/main/python/apache/thermos/testing/runner.py
+++ b/src/main/python/apache/thermos/testing/runner.py
@@ -24,14 +24,13 @@ import tempfile
import time
from thrift.TSerialization import deserialize as thrift_deserialize
-from twitter.common import log
from twitter.common.contextutil import environment_as, temporary_file
from apache.thermos.common.ckpt import CheckpointDispatcher
from apache.thermos.common.path import TaskPath
from apache.thermos.config.loader import ThermosTaskWrapper
-from gen.apache.thermos.ttypes import RunnerCkpt, RunnerState, TaskState
+from gen.apache.thermos.ttypes import RunnerState
class Runner(object):
@@ -98,11 +97,11 @@ with open('%(state_filename)s', 'w') as fp:
self.state_filename = tempfile.mktemp()
self.tempdir = tempfile.mkdtemp()
- self.task_id = '%s-runner-base' % int(time.time()*1000000)
+ self.task_id = '%s-runner-base' % int(time.time() * 1000000)
self.sandbox = os.path.join(self.tempdir, 'sandbox')
self.portmap = portmap
self.cleaned = False
- self.pathspec = TaskPath(root = self.tempdir, task_id = self.task_id)
+ self.pathspec = TaskPath(root=self.tempdir, task_id=self.task_id)
self.script_filename = None
self.success_rate = success_rate
self.random_seed = random_seed
@@ -150,7 +149,8 @@ with open('%(state_filename)s', 'w') as fp:
rc = self.po.returncode
if rc != 0:
if os.path.exists(self.job_filename):
- config = open(self.job_filename).read()
+ with open(self.job_filename) as fp:
+ config = fp.read()
else:
config = 'Nonexistent!'
if 'THERMOS_DEBUG' in os.environ:
@@ -167,8 +167,9 @@ with open('%(state_filename)s', 'w') as fp:
try:
self.reconstructed_state = CheckpointDispatcher.from_file(
- self.pathspec.getpath('runner_checkpoint'))
- except:
+ self.pathspec.getpath('runner_checkpoint'))
+ except Exception as e:
+ print('Failed to replay checkpoint: %s' % e, file=sys.stderr)
self.reconstructed_state = None
self.initialized = True
return rc
@@ -178,7 +179,8 @@ with open('%(state_filename)s', 'w') as fp:
if hasattr(self, 'po'):
try:
self.po.kill()
- except:
+ except Exception as e:
+ print('Failed to kill runner: %s' % e, file=sys.stderr)
pass
os.unlink(self.job_filename)
os.unlink(self.script_filename)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/admin/test_host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/admin/test_host_maintenance.py b/src/test/python/apache/aurora/admin/test_host_maintenance.py
index ed0782b..176076f 100644
--- a/src/test/python/apache/aurora/admin/test_host_maintenance.py
+++ b/src/test/python/apache/aurora/admin/test_host_maintenance.py
@@ -15,7 +15,6 @@
import unittest
import mock
-import pytest
from apache.aurora.admin.host_maintenance import HostMaintenance
from apache.aurora.client.base import add_grouping, remove_grouping
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_disambiguator.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_disambiguator.py b/src/test/python/apache/aurora/client/api/test_disambiguator.py
index fb6380d..e9523ac 100644
--- a/src/test/python/apache/aurora/client/api/test_disambiguator.py
+++ b/src/test/python/apache/aurora/client/api/test_disambiguator.py
@@ -25,7 +25,7 @@ from apache.aurora.common.cluster import Cluster
from gen.apache.aurora.api.constants import ResponseCode
from gen.apache.aurora.api.ttypes import GetJobsResult, JobConfiguration, JobKey, Response, Result
-TEST_CLUSTER = Cluster(name = 'smf1')
+TEST_CLUSTER = Cluster(name='smf1')
class LiveJobDisambiguatorTest(mox.MoxTestBase):
@@ -51,7 +51,7 @@ class LiveJobDisambiguatorTest(mox.MoxTestBase):
self._api.get_jobs(self.ROLE).AndReturn(Response(
responseCode=ResponseCode.OK,
messageDEPRECATED='Mock OK',
- result = Result(getJobsResult=GetJobsResult(
+ result=Result(getJobsResult=GetJobsResult(
configs=set(JobConfiguration(key=JobKey(role=self.ROLE, environment=env, name=self.NAME))
for env in envs)))))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_health_check.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_health_check.py b/src/test/python/apache/aurora/client/api/test_health_check.py
index 52cfcfe..af005be 100644
--- a/src/test/python/apache/aurora/client/api/test_health_check.py
+++ b/src/test/python/apache/aurora/client/api/test_health_check.py
@@ -15,7 +15,6 @@
import unittest
import mox
-import pytest
from apache.aurora.client.api.health_check import (
HealthCheck,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_instance_watcher.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_instance_watcher.py b/src/test/python/apache/aurora/client/api/test_instance_watcher.py
index b2d0c80..723a5b6 100644
--- a/src/test/python/apache/aurora/client/api/test_instance_watcher.py
+++ b/src/test/python/apache/aurora/client/api/test_instance_watcher.py
@@ -21,7 +21,19 @@ from apache.aurora.client.api.health_check import HealthCheck
from apache.aurora.client.api.instance_watcher import InstanceWatcher
from gen.apache.aurora.api.AuroraSchedulerManager import Client as scheduler_client
-from gen.apache.aurora.api.ttypes import *
+from gen.apache.aurora.api.ttypes import (
+ AssignedTask,
+ Identity,
+ JobKey,
+ Response,
+ ResponseCode,
+ Result,
+ ScheduledTask,
+ ScheduleStatus,
+ ScheduleStatusResult,
+ TaskConfig,
+ TaskQuery
+)
class FakeClock(object):
@@ -82,7 +94,8 @@ class InstanceWatcherTest(unittest.TestCase):
for x in range(int(num_calls)):
self._scheduler.getTasksStatus(query).AndReturn(response)
- def expect_io_error_in_get_statuses(self, instance_ids=WATCH_INSTANCES, num_calls=EXPECTED_CYCLES):
+ def expect_io_error_in_get_statuses(self, instance_ids=WATCH_INSTANCES,
+ num_calls=EXPECTED_CYCLES):
tasks = [self.create_task(instance_id) for instance_id in instance_ids]
response = Response(responseCode=ResponseCode.OK, messageDEPRECATED='test')
response.result = Result()
@@ -92,7 +105,6 @@ class InstanceWatcherTest(unittest.TestCase):
for x in range(int(num_calls)):
self._scheduler.getTasksStatus(query).AndRaise(IOError('oops'))
-
def mock_health_check(self, task, status, retry):
self._health_check.health(task).InAnyOrder().AndReturn((status, retry))
@@ -103,7 +115,8 @@ class InstanceWatcherTest(unittest.TestCase):
def assert_watch_result(self, expected_failed_instances, instances_to_watch=WATCH_INSTANCES):
instances_returned = self._watcher.watch(instances_to_watch, self._health_check)
assert set(expected_failed_instances) == instances_returned, (
- 'Expected instances (%s) : Returned instances (%s)' % (expected_failed_instances, instances_returned))
+ 'Expected instances (%s) : Returned instances (%s)' % (
+ expected_failed_instances, instances_returned))
def replay_mocks(self):
mox.Replay(self._scheduler)
@@ -142,7 +155,6 @@ class InstanceWatcherTest(unittest.TestCase):
self.assert_watch_result([0, 1, 2])
self.verify_mocks()
-
def test_all_instance_failure(self):
"""All failed instance in a batch of instances"""
self.expect_get_statuses()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_job_monitor.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_job_monitor.py b/src/test/python/apache/aurora/client/api/test_job_monitor.py
index 665db74..3aa9607 100644
--- a/src/test/python/apache/aurora/client/api/test_job_monitor.py
+++ b/src/test/python/apache/aurora/client/api/test_job_monitor.py
@@ -15,11 +15,9 @@ import unittest
from mock import Mock
-from apache.aurora.client.api import AuroraClientAPI
from apache.aurora.client.api.job_monitor import JobMonitor
from apache.aurora.common.aurora_job_key import AuroraJobKey
-from gen.apache.aurora.api.AuroraSchedulerManager import Client
from gen.apache.aurora.api.ttypes import (
AssignedTask,
Identity,
@@ -56,6 +54,7 @@ class JobMonitorTest(unittest.TestCase):
status=status,
timestamp=10)]
)
+
def mock_get_tasks(self, tasks, response_code=None):
response_code = ResponseCode.OK if response_code is None else response_code
resp = Response(responseCode=response_code, messageDEPRECATED='test')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_quota_check.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_quota_check.py b/src/test/python/apache/aurora/client/api/test_quota_check.py
index 6c241b7..d75bd1b 100644
--- a/src/test/python/apache/aurora/client/api/test_quota_check.py
+++ b/src/test/python/apache/aurora/client/api/test_quota_check.py
@@ -19,7 +19,6 @@ from mock import Mock
from apache.aurora.client.api.quota_check import CapacityRequest, QuotaCheck
-from gen.apache.aurora.api.AuroraSchedulerManager import Client as scheduler_client
from gen.apache.aurora.api.ttypes import (
GetQuotaResult,
JobKey,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_restarter.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_restarter.py b/src/test/python/apache/aurora/client/api/test_restarter.py
index e31fd6d..97add6f 100644
--- a/src/test/python/apache/aurora/client/api/test_restarter.py
+++ b/src/test/python/apache/aurora/client/api/test_restarter.py
@@ -21,22 +21,28 @@ from apache.aurora.client.fake_scheduler_proxy import FakeSchedulerProxy
from apache.aurora.common.aurora_job_key import AuroraJobKey
from gen.apache.aurora.api.AuroraSchedulerManager import Client as scheduler_client
-from gen.apache.aurora.api.ttypes import *
-
-# test space
-
+from gen.apache.aurora.api.ttypes import (
+ AssignedTask,
+ Response,
+ ResponseCode,
+ Result,
+ ScheduledTask,
+ ScheduleStatus,
+ ScheduleStatusResult,
+ TaskConfig
+)
SESSION_KEY = 'test_session'
-CLUSTER='smfd'
+CLUSTER = 'smfd'
JOB = AuroraJobKey(CLUSTER, 'johndoe', 'test', 'test_job')
HEALTH_CHECK_INTERVAL_SECONDS = 5
UPDATER_CONFIG = UpdaterConfig(
- 2, # batch_size
- 23, # restart_threshold
- 45, #watch_secs
- 0, # max_per_instance_failures
- 0, # max_total_failures
- True # rollback_on_failure
+ batch_size=2,
+ restart_threshold=23,
+ watch_secs=45,
+ max_per_shard_failures=0,
+ max_total_failures=0,
+ rollback_on_failure=True,
)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_scheduler_client.py b/src/test/python/apache/aurora/client/api/test_scheduler_client.py
index 402b426..6b23a4a 100644
--- a/src/test/python/apache/aurora/client/api/test_scheduler_client.py
+++ b/src/test/python/apache/aurora/client/api/test_scheduler_client.py
@@ -29,24 +29,39 @@ from apache.aurora.common.cluster import Cluster
import gen.apache.aurora.api.AuroraAdmin as AuroraAdmin
import gen.apache.aurora.api.AuroraSchedulerManager as AuroraSchedulerManager
from gen.apache.aurora.api.constants import CURRENT_API_VERSION, DEFAULT_ENVIRONMENT
-from gen.apache.aurora.api.ttypes import *
+from gen.apache.aurora.api.ttypes import (
+ Hosts,
+ JobConfiguration,
+ JobKey,
+ Lock,
+ LockValidation,
+ ResourceAggregate,
+ Response,
+ ResponseCode,
+ Result,
+ RewriteConfigsRequest,
+ ScheduleStatus,
+ SessionKey,
+ TaskQuery
+)
ROLE = 'foorole'
JOB_NAME = 'barjobname'
JOB_KEY = JobKey(role=ROLE, environment=DEFAULT_ENVIRONMENT, name=JOB_NAME)
-def test_testCoverage():
+def test_coverage():
"""Make sure a new thrift RPC doesn't get added without minimal test coverage."""
for name, klass in inspect.getmembers(AuroraAdmin) + inspect.getmembers(AuroraSchedulerManager):
if name.endswith('_args'):
rpc_name = name[:-len('_args')]
assert hasattr(TestSchedulerProxyAdminInjection, 'test_%s' % rpc_name), (
- 'No test defined for RPC %s' % rpc_name)
+ 'No test defined for RPC %s' % rpc_name)
class TestSchedulerProxy(scheduler_client.SchedulerProxy):
"""In testing we shouldn't use the real SSHAgentAuthenticator."""
+
def session_key(self):
return self.create_session('SOME_USER')
@@ -70,7 +85,7 @@ class TestSchedulerProxyInjection(unittest.TestCase):
self.mock_scheduler_client.get_thrift_client().AndReturn(self.mock_thrift_client)
version_resp = Response(responseCode=ResponseCode.OK)
- version_resp.result = Result(getVersionResult = CURRENT_API_VERSION)
+ version_resp.result = Result(getVersionResult=CURRENT_API_VERSION)
self.mock_thrift_client.getVersion().AndReturn(version_resp)
@@ -322,15 +337,15 @@ class TestZookeeperSchedulerClient(unittest.TestCase):
return mock_zk, [ServiceInstance.unpack(service_json)]
- class TestZookeeperSchedulerClient(scheduler_client.ZookeeperSchedulerClient):
+ class ZookeeperSchedulerClientTestImpl(scheduler_client.ZookeeperSchedulerClient):
SERVERSET_TIMEOUT = Amount(10, Time.MILLISECONDS)
- original_method = TestZookeeperSchedulerClient.get_scheduler_serverset
+ original_method = ZookeeperSchedulerClientTestImpl.get_scheduler_serverset
try:
- TestZookeeperSchedulerClient.get_scheduler_serverset = mock_get_serverset
+ ZookeeperSchedulerClientTestImpl.get_scheduler_serverset = mock_get_serverset
- zk_scheduler_client = TestZookeeperSchedulerClient(Cluster(proxy_url=None))
+ zk_scheduler_client = ZookeeperSchedulerClientTestImpl(Cluster(proxy_url=None))
self.mox.StubOutWithMock(zk_scheduler_client, '_connect_scheduler')
mock_zk.stop()
zk_scheduler_client._connect_scheduler(host, port)
@@ -339,7 +354,8 @@ class TestZookeeperSchedulerClient(unittest.TestCase):
assert zk_scheduler_client.url == 'http://%s:%d' % (host, port)
finally:
- TestZookeeperSchedulerClient.get_scheduler_serverset = original_method
+ ZookeeperSchedulerClientTestImpl.get_scheduler_serverset = original_method
+
class TestSchedulerClient(unittest.TestCase):
@mock.patch('thrift.transport.THttpClient.THttpClient', spec=THttpClient.THttpClient)
@@ -348,4 +364,5 @@ class TestSchedulerClient(unittest.TestCase):
mock_time = mock.Mock(spec=time)
scheduler_client.SchedulerClient._connect_scheduler('scheduler.example.com', 1337, mock_time)
assert MockTHttpClient.return_value.open.call_count is 2
- mock_time.sleep.assert_called_once_with(scheduler_client.SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
+ mock_time.sleep.assert_called_once_with(
+ scheduler_client.SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_sla.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_sla.py b/src/test/python/apache/aurora/client/api/test_sla.py
index 9c3bb6d..9ca8102 100644
--- a/src/test/python/apache/aurora/client/api/test_sla.py
+++ b/src/test/python/apache/aurora/client/api/test_sla.py
@@ -14,16 +14,15 @@
import time
import unittest
-
from contextlib import contextmanager
+
from mock import call, Mock, patch
-from apache.aurora.client.api.sla import DomainUpTimeSlaVector, JobUpTimeSlaVector, Sla, task_query
+from apache.aurora.client.api.sla import JobUpTimeLimit, Sla, task_query
from apache.aurora.client.base import add_grouping, DEFAULT_GROUPING, remove_grouping
from apache.aurora.common.aurora_job_key import AuroraJobKey
from apache.aurora.common.cluster import Cluster
-from gen.apache.aurora.api.AuroraSchedulerManager import Client as scheduler_client
from gen.apache.aurora.api.constants import LIVE_STATES
from gen.apache.aurora.api.ttypes import (
AssignedTask,
@@ -192,7 +191,6 @@ class SlaTest(unittest.TestCase):
self.mock_get_tasks(self.create_tasks([100, 200, 300, 400, 500]))
self.assert_count_result(100, 50)
-
def test_uptime_empty(self):
self.mock_get_tasks([])
self.assert_uptime_result(0, 50)
@@ -217,7 +215,6 @@ class SlaTest(unittest.TestCase):
self.mock_get_tasks(self.create_tasks([100, 200, 300, 400]))
self.assert_uptime_result(None, 100)
-
def test_wait_time_empty(self):
self.mock_get_tasks([])
self.assert_wait_time_result(None, 50, 200)
@@ -246,7 +243,6 @@ class SlaTest(unittest.TestCase):
self.mock_get_tasks(self.create_tasks([100, 200, 300, 400]))
self.assert_wait_time_result(150, 80, 250)
-
def test_domain_uptime_no_tasks(self):
self.mock_get_tasks([])
vector = self._sla.get_domain_uptime_vector(self._cluster, self._min_count)
@@ -289,11 +285,7 @@ class SlaTest(unittest.TestCase):
])
job_override = {
- self._job_key:
- DomainUpTimeSlaVector.JobUpTimeLimit(
- job=self._job_key,
- percentage=50,
- duration_secs=100)
+ self._job_key: JobUpTimeLimit(job=self._job_key, percentage=50, duration_secs=100)
}
self.assert_safe_domain_result('h1', 50, 400, in_limit=job_override)
@@ -337,7 +329,6 @@ class SlaTest(unittest.TestCase):
assert 0 == len(vector.get_safe_hosts(50, 150, None, 'by_rack')), 'Length must be empty.'
self.expect_task_status_call_cluster_scoped()
-
def test_probe_hosts_no_hosts(self):
self.mock_get_tasks([])
vector = self._sla.get_domain_uptime_vector(self._cluster, self._min_count)
@@ -431,7 +422,6 @@ class SlaTest(unittest.TestCase):
self.assert_probe_host_job_details(result, 'cl-r2-h03', 25.0, False, 100)
self.assert_probe_host_job_details(result, 'cl-r2-h04', 25.0, False, 100)
-
def test_get_domain_uptime_vector_with_hosts(self):
with patch('apache.aurora.client.api.sla.task_query', return_value=TaskQuery()) as (mock_query):
self.mock_get_tasks([
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/api/test_updater.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_updater.py b/src/test/python/apache/aurora/client/api/test_updater.py
index ba783da..0ee342c 100644
--- a/src/test/python/apache/aurora/client/api/test_updater.py
+++ b/src/test/python/apache/aurora/client/api/test_updater.py
@@ -38,7 +38,6 @@ from gen.apache.aurora.api.ttypes import (
JobConfiguration,
JobKey,
LimitConstraint,
- Lock,
LockKey,
LockValidation,
Metadata,
@@ -63,6 +62,7 @@ if 'UPDATER_DEBUG' in environ:
LogOptions.set_stderr_log_level('DEBUG')
log.init('test_updater')
+
class FakeConfig(object):
def __init__(self, role, name, env, update_config):
self._role = role
@@ -104,12 +104,12 @@ class FakeConfig(object):
class UpdaterTest(TestCase):
UPDATE_CONFIG = {
- 'batch_size': 3,
- 'restart_threshold': 50,
- 'watch_secs': 50,
- 'max_per_shard_failures': 0,
- 'max_total_failures': 0,
- 'rollback_on_failure': True,
+ 'batch_size': 3,
+ 'restart_threshold': 50,
+ 'watch_secs': 50,
+ 'max_per_shard_failures': 0,
+ 'max_total_failures': 0,
+ 'rollback_on_failure': True,
}
def setUp(self):
@@ -459,7 +459,6 @@ class UpdaterTest(TestCase):
self.update_and_expect_ok(instances=[2, 3, 4])
self.verify_mocks()
-
def test_patch_hole_with_instance_option(self):
"""Patching an instance ID gap created by a terminated update."""
old_configs = self.make_task_configs(8)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_command_hooks.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_command_hooks.py b/src/test/python/apache/aurora/client/cli/test_command_hooks.py
index 1ee11ff..65bce5f 100644
--- a/src/test/python/apache/aurora/client/cli/test_command_hooks.py
+++ b/src/test/python/apache/aurora/client/cli/test_command_hooks.py
@@ -215,8 +215,8 @@ class TestClientCreateCommand(AuroraClientCommandTest):
def test_dynamic_hook_syntax_error(self):
with patch("logging.warn") as log_patch:
GlobalCommandHookRegistry.reset()
- hook_locals = GlobalCommandHookRegistry.load_project_hooks(
- "./src/test/python/apache/aurora/client/cli/hook_test_data/bad_syntax")
+ GlobalCommandHookRegistry.load_project_hooks(
+ "./src/test/python/apache/aurora/client/cli/hook_test_data/bad_syntax")
log_patch.assert_called_with("Error compiling hooks file "
"./src/test/python/apache/aurora/client/cli/hook_test_data/bad_syntax/AuroraHooks: "
"invalid syntax (AuroraHooks, line 1)")
@@ -224,16 +224,15 @@ class TestClientCreateCommand(AuroraClientCommandTest):
def test_dynamic_hook_exec_error(self):
with patch("logging.warn") as log_patch:
GlobalCommandHookRegistry.reset()
- hook_locals = GlobalCommandHookRegistry.load_project_hooks(
- "./src/test/python/apache/aurora/client/cli/hook_test_data/exec_error")
+ GlobalCommandHookRegistry.load_project_hooks(
+ "./src/test/python/apache/aurora/client/cli/hook_test_data/exec_error")
log_patch.assert_called_with("Warning: error loading hooks file "
"./src/test/python/apache/aurora/client/cli/hook_test_data/exec_error/AuroraHooks: "
"integer division or modulo by zero")
def assert_skip_allowed(self, context, skip_opt, user, noun, verb, args):
"""Checks that a hook would be allowed to be skipped in a command invocation"""
- required_hooks = GlobalCommandHookRegistry.get_required_hooks(context, skip_opt, noun,
- verb, user)
+ GlobalCommandHookRegistry.get_required_hooks(context, skip_opt, noun, verb, user)
def assert_skip_forbidden(self, context, skip_opt, user, noun, verb, args):
"""Checks that a hook would NOT be allowed to be skipped in a command invocation"""
@@ -362,7 +361,6 @@ class TestClientCreateCommand(AuroraClientCommandTest):
}
}
- mock_query = self.create_mock_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.INIT))
mock_context.add_expected_status_query_result(
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_config_noun.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_config_noun.py b/src/test/python/apache/aurora/client/cli/test_config_noun.py
index 7a9d733..c55c9fe 100644
--- a/src/test/python/apache/aurora/client/cli/test_config_noun.py
+++ b/src/test/python/apache/aurora/client/cli/test_config_noun.py
@@ -12,14 +12,13 @@
# limitations under the License.
#
-import contextlib
import textwrap
+from mock import patch
from twitter.common.contextutil import temporary_file
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
-from mock import patch
class TestClientCreateCommand(AuroraClientCommandTest):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_create.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_create.py b/src/test/python/apache/aurora/client/cli/test_create.py
index b186b52..224adc3 100644
--- a/src/test/python/apache/aurora/client/cli/test_create.py
+++ b/src/test/python/apache/aurora/client/cli/test_create.py
@@ -20,7 +20,6 @@ from twitter.common.contextutil import temporary_file
from apache.aurora.client.cli import EXIT_COMMAND_FAILURE, EXIT_INVALID_CONFIGURATION
from apache.aurora.client.cli.client import AuroraCommandLine
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
-from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.config import AuroraConfig
from gen.apache.aurora.api.ttypes import (
@@ -164,7 +163,6 @@ class TestClientCreateCommand(AuroraClientCommandTest):
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
-
def test_create_job_failed_invalid_config(self):
"""Run a test of the "create" command against a mocked-out API, with a configuration
containing a syntax error"""
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/test/python/apache/aurora/client/cli/test_cron.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_cron.py b/src/test/python/apache/aurora/client/cli/test_cron.py
index 049405a..6066a3b 100644
--- a/src/test/python/apache/aurora/client/cli/test_cron.py
+++ b/src/test/python/apache/aurora/client/cli/test_cron.py
@@ -15,7 +15,9 @@
#
import contextlib
+
from mock import Mock, patch
+from twitter.common.contextutil import temporary_file
from apache.aurora.client.cli import EXIT_COMMAND_FAILURE, EXIT_INVALID_CONFIGURATION, EXIT_OK
from apache.aurora.client.cli.client import AuroraCommandLine
@@ -25,9 +27,6 @@ from apache.aurora.config import AuroraConfig
from gen.apache.aurora.api.ttypes import JobKey
-from twitter.common.contextutil import temporary_file
-
-
class TestCronNoun(AuroraClientCommandTest):
def test_successful_schedule(self):