You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wi...@apache.org on 2014/06/05 18:49:39 UTC
[4/4] git commit: Make style consistent with
build-support/python/checkstyle-check
Make style consistent with build-support/python/checkstyle-check
Reviewed at https://reviews.apache.org/r/22168/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/467f8428
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/467f8428
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/467f8428
Branch: refs/heads/master
Commit: 467f84285d7fd0c9f861b839f25449b194e391e4
Parents: ca4adf9
Author: Brian Wickman <wi...@apache.org>
Authored: Thu Jun 5 09:48:43 2014 -0700
Committer: Brian Wickman <wi...@apache.org>
Committed: Thu Jun 5 09:48:43 2014 -0700
----------------------------------------------------------------------
.../apache/aurora/admin/host_maintenance.py | 8 +-
.../python/apache/aurora/client/api/__init__.py | 17 +--
.../apache/aurora/client/api/command_runner.py | 21 ++-
.../apache/aurora/client/api/health_check.py | 6 +-
.../aurora/client/api/scheduler_client.py | 29 +++--
src/main/python/apache/aurora/client/api/sla.py | 23 ++--
.../python/apache/aurora/client/api/updater.py | 37 +++---
.../apache/aurora/client/api/updater_util.py | 2 +-
src/main/python/apache/aurora/client/base.py | 36 ++++--
.../apache/aurora/client/bin/aurora_admin.py | 5 +-
.../apache/aurora/client/bin/aurora_client.py | 5 +-
.../apache/aurora/client/binding_helper.py | 10 +-
.../python/apache/aurora/client/cli/__init__.py | 21 +--
.../python/apache/aurora/client/cli/bridge.py | 1 -
.../python/apache/aurora/client/cli/client.py | 8 +-
.../apache/aurora/client/cli/command_hooks.py | 6 +-
.../python/apache/aurora/client/cli/config.py | 12 +-
.../python/apache/aurora/client/cli/context.py | 4 +-
.../python/apache/aurora/client/cli/cron.py | 4 +-
.../python/apache/aurora/client/cli/jobs.py | 15 +--
.../python/apache/aurora/client/cli/logsetup.py | 9 +-
.../python/apache/aurora/client/cli/options.py | 7 +-
.../python/apache/aurora/client/cli/quota.py | 11 +-
.../aurora/client/cli/standalone_client.py | 3 +-
.../python/apache/aurora/client/cli/task.py | 45 ++-----
.../apache/aurora/client/commands/admin.py | 10 +-
.../apache/aurora/client/commands/core.py | 7 +-
.../apache/aurora/client/commands/help.py | 4 +-
.../aurora/client/commands/maintenance.py | 1 -
src/main/python/apache/aurora/client/config.py | 7 +-
src/main/python/apache/aurora/client/factory.py | 1 -
.../apache/aurora/client/hooks/hooked_api.py | 4 +-
.../apache/aurora/common/auth/__init__.py | 12 +-
.../apache/aurora/common/auth/auth_module.py | 2 -
src/main/python/apache/aurora/common/cluster.py | 2 +-
.../apache/aurora/common/cluster_option.py | 2 +-
.../python/apache/aurora/common/clusters.py | 4 +-
.../apache/aurora/common/http_signaler.py | 1 -
src/main/python/apache/aurora/config/loader.py | 1 -
.../python/apache/aurora/config/schema/base.py | 3 +
src/main/python/apache/aurora/config/thrift.py | 6 +-
.../apache/aurora/executor/aurora_executor.py | 2 +-
.../aurora/executor/bin/thermos_runner_main.py | 2 +-
.../aurora/executor/common/health_checker.py | 1 +
.../aurora/executor/common/kill_manager.py | 1 +
.../apache/aurora/executor/common/sandbox.py | 1 +
.../apache/aurora/executor/gc_executor.py | 26 ++--
.../apache/aurora/executor/thermos_runner.py | 2 +-
.../aurora/tools/java/organize_imports.py | 128 ++++++++++---------
.../aurora/tools/java/thrift_wrapper_codegen.py | 8 +-
src/main/python/apache/thermos/bin/thermos.py | 43 +++----
.../python/apache/thermos/bin/thermos_ckpt.py | 47 ++++---
src/main/python/apache/thermos/common/ckpt.py | 6 +-
.../python/apache/thermos/common/options.py | 2 +
src/main/python/apache/thermos/common/path.py | 20 +--
.../python/apache/thermos/common/planner.py | 33 ++---
.../apache/thermos/config/bin/config_load.py | 29 ++---
.../apache/thermos/config/bin/config_repl.py | 2 +-
src/main/python/apache/thermos/config/loader.py | 30 ++---
src/main/python/apache/thermos/config/schema.py | 4 +-
.../python/apache/thermos/config/schema_base.py | 2 +
.../apache/thermos/config/schema_helpers.py | 5 +
.../python/apache/thermos/core/inspector.py | 4 +-
src/main/python/apache/thermos/core/muxer.py | 23 ++--
src/main/python/apache/thermos/core/process.py | 4 +-
src/main/python/apache/thermos/core/runner.py | 44 +++----
src/main/python/apache/thermos/monitoring/BUILD | 2 +-
.../apache/thermos/monitoring/detector.py | 6 +-
.../python/apache/thermos/monitoring/disk.py | 7 +-
.../python/apache/thermos/monitoring/garbage.py | 6 +-
.../python/apache/thermos/monitoring/process.py | 35 ++---
.../monitoring/process_collector_psutil.py | 3 +-
.../apache/thermos/monitoring/resource.py | 11 +-
.../thermos/observer/bin/thermos_observer.py | 2 -
.../thermos/observer/http/file_browser.py | 2 -
.../thermos/observer/http/http_observer.py | 25 ++--
.../thermos/observer/http/static_assets.py | 1 +
.../python/apache/thermos/testing/runner.py | 18 +--
.../aurora/admin/test_host_maintenance.py | 1 -
.../aurora/client/api/test_disambiguator.py | 4 +-
.../aurora/client/api/test_health_check.py | 1 -
.../aurora/client/api/test_instance_watcher.py | 22 +++-
.../aurora/client/api/test_job_monitor.py | 3 +-
.../aurora/client/api/test_quota_check.py | 1 -
.../apache/aurora/client/api/test_restarter.py | 28 ++--
.../aurora/client/api/test_scheduler_client.py | 37 ++++--
.../python/apache/aurora/client/api/test_sla.py | 16 +--
.../apache/aurora/client/api/test_updater.py | 15 +--
.../aurora/client/cli/test_command_hooks.py | 12 +-
.../aurora/client/cli/test_config_noun.py | 3 +-
.../apache/aurora/client/cli/test_create.py | 2 -
.../apache/aurora/client/cli/test_cron.py | 5 +-
.../apache/aurora/client/cli/test_kill.py | 26 ++--
.../apache/aurora/client/cli/test_open.py | 6 +-
.../apache/aurora/client/cli/test_plugins.py | 18 +--
.../apache/aurora/client/cli/test_restart.py | 15 +--
.../apache/aurora/client/cli/test_status.py | 2 +-
.../apache/aurora/client/cli/test_task_run.py | 8 +-
.../apache/aurora/client/cli/test_update.py | 4 +-
.../python/apache/aurora/client/cli/util.py | 5 +-
.../aurora/client/commands/test_admin_sla.py | 24 ++--
.../client/commands/test_cancel_update.py | 5 -
.../aurora/client/commands/test_create.py | 6 -
.../apache/aurora/client/commands/test_hooks.py | 24 +---
.../apache/aurora/client/commands/test_kill.py | 48 ++-----
.../aurora/client/commands/test_listjobs.py | 3 -
.../aurora/client/commands/test_restart.py | 7 +-
.../aurora/client/commands/test_status.py | 9 +-
.../aurora/client/commands/test_update.py | 7 +-
.../apache/aurora/client/test_binding_helper.py | 2 -
.../python/apache/aurora/client/test_config.py | 23 +++-
.../python/apache/aurora/common/test_cluster.py | 18 +--
.../apache/aurora/common/test_cluster_option.py | 4 +-
.../apache/aurora/common/test_clusters.py | 3 +-
.../apache/aurora/common/test_http_signaler.py | 2 +-
.../apache/aurora/common/test_shellify.py | 3 +-
.../python/apache/aurora/config/test_base.py | 86 ++++++-------
.../aurora/config/test_constraint_parsing.py | 12 +-
.../python/apache/aurora/config/test_loader.py | 6 +-
.../python/apache/aurora/config/test_thrift.py | 70 +++++-----
.../executor/common/test_directory_sandbox.py | 2 +-
.../executor/common/test_health_checker.py | 1 +
.../aurora/executor/common/test_task_info.py | 19 +--
.../aurora/executor/test_executor_vars.py | 2 +-
.../apache/aurora/executor/test_gc_executor.py | 38 ++++--
.../aurora/executor/test_thermos_executor.py | 40 +++---
.../python/apache/thermos/bin/test_thermos.py | 2 +-
.../apache/thermos/common/test_planner.py | 5 +-
.../apache/thermos/common/test_task_planner.py | 54 ++++----
.../python/apache/thermos/config/test_schema.py | 7 +-
.../python/apache/thermos/core/test_angry.py | 16 +--
.../apache/thermos/core/test_ephemerals.py | 12 +-
.../apache/thermos/core/test_failing_runner.py | 34 ++---
.../apache/thermos/core/test_failure_limit.py | 27 ++--
.../apache/thermos/core/test_finalization.py | 2 +-
.../python/apache/thermos/core/test_helper.py | 5 +-
.../python/apache/thermos/core/test_process.py | 25 ++--
.../thermos/core/test_runner_integration.py | 128 +++++++++----------
.../apache/thermos/core/test_staged_kill.py | 21 ++-
.../apache/thermos/monitoring/test_disk.py | 4 +-
.../sh/org/apache/aurora/e2e/http_example.py | 4 +-
141 files changed, 1000 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/admin/host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/host_maintenance.py b/src/main/python/apache/aurora/admin/host_maintenance.py
index ca26de1..f223c5e 100644
--- a/src/main/python/apache/aurora/admin/host_maintenance.py
+++ b/src/main/python/apache/aurora/admin/host_maintenance.py
@@ -13,18 +13,12 @@
#
import time
-from collections import defaultdict
from twitter.common import log
from twitter.common.quantity import Amount, Time
from apache.aurora.client.api import AuroraClientAPI
-from apache.aurora.client.base import (
- check_and_log_response,
- DEFAULT_GROUPING,
- group_hosts,
- GROUPING_FUNCTIONS
-)
+from apache.aurora.client.base import check_and_log_response, DEFAULT_GROUPING, group_hosts
from gen.apache.aurora.api.ttypes import Hosts, MaintenanceMode
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/__init__.py b/src/main/python/apache/aurora/client/api/__init__.py
index 1af0f8f..a73f23f 100644
--- a/src/main/python/apache/aurora/client/api/__init__.py
+++ b/src/main/python/apache/aurora/client/api/__init__.py
@@ -24,13 +24,7 @@ from .sla import Sla
from .updater import Updater
from gen.apache.aurora.api.constants import LIVE_STATES
-from gen.apache.aurora.api.ttypes import (
- Identity,
- ResourceAggregate,
- Response,
- ResponseCode,
- TaskQuery
-)
+from gen.apache.aurora.api.ttypes import Identity, ResourceAggregate, ResponseCode, TaskQuery
class AuroraClientAPI(object):
@@ -139,9 +133,12 @@ class AuroraClientAPI(object):
return resp
def restart(self, job_key, instances, updater_config, health_check_interval_seconds):
- """Perform a rolling restart of the job. If instances is None or [], restart all instances. Returns
- the scheduler response for the last restarted batch of instances (which allows the client to
- show the job URL), or the status check response if no tasks were active.
+ """Perform a rolling restart of the job.
+
+ If instances is None or [], restart all instances. Returns the
+ scheduler response for the last restarted batch of instances (which
+ allows the client to show the job URL), or the status check response
+ if no tasks were active.
"""
self._assert_valid_job_key(job_key)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/command_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/command_runner.py b/src/main/python/apache/aurora/client/api/command_runner.py
index 8b29f35..a291fd9 100644
--- a/src/main/python/apache/aurora/client/api/command_runner.py
+++ b/src/main/python/apache/aurora/client/api/command_runner.py
@@ -12,6 +12,8 @@
# limitations under the License.
#
+from __future__ import print_function
+
import posixpath
import subprocess
from multiprocessing.pool import ThreadPool
@@ -29,8 +31,8 @@ from gen.apache.aurora.api.ttypes import Identity, ResponseCode, TaskQuery
class CommandRunnerTrait(Cluster.Trait):
- slave_root = Required(String)
- slave_run_directory = Required(String)
+ slave_root = Required(String) # noqa
+ slave_run_directory = Required(String) # noqa
class DistributedCommandRunner(object):
@@ -121,20 +123,25 @@ class DistributedCommandRunner(object):
def process_arguments(self, command, **kw):
for task in self.resolve():
host = task.assignedTask.slaveHost
- role = task.assignedTask.task.owner.role
yield (host, self._ssh_user, self.substitute(command, task, self._cluster, **kw))
def run(self, command, parallelism=1, **kw):
threadpool = ThreadPool(processes=parallelism)
for result in threadpool.imap_unordered(self.execute, self.process_arguments(command, **kw)):
- print result
+ print(result)
+
class InstanceDistributedCommandRunner(DistributedCommandRunner):
"""A distributed command runner that only runs on specified instances of a job."""
@classmethod
def query_from(cls, role, env, job, instances=None):
- return TaskQuery(statuses=LIVE_STATES, owner=Identity(role), jobName=job, environment=env, instanceIds=instances)
+ return TaskQuery(
+ statuses=LIVE_STATES,
+ owner=Identity(role),
+ jobName=job,
+ environment=env,
+ instanceIds=instances)
def __init__(self, cluster, role, env, job, ssh_user=None, instances=None):
super(InstanceDistributedCommandRunner, self).__init__(cluster, role, env, [job], ssh_user)
@@ -148,6 +155,6 @@ class InstanceDistributedCommandRunner(DistributedCommandRunner):
for task in resp.result.scheduleStatusResult.tasks:
yield task
else:
- print_aurora_log(logging.ERROR,
- "Error: could not retrieve task information for run command: %s" % resp.messageDEPRECATED)
+ # This should be fixed by AURORA-503.
+ log.error('could not retrieve task information for run command: %s' % resp.messageDEPRECATED)
raise ValueError("Could not retrieve task information: %s" % resp.messageDEPRECATED)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/health_check.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/health_check.py b/src/main/python/apache/aurora/client/api/health_check.py
index d6ef596..0503c0b 100644
--- a/src/main/python/apache/aurora/client/api/health_check.py
+++ b/src/main/python/apache/aurora/client/api/health_check.py
@@ -58,6 +58,7 @@ class StatusHealthCheck(HealthCheck):
1. A task is in state RUNNING
2. A task that satisfies (1) and is already known has the same task id.
"""
+
def __init__(self):
self._task_ids = {}
@@ -68,7 +69,10 @@ class StatusHealthCheck(HealthCheck):
if status == ScheduleStatus.RUNNING:
if instance_id in self._task_ids:
- return Retriable.alive() if task_id == self._task_ids.get(instance_id) else NotRetriable.dead()
+ if task_id == self._task_ids.get(instance_id):
+ return Retriable.alive()
+ else:
+ return NotRetriable.dead()
else:
log.info('Detected RUNNING instance %s' % instance_id)
self._task_ids[instance_id] = task_id
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/scheduler_client.py b/src/main/python/apache/aurora/client/api/scheduler_client.py
index 7be974e..86ccdd9 100644
--- a/src/main/python/apache/aurora/client/api/scheduler_client.py
+++ b/src/main/python/apache/aurora/client/api/scheduler_client.py
@@ -17,7 +17,7 @@ import threading
import time
import traceback
-from pystachio import Boolean, Default, Integer, String
+from pystachio import Default, Integer, String
from thrift.protocol import TJSONProtocol
from thrift.transport import THttpClient, TTransport
from twitter.common import log
@@ -33,19 +33,20 @@ from gen.apache.aurora.api.constants import CURRENT_API_VERSION
class SchedulerClientTrait(Cluster.Trait):
- zk = String
- zk_port = Default(Integer, 2181)
- scheduler_zk_path = String
- scheduler_uri = String
- proxy_url = String
- auth_mechanism = Default(String, 'UNAUTHENTICATED')
+ zk = String # noqa
+ zk_port = Default(Integer, 2181) # noqa
+ scheduler_zk_path = String # noqa
+ scheduler_uri = String # noqa
+ proxy_url = String # noqa
+ auth_mechanism = Default(String, 'UNAUTHENTICATED') # noqa
class SchedulerClient(object):
THRIFT_RETRIES = 5
RETRY_TIMEOUT = Amount(1, Time.SECONDS)
- class CouldNotConnect(Exception): pass
+ class Error(Exception): pass
+ class CouldNotConnect(Error): pass
# TODO(wickman) Refactor per MESOS-3005 into two separate classes with separate traits:
# ZookeeperClientTrait
@@ -81,23 +82,23 @@ class SchedulerClient(object):
def _connect(self):
return None
- @staticmethod
- def _connect_scheduler(host, port, clock=time):
+ @classmethod
+ def _connect_scheduler(cls, host, port, clock=time):
transport = THttpClient.THttpClient('http://%s:%s/api' % (host, port))
protocol = TJSONProtocol.TJSONProtocol(transport)
schedulerClient = AuroraAdmin.Client(protocol)
- for _ in range(SchedulerClient.THRIFT_RETRIES):
+ for _ in range(cls.THRIFT_RETRIES):
try:
transport.open()
return schedulerClient
except TTransport.TTransportException:
- clock.sleep(SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
+ clock.sleep(cls.RETRY_TIMEOUT.as_(Time.SECONDS))
continue
except Exception as e:
# Monkey-patched proxies, like socks, can generate a proxy error here.
# without adding a dependency, we can't catch those in a more specific way.
- raise SchedulerClient.CouldNotConnect('Connection to scheduler failed: %s' % e)
- raise SchedulerClient.CouldNotConnect('Could not connect to %s:%s' % (host, port))
+ raise cls.CouldNotConnect('Connection to scheduler failed: %s' % e)
+ raise cls.CouldNotConnect('Could not connect to %s:%s' % (host, port))
class ZookeeperSchedulerClient(SchedulerClient):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/sla.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/sla.py b/src/main/python/apache/aurora/client/api/sla.py
index d15491a..a7b8760 100644
--- a/src/main/python/apache/aurora/client/api/sla.py
+++ b/src/main/python/apache/aurora/client/api/sla.py
@@ -15,13 +15,12 @@
import math
import time
from collections import defaultdict, namedtuple
-from copy import deepcopy
from apache.aurora.client.base import DEFAULT_GROUPING, group_hosts, log_response
from apache.aurora.common.aurora_job_key import AuroraJobKey
from gen.apache.aurora.api.constants import LIVE_STATES
-from gen.apache.aurora.api.ttypes import Identity, Response, ResponseCode, ScheduleStatus, TaskQuery
+from gen.apache.aurora.api.ttypes import Identity, ResponseCode, ScheduleStatus, TaskQuery
def job_key_from_scheduled(task, cluster):
@@ -99,7 +98,6 @@ class JobUpTimeSlaVector(object):
else:
return duration - sorted(self._uptime_map.values())[index]
-
def get_task_up_count(self, duration, total_tasks=None):
"""Returns the percentage of job tasks that stayed up longer than duration.
@@ -130,11 +128,19 @@ class JobUpTimeSlaVector(object):
for task in self._tasks:
for event in task.taskEvents:
if event.status == ScheduleStatus.RUNNING:
- instance_map[task.assignedTask.instanceId] = math.floor(self._now - event.timestamp / 1000)
+ instance_map[task.assignedTask.instanceId] = math.floor(
+ self._now - event.timestamp / 1000)
break
return instance_map
+JobUpTimeLimit = namedtuple('JobUpTimeLimit', ['job', 'percentage', 'duration_secs'])
+
+
+JobUpTimeDetails = namedtuple('JobUpTimeDetails',
+ ['job', 'predicted_percentage', 'safe', 'safe_in_secs'])
+
+
class DomainUpTimeSlaVector(object):
"""A grouping of all active tasks in the cluster by:
- job: Map of job_key -> task. Provides logical mapping between jobs and their active tasks.
@@ -143,10 +149,6 @@ class DomainUpTimeSlaVector(object):
"""
DEFAULT_MIN_INSTANCE_COUNT = 2
- JobUpTimeLimit = namedtuple('JobUpTimeLimit', ['job', 'percentage', 'duration_secs'])
- JobUpTimeDetails = namedtuple('JobUpTimeDetails',
- ['job', 'predicted_percentage', 'safe', 'safe_in_secs'])
-
def __init__(self, cluster, tasks, min_instance_count=DEFAULT_MIN_INSTANCE_COUNT, hosts=None):
self._cluster = cluster
self._tasks = tasks
@@ -188,7 +190,7 @@ class DomainUpTimeSlaVector(object):
break
for host in job_hosts:
- safe_hosts[host].append(self.JobUpTimeLimit(job_key, filtered_percentage, job_duration))
+ safe_hosts[host].append(JobUpTimeLimit(job_key, filtered_percentage, job_duration))
else:
safe_groups.append(safe_hosts)
@@ -226,7 +228,7 @@ class DomainUpTimeSlaVector(object):
for host in job_hosts:
probed_hosts[host].append(
- self.JobUpTimeDetails(job_key, filtered_percentage, safe, wait_to_sla))
+ JobUpTimeDetails(job_key, filtered_percentage, safe, wait_to_sla))
if probed_hosts:
probed_groups.append(probed_hosts)
@@ -248,7 +250,6 @@ class DomainUpTimeSlaVector(object):
for host in group.keys():
result[host].append(uptime_details)
-
def _simulate_hosts_down(self, job_key, hosts, duration):
unfiltered_tasks = self._tasks_by_job[job_key]
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/updater.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater.py b/src/main/python/apache/aurora/client/api/updater.py
index ea7285a..de7912c 100644
--- a/src/main/python/apache/aurora/client/api/updater.py
+++ b/src/main/python/apache/aurora/client/api/updater.py
@@ -39,19 +39,23 @@ from gen.apache.aurora.api.ttypes import (
TaskQuery
)
+InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
+
+
+OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
+
+
+InstanceConfigs = namedtuple(
+ 'InstanceConfigs',
+ ['remote_config_map', 'local_config_map', 'instances_to_process']
+)
+
class Updater(object):
"""Update the instances of a job in batches."""
class Error(Exception): pass
- InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
- OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
- InstanceConfigs = namedtuple(
- 'InstanceConfigs',
- ['remote_config_map', 'local_config_map', 'instances_to_process']
- )
-
def __init__(self,
config,
health_check_interval_seconds,
@@ -116,19 +120,19 @@ class Updater(object):
self._update_config.max_total_failures
)
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.remote_config_map,
to_config=instance_configs.local_config_map
)
remaining_instances = [
- self.InstanceState(instance_id, is_updated=False)
+ InstanceState(instance_id, is_updated=False)
for instance_id in instance_configs.instances_to_process
]
log.info('Starting job update.')
while remaining_instances and not failure_threshold.is_failed_update():
- batch_instances = remaining_instances[0 : self._update_config.batch_size]
+ batch_instances = remaining_instances[0:self._update_config.batch_size]
remaining_instances = list(set(remaining_instances) - set(batch_instances))
instances_to_restart = [s.instance_id for s in batch_instances if s.is_updated]
instances_to_update = [s.instance_id for s in batch_instances if not s.is_updated]
@@ -152,7 +156,7 @@ class Updater(object):
(unretryable_instances, self._update_config.max_per_instance_failures))
retryable_instances = list(set(failed_instances) - set(unretryable_instances))
remaining_instances += [
- self.InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
+ InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
]
remaining_instances.sort(key=lambda tup: tup.instance_id)
@@ -177,14 +181,14 @@ class Updater(object):
return
log.info('Reverting update for %s' % instances_to_rollback)
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.local_config_map,
to_config=instance_configs.remote_config_map
)
instances_to_rollback.sort(reverse=True)
failed_instances = []
while instances_to_rollback:
- batch_instances = instances_to_rollback[0 : self._update_config.batch_size]
+ batch_instances = instances_to_rollback[0:self._update_config.batch_size]
instances_to_rollback = list(set(instances_to_rollback) - set(batch_instances))
instances_to_rollback.sort(reverse=True)
instances_to_watch = self._update_instances(batch_instances, instance_operation)
@@ -236,7 +240,8 @@ class Updater(object):
if from_config and to_config:
diff_output = self._diff_configs(from_config, to_config)
if diff_output:
- log.debug('Task configuration changed for instance [%s]:\n%s' % (instance_id, diff_output))
+ log.debug('Task configuration changed for instance [%s]:\n%s' % (
+ instance_id, diff_output))
to_kill.append(instance_id)
to_add.append(instance_id)
elif from_config and not to_config:
@@ -318,7 +323,7 @@ class Updater(object):
Returns Response.OK if quota check was successful.
"""
- instance_operation = self.OperationConfigs(
+ instance_operation = OperationConfigs(
from_config=instance_configs.remote_config_map,
to_config=instance_configs.local_config_map
)
@@ -379,7 +384,7 @@ class Updater(object):
# Populate local config map
local_config_map = dict.fromkeys(job_config_instances, local_task_config)
- return self.InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
+ return InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
def _get_existing_tasks(self):
"""Loads all existing tasks from the scheduler.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/api/updater_util.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater_util.py b/src/main/python/apache/aurora/client/api/updater_util.py
index 04105de..6b689c1 100644
--- a/src/main/python/apache/aurora/client/api/updater_util.py
+++ b/src/main/python/apache/aurora/client/api/updater_util.py
@@ -30,6 +30,7 @@ class UpdaterConfig(object):
An instance is also expected to remain healthy for at least watch_secs. If these conditions are
not satisfied, the instance is deemed unhealthy.
"""
+
def __init__(self,
batch_size,
restart_threshold,
@@ -58,7 +59,6 @@ class FailureThreshold(object):
self._max_total_failures = max_total_failures
self._failures_by_instance = collections.defaultdict(int)
-
def update_failure_counts(self, failed_instances):
"""Update the failure counts metrics based upon a batch of failed instances.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/base.py b/src/main/python/apache/aurora/client/base.py
index ef0855d..6716ba6 100644
--- a/src/main/python/apache/aurora/client/base.py
+++ b/src/main/python/apache/aurora/client/base.py
@@ -34,14 +34,17 @@ After checking on the above, you may release the update lock on the job by
invoking cancel_update.
"""
+
def die(msg):
log.fatal(msg)
sys.exit(1)
+
def log_response(resp):
log.info('Response from scheduler: %s (message: %s)'
% (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.messageDEPRECATED))
+
def check_and_log_response(resp):
log_response(resp)
if resp.responseCode != ResponseCode.OK:
@@ -49,6 +52,12 @@ def check_and_log_response(resp):
log.info(LOCKED_WARNING)
sys.exit(1)
+
+def check_and_log_locked_response(resp):
+ if resp.responseCode == ResponseCode.LOCK_ERROR:
+ log.info(LOCKED_WARNING)
+
+
def deprecation_warning(text):
log.warning('')
log.warning('*' * 80)
@@ -59,9 +68,9 @@ def deprecation_warning(text):
log.warning('')
-class requires(object):
- @staticmethod
- def wrap_function(fn, fnargs, comparator):
+class requires(object): # noqa
+ @classmethod
+ def wrap_function(cls, fn, fnargs, comparator):
@functools.wraps(fn)
def wrapped_function(args):
if not comparator(args, fnargs):
@@ -72,20 +81,20 @@ class requires(object):
return fn(*args)
return wrapped_function
- @staticmethod
- def exactly(*args):
+ @classmethod
+ def exactly(cls, *args):
def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
+ return cls.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
return wrap
- @staticmethod
- def at_least(*args):
+ @classmethod
+ def at_least(cls, *args):
def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
+ return cls.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
return wrap
- @staticmethod
- def nothing(fn):
+ @classmethod
+ def nothing(cls, fn):
@functools.wraps(fn)
def real_fn(line):
return fn(*line)
@@ -109,17 +118,21 @@ HOSTS_OPTION = optparse.Option(
def group_by_host(hostname):
return hostname
+
DEFAULT_GROUPING = 'by_host'
GROUPING_FUNCTIONS = {
'by_host': group_by_host,
}
+
def add_grouping(name, function):
GROUPING_FUNCTIONS[name] = function
+
def remove_grouping(name):
GROUPING_FUNCTIONS.pop(name)
+
def get_grouping_or_die(grouping_function):
try:
return GROUPING_FUNCTIONS[grouping_function]
@@ -127,6 +140,7 @@ def get_grouping_or_die(grouping_function):
die('Unknown grouping function %s. Must be one of: %s'
% (grouping_function, GROUPING_FUNCTIONS.keys()))
+
def group_hosts(hostnames, grouping_function=DEFAULT_GROUPING):
grouping_function = get_grouping_or_die(grouping_function)
groups = defaultdict(set)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/bin/aurora_admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_admin.py b/src/main/python/apache/aurora/client/bin/aurora_admin.py
index d1247e6..136cf60 100644
--- a/src/main/python/apache/aurora/client/bin/aurora_admin.py
+++ b/src/main/python/apache/aurora/client/bin/aurora_admin.py
@@ -16,10 +16,11 @@ from twitter.common import app
from twitter.common.log.options import LogOptions
from apache.aurora.client.base import generate_terse_usage
-from apache.aurora.client.commands import admin, help, maintenance
+from apache.aurora.client.commands import help as help_commands
+from apache.aurora.client.commands import admin, maintenance
from apache.aurora.client.options import add_verbosity_options
-app.register_commands_from(admin, help, maintenance)
+app.register_commands_from(admin, help_commands, maintenance)
add_verbosity_options()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/bin/aurora_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_client.py b/src/main/python/apache/aurora/client/bin/aurora_client.py
index 1317fae..4999265 100644
--- a/src/main/python/apache/aurora/client/bin/aurora_client.py
+++ b/src/main/python/apache/aurora/client/bin/aurora_client.py
@@ -16,7 +16,8 @@ from twitter.common import app
from twitter.common.log.options import LogOptions
from apache.aurora.client.base import generate_terse_usage
-from apache.aurora.client.commands import core, help, run, ssh
+from apache.aurora.client.commands import help as help_commands
+from apache.aurora.client.commands import core, run, ssh
from apache.aurora.client.options import add_verbosity_options
# These are are side-effecting imports in that they register commands via
@@ -25,7 +26,7 @@ from apache.aurora.client.options import add_verbosity_options
# composition.
app.register_commands_from(core, run, ssh)
-app.register_commands_from(help)
+app.register_commands_from(help_commands)
add_verbosity_options()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/binding_helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/binding_helper.py b/src/main/python/apache/aurora/client/binding_helper.py
index d17d57e..6d6a067 100644
--- a/src/main/python/apache/aurora/client/binding_helper.py
+++ b/src/main/python/apache/aurora/client/binding_helper.py
@@ -12,9 +12,6 @@
# limitations under the License.
#
-import inspect
-import os
-import sys
from abc import abstractmethod, abstractproperty
from twitter.common.lang import Interface
@@ -63,10 +60,10 @@ class BindingHelper(Interface):
(because a human reader wants to know the version of the package, not the meaningless
HDFS URL.
"""
- @staticmethod
- def register(helper):
- _BINDING_HELPERS.append(helper)
+ @classmethod
+ def register(cls, helper):
+ _BINDING_HELPERS.append(helper)
def apply(self, config, env=None, binding_dict=None):
for match in self.matcher.match(config.raw()):
@@ -88,6 +85,7 @@ class BindingHelper(Interface):
class CachingBindingHelper(BindingHelper):
"""A binding helper implementation that caches binding results"""
+
def __init__(self):
self.cache = {}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/__init__.py b/src/main/python/apache/aurora/client/cli/__init__.py
index fd6f96e..fc63dbf 100644
--- a/src/main/python/apache/aurora/client/cli/__init__.py
+++ b/src/main/python/apache/aurora/client/cli/__init__.py
@@ -41,6 +41,7 @@ from twitter.common.python.pex import PexInfo
from .command_hooks import GlobalCommandHookRegistry
from .logsetup import TRANSCRIPT
+from .options import CommandOption
# Constants for standard return codes.
EXIT_OK = 0
@@ -78,6 +79,7 @@ def print_aurora_log(sev, msg, *args, **kwargs):
kwargs["extra"] = extra
logger.log(sev, msg, *args, **kwargs)
+
def get_client_version():
try:
pexpath = sys.argv[0]
@@ -90,12 +92,11 @@ def get_client_version():
class Context(object):
class Error(Exception): pass
-
class ArgumentException(Error): pass
class CommandError(Error):
def __init__(self, code, msg):
- super(Context.CommandError, self).__init__(msg)
+ super(Context.CommandError, self).__init__(msg) # noqa:T800
self.msg = msg
self.code = code
self.options = None
@@ -151,7 +152,7 @@ class ConfigurationPlugin(object):
class Error(Exception):
def __init__(self, msg, code=0):
- super(ConfigurationPlugin.Error, self).__init__(msg)
+ super(ConfigurationPlugin.Error, self).__init__(msg) # noqa:T800
self.code = code
self.msg = msg
@@ -295,7 +296,6 @@ class CommandLine(object):
result.append("\nRun 'help noun' or 'help noun verb' for help about a specific command")
return "\n".join(result)
-
def register_nouns(self):
"""This method should overridden by applications to register the collection of nouns
that they can manipulate.
@@ -309,7 +309,6 @@ class CommandLine(object):
etc. You wouldn't want to clutter the help output with AWS commands for users
that weren't using AWS. So you could have the command-line check the cluster.json
file, and only register the AWS noun if there was an AWS cluster.
-
"""
@property
@@ -320,7 +319,9 @@ class CommandLine(object):
def _setup(self, args):
GlobalCommandHookRegistry.setup(GLOBAL_HOOK_SKIP_RULES_URL)
- nouns = self.registered_nouns
+ # Accessing registered_nouns has the side-effect of registering them, but since
+ # nouns is unused, we must disable checkstyle.
+ nouns = self.registered_nouns # noqa
for plugin in self.plugins:
args = plugin.before_dispatch(args)
return args
@@ -346,8 +347,8 @@ class CommandLine(object):
for plugin in self.plugins:
plugin.before_execution(context)
except ConfigurationPlugin.Error as e:
- print("Error in configuration plugin before execution: %s" % c.msg, file=sys.stderr)
- return c.code
+ print("Error in configuration plugin before execution: %s" % e.msg, file=sys.stderr)
+ return e.code
plugin_result = GlobalCommandHookRegistry.run_pre_hooks(context, context.options.noun,
context.options.verb)
if plugin_result != EXIT_OK:
@@ -383,8 +384,8 @@ class CommandLine(object):
result = noun.execute(context)
if result == EXIT_OK:
print_aurora_log(logging.INFO, "Command terminated successfully")
- GlobalCommandHookRegistry.run_post_hooks(context, context.options.noun, context.options.verb,
- result)
+ GlobalCommandHookRegistry.run_post_hooks(context, context.options.noun,
+ context.options.verb, result)
else:
print_aurora_log(logging.INFO, "Command terminated with error code %s", result)
self._run_post_plugins(context, result)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/bridge.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/bridge.py b/src/main/python/apache/aurora/client/cli/bridge.py
index d5eec8a..6528d90 100644
--- a/src/main/python/apache/aurora/client/cli/bridge.py
+++ b/src/main/python/apache/aurora/client/cli/bridge.py
@@ -74,7 +74,6 @@ class Bridge(object):
if self.default is not None:
return self.default.execute(args)
-
def execute(self, args):
"""Dispatch a command line to the appropriate CommandProcessor"""
if len(args) == 1:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/client.py b/src/main/python/apache/aurora/client/cli/client.py
index fa21576..e4409e5 100644
--- a/src/main/python/apache/aurora/client/cli/client.py
+++ b/src/main/python/apache/aurora/client/cli/client.py
@@ -11,20 +11,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import logging
import sys
-from apache.aurora.client.cli import CommandLine
from apache.aurora.client.cli.bridge import Bridge, CommandProcessor
from apache.aurora.client.cli.logsetup import setup_default_log_handlers
from apache.aurora.client.cli.standalone_client import AuroraCommandLine
-import logging
# TODO(mchucarroll): the entire bridged executable mechanism here is
# intended to be deprecated once clientv2 has proven to be stable
# and adopted by most users. Once we reach that point, this should
# be replaced by the standalone executable defined in standalone_client.py.
-
class AuroraClientV2CommandProcessor(CommandProcessor):
def __init__(self):
self.commandline = AuroraCommandLine()
@@ -50,17 +48,19 @@ class AuroraClientV1CommandProcessor(CommandProcessor):
def get_commands(self):
return ["cancel_update", "create", "diff", "get_quota", "inspect", "kill", "list_jobs",
- "open", "restart", "run", "ssh", "start_cron", "status", "update", "version" ]
+ "open", "restart", "run", "ssh", "start_cron", "status", "update", "version"]
def execute(self, args):
from apache.aurora.client.bin.aurora_client import proxy_main as clientone_proxy_main
return clientone_proxy_main()
+
def proxy_main():
v2 = AuroraClientV2CommandProcessor()
v1 = AuroraClientV1CommandProcessor()
bridge = Bridge([v2, v1], default=v1)
bridge.execute(sys.argv)
+
if __name__ == '__main__':
proxy_main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/command_hooks.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/command_hooks.py b/src/main/python/apache/aurora/client/cli/command_hooks.py
index c349824..aa850bf 100644
--- a/src/main/python/apache/aurora/client/cli/command_hooks.py
+++ b/src/main/python/apache/aurora/client/cli/command_hooks.py
@@ -137,7 +137,6 @@ class GlobalCommandHookRegistry(object):
except ValueError as e:
logging.error("Client could not decode hook skip rules: %s" % e)
-
@classmethod
def load_hooks_file(cls, path):
"""Load a file containing hooks. If there are any errors compiling or executing the file,
@@ -177,7 +176,7 @@ class GlobalCommandHookRegistry(object):
# it's an HG root if it contains a directory named ".hg"
return any(os.path.isdir(os.path.join(dir, rootname)) for rootname in [".git", ".hg"])
- filepath = os.path.join(dir, cls.HOOKS_FILE_NAME)
+ filepath = os.path.join(dir, cls.HOOKS_FILE_NAME)
if os.path.exists(filepath):
return filepath
elif is_repos_root(dir):
@@ -282,11 +281,12 @@ class GlobalCommandHookRegistry(object):
context.print_err("Error executing command hook %s: %s; aborting" % hook.name, c.msg)
return c.code
+
class CommandHook(object):
"""A hook which contains code that should be run before certain commands."""
class Error(Exception):
def __init__(self, code, msg):
- super(CommandHook.Error, self).__init__(msg)
+ super(CommandHook.Error, self).__init__(msg) # noqa:T800
self.code = code
self.msg = msg
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/config.py b/src/main/python/apache/aurora/client/cli/config.py
index 034c68e..637e7ac 100644
--- a/src/main/python/apache/aurora/client/cli/config.py
+++ b/src/main/python/apache/aurora/client/cli/config.py
@@ -19,17 +19,9 @@ operate in on configuration files.
from __future__ import print_function
-from apache.aurora.client.cli import (
- EXIT_COMMAND_FAILURE,
- EXIT_OK,
- Noun,
- Verb,
-)
+from apache.aurora.client.cli import EXIT_COMMAND_FAILURE, EXIT_OK, Noun, Verb
from apache.aurora.client.cli.context import AuroraCommandContext
-from apache.aurora.client.cli.options import (
- BIND_OPTION,
- CONFIG_ARGUMENT,
-)
+from apache.aurora.client.cli.options import BIND_OPTION, CONFIG_ARGUMENT
from apache.aurora.client.config import AuroraConfig
from apache.aurora.config.loader import AuroraConfigLoader
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/context.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/context.py b/src/main/python/apache/aurora/client/cli/context.py
index d1f1f3f..669add7 100644
--- a/src/main/python/apache/aurora/client/cli/context.py
+++ b/src/main/python/apache/aurora/client/cli/context.py
@@ -15,16 +15,14 @@
from __future__ import print_function
import logging
-import sys
from collections import namedtuple
from fnmatch import fnmatch
-from twitter.common import log
-
from apache.aurora.client.base import synthesize_url
from apache.aurora.client.cli import (
Context,
EXIT_API_ERROR,
+ EXIT_COMMAND_FAILURE,
EXIT_INVALID_CONFIGURATION,
EXIT_INVALID_PARAMETER
)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/cron.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/cron.py b/src/main/python/apache/aurora/client/cli/cron.py
index c30a0a6..cefeca4 100644
--- a/src/main/python/apache/aurora/client/cli/cron.py
+++ b/src/main/python/apache/aurora/client/cli/cron.py
@@ -21,7 +21,7 @@ from apache.aurora.client.cli import (
EXIT_INVALID_PARAMETER,
EXIT_OK,
Noun,
- Verb,
+ Verb
)
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import (
@@ -30,7 +30,7 @@ from apache.aurora.client.cli.options import (
CONFIG_ARGUMENT,
CONFIG_OPTION,
JOBSPEC_ARGUMENT,
- JSON_READ_OPTION,
+ JSON_READ_OPTION
)
from gen.apache.aurora.api.ttypes import ResponseCode
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/jobs.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/jobs.py b/src/main/python/apache/aurora/client/cli/jobs.py
index 8020c35..cd5070e 100644
--- a/src/main/python/apache/aurora/client/cli/jobs.py
+++ b/src/main/python/apache/aurora/client/cli/jobs.py
@@ -19,12 +19,10 @@ import logging
import os
import pprint
import subprocess
-import sys
import time
from datetime import datetime
from tempfile import NamedTemporaryFile
-from pystachio.config import Config
from thrift.protocol import TJSONProtocol
from thrift.TSerialization import serialize
@@ -403,8 +401,10 @@ class ListJobsCommand(Verb):
context.print_out('%s/%s/%s/%s' % (j.cluster, j.role, j.env, j.name))
return EXIT_OK
+
class OpenCommand(Verb):
# TODO(mchucarroll): this is awkward in the noun/verb framework: where does it actually belong?
+
@property
def name(self):
return 'open'
@@ -424,7 +424,6 @@ class OpenCommand(Verb):
while len(key_parts) < 4:
key_parts.append(None)
(cluster, role, env, name) = key_parts
- api = context.get_api(cluster)
context.open_scheduler_page(cluster, role, env, name)
return EXIT_OK
@@ -438,11 +437,11 @@ class RestartCommand(Verb):
return [BATCH_OPTION, BIND_OPTION, BROWSER_OPTION, FORCE_OPTION, HEALTHCHECK_OPTION,
JSON_READ_OPTION, WATCH_OPTION,
CommandOption('--max-per-instance-failures', type=int, default=0,
- help='Maximum number of restarts per instance during restart. Increments total failure '
- 'count when this limit is exceeded.'),
+ help='Maximum number of restarts per instance during restart. Increments total '
+ 'failure count when this limit is exceeded.'),
CommandOption('--restart-threshold', type=int, default=60,
help='Maximum number of seconds before a shard must move into the RUNNING state '
- 'before considered a failure.'),
+ 'before considered a failure.'),
MAX_TOTAL_FAILURES_OPTION,
STRICT_OPTION,
CommandOption('--rollback-on-failure', default=True, action='store_false',
@@ -467,7 +466,7 @@ Restarts are fully controlled client-side, so aborting halts the restart."""
job = context.options.instance_spec.jobkey
instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else
context.options.instance_spec.instance)
- if instances != None and context.options.strict:
+ if instances is not None and context.options.strict:
context.verify_shards_option_validity(job, instances)
api = context.get_api(job.cluster)
config = (context.get_job_config(job, context.options.config_file)
@@ -628,7 +627,7 @@ to preview what changes will take effect.
job = context.options.instance_spec.jobkey
instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else
context.options.instance_spec.instance)
- if instances != None and context.options.strict:
+ if instances is not None and context.options.strict:
context.verify_shards_option_validity(job, instances)
config = context.get_job_config(job, context.options.config_file)
api = context.get_api(config.cluster())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/logsetup.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/logsetup.py b/src/main/python/apache/aurora/client/cli/logsetup.py
index 15fb306..55d99c4 100644
--- a/src/main/python/apache/aurora/client/cli/logsetup.py
+++ b/src/main/python/apache/aurora/client/cli/logsetup.py
@@ -21,6 +21,7 @@ import sys
TRANSCRIPT = logging.DEBUG + 1
logging.addLevelName(TRANSCRIPT, "TRANSCRIPT")
+
class PlainFormatter(logging.Formatter):
"""
Format a log in a simple style:
@@ -31,8 +32,8 @@ class PlainFormatter(logging.Formatter):
LEVEL_MAP = {
logging.FATAL: "FATAL",
logging.ERROR: "ERROR",
- logging.WARN: "WARN",
- logging.INFO: "info",
+ logging.WARN: "WARN",
+ logging.INFO: "info",
TRANSCRIPT: "transcript",
logging.DEBUG: "debug"
}
@@ -46,8 +47,8 @@ class PlainFormatter(logging.Formatter):
except TypeError:
record_message = record.msg
try:
- level = PlainFormatter.LEVEL_MAP[record.levelno]
- except:
+ level = self.LEVEL_MAP[record.levelno]
+ except Exception:
level = "?????"
record_message = "log(%s): %s" % (level, record_message)
record.getMessage = lambda: record_message
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/options.py b/src/main/python/apache/aurora/client/cli/options.py
index 0d49bac..f10ef4c 100644
--- a/src/main/python/apache/aurora/client/cli/options.py
+++ b/src/main/python/apache/aurora/client/cli/options.py
@@ -39,11 +39,9 @@ class CommandOption(object):
self.type = kwargs.get('type')
self.help = kwargs.get('help', '')
-
def is_mandatory(self):
return self.kwargs.get('required', not self.name.startswith('--'))
-
def get_displayname(self):
"""Get a display name for a the expected format of a parameter value"""
if 'metavar' in self.kwargs:
@@ -58,7 +56,6 @@ class CommandOption(object):
displayname = "value"
return displayname
-
def render_usage(self):
"""Create a usage string for this option"""
if not self.name.startswith('--'):
@@ -73,7 +70,6 @@ class CommandOption(object):
else:
return "[%s=%s]" % (self.name, self.get_displayname())
-
def render_help(self):
"""Render a full help message for this option"""
result = ""
@@ -153,7 +149,7 @@ def parse_percentiles(percentiles):
return sorted(map(parse_percentile, percentiles.split(','))) if percentiles else None
-TaskInstanceKey = namedtuple('TaskInstanceKey', [ 'jobkey', 'instance' ])
+TaskInstanceKey = namedtuple('TaskInstanceKey', ['jobkey', 'instance'])
def parse_task_instance_key(key):
@@ -266,7 +262,6 @@ STRICT_OPTION = CommandOption('--strict', default=False, action='store_true',
"that are larger than the actual set of instances in the job"))
-
TASK_INSTANCE_ARGUMENT = CommandOption('task_instance', type=parse_task_instance_key,
help='A task instance specifier, in the form CLUSTER/ROLE/ENV/NAME/INSTANCE')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/quota.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/quota.py b/src/main/python/apache/aurora/client/cli/quota.py
index af07d83..5811825 100644
--- a/src/main/python/apache/aurora/client/cli/quota.py
+++ b/src/main/python/apache/aurora/client/cli/quota.py
@@ -17,7 +17,13 @@ from __future__ import print_function
from thrift.protocol import TJSONProtocol
from thrift.TSerialization import serialize
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, EXIT_OK, Noun, Verb
+from apache.aurora.client.cli import (
+ EXIT_COMMAND_FAILURE,
+ EXIT_INVALID_PARAMETER,
+ EXIT_OK,
+ Noun,
+ Verb
+)
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import JSON_WRITE_OPTION, ROLE_ARGUMENT
@@ -75,7 +81,8 @@ class GetQuotaCmd(Verb):
elif resp.responseCode == ResponseCode.INVALID_REQUEST:
raise context.CommandError(EXIT_COMMAND_FAILURE, 'Error retrieving quota for role %s' % role)
elif resp.responseCode == ResponseCode.AUTH_FAILED:
- raise context.CommandError(EXIT_COMMAND_FAILURE, 'Invalid authorization to retrieve quota for role %s' % role)
+ raise context.CommandError(EXIT_COMMAND_FAILURE,
+ 'Invalid authorization to retrieve quota for role %s' % role)
context.print_out(self.render_quota(context.options.write_json, resp))
return EXIT_OK
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/standalone_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/standalone_client.py b/src/main/python/apache/aurora/client/cli/standalone_client.py
index 1e63ead..5d1a4a5 100644
--- a/src/main/python/apache/aurora/client/cli/standalone_client.py
+++ b/src/main/python/apache/aurora/client/cli/standalone_client.py
@@ -11,11 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import logging
import sys
from apache.aurora.client.cli import CommandLine
from apache.aurora.client.cli.logsetup import setup_default_log_handlers
-import logging
class AuroraCommandLine(CommandLine):
@@ -50,4 +50,3 @@ def proxy_main():
if __name__ == '__main__':
proxy_main()
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/cli/task.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/task.py b/src/main/python/apache/aurora/client/cli/task.py
index fe11f38..7fb3f55 100644
--- a/src/main/python/apache/aurora/client/cli/task.py
+++ b/src/main/python/apache/aurora/client/cli/task.py
@@ -17,54 +17,24 @@
from __future__ import print_function
-import json
-import os
-import pprint
import subprocess
-import sys
-import time
-from datetime import datetime
-
-from pystachio.config import Config
-from thrift.protocol import TJSONProtocol
-from thrift.TSerialization import serialize
from apache.aurora.client.api.command_runner import (
DistributedCommandRunner,
InstanceDistributedCommandRunner
)
-from apache.aurora.client.api.updater_util import UpdaterConfig
-from apache.aurora.client.cli import (
- EXIT_COMMAND_FAILURE,
- EXIT_INVALID_CONFIGURATION,
- EXIT_INVALID_PARAMETER,
- EXIT_OK,
- Noun,
- Verb
-)
+from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, Noun, Verb
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import (
- BATCH_OPTION,
- BIND_OPTION,
- BROWSER_OPTION,
CommandOption,
- CONFIG_ARGUMENT,
EXECUTOR_SANDBOX_OPTION,
- FORCE_OPTION,
- HEALTHCHECK_OPTION,
INSTANCES_SPEC_ARGUMENT,
- JOBSPEC_ARGUMENT,
- JSON_READ_OPTION,
- JSON_WRITE_OPTION,
SSH_USER_OPTION,
- TASK_INSTANCE_ARGUMENT,
- WATCH_OPTION
+ TASK_INSTANCE_ARGUMENT
)
-from apache.aurora.common.aurora_job_key import AuroraJobKey
from apache.aurora.common.clusters import CLUSTERS
-from gen.apache.aurora.api.constants import ACTIVE_STATES, AURORA_EXECUTOR_NAME
-from gen.apache.aurora.api.ttypes import ExecutorConfig, ResponseCode, ScheduleStatus
+from gen.apache.aurora.api.ttypes import ResponseCode
class RunCommand(Verb):
@@ -98,7 +68,8 @@ class RunCommand(Verb):
# TODO(mchucarroll): add options to specify which instances to run on (AURORA-198)
(cluster_name, role, env, name), instances = context.options.instance_spec
cluster = CLUSTERS[cluster_name]
- dcr = InstanceDistributedCommandRunner(cluster, role, env, name, context.options.ssh_user, instances)
+ dcr = InstanceDistributedCommandRunner(
+ cluster, role, env, name, context.options.ssh_user, instances)
dcr.run(context.options.cmd, parallelism=context.options.num_threads,
executor_sandbox=context.options.executor_sandbox)
@@ -150,9 +121,11 @@ class SshCommand(Verb):
port, name = tunnel.split(':')
port = int(port)
except ValueError:
- die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
+ raise context.CommandError(EXIT_INVALID_PARAMETER,
+ 'Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in first_task.assignedTask.assignedPorts:
- die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
+ raise context.CommandError(EXIT_INVALID_PARAMETER,
+ 'Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/admin.py b/src/main/python/apache/aurora/client/commands/admin.py
index 919eea9..a3b5584 100644
--- a/src/main/python/apache/aurora/client/commands/admin.py
+++ b/src/main/python/apache/aurora/client/commands/admin.py
@@ -16,9 +16,6 @@ from __future__ import print_function
import json
import optparse
-import os
-import pipes
-import subprocess
import sys
from twitter.common import app, log
@@ -26,7 +23,7 @@ from twitter.common.quantity import Amount, Data, Time
from twitter.common.quantity.parse_simple import parse_data, parse_time
from apache.aurora.client.api import AuroraClientAPI
-from apache.aurora.client.api.sla import DomainUpTimeSlaVector
+from apache.aurora.client.api.sla import JobUpTimeLimit
from apache.aurora.client.base import (
check_and_log_response,
die,
@@ -395,7 +392,7 @@ def sla_list_safe_domain(cluster, percentage, duration):
if len(tokens) != 3:
die('Invalid line in %s:%s' % (filename, line))
job_key = AuroraJobKey.from_path(tokens[0])
- result[job_key] = DomainUpTimeSlaVector.JobUpTimeLimit(
+ result[job_key] = JobUpTimeLimit(
job=job_key,
percentage=parse_sla_percentage(tokens[1]),
duration_secs=parse_time(tokens[2]).as_(Time.SECONDS)
@@ -488,6 +485,7 @@ def sla_probe_hosts(cluster, percentage, duration):
print_results(results)
+
@app.command
@app.command_option('--sh', default=False, action="store_true",
help="Emit a shell script instead of JSON.")
@@ -506,5 +504,5 @@ def get_cluster_config(cluster):
if not options.sh:
json.dump(cluster, sys.stdout)
else:
- for line in shellify(cluster, options.export, prefix = "AURORA_CLUSTER_"):
+ for line in shellify(cluster, options.export, prefix="AURORA_CLUSTER_"):
print(line)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/core.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/core.py b/src/main/python/apache/aurora/client/commands/core.py
index 29e70a9..0939f05 100644
--- a/src/main/python/apache/aurora/client/commands/core.py
+++ b/src/main/python/apache/aurora/client/commands/core.py
@@ -17,7 +17,6 @@
from __future__ import print_function
-import collections
import json
import os
import pprint
@@ -385,7 +384,7 @@ def list_jobs(cluster_and_role):
# Take the cluster_and_role parameter, and split it into its two components.
if cluster_and_role.count('/') != 1:
die('list_jobs parameter must be in cluster/role format')
- (cluster,role) = cluster_and_role.split('/')
+ cluster, role = cluster_and_role.split('/')
api = make_client(cluster)
resp = api.get_jobs(role)
check_and_log_response(resp)
@@ -423,7 +422,6 @@ def kill(args, options):
wait_kill_tasks(api.scheduler_proxy, job_key, options.shards)
-
def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
""" Common behavior shared by kill and killAll for killing instances in
a sequence of batches.
@@ -437,7 +435,6 @@ def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
result.append(batch)
return result
-
resp = api.check_status(job_key)
if resp.responseCode is not ResponseCode.OK:
log.error("Job %s could not be found" % job_key)
@@ -465,7 +462,6 @@ def kill_in_batches(api, job_key, instances_arg, batch_size, max_failures):
return 1
-
@app.command
@app.command_option(CLUSTER_INVOKE_OPTION)
@app.command_option(OPEN_BROWSER_OPTION)
@@ -589,7 +585,6 @@ def update(job_spec, config_file):
def warn_if_dangerous_change(api, job_spec, config):
# Get the current job status, so that we can check if there's anything
# dangerous about this update.
- job_key = AuroraJobKey(config.cluster(), config.role(), config.environment(), config.name())
resp = api.query(api.build_query(config.role(), config.name(),
statuses=ACTIVE_STATES, env=config.environment()))
if resp.responseCode != ResponseCode.OK:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/help.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/help.py b/src/main/python/apache/aurora/client/commands/help.py
index d59b299..f79e971 100644
--- a/src/main/python/apache/aurora/client/commands/help.py
+++ b/src/main/python/apache/aurora/client/commands/help.py
@@ -47,8 +47,8 @@ def generate_full_usage():
return 'Available commands:\n\n' + '\n'.join(usage)
-@app.command
-def help(args):
+@app.command(name='help')
+def help_command(args):
"""usage: help [subcommand]
Prints help for using the aurora client, or one of its specific subcommands.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/commands/maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/maintenance.py b/src/main/python/apache/aurora/client/commands/maintenance.py
index f6ebe3b..e4d60d4 100644
--- a/src/main/python/apache/aurora/client/commands/maintenance.py
+++ b/src/main/python/apache/aurora/client/commands/maintenance.py
@@ -12,7 +12,6 @@
# limitations under the License.
#
-import optparse
import os
import subprocess
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py
index 3b01792..e440f58 100644
--- a/src/main/python/apache/aurora/client/config.py
+++ b/src/main/python/apache/aurora/client/config.py
@@ -19,17 +19,14 @@ from __future__ import print_function
import functools
import math
-import posixpath
import re
import sys
-from pystachio import Empty, Ref
-from twitter.common import app, log
+from pystachio import Empty
from apache.aurora.client import binding_helper
from apache.aurora.client.base import deprecation_warning, die
from apache.aurora.config import AuroraConfig
-from apache.thermos.config.schema_helpers import Tasks
from gen.apache.aurora.api.constants import DEFAULT_ENVIRONMENT
@@ -125,6 +122,7 @@ Based on your job size (%s) you should use max_total_failures >= %s.
See http://go/auroraconfig for details.
'''
+
WATCH_SECS_INSUFFICIENT_ERROR_FORMAT = '''
You have specified an insufficiently short watch period (%d seconds) in your update configuration.
Your update will always succeed. In order for the updater to detect health check failures,
@@ -132,6 +130,7 @@ UpdateConfig.watch_secs must be greater than %d seconds to account for an initia
health check interval (%d seconds) plus %d consecutive failures at a check interval of %d seconds.
'''
+
def _validate_update_config(config):
job_size = config.instances()
update_config = config.update_config()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/factory.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/factory.py b/src/main/python/apache/aurora/client/factory.py
index 22805f0..4b31e85 100644
--- a/src/main/python/apache/aurora/client/factory.py
+++ b/src/main/python/apache/aurora/client/factory.py
@@ -26,7 +26,6 @@ from .base import die
# TODO(wickman) Kill make_client and make_client_factory as part of MESOS-3801.
# These are currently necessary indirections for the LiveJobDisambiguator among
# other things but can go away once those are scrubbed.
-
def make_client_factory():
verbose = getattr(app.get_options(), 'verbosity', 'normal') == 'verbose'
class TwitterAuroraClientAPI(HookedAuroraClientAPI):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/client/hooks/hooked_api.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/hooks/hooked_api.py b/src/main/python/apache/aurora/client/hooks/hooked_api.py
index a205777..91efe52 100644
--- a/src/main/python/apache/aurora/client/hooks/hooked_api.py
+++ b/src/main/python/apache/aurora/client/hooks/hooked_api.py
@@ -19,7 +19,6 @@ from twitter.common import log
from apache.aurora.client.api import AuroraClientAPI
from apache.aurora.client.config import GlobalHookRegistry
-from apache.aurora.common.aurora_job_key import AuroraJobKey
from gen.apache.aurora.api.ttypes import ResponseCode
@@ -181,4 +180,5 @@ class HookedAuroraClientAPI(NonHookedAuroraClientAPI):
def update_job(self, config, health_check_interval_seconds=3, instances=None):
return self._hooked_call(config, None,
_partial(super(HookedAuroraClientAPI, self).update_job,
- config, health_check_interval_seconds=health_check_interval_seconds, instances=instances))
+ config, health_check_interval_seconds=health_check_interval_seconds,
+ instances=instances))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/auth/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/__init__.py b/src/main/python/apache/aurora/common/auth/__init__.py
index 95e56ad..3119fd6 100644
--- a/src/main/python/apache/aurora/common/auth/__init__.py
+++ b/src/main/python/apache/aurora/common/auth/__init__.py
@@ -12,5 +12,13 @@
# limitations under the License.
#
-from .auth_module import AuthModule, InsecureAuthModule
-from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError
+from .auth_module import AuthModule, InsecureAuthModule # noqa
+from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError # noqa
+
+__all__ = (
+ 'AuthModule',
+ 'InsecureAuthModule',
+ 'make_session_key',
+ 'register_auth_module',
+ 'SessionKeyError',
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/auth/auth_module.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/auth_module.py b/src/main/python/apache/aurora/common/auth/auth_module.py
index aacb1b9..5f4116e 100644
--- a/src/main/python/apache/aurora/common/auth/auth_module.py
+++ b/src/main/python/apache/aurora/common/auth/auth_module.py
@@ -12,8 +12,6 @@
# limitations under the License.
#
-import getpass
-import time
from abc import abstractmethod, abstractproperty
from twitter.common.lang import Interface
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/cluster.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster.py b/src/main/python/apache/aurora/common/cluster.py
index f04718f..285b890 100644
--- a/src/main/python/apache/aurora/common/cluster.py
+++ b/src/main/python/apache/aurora/common/cluster.py
@@ -43,7 +43,7 @@ class Cluster(dict):
# Will use the default if none is provided on Cluster.
cluster.with_trait(ResolverTrait).scheduler_zk_path
"""
- Trait = Struct
+ Trait = Struct # noqa
def __init__(self, **kwargs):
self._traits = ()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/cluster_option.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster_option.py b/src/main/python/apache/aurora/common/cluster_option.py
index 9ddd868..77202c2 100644
--- a/src/main/python/apache/aurora/common/cluster_option.py
+++ b/src/main/python/apache/aurora/common/cluster_option.py
@@ -13,7 +13,7 @@
#
from copy import copy
-from optparse import NO_DEFAULT, Option, OptionValueError
+from optparse import Option, OptionValueError
def _check_mesos_cluster(option, opt, value):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/clusters.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/clusters.py b/src/main/python/apache/aurora/common/clusters.py
index 389b6f9..e55aa77 100644
--- a/src/main/python/apache/aurora/common/clusters.py
+++ b/src/main/python/apache/aurora/common/clusters.py
@@ -17,7 +17,6 @@ from __future__ import print_function
import itertools
import json
import os
-import sys
from collections import Mapping, namedtuple
from contextlib import contextmanager
@@ -40,7 +39,7 @@ __all__ = (
class NameTrait(Cluster.Trait):
- name = Required(String)
+ name = Required(String) # noqa
Parser = namedtuple('Parser', 'loader exception')
@@ -127,7 +126,6 @@ class Clusters(Mapping):
name, ', '.join(self._clusters.keys())))
-
DEFAULT_SEARCH_PATHS = (
os.environ.get('AURORA_CONFIG_ROOT') or '/etc/aurora',
os.path.expanduser('~/.aurora')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/common/http_signaler.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/http_signaler.py b/src/main/python/apache/aurora/common/http_signaler.py
index 4e5d7b4..e3e819d 100644
--- a/src/main/python/apache/aurora/common/http_signaler.py
+++ b/src/main/python/apache/aurora/common/http_signaler.py
@@ -14,7 +14,6 @@
import contextlib
import os
-import sys
from socket import timeout as SocketTimeout
from twitter.common import log
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/loader.py b/src/main/python/apache/aurora/config/loader.py
index 942f149..c8b045e 100644
--- a/src/main/python/apache/aurora/config/loader.py
+++ b/src/main/python/apache/aurora/config/loader.py
@@ -14,7 +14,6 @@
import json
import pkgutil
-import textwrap
from pystachio.config import Config as PystachioConfig
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
index 43ae5cf..3b90ccb 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -12,6 +12,9 @@
# limitations under the License.
#
+# Disable checkstyle for this entire file as it is a pystachio schema.
+# checkstyle: noqa
+
from apache.thermos.config.schema import *
from gen.apache.aurora.api.constants import DEFAULT_ENVIRONMENT
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
index 419625d..288fb40 100644
--- a/src/main/python/apache/aurora/config/thrift.py
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -51,8 +51,8 @@ def constraints_to_thrift(constraints):
result = set()
for attribute, constraint_value in constraints.items():
assert isinstance(attribute, Compatibility.string) and (
- isinstance(constraint_value, Compatibility.string)), (
- "Both attribute name and value in constraints must be string")
+ isinstance(constraint_value, Compatibility.string)), (
+ "Both attribute name and value in constraints must be string")
constraint = Constraint()
constraint.name = attribute
task_constraint = TaskConstraint()
@@ -230,7 +230,7 @@ def convert(job, metadata=frozenset(), ports=frozenset()):
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
- underlying_checked = underlying.bind(mesos = {'instance': 31337})
+ underlying_checked = underlying.bind(mesos={'instance': 31337})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/aurora_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/aurora_executor.py b/src/main/python/apache/aurora/executor/aurora_executor.py
index a71f1eb..8985aeb 100644
--- a/src/main/python/apache/aurora/executor/aurora_executor.py
+++ b/src/main/python/apache/aurora/executor/aurora_executor.py
@@ -233,7 +233,7 @@ class AuroraExecutor(ExecutorBase, Observable):
try:
assigned_task = assigned_task_from_mesos_task(task)
return assigned_task
- except Exception as e:
+ except Exception:
log.fatal('Could not deserialize AssignedTask')
log.fatal(traceback.format_exc())
return None
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
index 38a4c1d..f61caf0 100644
--- a/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
+++ b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
@@ -21,6 +21,6 @@ LogOptions.set_simple(True)
def proxy_main():
- main = runner_proxy_main
+ main = runner_proxy_main # noqa
app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/health_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/health_checker.py b/src/main/python/apache/aurora/executor/common/health_checker.py
index 05a89ee..671f497 100644
--- a/src/main/python/apache/aurora/executor/common/health_checker.py
+++ b/src/main/python/apache/aurora/executor/common/health_checker.py
@@ -31,6 +31,7 @@ class HealthCheckerThread(StatusChecker, ExceptionalThread):
respectively the health of the service and the reason for its failure (or None if the service is
still healthy).
"""
+
def __init__(self,
health_checker,
interval_secs=10,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/kill_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/kill_manager.py b/src/main/python/apache/aurora/executor/common/kill_manager.py
index 2bfb5dc..6e482cf 100644
--- a/src/main/python/apache/aurora/executor/common/kill_manager.py
+++ b/src/main/python/apache/aurora/executor/common/kill_manager.py
@@ -19,6 +19,7 @@ class KillManager(StatusChecker):
"""
A health interface that provides a kill-switch for a task monitored by the status manager.
"""
+
def __init__(self):
self._killed = False
self._reason = None
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/common/sandbox.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/sandbox.py b/src/main/python/apache/aurora/executor/common/sandbox.py
index b4125f4..f47a32b 100644
--- a/src/main/python/apache/aurora/executor/common/sandbox.py
+++ b/src/main/python/apache/aurora/executor/common/sandbox.py
@@ -57,6 +57,7 @@ class SandboxProvider(Interface):
class DirectorySandbox(SandboxInterface):
""" Basic sandbox implementation using a directory on the filesystem """
+
def __init__(self, root, user=getpass.getuser()):
self._root = root
self._user = user
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/gc_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/gc_executor.py b/src/main/python/apache/aurora/executor/gc_executor.py
index 1800ee7..c45d5eb 100644
--- a/src/main/python/apache/aurora/executor/gc_executor.py
+++ b/src/main/python/apache/aurora/executor/gc_executor.py
@@ -41,7 +41,7 @@ from apache.thermos.core.inspector import CheckpointInspector
from apache.thermos.monitoring.detector import TaskDetector
from apache.thermos.monitoring.garbage import TaskGarbageCollector
-from .common.sandbox import DirectorySandbox, SandboxInterface
+from .common.sandbox import DirectorySandbox
from .executor_base import ExecutorBase
from .executor_detector import ExecutorDetector
@@ -109,15 +109,17 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
ExceptionalThread.__init__(self)
self.daemon = True
self._stop_event = threading.Event()
- self._gc_task_queue = OrderedDict() # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in
- # the order in which they were received via a launchTask.
- self._driver = None # cache the ExecutorDriver provided by the slave, so we can use it
- # out of band from slave-initiated callbacks. This should be supplied by
- # ExecutorBase.registered() when the executor first registers with
- # the slave.
- self._slave_id = None # cache the slave ID provided by the slave
+ # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in the order in
+ # which they were received via a launchTask.
+ self._gc_task_queue = OrderedDict()
+ # cache the ExecutorDriver provided by the slave, so we can use it out
+ # of band from slave-initiated callbacks. This should be supplied by
+ # ExecutorBase.registered() when the executor first registers with the
+ # slave.
+ self._driver = None
+ self._slave_id = None # cache the slave ID provided by the slave
self._task_id = None # the task_id currently being executed by the ThermosGCExecutor, if any
- self._start_time = None # the start time of a task currently being executed, if any
+ self._start_time = None # the start time of a task currently being executed, if any
self._detector = executor_detector()
self._collector = task_garbage_collector(root=checkpoint_root)
self._clock = clock
@@ -465,7 +467,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
_, (task, retain_tasks, retain_start) = self._gc_task_queue.popitem(0)
last_gc_run = retain_start
self._run_gc(task, retain_tasks, retain_start)
- except KeyError: # no enqueued GC tasks
+ except KeyError: # no enqueued GC tasks
pass
if self._driver is not None:
self.clean_orphans(self._driver)
@@ -475,7 +477,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
if self._driver is not None:
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks
+ except KeyError: # no enqueued GC tasks
pass
else:
self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
@@ -509,7 +511,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable):
return
try:
prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks - reset counter
+ except KeyError: # no enqueued GC tasks - reset counter
self._dropped_tasks.write(0)
else:
self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/467f8428/src/main/python/apache/aurora/executor/thermos_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_runner.py b/src/main/python/apache/aurora/executor/thermos_runner.py
index aef45d7..396753e 100644
--- a/src/main/python/apache/aurora/executor/thermos_runner.py
+++ b/src/main/python/apache/aurora/executor/thermos_runner.py
@@ -80,7 +80,7 @@ app.add_option(
callback=add_port_to('prebound_ports'),
dest='prebound_ports',
default={},
- metavar = "NAME:PORT",
+ metavar="NAME:PORT",
help="bind a numbered port PORT to name NAME")