You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by mc...@apache.org on 2014/02/20 22:39:01 UTC
git commit: Add configuration plugins to the clientv2 framework.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 56a8cc379 -> cb549590a
Add configuration plugins to the clientv2 framework.
A configuration plugin is a component that allows a commandline
to have objects which perform some kind of configuration option
ahead of actually invoking commands. For example, a configuration
plugin could add options to perform a security handshake before
connecting to a production cluster.
Bugs closed: aurora-220
Reviewed at https://reviews.apache.org/r/18265/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/cb549590
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/cb549590
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/cb549590
Branch: refs/heads/master
Commit: cb549590a79af3403f44de471a54daa7d9170efb
Parents: 56a8cc3
Author: Mark Chu-Carroll <mc...@twopensource.com>
Authored: Thu Feb 20 16:38:25 2014 -0500
Committer: Mark Chu-Carroll <mc...@twitter.com>
Committed: Thu Feb 20 16:38:25 2014 -0500
----------------------------------------------------------------------
.../python/apache/aurora/client/cli/__init__.py | 44 +++++
src/main/python/apache/aurora/client/cli/sla.py | 12 +-
src/test/python/apache/aurora/client/cli/BUILD | 17 +-
.../apache/aurora/client/cli/test_plugins.py | 159 +++++++++++++++++++
4 files changed, 226 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/cb549590/src/main/python/apache/aurora/client/cli/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/__init__.py b/src/main/python/apache/aurora/client/cli/__init__.py
index d3bd119..4a6a7ee 100644
--- a/src/main/python/apache/aurora/client/cli/__init__.py
+++ b/src/main/python/apache/aurora/client/cli/__init__.py
@@ -72,6 +72,29 @@ class Context(object):
self.options = options
+class ConfigurationPlugin(object):
+ """A component that can be plugged in to a command-line.
+ The component can add a set of options to the command-line.
+ After a context is created, but before a call is dispatched
+ to the noun/verb for execution, the plugin will have the opportunity
+ to process its parameters and perform whatever initialization it
+ requires on the context.
+
+ For example, if a production environment is protected behind some
+ kind of gateway, a ConfigurationPlugin could be created that
+ performs a security handshake with the gateway before any of the commands
+ attempt to communicate with the environment.
+ """
+
+ @abstractmethod
+ def get_options(self):
+ """Return the set of options processed by this plugin"""
+
+ @abstractmethod
+ def execute(self, context):
+ """Run the context/command line initialization code for this plugin."""
+
+
class AuroraCommand(object):
def setup_options_parser(self, argparser):
"""Sets up command line options parsing for this command.
@@ -115,6 +138,7 @@ class CommandLine(object):
def __init__(self):
self.nouns = None
self.parser = None
+ self.plugins = []
def register_noun(self, noun):
"""Adds a noun to the application"""
@@ -123,6 +147,10 @@ class CommandLine(object):
if not isinstance(noun, Noun):
raise TypeError('register_noun requires a Noun argument')
self.nouns[noun.name] = noun
+ noun.set_commandline(self)
+
+ def register_plugin(self, plugin):
+ self.plugins.append(plugin)
def setup_options_parser(self):
""" Builds the options parsing for the application."""
@@ -216,6 +244,12 @@ class CommandLine(object):
context = noun.create_context()
context.set_options(options)
try:
+ for plugin in self.plugins:
+ plugin.execute(context)
+ except Context.CommandError as c:
+ print('Error in configuration plugin: %s' % c.msg, file=sys.stderr)
+ return c.code
+ try:
return noun.execute(context)
except Context.CommandError as c:
print('Error executing command: %s' % c.msg, file=sys.stderr)
@@ -229,6 +263,10 @@ class Noun(AuroraCommand):
def __init__(self):
super(Noun, self).__init__()
self.verbs = {}
+ self.commandline = None
+
+ def set_commandline(self, commandline):
+ self.commandline = commandline
def register_verb(self, verb):
"""Add an operation supported for this noun."""
@@ -245,6 +283,9 @@ class Noun(AuroraCommand):
vparser = subparser.add_parser(name, help=verb.help)
for opt in verb.get_options():
opt.add_to_parser(vparser)
+ for plugin in self.commandline.plugins:
+ for opt in plugin.get_options():
+ opt.add_to_parser(vparser)
@property
def usage(self):
@@ -300,6 +341,9 @@ class Verb(AuroraCommand):
result += ["Options:"]
for opt in self.get_options():
result += [" " + s for s in opt.render_help()]
+ for plugin in self.noun.commandline.plugins:
+ for opt in plugin.get_options():
+ result += [" " + s for s in opt.render_help()]
result += ["", self.help]
return "\n".join(result)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/cb549590/src/main/python/apache/aurora/client/cli/sla.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/sla.py b/src/main/python/apache/aurora/client/cli/sla.py
index 9dd766b..62d1385 100644
--- a/src/main/python/apache/aurora/client/cli/sla.py
+++ b/src/main/python/apache/aurora/client/cli/sla.py
@@ -21,9 +21,9 @@ from apache.aurora.client.cli import (
)
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.cli.options import (
- CommandOption,
- JOBSPEC_ARGUMENT,
- parse_percentiles,
+ CommandOption,
+ JOBSPEC_ARGUMENT,
+ parse_percentiles,
parse_time_values
)
@@ -41,7 +41,7 @@ class GetTaskUpCountCmd(Verb):
If duration is not specified prints a histogram-like log-scale distribution
of task uptime percentages.
"""
-
+
@classmethod
def render_get_task_up_count(cls, context, vector):
def format_output(durations):
@@ -56,7 +56,9 @@ of task uptime percentages.
CommandOption('--durations', type=parse_time_values, default=None,
help="""Durations to report uptime for.
Format: XdYhZmWs (each field optional but must be in that order.)
-Example: --durations=3m,10s,1h3m10s"""),
+Examples:
+ --durations=1d'
+ --durations=3m,10s,1h3m10s"""),
JOBSPEC_ARGUMENT]
def execute(self, context):
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/cb549590/src/test/python/apache/aurora/client/cli/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/BUILD b/src/test/python/apache/aurora/client/cli/BUILD
index f9e6749..a48284c 100644
--- a/src/test/python/apache/aurora/client/cli/BUILD
+++ b/src/test/python/apache/aurora/client/cli/BUILD
@@ -18,10 +18,12 @@ python_test_suite(
name = 'all',
dependencies = [
pants(':bridge'),
+ pants(':help'),
pants(':job'),
+ pants(':plugins'),
pants(':quota'),
pants(':sla'),
- pants(':task')
+ pants(':task'),
]
)
@@ -112,3 +114,16 @@ python_tests(
pants('src/test/python/apache/aurora/client/commands:util')
]
)
+
+python_tests(
+ name = 'plugins',
+ sources = [ 'test_plugins.py' ],
+ dependencies = [
+ pants(':util'),
+ pants('3rdparty/python:mock'),
+ pants('3rdparty/python:twitter.common.contextutil'),
+ pants('src/main/python/apache/aurora/client/cli'),
+ pants('src/main/python/apache/aurora/client/cli:client'),
+ pants('src/test/python/apache/aurora/client/commands:util')
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/cb549590/src/test/python/apache/aurora/client/cli/test_plugins.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_plugins.py b/src/test/python/apache/aurora/client/cli/test_plugins.py
new file mode 100644
index 0000000..b36e5f1
--- /dev/null
+++ b/src/test/python/apache/aurora/client/cli/test_plugins.py
@@ -0,0 +1,159 @@
+#
+# Copyright 2013 Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import contextlib
+
+from twitter.common.contextutil import temporary_file
+
+from gen.apache.aurora.ttypes import (
+ AssignedTask,
+ Identity,
+ ScheduledTask,
+ ScheduleStatus,
+ ScheduleStatusResult,
+ TaskEvent,
+ TaskQuery,
+)
+
+from apache.aurora.client.cli import (
+ ConfigurationPlugin,
+ EXIT_COMMAND_FAILURE,
+ EXIT_INVALID_CONFIGURATION,
+ EXIT_OK,
+)
+from apache.aurora.client.cli.client import AuroraCommandLine
+from apache.aurora.client.cli.options import CommandOption
+from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext
+from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
+from apache.aurora.config import AuroraConfig
+from mock import Mock, patch
+
+
+class BogusPlugin(ConfigurationPlugin):
+ def get_options(self):
+ return [CommandOption('--bogosity', type=str, help='Permitted bogosity level')]
+
+ def execute(self, context):
+ context.bogosity = context.options.bogosity
+
+class TestPlugins(AuroraClientCommandTest):
+
+ @classmethod
+ def create_mock_task(cls, task_id, instance_id, initial_time, status):
+ mock_task = Mock(spec=ScheduledTask)
+ mock_task.assignedTask = Mock(spec=AssignedTask)
+ mock_task.assignedTask.taskId = task_id
+ mock_task.assignedTask.instanceId = instance_id
+ mock_task.status = status
+ mock_task_event = Mock(spec=TaskEvent)
+ mock_task_event.timestamp = initial_time
+ mock_task.taskEvents = [mock_task_event]
+ return mock_task
+
+ @classmethod
+ def create_mock_status_query_result(cls, scheduleStatus):
+ mock_query_result = cls.create_simple_success_response()
+ mock_query_result.result.scheduleStatusResult = Mock(spec=ScheduleStatusResult)
+ if scheduleStatus == ScheduleStatus.INIT:
+ # status query result for before job is launched.
+ mock_query_result.result.scheduleStatusResult.tasks = []
+ else:
+ mock_task_one = cls.create_mock_task('hello', 0, 1000, scheduleStatus)
+ mock_task_two = cls.create_mock_task('hello', 1, 1004, scheduleStatus)
+ mock_query_result.result.scheduleStatusResult.tasks = [mock_task_one, mock_task_two]
+ return mock_query_result
+
+ @classmethod
+ def create_mock_query(cls):
+ return TaskQuery(owner=Identity(role=cls.TEST_ROLE), environment=cls.TEST_ENV,
+ jobName=cls.TEST_JOB)
+
+ @classmethod
+ def get_createjob_response(cls):
+ # Then, we call api.create_job(config)
+ return cls.create_simple_success_response()
+
+ @classmethod
+ def assert_create_job_called(cls, mock_api):
+ # Check that create_job was called exactly once, with an AuroraConfig parameter.
+ assert mock_api.create_job.call_count == 1
+ assert isinstance(mock_api.create_job.call_args_list[0][0][0], AuroraConfig)
+
+ @classmethod
+ def assert_scheduler_called(cls, mock_api, mock_query, num_queries):
+ assert mock_api.scheduler_proxy.getTasksStatus.call_count == num_queries
+ mock_api.scheduler_proxy.getTasksStatus.assert_called_with(mock_query)
+
+ def test_plugin_runs_in_create_job(self):
+ """Run a test of the "create" command against a mocked-out API:
+ Verifies that the creation command sends the right API RPCs, and performs the correct
+ tests on the result."""
+
+ # We'll patch out create_context, which will give us a fake context
+ # object, and everything can be stubbed through that.
+ mock_context = FakeAuroraCommandContext()
+ with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
+ # After making the client, create sets up a job monitor.
+ # The monitor uses TaskQuery to get the tasks. It's called at least twice:once before
+ # the job is created, and once after. So we need to set up mocks for the query results.
+ mock_query = self.create_mock_query()
+ mock_context.add_expected_status_query_result(
+ self.create_mock_status_query_result(ScheduleStatus.INIT))
+ mock_context.add_expected_status_query_result(
+ self.create_mock_status_query_result(ScheduleStatus.RUNNING))
+ api = mock_context.get_api('west')
+ api.create_job.return_value = self.get_createjob_response()
+
+ # This is the real test: invoke create as if it had been called by the command line.
+ with temporary_file() as fp:
+ fp.write(self.get_valid_config())
+ fp.flush()
+ cmd = AuroraCommandLine()
+ cmd.register_plugin(BogusPlugin())
+ cmd.execute(['job', 'create', '--bogosity=maximum', '--wait_until=RUNNING',
+ 'west/bozo/test/hello', fp.name])
+
+ # Now check that the right API calls got made.
+ # Check that create_job was called exactly once, with an AuroraConfig parameter.
+ self.assert_create_job_called(api)
+ self.assert_scheduler_called(api, mock_query, 2)
+ # Check that the plugin did its job.
+ assert mock_context.bogosity == "maximum"
+
+ def mock_print(self, str):
+ for str in str.split('\n'):
+ self.transcript.append(str)
+
+ def mock_print_err(self, str):
+ for str in str.split('\n'):
+ self.err_transcript.append(str)
+
+
+ def test_plugin_options_in_help(self):
+ cmd = AuroraCommandLine()
+ cmd.register_plugin(BogusPlugin())
+ self.transcript = []
+ self.err_transcript = []
+ with patch('apache.aurora.client.cli.client.AuroraCommandLine.print_out',
+ side_effect=self.mock_print):
+ assert cmd.execute(['help', 'job', 'status']) == EXIT_OK
+ assert len(self.transcript) > 5
+ assert self.transcript[0] == 'Usage for verb "job status":' in self.transcript
+ assert not any('quota' in t for t in self.transcript)
+ assert not any('list' in t for t in self.transcript)
+ assert "Options:" in self.transcript
+ assert any('bogosity' in t for t in self.transcript)
+