You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by kl...@apache.org on 2017/06/12 16:04:30 UTC
[1/2] mesos git commit: Updated 'config.py' in the new Mesos CLI to
take settings as parameter.
Repository: mesos
Updated Branches:
refs/heads/master 9e61b3c7a -> 3644defdd
Updated 'config.py' in the new Mesos CLI to take settings as parameter.
Previously, we were importing settings directly from the `bin`
directory, which is obviously not portable. Instead, we need to be
passing the settings into the constructor for the config so that it
can access it from there.
Review: https://reviews.apache.org/r/59634/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a2368db1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a2368db1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a2368db1
Branch: refs/heads/master
Commit: a2368db102f281f55e1531461ec36889e3f5ecf3
Parents: 9e61b3c
Author: Kevin Klues <kl...@gmail.com>
Authored: Mon Jun 12 08:47:40 2017 -0700
Committer: Kevin Klues <kl...@gmail.com>
Committed: Mon Jun 12 08:47:40 2017 -0700
----------------------------------------------------------------------
src/cli_new/bin/main.py | 2 +-
src/cli_new/lib/cli/config.py | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a2368db1/src/cli_new/bin/main.py
----------------------------------------------------------------------
diff --git a/src/cli_new/bin/main.py b/src/cli_new/bin/main.py
index d0ec2bd..5313038 100644
--- a/src/cli_new/bin/main.py
+++ b/src/cli_new/bin/main.py
@@ -84,7 +84,7 @@ def main(argv):
"""
# Load the CLI config.
- config = cli.config.Config()
+ config = cli.config.Config(settings)
plugins = cli.util.import_modules(
cli.util.join_plugin_paths(settings, config),
http://git-wip-us.apache.org/repos/asf/mesos/blob/a2368db1/src/cli_new/lib/cli/config.py
----------------------------------------------------------------------
diff --git a/src/cli_new/lib/cli/config.py b/src/cli_new/lib/cli/config.py
index 353785b..36a32f9 100644
--- a/src/cli_new/lib/cli/config.py
+++ b/src/cli_new/lib/cli/config.py
@@ -21,7 +21,6 @@ Config class to manage the configuration file.
import os
import toml
-import settings
from cli.exceptions import CLIException
@@ -31,7 +30,7 @@ class Config(object):
one method for each element that can be specified in the config file.
"""
- def __init__(self):
+ def __init__(self, settings):
# Load the configuration file path for the CLI.
if os.environ.get("MESOS_CLI_CONFIG"):
self.path = os.environ["MESOS_CLI_CONFIG"]
[2/2] mesos git commit: Extended the unit test infrastructure in the
new Mesos CLI.
Posted by kl...@apache.org.
Extended the unit test infrastructure in the new Mesos CLI.
This infrastructure includes the ability to bring up a test cluster to
run the CLI against. Future unit tests will use this infrastructure to
test their commands against a running mesos cluster. The tests require
some binaries created when building Mesos.
Review: https://reviews.apache.org/r/58720/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3644defd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3644defd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3644defd
Branch: refs/heads/master
Commit: 3644defdd5e192a6d086ac11851ab7a88489d66b
Parents: a2368db
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Mon Jun 12 08:49:54 2017 -0700
Committer: Kevin Klues <kl...@gmail.com>
Committed: Mon Jun 12 09:01:51 2017 -0700
----------------------------------------------------------------------
src/cli_new/lib/cli/tests/__init__.py | 23 ++
src/cli_new/lib/cli/tests/base.py | 422 ++++++++++++++++++++++++++++
src/cli_new/lib/cli/tests/constants.py | 25 ++
src/cli_new/lib/cli/tests/tests.py | 65 +++++
src/cli_new/pip-requirements.txt | 1 +
src/cli_new/tests/main.py | 7 +
6 files changed, 543 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/lib/cli/tests/__init__.py
----------------------------------------------------------------------
diff --git a/src/cli_new/lib/cli/tests/__init__.py b/src/cli_new/lib/cli/tests/__init__.py
new file mode 100644
index 0000000..0daf288
--- /dev/null
+++ b/src/cli_new/lib/cli/tests/__init__.py
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Mesos CLI unit tests module.
+"""
+
+# pylint: disable=W0401
+from .base import *
+from .tests import *
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/lib/cli/tests/base.py
----------------------------------------------------------------------
diff --git a/src/cli_new/lib/cli/tests/base.py b/src/cli_new/lib/cli/tests/base.py
new file mode 100644
index 0000000..62a4ee8
--- /dev/null
+++ b/src/cli_new/lib/cli/tests/base.py
@@ -0,0 +1,422 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Set of classes and helper functions for building unit tests for the Mesos CLI.
+"""
+
+import os
+import shutil
+import StringIO
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import parse
+
+import cli.http as http
+
+from cli.tests.constants import DEFAULT_AGENT_IP
+from cli.tests.constants import DEFAULT_AGENT_PORT
+from cli.tests.constants import DEFAULT_MASTER_IP
+from cli.tests.constants import DEFAULT_MASTER_PORT
+
+from cli.exceptions import CLIException
+
+# Timeout used when creating, killing, and getting data from objects that are
+# part of our test infrastructure.
+TIMEOUT = 5
+
+
+class CLITestCase(unittest.TestCase):
+ """
+ Base class for CLI TestCases.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ print "\n{class_name}".format(class_name=cls.__name__)
+
+ @staticmethod
+ def default_mesos_build_dir():
+ """
+ Returns the default path of the Mesos build directory. Useful when
+ we wish to use some binaries such as mesos-execute.
+ """
+ tests_dir = os.path.dirname(__file__)
+ cli_dir = os.path.dirname(tests_dir)
+ lib_dir = os.path.dirname(cli_dir)
+ cli_new_dir = os.path.dirname(lib_dir)
+ src_dir = os.path.dirname(cli_new_dir)
+ mesos_dir = os.path.dirname(src_dir)
+ build_dir = os.path.join(mesos_dir, "build")
+
+ if os.path.isdir(build_dir):
+ return build_dir
+ else:
+ raise CLIException("The Mesos build directory"
+ " does not exist: {path}"
+ .format(path=build_dir))
+
+# This value is set to the correct path when running tests/main.py. We
+# set it here to make sure that CLITestCase has a MESOS_BUILD_DIR member.
+CLITestCase.MESOS_BUILD_DIR = ""
+
+
+class Executable(object):
+ """
+ This class defines the base class for launching an executable for
+ the CLI unit tests. It will be subclassed by (at least) a
+ 'Master', 'Agent', and 'Task' subclass.
+ """
+ def __init__(self):
+ self.name = ""
+ self.executable = ""
+ self.flags = {}
+ self.proc = None
+
+ def __del__(self):
+ if hasattr(self, "proc"):
+ self.kill()
+
+ def launch(self):
+ """
+ Launch 'self.executable'. We assume it is in the system PATH.
+ """
+ if self.proc is not None:
+ raise CLIException("{name} already launched"
+ .format(name=self.name.capitalize()))
+
+ try:
+ flags = ["--{key}={value}".format(key=key, value=value)
+ for key, value in self.flags.iteritems()]
+
+ proc = subprocess.Popen(
+ [self.executable] + flags,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ except Exception as exception:
+ raise CLIException("Failed to launch '{executable}': {error}"
+ .format(executable=self.executable,
+ error=exception))
+
+ if proc.poll():
+ raise CLIException("Failed to launch '{executable}': {error}"
+ .format(executable=self.executable,
+ error=proc.stdout.read()))
+
+ self.proc = proc
+
+ def kill(self):
+ """
+ Kills a previously launched executable.
+ """
+ if self.proc is None:
+ return
+
+ try:
+ self.proc.kill()
+ self.proc.wait()
+ self.proc = None
+ except Exception as exception:
+ raise CLIException("Could not kill {name}: {error}"
+ .format(name=self.name, error=exception))
+
+
+class Master(Executable):
+ """
+ This class defines the functions necessary to launch a master in
+ the CLI unit tests.
+ """
+ count = 0
+
+ def __init__(self, flags=None):
+ super(Master, self).__init__()
+
+ if Master.count > 0:
+ raise CLIException("Creating more than one master"
+ " is currently not possible")
+
+ Master.count += 1
+
+ if flags is None:
+ flags = {}
+
+ if "ip" not in flags:
+ flags["ip"] = DEFAULT_MASTER_IP
+ if "port" not in flags:
+ flags["port"] = DEFAULT_MASTER_PORT
+ if "work_dir" not in flags:
+ flags["work_dir"] = tempfile.mkdtemp()
+
+ self.flags = flags
+ self.name = "master"
+ self.addr = "{ip}:{port}".format(ip=flags["ip"], port=flags["port"])
+ self.executable = os.path.join(
+ CLITestCase.MESOS_BUILD_DIR,
+ "bin",
+ "mesos-{name}.sh".format(name=self.name))
+
+ def __del__(self):
+ super(Master, self).__del__()
+
+ if hasattr(self, "flags"):
+ shutil.rmtree(self.flags["work_dir"])
+
+ Master.count -= 1
+
+
+class Agent(Executable):
+ """
+ This class defines the functions necessary to launch an agent in
+ the CLI unit tests.
+ """
+ count = 0
+
+ def __init__(self, flags=None):
+ super(Agent, self).__init__()
+
+ if Agent.count > 0:
+ raise CLIException("Creating more than one agent"
+ " is currently not possible")
+
+ Agent.count += 1
+
+ if flags is None:
+ flags = {}
+
+ if "ip" not in flags:
+ flags["ip"] = DEFAULT_AGENT_IP
+ if "port" not in flags:
+ flags["port"] = DEFAULT_AGENT_PORT
+ if "master" not in flags:
+ flags["master"] = "{ip}:{port}".format(
+ ip=DEFAULT_MASTER_IP,
+ port=DEFAULT_MASTER_PORT)
+ if "work_dir" not in flags:
+ flags["work_dir"] = tempfile.mkdtemp()
+ if "runtime_dir" not in flags:
+ flags["runtime_dir"] = tempfile.mkdtemp()
+ # Disabling systemd support on Linux to run without sudo.
+ if "linux" in sys.platform and "systemd_enable_support" not in flags:
+ flags["systemd_enable_support"] = "false"
+
+ self.flags = flags
+ self.name = "agent"
+ self.addr = "{ip}:{port}".format(ip=flags["ip"], port=flags["port"])
+ self.executable = os.path.join(
+ CLITestCase.MESOS_BUILD_DIR,
+ "bin",
+ "mesos-{name}.sh".format(name=self.name))
+
+ def __del__(self):
+ super(Agent, self).__del__()
+
+ if hasattr(self, "flags"):
+ shutil.rmtree(self.flags["work_dir"])
+ shutil.rmtree(self.flags["runtime_dir"])
+
+ Agent.count -= 1
+
+ # pylint: disable=arguments-differ
+ def launch(self, timeout=TIMEOUT):
+ """
+ After starting the agent, we need to make sure it has
+ successfully registered with the master before proceeding.
+ """
+ super(Agent, self).launch()
+
+ try:
+ # pylint: disable=missing-docstring
+ def single_slave(data):
+ return len(data["slaves"]) == 1
+
+ http.get_json(self.flags["master"], "slaves", single_slave, timeout)
+ except Exception as exception:
+ stdout = ""
+ if self.proc.poll():
+ stdout = "\n{output}".format(output=self.proc.stdout.read())
+ self.proc = None
+
+ raise CLIException("Could not get '/slaves' endpoint as JSON with"
+ " only 1 agent in it: {error}{stdout}"
+ .format(error=exception, stdout=stdout))
+
+ # pylint: disable=arguments-differ
+ def kill(self, timeout=TIMEOUT):
+ """
+ After killing the agent, we need to make sure it has
+ successfully unregistered from the master before proceeding.
+ """
+ super(Agent, self).kill()
+
+ if self.proc is None:
+ return
+
+ try:
+ # pylint: disable=missing-docstring
+ def no_slaves(data):
+ return len(data["slaves"]) == 0
+
+ http.get_json(self.flags["master"], "slaves", no_slaves, timeout)
+ except Exception as exception:
+ raise CLIException("Could not get '/slaves' endpoint as"
+ " JSON with 0 agents in it: {error}"
+ .format(error=exception))
+
+
+class Task(Executable):
+ """
+ This class defines the functions necessary to launch a task in
+ the CLI unit tests.
+ """
+ count = 0
+
+ def __init__(self, flags=None):
+ super(Task, self).__init__()
+
+ if flags is None:
+ flags = {}
+
+ if "master" not in flags:
+ flags["master"] = "{ip}:{port}".format(
+ ip=DEFAULT_MASTER_IP,
+ port=DEFAULT_MASTER_PORT)
+ if "name" not in flags:
+ flags["name"] = "task-{id}".format(id=Task.count)
+ Task.count += 1
+ if "command" not in flags:
+ raise CLIException("No command supplied when creating task")
+
+ self.flags = flags
+ self.name = "task"
+ self.executable = os.path.join(
+ CLITestCase.MESOS_BUILD_DIR,
+ "src",
+ "mesos-execute")
+
+ def __wait_for_containers(self, condition, timeout=TIMEOUT):
+ """
+ Wait for the agent's '/containers' endpoint
+ to return data subject to 'condition'.
+ """
+ try:
+ data = http.get_json(self.flags["master"], "slaves")
+ except Exception as exception:
+ raise CLIException("Could not get '/slaves' endpoint"
+ " as JSON: {error}"
+ .format(error=exception))
+
+ if len(data["slaves"]) != 1:
+ raise CLIException("More than one agent detected when"
+ " reading from '/slaves' endpoint")
+
+ try:
+ agent = parse.parse(
+ "slave({id})@{addr}",
+ data["slaves"][0]["pid"])
+ except Exception as exception:
+ raise CLIException("Unable to parse agent info: {error}"
+ .format(error=exception))
+
+ try:
+ data = http.get_json(
+ agent["addr"],
+ "containers",
+ condition,
+ timeout)
+ except Exception as exception:
+ raise CLIException("Could not get '/containers' endpoint as"
+ " JSON subject to condition: {error}"
+ .format(error=exception))
+
+ # pylint: disable=arguments-differ
+ def launch(self, timeout=TIMEOUT):
+ """
+ After starting the task, we need to make sure its container
+ has actually been added to the agent before proceeding.
+ """
+ super(Task, self).launch()
+
+ try:
+ # pylint: disable=missing-docstring
+ def container_exists(data):
+ return any(container["executor_id"] == self.flags["name"]
+ for container in data)
+
+ self.__wait_for_containers(container_exists, timeout)
+ except Exception as exception:
+ stdout = ""
+ if self.proc.poll():
+ stdout = "\n{output}".format(output=self.proc.stdout.read())
+ self.proc = None
+
+ raise CLIException("Waiting for container '{name}'"
+ " failed: {error}{stdout}"
+ .format(name=self.flags["name"],
+ error=exception,
+ stdout=stdout))
+
+ # pylint: disable=arguments-differ
+ def kill(self, timeout=TIMEOUT):
+ """
+ After killing the task, we need to make sure its container has
+ actually been removed from the agent before proceeding.
+ """
+ super(Task, self).kill()
+
+ if self.proc is None:
+ return
+
+ try:
+ # pylint: disable=missing-docstring
+ def container_does_not_exist(data):
+ return not any(container["executor_id"] == self.flags["name"]
+ for container in data)
+
+ self.__wait_for_containers(container_does_not_exist, timeout)
+ except Exception as exception:
+ raise CLIException("Container with name '{name}' still"
+ " exists after timeout: {error}"
+ .format(name=self.flags["name"],
+ error=exception))
+
+
+def capture_output(command, argv, extra_args=None):
+ """
+ Redirect the output of a command to a string and return it.
+ """
+ if not extra_args:
+ extra_args = {}
+
+ stdout = sys.stdout
+ sys.stdout = StringIO.StringIO()
+
+ try:
+ command(argv, **extra_args)
+ except Exception as exception:
+ # Fix stdout in case something goes wrong
+ sys.stdout = stdout
+ raise CLIException("Could not get command output: {error}"
+ .format(error=exception))
+
+ sys.stdout.seek(0)
+ output = sys.stdout.read().strip()
+ sys.stdout = stdout
+
+ return output
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/lib/cli/tests/constants.py
----------------------------------------------------------------------
diff --git a/src/cli_new/lib/cli/tests/constants.py b/src/cli_new/lib/cli/tests/constants.py
new file mode 100644
index 0000000..5d45d94
--- /dev/null
+++ b/src/cli_new/lib/cli/tests/constants.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A collection of constants useful for the CLI unit tests.
+"""
+
+DEFAULT_MASTER_IP = "127.0.0.1"
+DEFAULT_MASTER_PORT = "9090"
+
+DEFAULT_AGENT_IP = "127.0.0.1"
+DEFAULT_AGENT_PORT = "9091"
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/lib/cli/tests/tests.py
----------------------------------------------------------------------
diff --git a/src/cli_new/lib/cli/tests/tests.py b/src/cli_new/lib/cli/tests/tests.py
new file mode 100644
index 0000000..79e1036
--- /dev/null
+++ b/src/cli_new/lib/cli/tests/tests.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tests that test the test infrastructure itself.
+"""
+
+from cli.tests import capture_output
+from cli.tests import CLITestCase
+from cli.tests import Agent
+from cli.tests import Master
+from cli.tests import Task
+
+
+class TestInfrastructure(CLITestCase):
+ """
+ This class tests the functionality of the test
+ infrastructure used to run all other CLI tests.
+ """
+ def test_launch_binaries(self):
+ """
+ Tests the launching and killing of
+ a mesos master, agent, and task.
+ """
+ master = Master()
+ master.launch()
+
+ agent = Agent()
+ agent.launch()
+
+ task = Task({"command": "sleep 1000"})
+ task.launch()
+
+ task.kill()
+ agent.kill()
+ master.kill()
+
+ def test_capture_output(self):
+ """
+ Tests the ability to capture the output
+ from an arbitrary CLI sub-command.
+ """
+ template = "Arguments: {args}"
+ arguments = ["argument1", "argument2"]
+
+ # pylint: disable=missing-docstring
+ def command(args):
+ print template.format(args=args)
+
+ output = capture_output(command, arguments)
+
+ self.assertEqual(output, template.format(args=arguments))
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/pip-requirements.txt
----------------------------------------------------------------------
diff --git a/src/cli_new/pip-requirements.txt b/src/cli_new/pip-requirements.txt
index 28613e5..cb657e1 100644
--- a/src/cli_new/pip-requirements.txt
+++ b/src/cli_new/pip-requirements.txt
@@ -5,6 +5,7 @@ docopt==0.6.2
isort==4.2.5
lazy-object-proxy==1.2.2
mccabe==0.5.2
+parse==1.8.0
Pygments==2.1.3
PyInstaller==3.1.1
pylint==1.6.4
http://git-wip-us.apache.org/repos/asf/mesos/blob/3644defd/src/cli_new/tests/main.py
----------------------------------------------------------------------
diff --git a/src/cli_new/tests/main.py b/src/cli_new/tests/main.py
index dff5d48..3e4d2e4 100644
--- a/src/cli_new/tests/main.py
+++ b/src/cli_new/tests/main.py
@@ -22,7 +22,14 @@ import unittest
from termcolor import colored
+from cli.tests import CLITestCase
+
+# pylint: disable=unused-import
+# We import the tests that we want to run.
+from cli.tests import TestInfrastructure
if __name__ == '__main__':
+ CLITestCase.MESOS_BUILD_DIR = CLITestCase.default_mesos_build_dir()
+
print colored("Running the Mesos CLI unit tests", "yellow")
unittest.main(verbosity=2, testRunner=unittest.TextTestRunner)