You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2020/07/30 09:12:16 UTC
[ignite] branch ignite-ducktape updated: Ducktape codestyle (#8098)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new 330c17b Ducktape codestyle (#8098)
330c17b is described below
commit 330c17b084aaf91d19591178885dd2637dc8fa21
Author: Ivan Daschinskiy <iv...@gmail.com>
AuthorDate: Thu Jul 30 12:11:59 2020 +0300
Ducktape codestyle (#8098)
---
.../services/utils/decorators.py => .pylintrc} | 24 ++----
.../utils/decorators.py => check_style.sh} | 24 +++---
modules/ducktests/tests/ignitetest/__init__.py | 1 +
.../ducktests/tests/ignitetest/services/ignite.py | 27 +++++--
.../tests/ignitetest/services/ignite_app.py | 12 ++-
.../tests/ignitetest/services/ignite_spark_app.py | 6 +-
.../ducktests/tests/ignitetest/services/spark.py | 40 +++++++---
.../tests/ignitetest/services/utils/decorators.py | 8 ++
.../ignitetest/services/utils/ignite_aware.py | 59 +++++++++++++--
.../ignitetest/services/utils/ignite_aware_app.py | 32 ++++++--
.../ignitetest/services/utils/ignite_config.py | 21 +++++-
.../tests/ignitetest/services/utils/ignite_path.py | 45 +++++++-----
.../tests/ignitetest/services/utils/jmx_utils.py | 85 ++++++++++++++++++++--
.../tests/ignitetest/services/zk/zookeeper.py | 39 +++++++++-
.../ignitetest/tests/add_node_rebalance_test.py | 12 ++-
.../tests/ignitetest/tests/discovery_test.py | 44 ++++++++---
.../tests/ignitetest/tests/pme_free_switch_test.py | 18 ++++-
.../ignitetest/tests/spark_integration_test.py | 11 ++-
.../tests/ignitetest/tests/utils/ignite_test.py | 12 +++
.../tests/ignitetest/tests/utils/version.py | 10 ++-
20 files changed, 414 insertions(+), 116 deletions(-)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/.pylintrc
similarity index 65%
copy from modules/ducktests/tests/ignitetest/services/utils/decorators.py
copy to modules/ducktests/tests/.pylintrc
index 788ddba..73536f3 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py
+++ b/modules/ducktests/tests/.pylintrc
@@ -12,21 +12,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import functools
-from threading import RLock
+[BASIC]
+min-public-methods=0
+# TODO: Remove after migrating to python 3
+disable=R0205
+[SIMILARITIES]
+ignore-imports=yes
-def memoize(func):
- cache = func.cache = {}
- lock = RLock()
-
- @functools.wraps(func)
- def memoized_func(*args, **kwargs):
- key = str(args) + str(kwargs)
- if key not in cache:
- with lock:
- if key not in cache:
- cache[key] = func(*args, **kwargs)
- return cache[key]
-
- return memoized_func
+[FORMAT]
+max-line-length=120
diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/check_style.sh
old mode 100644
new mode 100755
similarity index 65%
copy from modules/ducktests/tests/ignitetest/services/utils/decorators.py
copy to modules/ducktests/tests/check_style.sh
index 788ddba..39405a7
--- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py
+++ b/modules/ducktests/tests/check_style.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,21 +14,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import functools
-from threading import RLock
-
-def memoize(func):
- cache = func.cache = {}
- lock = RLock()
+if ! command -v pylint &> /dev/null
+then
+ echo "Please, install pylint first"
+ exit 1
+fi
- @functools.wraps(func)
- def memoized_func(*args, **kwargs):
- key = str(args) + str(kwargs)
- if key not in cache:
- with lock:
- if key not in cache:
- cache[key] = func(*args, **kwargs)
- return cache[key]
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
- return memoized_func
+pylint --rcfile="$SCRIPT_DIR"/.pylintrc "$SCRIPT_DIR"/ignitetest
diff --git a/modules/ducktests/tests/ignitetest/__init__.py b/modules/ducktests/tests/ignitetest/__init__.py
index 1c2357f..8cbaed8 100644
--- a/modules/ducktests/tests/ignitetest/__init__.py
+++ b/modules/ducktests/tests/ignitetest/__init__.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# pylint: disable=missing-module-docstring
# This determines the version of ignitetest that can be published to PyPi and installed with pip
#
# Note that in development, this version name can't follow Ignite's convention of having a trailing "-SNAPSHOT"
diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py
index 05f108e..bfc96ad 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite.py
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+This module contains class to start ignite cluster node.
+"""
+
import os.path
import signal
@@ -20,11 +24,13 @@ from ducktape.cluster.remoteaccount import RemoteCommandError
from ducktape.utils.util import wait_until
from ignitetest.services.utils.ignite_aware import IgniteAwareService
-from ignitetest.services.utils.ignite_config import IgniteServerConfig, IgniteClientConfig
from ignitetest.tests.utils.version import DEV_BRANCH
class IgniteService(IgniteAwareService):
+ """
+ Ignite node service.
+ """
APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup"
HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin")
@@ -38,9 +44,11 @@ class IgniteService(IgniteAwareService):
"collect_default": False}
}
+ # pylint: disable=R0913
def __init__(self, context, num_nodes, client_mode=False, version=DEV_BRANCH, properties=""):
super(IgniteService, self).__init__(context, num_nodes, client_mode, version, properties)
+ # pylint: disable=W0221
def start(self, timeout_sec=180):
super(IgniteService, self).start()
@@ -66,11 +74,17 @@ class IgniteService(IgniteAwareService):
return cmd
def await_node_started(self, node, timeout_sec):
+ """
+ Await topology ready event on node start.
+ :param node: Node to wait
+ :param timeout_sec: Number of seconds to wait event.
+ """
self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True)
if len(self.pids(node)) == 0:
raise Exception("No process ids recorded on node %s" % node.account.hostname)
+ # pylint: disable=W0221
def stop_node(self, node, clean_shutdown=True, timeout_sec=60):
pids = self.pids(node)
sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL
@@ -90,17 +104,20 @@ class IgniteService(IgniteAwareService):
node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False)
def thread_dump(self, node):
+ """
+ Generate thread dump on node.
+ :param node: Ignite service node.
+ """
for pid in self.pids(node):
try:
node.account.signal(pid, signal.SIGQUIT, allow_fail=True)
- except:
+ except RemoteCommandError:
self.logger.warn("Could not dump threads on node")
def pids(self, node):
- """Return process ids associated with running processes on the given node."""
try:
cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.APP_SERVICE_CLASS
- pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
+ pid_arr = list(node.account.ssh_capture(cmd, allow_fail=True, callback=int))
return pid_arr
- except (RemoteCommandError, ValueError) as e:
+ except (RemoteCommandError, ValueError):
return []
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py
index 64de176..0702221 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_app.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py
@@ -13,17 +13,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService
-from ignitetest.tests.utils.version import DEV_BRANCH
-
"""
-The Ignite application service allows to perform custom logic writen on java.
+This module contains the ignite application service allows to perform custom logic writen on java.
"""
+from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService
+from ignitetest.tests.utils.version import DEV_BRANCH
+
class IgniteApplicationService(IgniteAwareApplicationService):
+ """
+ The Ignite application service allows to perform custom logic writen on java.
+ """
service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService"
+ # pylint: disable=R0913
def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="",
timeout_sec=60):
super(IgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties,
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
index 775f19c1..71014b7 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
@@ -14,13 +14,17 @@
# limitations under the License.
"""
-The Ignite-Spark application service.
+This module contains the Ignite-Spark application service.
"""
from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService
from ignitetest.tests.utils.version import DEV_BRANCH
class SparkIgniteApplicationService(IgniteAwareApplicationService):
+ """
+ The Ignite-Spark application service.
+ """
+ # pylint: disable=R0913
def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="",
timeout_sec=60):
super(SparkIgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties,
diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py
index fdd5c73..657a6b6 100644
--- a/modules/ducktests/tests/ignitetest/services/spark.py
+++ b/modules/ducktests/tests/ignitetest/services/spark.py
@@ -13,18 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+This module contains spark service class.
+"""
+
import os.path
from ducktape.cluster.remoteaccount import RemoteCommandError
from ducktape.services.service import Service
from ignitetest.services.utils.ignite_aware import IgniteAwareService
-from ignitetest.services.utils.ignite_config import IgniteServerConfig
-
from ignitetest.tests.utils.version import DEV_BRANCH
class SparkService(IgniteAwareService):
+ """
+ Start a spark node.
+ """
INSTALL_DIR = "/opt/spark-{version}".format(version="2.3.4")
SPARK_PERSISTENT_ROOT = "/mnt/spark"
@@ -68,6 +73,7 @@ class SparkService(IgniteAwareService):
return cmd
+ # pylint: disable=W0221
def start_node(self, node, timeout_sec=30):
self.init_persistent(node)
@@ -91,7 +97,7 @@ class SparkService(IgniteAwareService):
if len(self.pids(node)) == 0:
raise Exception("No process ids recorded on node %s" % node.account.hostname)
- def stop_node(self, node, clean_shutdown=True, timeout_sec=60):
+ def stop_node(self, node):
if node == self.nodes[0]:
node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-master.sh"))
else:
@@ -103,28 +109,40 @@ class SparkService(IgniteAwareService):
node.account.ssh("sudo rm -rf -- %s" % SparkService.SPARK_PERSISTENT_ROOT, allow_fail=False)
def pids(self, node):
- """Return process ids associated with running processes on the given node."""
try:
cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node)
- pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
- return pid_arr
- except (RemoteCommandError, ValueError) as e:
+ return list(node.account.ssh_capture(cmd, allow_fail=True, callback=int))
+ except (RemoteCommandError, ValueError):
return []
def java_class_name(self, node):
+ """
+ :param node: Spark node.
+ :return: Class name depending on node type (master or slave).
+ """
if node == self.nodes[0]:
return "org.apache.spark.deploy.master.Master"
- else:
- return "org.apache.spark.deploy.worker.Worker"
- def master_log_path(self, node):
+ return "org.apache.spark.deploy.worker.Worker"
+
+ @staticmethod
+ def master_log_path(node):
+ """
+ :param node: Spark master node.
+ :return: Path to log file.
+ """
return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format(
SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT,
userID=node.account.user,
instance=1,
host=node.account.hostname)
- def slave_log_path(self, node):
+ @staticmethod
+ def slave_log_path(node):
+ """
+ :param node: Spark slave node.
+ :return: Path to log file.
+ """
return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format(
SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT,
userID=node.account.user,
diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py
index 788ddba..9c767f4 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py
@@ -12,11 +12,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains various useful decorators.
+"""
+
import functools
from threading import RLock
def memoize(func):
+ """
+ Decorate function to memoize first call to thread safe cache.
+ """
cache = func.cache = {}
lock = RLock()
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index 578d728..b4d12f0 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -12,6 +12,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains the base class to build services aware of Ignite.
+"""
+
import os
from abc import abstractmethod
@@ -22,12 +27,11 @@ from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteSe
from ignitetest.services.utils.ignite_path import IgnitePath
from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin
-"""
-The base class to build services aware of Ignite.
-"""
-
class IgniteAwareService(BackgroundThreadService):
+ """
+ The base class to build services aware of Ignite.
+ """
# Root directory for persistent output
PERSISTENT_ROOT = "/mnt/service"
STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log")
@@ -41,6 +45,7 @@ class IgniteAwareService(BackgroundThreadService):
"collect_default": True}
}
+ # pylint: disable=R0913
def __init__(self, context, num_nodes, client_mode, version, properties):
super(IgniteAwareService, self).__init__(context, num_nodes)
@@ -66,6 +71,10 @@ class IgniteAwareService(BackgroundThreadService):
ignite_jmx_mixin(node, self.pids(node))
def init_persistent(self, node):
+ """
+ Init persistent directory.
+ :param node: Ignite service node.
+ """
node.account.mkdirs(self.PERSISTENT_ROOT)
node.account.create_file(self.CONFIG_FILE, self.config().render(
config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties))
@@ -73,18 +82,30 @@ class IgniteAwareService(BackgroundThreadService):
@abstractmethod
def start_cmd(self, node):
+ """
+ Start up command for service.
+ :param node: Ignite service node.
+ """
raise NotImplementedError
@abstractmethod
def pids(self, node):
+ """
+ :param node: Ignite service node.
+ :return: List of service's pids.
+ """
raise NotImplementedError
def config(self):
+ """
+ :return: Ignite node configuration.
+ """
if self.client_mode:
return IgniteClientConfig(self.context)
- else:
- return IgniteServerConfig(self.context)
+ return IgniteServerConfig(self.context)
+
+ # pylint: disable=W0613
def _worker(self, idx, node):
cmd = self.start_cmd(node)
@@ -93,9 +114,23 @@ class IgniteAwareService(BackgroundThreadService):
node.account.ssh(cmd)
def alive(self, node):
+ """
+ :param node: Ignite service node.
+ :return: True if node is alive.
+ """
return len(self.pids(node)) > 0
+ # pylint: disable=R0913
def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5):
+ """
+ Await for specific event message in a node's log file.
+ :param evt_message: Event message.
+ :param node: Ignite service node.
+ :param timeout_sec: Number of seconds to check the condition for before failing.
+ :param from_the_beginning: If True, search for message from the beggining of log file.
+ :param backoff_sec: Number of seconds to back off between each failure to meet the condition
+ before checking again.
+ """
with node.account.monitor_log(self.STDOUT_STDERR_CAPTURE) as monitor:
if from_the_beginning:
monitor.offset = 0
@@ -104,12 +139,24 @@ class IgniteAwareService(BackgroundThreadService):
err_msg="Event [%s] was not triggered in %d seconds" % (evt_message, timeout_sec))
def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backoff_sec=5):
+ """
+ Await for specific event messages on all nodes.
+ :param evt_message: Event message.
+ :param timeout_sec: Number of seconds to check the condition for before failing.
+ :param from_the_beginning: If True, search for message from the beggining of log file.
+ :param backoff_sec: Number of seconds to back off between each failure to meet the condition
+ before checking again.
+ """
assert len(self.nodes) == 1
self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning,
backoff_sec=backoff_sec)
def execute(self, command):
+ """
+ Execute command on all nodes.
+ :param command: Command to execute.
+ """
for node in self.nodes:
cmd = "%s 1>> %s 2>> %s" % \
(self.path.script(command, node),
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
index fd2c860..152b5fe 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
@@ -12,19 +12,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import re
-
-from ducktape.services.service import Service
-
-from ignitetest.services.utils.ignite_aware import IgniteAwareService
-from ignitetest.services.utils.ignite_config import IgniteClientConfig
"""
-The base class to build Ignite aware application written on java.
+This module contains the base class to build Ignite aware application written on java.
"""
+import re
+
+from ignitetest.services.utils.ignite_aware import IgniteAwareService
+
class IgniteAwareApplicationService(IgniteAwareService):
+ """
+ The base class to build Ignite aware application written on java.
+ """
+ # pylint: disable=R0913
def __init__(self, context, java_class_name, client_mode, version, properties, params, timeout_sec,
service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"):
super(IgniteAwareApplicationService, self).__init__(context, 1, client_mode, version, properties)
@@ -53,9 +55,10 @@ class IgniteAwareApplicationService(IgniteAwareService):
self.STDOUT_STDERR_CAPTURE)
return cmd
+ # pylint: disable=W0221
def stop_node(self, node, clean_shutdown=True, timeout_sec=20):
self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account)))
- node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=True, allow_fail=True)
+ node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True)
stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec)
assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \
@@ -73,6 +76,9 @@ class IgniteAwareApplicationService(IgniteAwareService):
node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False)
def app_args(self):
+ """
+ :return: Application arguments.
+ """
args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE
if self.params != "":
@@ -84,6 +90,9 @@ class IgniteAwareApplicationService(IgniteAwareService):
return node.account.java_pids(self.servicejava_class_name)
def jvm_opts(self):
+ """
+ :return: Application JVM options.
+ """
return "-J-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file " + \
"-J-Dlog4j.configDebug=true " \
"-J-Xmx1G " \
@@ -91,6 +100,9 @@ class IgniteAwareApplicationService(IgniteAwareService):
"-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + self.jvm_options
def env(self):
+ """
+ :return: Export string of additional environment variables.
+ """
return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \
"export EXCLUDE_TEST_CLASSES=true; " + \
"export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \
@@ -98,6 +110,10 @@ class IgniteAwareApplicationService(IgniteAwareService):
% self.path.home(self.version)
def extract_result(self, name):
+ """
+ :param name: Result parameter's name.
+ :return: Extracted result of application run.
+ """
res = ""
output = self.nodes[0].account.ssh_capture(
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py
index 077c39b..90b3bfa 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py
@@ -14,18 +14,20 @@
# limitations under the License.
"""
-This module renders Ignite config and all related artifacts
+This module contains ignite config classes and utilities.
"""
+import os
from jinja2 import FileSystemLoader, Environment
-import os
-
DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/config"
DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2"
class Config(object):
+ """
+ Basic configuration.
+ """
def __init__(self, path):
tmpl_dir = os.path.dirname(path)
tmpl_file = os.path.basename(path)
@@ -37,12 +39,18 @@ class Config(object):
self.default_params = {}
def render(self, **kwargs):
+ """
+ Render configuration.
+ """
kwargs.update(self.default_params)
res = self.template.render(**kwargs)
return res
class IgniteServerConfig(Config):
+ """
+ Ignite server node configuration.
+ """
def __init__(self, context):
path = DEFAULT_IGNITE_CONF
if 'ignite_server_config_path' in context.globals:
@@ -51,6 +59,9 @@ class IgniteServerConfig(Config):
class IgniteClientConfig(Config):
+ """
+ Ignite client node configuration.
+ """
def __init__(self, context):
path = DEFAULT_IGNITE_CONF
if 'ignite_client_config_path' in context.globals:
@@ -60,6 +71,8 @@ class IgniteClientConfig(Config):
class IgniteLoggerConfig(Config):
+ """
+ Ignite logger configuration.
+ """
def __init__(self):
super(IgniteLoggerConfig, self).__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2")
-
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
index 4eab4a4..4d24d36 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
@@ -13,32 +13,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+This module contains ignite path resolve utilities.
+"""
+
import os
from ignitetest.tests.utils.version import get_version, IgniteVersion
-"""
-This module provides Ignite path methods
-"""
-
class IgnitePath:
- SCRATCH_ROOT = "/mnt"
- IGNITE_INSTALL_ROOT = "/opt"
-
"""Path resolver for Ignite system tests which assumes the following layout:
- /opt/ignite-dev # Current version of Ignite under test
- /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball
- /opt/ignite-<version> # Other previous versions of Ignite
- ...
- """
+ /opt/ignite-dev # Current version of Ignite under test
+ /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball
+ /opt/ignite-<version> # Other previous versions of Ignite
+ ...
+ """
+ SCRATCH_ROOT = "/mnt"
+ IGNITE_INSTALL_ROOT = "/opt"
def __init__(self, context):
self.project = context.globals.get("project", "ignite")
def home(self, node_or_version, project=None):
- version = self._version(node_or_version)
+ """
+ :param node_or_version: Ignite service node or IgniteVersion instance.
+ :param project: Project name.
+ :return: Home directory.
+ """
+ version = self.__version(node_or_version)
home_dir = project or self.project
if version is not None:
home_dir += "-%s" % str(version)
@@ -46,11 +50,18 @@ class IgnitePath:
return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir)
def script(self, script_name, node_or_version, project=None):
- version = self._version(node_or_version)
+ """
+ :param script_name: Script name.
+ :param node_or_version: Ignite service node or IgniteVersion instance.
+ :param project: Project name.
+ :return: Full path to script.
+ """
+ version = self.__version(node_or_version)
return os.path.join(self.home(version, project=project), "bin", script_name)
- def _version(self, node_or_version):
+ @staticmethod
+ def __version(node_or_version):
if isinstance(node_or_version, IgniteVersion):
return node_or_version
- else:
- return get_version(node_or_version)
+
+ return get_version(node_or_version)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
index 6843c94..bfbab25 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
@@ -12,12 +12,23 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains JMX Console client and different utilities and mixins to retrieve ignite node parameters
+and attributes.
+"""
+
import re
from ignitetest.services.utils.decorators import memoize
def ignite_jmx_mixin(node, pids):
+ """
+ Dynamically mixin JMX attributes to Ignite service node.
+ :param node: Ignite service node.
+ :param pids: Ignite service node pids.
+ """
setattr(node, 'pids', pids)
base_cls = node.__class__
base_cls_name = node.__class__.__name__
@@ -25,15 +36,25 @@ def ignite_jmx_mixin(node, pids):
class JmxMBean(object):
+ """
+ Dynamically exposes JMX MBean attributes.
+ """
def __init__(self, client, name):
self.client = client
self.name = name
def __getattr__(self, attr):
+ """
+ Retrieves through JMX client MBean attributes.
+ :param attr: Attribute name.
+ :return: Attribute value.
+ """
return self.client.mbean_attribute(self.name, attr)
class JmxClient(object):
+ """JMX client, invokes jmxterm on node locally.
+ """
jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n'
def __init__(self, node):
@@ -42,50 +63,82 @@ class JmxClient(object):
@memoize
def find_mbean(self, pattern, domain='org.apache'):
+ """
+ Find mbean by specified pattern and domain on node.
+ :param pattern: MBean name pattern.
+ :param domain: Domain of MBean
+ :return: JmxMBean instance
+ """
cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \
% (self.pid, domain, self.jmx_util_cmd, pattern)
- name = next(self.run_cmd(cmd)).strip()
+ name = next(self.__run_cmd(cmd)).strip()
return JmxMBean(self, name)
def mbean_attribute(self, mbean, attr):
+ """
+ Get MBean attribute.
+ :param mbean: MBean name
+ :param attr: Attribute name
+ :return: Attribute value
+ """
cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \
% (self.pid, mbean, attr, self.jmx_util_cmd, attr)
- return iter(s.strip() for s in self.run_cmd(cmd))
+ return iter(s.strip() for s in self.__run_cmd(cmd))
- def run_cmd(self, cmd):
+ def __run_cmd(self, cmd):
return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str)
class DiscoveryInfo(object):
+ """ Ignite service node discovery info, obtained from DiscoverySpi mbean.
+ """
def __init__(self, coordinator, local_raw):
self._local_raw = local_raw
self._coordinator = coordinator
@property
- def id(self):
+ def node_id(self):
+ """
+ :return: Local node id.
+ """
return self.__find__("id=([^\\s]+),")
@property
def coordinator(self):
+ """
+ :return: Coordinator node id.
+ """
return self._coordinator
@property
def consistent_id(self):
+ """
+ :return: Node consistent id, if presents (only in TcpDiscovery).
+ """
return self.__find__("consistentId=([^\\s]+),")
@property
def is_client(self):
+ """
+ :return: True if node is client.
+ """
return self.__find__("isClient=([^\\s]+),") == "true"
@property
def order(self):
+ """
+ :return: Topology order.
+ """
return int(self.__find__("order=(\\d+),"))
@property
def int_order(self):
+ """
+ :return: Internal order (TcpDiscovery).
+ """
val = self.__find__("intOrder=(\\d+),")
return int(val) if val else -1
@@ -95,16 +148,28 @@ class DiscoveryInfo(object):
class IgniteJmxMixin(object):
+ """
+ Mixin to IgniteService node, exposing useful properties, obtained from JMX.
+ """
@memoize
def jmx_client(self):
+ """
+ :return: JmxClient instance.
+ """
# noinspection PyTypeChecker
return JmxClient(self)
@memoize
- def id(self):
+ def node_id(self):
+ """
+ :return: Local node id.
+ """
return next(self.kernal_mbean().LocalNodeId).strip()
def discovery_info(self):
+ """
+ :return: DiscoveryInfo instance.
+ """
disco_mbean = self.disco_mbean()
crd = next(disco_mbean.Coordinator).strip()
local = next(disco_mbean.LocalNodeFormatted).strip()
@@ -112,13 +177,19 @@ class IgniteJmxMixin(object):
return DiscoveryInfo(crd, local)
def kernal_mbean(self):
+ """
+ :return: IgniteKernal MBean.
+ """
return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal')
@memoize
def disco_mbean(self):
+ """
+ :return: DiscoverySpi MBean.
+ """
disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip()
if 'ZookeeperDiscoverySpi' in disco_spi:
return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi')
- else:
- return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi')
+
+ return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi')
diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
index d32cbbc..5c29c66 100644
--- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
+++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
@@ -12,12 +12,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains classes and utilities to start zookeeper cluster for testing ZookeeperDiscovery.
+"""
+
import os.path
from ducktape.services.service import Service
class ZookeeperSettings:
+ """
+ Settings for zookeeper quorum nodes.
+ """
def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181):
self.tick_time = tick_time
self.init_limit = init_limit
@@ -26,6 +34,9 @@ class ZookeeperSettings:
class ZookeeperService(Service):
+ """
+ Zookeeper service.
+ """
PERSISTENT_ROOT = "/mnt/zookeeper"
CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf")
LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log")
@@ -41,16 +52,17 @@ class ZookeeperService(Service):
}
}
- def __init__(self, context, num_nodes, settings=ZookeeperSettings()):
+ def __init__(self, context, num_nodes, settings=ZookeeperSettings(), start_timeout_sec=60):
super(ZookeeperService, self).__init__(context, num_nodes)
self.settings = settings
+ self.start_timeout_sec = start_timeout_sec
- def start(self, timeout_sec=60):
- Service.start(self)
+ def start(self):
+ super(ZookeeperService, self).start()
self.logger.info("Waiting for Zookeeper quorum...")
for node in self.nodes:
- self.await_quorum(node, timeout_sec)
+ self.await_quorum(node, self.start_timeout_sec)
self.logger.info("Zookeeper quorum is formed.")
@@ -89,6 +101,11 @@ class ZookeeperService(Service):
self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname)
def await_quorum(self, node, timeout):
+ """
+ Await quorum formed on node (leader election ready).
+ :param node: Zookeeper service node.
+ :param timeout: Wait timeout.
+ """
with node.account.monitor_log(self.LOG_FILE) as monitor:
monitor.offset = 0
monitor.wait_until(
@@ -103,12 +120,26 @@ class ZookeeperService(Service):
return "org.apache.zookeeper.server.quorum.QuorumPeerMain"
def pids(self, node):
+ """
+ Get pids of zookeeper service node.
+ :param node: Zookeeper service node.
+ :return: List of pids.
+ """
return node.account.java_pids(self.java_class_name())
def alive(self, node):
+ """
+ Check if zookeeper service node is alive.
+ :param node: Zookeeper service node.
+ :return: True if node is alive
+ """
return len(self.pids(node)) > 0
def connection_string(self):
+ """
+ Form a connection string to zookeeper cluster.
+ :return: Connection string.
+ """
return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes])
def stop_node(self, node):
diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
index a3c9546..d836681 100644
--- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Module contains node rebalance tests.
+"""
+
from ducktape.mark import parametrize
from ducktape.mark.resource import cluster
@@ -22,16 +26,16 @@ from ignitetest.tests.utils.ignite_test import IgniteTest
from ignitetest.tests.utils.version import DEV_BRANCH, IgniteVersion, LATEST
+# pylint: disable=W0223
class AddNodeRebalanceTest(IgniteTest):
+ """
+ Test basic rebalance scenarios.
+ """
NUM_NODES = 4
PRELOAD_TIMEOUT = 60
DATA_AMOUNT = 1000000
REBALANCE_TIMEOUT = 60
- """
- Test performs rebalance tests.
- """
-
def __init__(self, test_context):
super(AddNodeRebalanceTest, self).__init__(test_context=test_context)
diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
index 01aebe4..d8d4e0b 100644
--- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
@@ -12,6 +12,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+Module contains discovery tests.
+"""
+
import random
from ducktape.mark import parametrize
@@ -24,7 +29,14 @@ from ignitetest.tests.utils.ignite_test import IgniteTest
from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7
+# pylint: disable=W0223
class DiscoveryTest(IgniteTest):
+ """
+ Test basic discovery scenarious (TCP and Zookeeper).
+ 1. Start of ignite cluster.
+ 2. Kill random node.
+ 3. Wait that survived node detects node failure.
+ """
NUM_NODES = 7
CONFIG_TEMPLATE = """
@@ -44,11 +56,15 @@ class DiscoveryTest(IgniteTest):
def __init__(self, test_context):
super(DiscoveryTest, self).__init__(test_context=test_context)
- self.zk = None
+ self.zk_quorum = None
self.servers = None
@staticmethod
def properties(zookeeper_settings=None):
+ """
+ :param zookeeper_settings: ZookeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used.
+ :return: Rendered node's properties.
+ """
return Template(DiscoveryTest.CONFIG_TEMPLATE) \
.render(zookeeper_settings=zookeeper_settings)
@@ -56,8 +72,8 @@ class DiscoveryTest(IgniteTest):
pass
def teardown(self):
- if self.zk:
- self.zk.stop()
+ if self.zk_quorum:
+ self.zk_quorum.stop()
if self.servers:
self.servers.stop()
@@ -66,20 +82,26 @@ class DiscoveryTest(IgniteTest):
@parametrize(version=str(DEV_BRANCH))
@parametrize(version=str(LATEST_2_7))
def test_tcp(self, version):
+ """
+ Test basic discovery scenario with TcpDiscoverySpi.
+ """
return self.__basic_test__(version, False)
@cluster(num_nodes=NUM_NODES + 3)
@parametrize(version=str(DEV_BRANCH))
@parametrize(version=str(LATEST_2_7))
def test_zk(self, version):
+ """
+ Test basic discovery scenario with ZookeeperDiscoverySpi.
+ """
return self.__basic_test__(version, True)
def __basic_test__(self, version, with_zk=False):
if with_zk:
- self.zk = ZookeeperService(self.test_context, 3)
+ self.zk_quorum = ZookeeperService(self.test_context, 3)
self.stage("Starting Zookeper quorum")
- self.zk.start()
- properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()})
+ self.zk_quorum.start()
+ properties = self.properties(zookeeper_settings={'connection_string': self.zk_quorum.connection_string()})
self.stage("Zookeper quorum started")
else:
properties = self.properties()
@@ -94,19 +116,19 @@ class DiscoveryTest(IgniteTest):
start = self.monotonic()
self.servers.start()
- data = {'Ignite cluster start time (s)': self.monotonic() - start }
+ data = {'Ignite cluster start time (s)': self.monotonic() - start}
self.stage("Topology is ready")
# Node failure detection
fail_node, survived_node = self.choose_random_node_to_kill(self.servers)
- data["nodes"] = [node.id() for node in self.servers.nodes]
+ data["nodes"] = [node.node_id() for node in self.servers.nodes]
disco_infos = []
for node in self.servers.nodes:
disco_info = node.discovery_info()
disco_infos.append({
- "id": disco_info.id,
+ "id": disco_info.node_id,
"consistent_id": disco_info.consistent_id,
"coordinator": disco_info.coordinator,
"order": disco_info.order,
@@ -127,6 +149,10 @@ class DiscoveryTest(IgniteTest):
@staticmethod
def choose_random_node_to_kill(service):
+ """
+ :param service: Service nodes to process.
+ :return: Tuple of random node to kill and survived nodes
+ """
idx = random.randint(0, len(service.nodes) - 1)
survive = [node for i, node in enumerate(service.nodes) if i != idx]
diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
index 0df2f84d..f0cfd77 100644
--- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
@@ -12,6 +12,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains PME free switch tests.
+"""
+
import time
from ducktape.mark import parametrize
@@ -23,11 +28,18 @@ from ignitetest.tests.utils.ignite_test import IgniteTest
from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion
+# pylint: disable=W0223
class PmeFreeSwitchTest(IgniteTest):
+ """
+ Tests PME free switch scenarios.
+ """
NUM_NODES = 3
@staticmethod
def properties():
+ """
+ :return: Rendered configuration properties.
+ """
return """
<property name="cacheConfiguration">
<list>
@@ -53,6 +65,9 @@ class PmeFreeSwitchTest(IgniteTest):
@parametrize(version=str(DEV_BRANCH))
@parametrize(version=str(LATEST_2_7))
def test(self, version):
+ """
+ Test PME free scenario (node stop).
+ """
data = {}
self.stage("Starting nodes")
@@ -82,7 +97,8 @@ class PmeFreeSwitchTest(IgniteTest):
single_key_tx_streamer = IgniteApplicationService(
self.test_context,
- java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.SingleKeyTxStreamerApplication",
+ java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test."
+ "SingleKeyTxStreamerApplication",
properties=self.properties(),
params="test-cache,1000",
version=ignite_version)
diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py
index 7e2b4fc..a5f188b 100644
--- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+This module contains spark integration test.
+"""
+
from ducktape.mark import parametrize
from ignitetest.services.ignite import IgniteService
@@ -23,6 +27,7 @@ from ignitetest.tests.utils.ignite_test import IgniteTest
from ignitetest.tests.utils.version import DEV_BRANCH
+# pylint: disable=W0223
class SparkIntegrationTest(IgniteTest):
"""
Test performs:
@@ -45,6 +50,9 @@ class SparkIntegrationTest(IgniteTest):
@parametrize(version=str(DEV_BRANCH))
def test_spark_client(self, version):
+ """
+ Performs test scenario.
+ """
self.spark = SparkService(self.test_context, version=version, num_nodes=2)
self.spark.start()
@@ -55,7 +63,8 @@ class SparkIntegrationTest(IgniteTest):
IgniteApplicationService(
self.test_context,
- java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test.SampleDataStreamerApplication",
+ java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test."
+ "SampleDataStreamerApplication",
params="cache,1000",
version=version).run()
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
index a9c13d9..df0d30e 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
@@ -13,15 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+This module contains basic ignite test.
+"""
+
from ducktape.tests.test import Test
from monotonic import monotonic
+# pylint: disable=W0223
class IgniteTest(Test):
+ """
+ Basic ignite test.
+ """
def __init__(self, test_context):
super(IgniteTest, self).__init__(test_context=test_context)
def stage(self, msg):
+ """
+ Print stage mark.
+ :param msg: Stage mark message.
+ """
self.logger.info("[TEST_STAGE] " + msg + "...")
@staticmethod
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py
index 2f1f003..5173188 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/version.py
+++ b/modules/ducktests/tests/ignitetest/tests/utils/version.py
@@ -13,6 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Module contains ignite version utility class.
+"""
from distutils.version import LooseVersion
from ducktape.cluster.cluster import ClusterNode
@@ -49,8 +52,8 @@ class IgniteVersion(LooseVersion):
def __str__(self):
if self.is_dev:
return "dev"
- else:
- return LooseVersion.__str__(self)
+
+ return LooseVersion.__str__(self)
def get_version(node=None):
@@ -61,7 +64,8 @@ def get_version(node=None):
if isinstance(node, ClusterNode) and hasattr(node, 'version'):
return getattr(node, 'version')
- if isinstance(node, str) or isinstance(node, unicode):
+ # pylint: disable=E0602
+ if isinstance(node, (str, unicode)):
return node
return DEV_BRANCH