You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2020/07/21 09:41:00 UTC
[ignite] branch ignite-ducktape updated: Add zookeeper service and
basic topology test to ignite-ducktests (#8059)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new ea7fcd5 Add zookeeper service and basic topology test to ignite-ducktests (#8059)
ea7fcd5 is described below
commit ea7fcd548743171e11d74ae27b9154793b8f871f
Author: Ivan Daschinskiy <iv...@gmail.com>
AuthorDate: Tue Jul 21 12:40:33 2020 +0300
Add zookeeper service and basic topology test to ignite-ducktests (#8059)
---
modules/ducktests/pom.xml | 36 ++++++
modules/ducktests/tests/docker/Dockerfile | 23 +++-
modules/ducktests/tests/docker/run_tests.sh | 2 +-
.../ignite_test.py => services/zk/__init__.py} | 10 --
.../services/zk/templates/log4j.properties.j2 | 33 ++++++
.../services/zk/templates/zookeeper.properties.j2 | 25 ++++
.../tests/ignitetest/services/zk/zookeeper.py | 127 +++++++++++++++++++++
.../tests/benchmarks/add_node_rebalance_test.py | 5 +-
.../tests/ignitetest/tests/discovery_test.py | 120 +++++++++++++++++++
.../tests/ignitetest/tests/utils/ignite_test.py | 13 +++
modules/ducktests/tests/setup.py | 2 +-
11 files changed, 376 insertions(+), 20 deletions(-)
diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml
index 81c73ca..7bfdc2c 100644
--- a/modules/ducktests/pom.xml
+++ b/modules/ducktests/pom.xml
@@ -51,12 +51,48 @@
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-spark</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile
index b199c37..0baefda 100644
--- a/modules/ducktests/tests/docker/Dockerfile
+++ b/modules/ducktests/tests/docker/Dockerfile
@@ -35,7 +35,7 @@ LABEL ducker.creator=$ducker_creator
RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list
RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean
RUN python -m pip install -U pip==9.0.3;
-RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7
+RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.7
# Set up ssh
COPY ./ssh-config /root/.ssh/config
@@ -47,12 +47,25 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/"
ARG APACHE_ARCHIVE="https://archive.apache.org/dist/"
# Install binary test dependencies.
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.7.6/apache-ignite-2.7.6-bin.zip && unzip apache-ignite-2.7.6-bin.zip && mv /opt/apache-ignite-2.7.6-bin /opt/ignite-2.7.6
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.0/apache-ignite-2.8.0-bin.zip && unzip apache-ignite-2.8.0-bin.zip && mv /opt/apache-ignite-2.8.0-bin /opt/ignite-2.8.0
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.1/apache-ignite-2.8.1-bin.zip && unzip apache-ignite-2.8.1-bin.zip && mv /opt/apache-ignite-2.8.1-bin /opt/ignite-2.8.1
+RUN for v in "2.7.6" "2.8.0" "2.8.1"; \
+ do cd /opt; \
+ curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\
+ unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\
+ cp -r ./ignite-$v/libs/optional/ignite-zookeeper ./ignite-$v/libs; \
+ done
RUN rm /opt/apache-ignite-*-bin.zip
+#Install zookeeper.
+ARG ZOOKEEPER_VERSION="3.5.8"
+ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION"
+ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin"
+ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz"
+RUN echo $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT
+RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \
+ && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT
+RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME
+
# Install spark
ARG SPARK_VERSION="2.3.4"
ARG SPARK_NAME="spark-$SPARK_VERSION"
@@ -77,4 +90,4 @@ USER ducker
CMD sudo service ssh start && tail -f /dev/null
# Container port exposure
-EXPOSE 11211 47100 47500 49112 10800 8080
+EXPOSE 11211 47100 47500 49112 10800 8080 2888 3888 2181
diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh
index 477ec82..e392e3d 100755
--- a/modules/ducktests/tests/docker/run_tests.sh
+++ b/modules/ducktests/tests/docker/run_tests.sh
@@ -16,7 +16,7 @@
# limitations under the License.
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-6}
+IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-11}
TC_PATHS=${TC_PATHS:-./ignitetest/}
die() {
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/services/zk/__init__.py
similarity index 75%
copy from modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
copy to modules/ducktests/tests/ignitetest/services/zk/__init__.py
index 1ec2132..ec20143 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/services/zk/__init__.py
@@ -12,13 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from ducktape.tests.test import Test
-
-
-class IgniteTest(Test):
- def __init__(self, test_context):
- super(IgniteTest, self).__init__(test_context=test_context)
-
- def stage(self, msg):
- self.logger.info("[TEST_STAGE] " + msg + "...")
diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2
new file mode 100644
index 0000000..507ec98
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2
@@ -0,0 +1,33 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+zookeeper.root.logger=INFO, FILE
+zookeeper.console.threshold=INFO
+
+zookeeper.log.dir={{ PERSISTENT_ROOT }}
+zookeeper.log.file=zookeeper.log
+zookeeper.log.threshold=INFO
+zookeeper.log.maxfilesize=256MB
+zookeeper.log.maxbackupindex=20
+
+log4j.rootLogger=${zookeeper.root.logger}
+
+log4j.appender.FILE=org.apache.log4j.FileAppender
+log4j.appender.FILE.Threshold=${zookeeper.log.threshold}
+log4j.appender.FILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
+log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2
new file mode 100644
index 0000000..2c80e84
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2
@@ -0,0 +1,25 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+tickTime={{ settings.tick_time }}
+initLimit={{ settings.init_limit }}
+syncLimit={{ settings.sync_limit }}
+dataDir={{ DATA_DIR }}
+clientPort={{ settings.client_port }}
+{% for node in nodes %}
+server.{{ loop.index }}={{ node.account.hostname }}:2888:3888
+{% endfor %}
diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
new file mode 100644
index 0000000..7767ce9
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+from ducktape.services.service import Service
+
+
+class ZookeeperSettings:
+ def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181):
+ self.tick_time = tick_time
+ self.init_limit = init_limit
+ self.sync_limit = sync_limit
+ self.client_port = client_port
+
+
+class ZookeeperService(Service):
+ PERSISTENT_ROOT = "/mnt/zookeeper"
+ CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf")
+ LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log")
+ DATA_DIR = os.path.join(PERSISTENT_ROOT, "data")
+ CONFIG_FILE = os.path.join(CONFIG_ROOT, "zookeeper.properties")
+ LOG_CONFIG_FILE = os.path.join(CONFIG_ROOT, "log4j.properties")
+ ZK_LIB_DIR = "/opt/zookeeper-3.5.8/lib"
+
+ logs = {
+ "zk_log": {
+ "path": LOG_FILE,
+ "collect_default": True
+ }
+ }
+
+ def __init__(self, context, num_nodes, settings=ZookeeperSettings()):
+ super(ZookeeperService, self).__init__(context, num_nodes)
+ self.settings = settings
+
+ def start(self, timeout_sec=60):
+ Service.start(self)
+ self.logger.info("Waiting for Zookeeper quorum...")
+
+ for node in self.nodes:
+ self.await_quorum(node, timeout_sec)
+
+ self.logger.info("Zookeeper quorum is formed.")
+
+ def start_node(self, node):
+ idx = self.idx(node)
+
+ self.logger.info("Starting Zookeeper node %d on %s", idx, node.account.hostname)
+
+ node.account.ssh("mkdir -p %s" % self.DATA_DIR)
+ node.account.ssh("mkdir -p %s" % self.CONFIG_ROOT)
+ node.account.ssh("echo %d > %s/myid" % (idx, self.DATA_DIR))
+
+ config_file = self.render('zookeeper.properties.j2', settings=self.settings)
+ node.account.create_file(self.CONFIG_FILE, config_file)
+ self.logger.info("ZK config %s", config_file)
+
+ log_config_file = self.render('log4j.properties.j2')
+ node.account.create_file(self.LOG_CONFIG_FILE, log_config_file)
+
+ start_cmd = "nohup java -cp %s/*:%s org.apache.zookeeper.server.quorum.QuorumPeerMain %s >/dev/null 2>&1 &" % \
+ (self.ZK_LIB_DIR, self.CONFIG_ROOT, self.CONFIG_FILE)
+
+ node.account.ssh(start_cmd)
+
+ def wait_node(self, node, timeout_sec=20):
+ idx = self.idx(node)
+
+ with node.account.monitor_log(self.LOG_FILE) as monitor:
+ monitor.offset = 0
+ monitor.wait_until(
+ "binding to port",
+ timeout_sec=timeout_sec,
+ err_msg="Zookeeper service didn't finish startup on %s" % node.account.hostname
+ )
+
+ self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname)
+
+ def await_quorum(self, node, timeout):
+ with node.account.monitor_log(self.LOG_FILE) as monitor:
+ monitor.offset = 0
+ monitor.wait_until(
+ "LEADER ELECTION TOOK",
+ timeout_sec=timeout,
+ err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname
+ )
+
+ def pids(self, node):
+ try:
+ cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'"
+ pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
+ return pid_arr
+ except (RemoteCommandError, ValueError) as e:
+ return []
+
+ def alive(self, node):
+ return len(self.pids(node)) > 0
+
+ def connection_string(self):
+ return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes])
+
+ def stop_node(self, node):
+ idx = self.idx(node)
+ self.logger.info("Stopping %s node %d on %s" % (type(self).__name__, idx, node.account.hostname))
+ node.account.kill_process("zookeeper", allow_fail=False)
+
+ def clean_node(self, node):
+ self.logger.info("Cleaning Zookeeper node %d on %s", self.idx(node), node.account.hostname)
+ if self.alive(node):
+ self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." %
+ (self.__class__.__name__, node.account))
+ node.account.kill_process("zookeeper", clean_shutdown=False, allow_fail=True)
+ node.account.ssh("rm -rf %s %s %s" % (self.CONFIG_ROOT, self.DATA_DIR, self.LOG_FILE), allow_fail=False)
diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
index 2c725d7..ba5b324 100644
--- a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import time
from ducktape.mark import parametrize
from ducktape.mark.resource import cluster
@@ -82,13 +81,13 @@ class AddNodeRebalanceTest(IgniteTest):
ignite.start()
- start = time.time()
+ start = self.monotonic()
ignite.await_event("rebalanced=true, wasRebalanced=false",
timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT,
from_the_beginning=True,
backoff_sec=1)
- data = {"Rebalanced in (sec)": time.time() - start}
+ data = {"Rebalanced in (sec)": self.monotonic() - start}
return data
diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
new file mode 100644
index 0000000..42228e3
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+
+from ducktape.mark import parametrize
+from ducktape.mark.resource import cluster
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.zk.zookeeper import ZookeeperService
+from ignitetest.version import DEV_BRANCH, LATEST_2_7
+from ignitetest.tests.utils.ignite_test import IgniteTest
+
+from jinja2 import Template
+
+class DiscoveryTest(IgniteTest):
+ NUM_NODES = 7
+
+ CONFIG_TEMPLATE = """
+ <property name="clientMode" value="{{ client_mode | lower }}"/>
+ {% if zookeeper_settings %}
+ {% with zk = zookeeper_settings %}
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi">
+ <property name="zkConnectionString" value="{{ zk.connection_string }}"/>
+ <property name="sessionTimeout" value="{{ zk.session_timeout or 3000 }}"/>
+ <property name="zkRootPath" value="{{ zk.root_path or '/apacheIgnite' }}"/>
+ <property name="joinTimeout" value="{{ zk.join_timeout or 10000 }}"/>
+ </bean>
+ </property>
+ {% endwith %}
+ {% endif %}
+ """
+
+ def __init__(self, test_context):
+ super(DiscoveryTest, self).__init__(test_context=test_context)
+ self.zk = None
+ self.servers = None
+
+ @staticmethod
+ def properties(client_mode="false", zookeeper_settings=None):
+ return Template(DiscoveryTest.CONFIG_TEMPLATE) \
+ .render(client_mode=client_mode, zookeeper_settings=zookeeper_settings)
+
+ def setUp(self):
+ pass
+
+ def teardown(self):
+ if self.zk:
+ self.zk.stop()
+
+ if self.servers:
+ self.servers.stop()
+
+ @cluster(num_nodes=NUM_NODES)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_7))
+ def test_tcp(self, version):
+ return self.__basic_test__(version, False)
+
+ @cluster(num_nodes=NUM_NODES + 3)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_7))
+ def test_zk(self, version):
+ return self.__basic_test__(version, True)
+
+ def __basic_test__(self, version, with_zk=False):
+ if with_zk:
+ self.zk = ZookeeperService(self.test_context, 3)
+ self.stage("Starting Zookeper quorum")
+ self.zk.start()
+ properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()})
+ self.stage("Zookeper quorum started")
+ else:
+ properties = self.properties()
+
+ self.servers = IgniteService(
+ self.test_context,
+ num_nodes=self.NUM_NODES,
+ properties=properties,
+ version=version)
+
+ self.stage("Starting ignite cluster")
+
+ start = self.monotonic()
+ self.servers.start()
+ data = {'Ignite cluster start time (s)': self.monotonic() - start }
+ self.stage("Topology is ready")
+
+ # Node failure detection
+ fail_node, survived_node = self.choose_random_node_to_kill(self.servers)
+ self.servers.stop_node(fail_node, clean_shutdown=False)
+
+ start = self.monotonic()
+ self.servers.await_event_on_node("Node FAILED", random.choice(survived_node), 60, True)
+
+ data['Failure of node detected in time (s)'] = self.monotonic() - start
+
+ return data
+
+ @staticmethod
+ def choose_random_node_to_kill(service):
+ idx = random.randint(0, len(service.nodes) - 1)
+
+ survive = [node for i, node in enumerate(service.nodes) if i != idx]
+ kill = service.nodes[idx]
+
+ return kill, survive
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
index 1ec2132..a9c13d9 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
@@ -14,6 +14,7 @@
# limitations under the License.
from ducktape.tests.test import Test
+from monotonic import monotonic
class IgniteTest(Test):
@@ -22,3 +23,15 @@ class IgniteTest(Test):
def stage(self, msg):
self.logger.info("[TEST_STAGE] " + msg + "...")
+
+ @staticmethod
+ def monotonic():
+ """
+ monotonic() -> float
+
+ :return:
+ The value (in fractional seconds) of a monotonic clock, i.e. a clock that cannot go backwards.
+ The clock is not affected by system clock updates. The reference point of the returned value is undefined,
+ so that only the difference between the results of consecutive calls is valid.
+ """
+ return monotonic()
diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py
index e82ed2b..2eef514 100644
--- a/modules/ducktests/tests/setup.py
+++ b/modules/ducktests/tests/setup.py
@@ -52,6 +52,6 @@ setup(name="ignitetest",
packages=find_packages(),
include_package_data=True,
install_requires=["ducktape==0.7.7", "requests==2.20.0"],
- tests_require=["pytest", "mock"],
+ tests_require=["pytest", "mock", "monotonic"],
cmdclass={'test': PyTest}
)