You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2020/07/21 09:41:00 UTC

[ignite] branch ignite-ducktape updated: Add zookeeper service and basic topology test to ignite-ducktests (#8059)

This is an automated email from the ASF dual-hosted git repository.

av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/ignite-ducktape by this push:
     new ea7fcd5  Add zookeeper service and basic topology test to ignite-ducktests (#8059)
ea7fcd5 is described below

commit ea7fcd548743171e11d74ae27b9154793b8f871f
Author: Ivan Daschinskiy <iv...@gmail.com>
AuthorDate: Tue Jul 21 12:40:33 2020 +0300

    Add zookeeper service and basic topology test to ignite-ducktests (#8059)
---
 modules/ducktests/pom.xml                          |  36 ++++++
 modules/ducktests/tests/docker/Dockerfile          |  23 +++-
 modules/ducktests/tests/docker/run_tests.sh        |   2 +-
 .../ignite_test.py => services/zk/__init__.py}     |  10 --
 .../services/zk/templates/log4j.properties.j2      |  33 ++++++
 .../services/zk/templates/zookeeper.properties.j2  |  25 ++++
 .../tests/ignitetest/services/zk/zookeeper.py      | 127 +++++++++++++++++++++
 .../tests/benchmarks/add_node_rebalance_test.py    |   5 +-
 .../tests/ignitetest/tests/discovery_test.py       | 120 +++++++++++++++++++
 .../tests/ignitetest/tests/utils/ignite_test.py    |  13 +++
 modules/ducktests/tests/setup.py                   |   2 +-
 11 files changed, 376 insertions(+), 20 deletions(-)

diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml
index 81c73ca..7bfdc2c 100644
--- a/modules/ducktests/pom.xml
+++ b/modules/ducktests/pom.xml
@@ -51,12 +51,48 @@
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-spark</artifactId>
             <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-recipes</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-framework</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.11</artifactId>
             <version>${spark.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-recipes</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-framework</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile
index b199c37..0baefda 100644
--- a/modules/ducktests/tests/docker/Dockerfile
+++ b/modules/ducktests/tests/docker/Dockerfile
@@ -35,7 +35,7 @@ LABEL ducker.creator=$ducker_creator
 RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list
 RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean
 RUN python -m pip install -U pip==9.0.3;
-RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7
+RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.7
 
 # Set up ssh
 COPY ./ssh-config /root/.ssh/config
@@ -47,12 +47,25 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/"
 ARG APACHE_ARCHIVE="https://archive.apache.org/dist/"
 
 # Install binary test dependencies.
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.7.6/apache-ignite-2.7.6-bin.zip && unzip apache-ignite-2.7.6-bin.zip && mv /opt/apache-ignite-2.7.6-bin /opt/ignite-2.7.6
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.0/apache-ignite-2.8.0-bin.zip && unzip apache-ignite-2.8.0-bin.zip && mv /opt/apache-ignite-2.8.0-bin /opt/ignite-2.8.0
-RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.1/apache-ignite-2.8.1-bin.zip && unzip apache-ignite-2.8.1-bin.zip && mv /opt/apache-ignite-2.8.1-bin /opt/ignite-2.8.1
+RUN for v in "2.7.6" "2.8.0" "2.8.1"; \
+    do cd /opt; \
+    curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\
+    unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\
+    cp -r ./ignite-$v/libs/optional/ignite-zookeeper ./ignite-$v/libs; \
+    done
 
 RUN rm /opt/apache-ignite-*-bin.zip
 
+#Install zookeeper.
+ARG ZOOKEEPER_VERSION="3.5.8"
+ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION"
+ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin"
+ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz"
+RUN echo $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT
+RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \
+ && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT
+RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME
+
 # Install spark
 ARG SPARK_VERSION="2.3.4"
 ARG SPARK_NAME="spark-$SPARK_VERSION"
@@ -77,4 +90,4 @@ USER ducker
 CMD sudo service ssh start && tail -f /dev/null
 
 # Container port exposure
-EXPOSE 11211 47100 47500 49112 10800 8080
+EXPOSE 11211 47100 47500 49112 10800 8080 2888 3888 2181
diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh
index 477ec82..e392e3d 100755
--- a/modules/ducktests/tests/docker/run_tests.sh
+++ b/modules/ducktests/tests/docker/run_tests.sh
@@ -16,7 +16,7 @@
 # limitations under the License.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-6}
+IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-11}
 TC_PATHS=${TC_PATHS:-./ignitetest/}
 
 die() {
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/services/zk/__init__.py
similarity index 75%
copy from modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
copy to modules/ducktests/tests/ignitetest/services/zk/__init__.py
index 1ec2132..ec20143 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/services/zk/__init__.py
@@ -12,13 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from ducktape.tests.test import Test
-
-
-class IgniteTest(Test):
-    def __init__(self, test_context):
-        super(IgniteTest, self).__init__(test_context=test_context)
-
-    def stage(self, msg):
-        self.logger.info("[TEST_STAGE] " + msg + "...")
diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2
new file mode 100644
index 0000000..507ec98
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2
@@ -0,0 +1,33 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+zookeeper.root.logger=INFO, FILE
+zookeeper.console.threshold=INFO
+
+zookeeper.log.dir={{ PERSISTENT_ROOT }}
+zookeeper.log.file=zookeeper.log
+zookeeper.log.threshold=INFO
+zookeeper.log.maxfilesize=256MB
+zookeeper.log.maxbackupindex=20
+
+log4j.rootLogger=${zookeeper.root.logger}
+
+log4j.appender.FILE=org.apache.log4j.FileAppender
+log4j.appender.FILE.Threshold=${zookeeper.log.threshold}
+log4j.appender.FILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
+log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2
new file mode 100644
index 0000000..2c80e84
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2
@@ -0,0 +1,25 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+tickTime={{ settings.tick_time }}
+initLimit={{ settings.init_limit }}
+syncLimit={{ settings.sync_limit }}
+dataDir={{ DATA_DIR }}
+clientPort={{ settings.client_port }}
+{% for node in nodes %}
+server.{{ loop.index }}={{ node.account.hostname }}:2888:3888
+{% endfor %}
diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
new file mode 100644
index 0000000..7767ce9
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+from ducktape.services.service import Service
+
+
+class ZookeeperSettings:
+    def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181):
+        self.tick_time = tick_time
+        self.init_limit = init_limit
+        self.sync_limit = sync_limit
+        self.client_port = client_port
+
+
+class ZookeeperService(Service):
+    PERSISTENT_ROOT = "/mnt/zookeeper"
+    CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf")
+    LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log")
+    DATA_DIR = os.path.join(PERSISTENT_ROOT, "data")
+    CONFIG_FILE = os.path.join(CONFIG_ROOT, "zookeeper.properties")
+    LOG_CONFIG_FILE = os.path.join(CONFIG_ROOT, "log4j.properties")
+    ZK_LIB_DIR = "/opt/zookeeper-3.5.8/lib"
+
+    logs = {
+        "zk_log": {
+            "path": LOG_FILE,
+            "collect_default": True
+        }
+    }
+
+    def __init__(self, context, num_nodes, settings=ZookeeperSettings()):
+        super(ZookeeperService, self).__init__(context, num_nodes)
+        self.settings = settings
+
+    def start(self, timeout_sec=60):
+        Service.start(self)
+        self.logger.info("Waiting for Zookeeper quorum...")
+
+        for node in self.nodes:
+            self.await_quorum(node, timeout_sec)
+
+        self.logger.info("Zookeeper quorum is formed.")
+
+    def start_node(self, node):
+        idx = self.idx(node)
+
+        self.logger.info("Starting Zookeeper node %d on %s", idx, node.account.hostname)
+
+        node.account.ssh("mkdir -p %s" % self.DATA_DIR)
+        node.account.ssh("mkdir -p %s" % self.CONFIG_ROOT)
+        node.account.ssh("echo %d > %s/myid" % (idx, self.DATA_DIR))
+
+        config_file = self.render('zookeeper.properties.j2', settings=self.settings)
+        node.account.create_file(self.CONFIG_FILE, config_file)
+        self.logger.info("ZK config %s", config_file)
+
+        log_config_file = self.render('log4j.properties.j2')
+        node.account.create_file(self.LOG_CONFIG_FILE, log_config_file)
+
+        start_cmd = "nohup java -cp %s/*:%s org.apache.zookeeper.server.quorum.QuorumPeerMain %s >/dev/null 2>&1 &" % \
+                    (self.ZK_LIB_DIR, self.CONFIG_ROOT, self.CONFIG_FILE)
+
+        node.account.ssh(start_cmd)
+
+    def wait_node(self, node, timeout_sec=20):
+        idx = self.idx(node)
+
+        with node.account.monitor_log(self.LOG_FILE) as monitor:
+            monitor.offset = 0
+            monitor.wait_until(
+                "binding to port",
+                timeout_sec=timeout_sec,
+                err_msg="Zookeeper service didn't finish startup on %s" % node.account.hostname
+            )
+
+        self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname)
+
+    def await_quorum(self, node, timeout):
+        with node.account.monitor_log(self.LOG_FILE) as monitor:
+            monitor.offset = 0
+            monitor.wait_until(
+                "LEADER ELECTION TOOK",
+                timeout_sec=timeout,
+                err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname
+            )
+
+    def pids(self, node):
+        try:
+            cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'"
+            pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
+            return pid_arr
+        except (RemoteCommandError, ValueError) as e:
+            return []
+
+    def alive(self, node):
+        return len(self.pids(node)) > 0
+
+    def connection_string(self):
+        return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes])
+
+    def stop_node(self, node):
+        idx = self.idx(node)
+        self.logger.info("Stopping %s node %d on %s" % (type(self).__name__, idx, node.account.hostname))
+        node.account.kill_process("zookeeper", allow_fail=False)
+
+    def clean_node(self, node):
+        self.logger.info("Cleaning Zookeeper node %d on %s", self.idx(node), node.account.hostname)
+        if self.alive(node):
+            self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." %
+                             (self.__class__.__name__, node.account))
+        node.account.kill_process("zookeeper", clean_shutdown=False, allow_fail=True)
+        node.account.ssh("rm -rf %s %s %s" % (self.CONFIG_ROOT, self.DATA_DIR, self.LOG_FILE), allow_fail=False)
diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
index 2c725d7..ba5b324 100644
--- a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import time
 
 from ducktape.mark import parametrize
 from ducktape.mark.resource import cluster
@@ -82,13 +81,13 @@ class AddNodeRebalanceTest(IgniteTest):
 
         ignite.start()
 
-        start = time.time()
+        start = self.monotonic()
 
         ignite.await_event("rebalanced=true, wasRebalanced=false",
                            timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT,
                            from_the_beginning=True,
                            backoff_sec=1)
 
-        data = {"Rebalanced in (sec)": time.time() - start}
+        data = {"Rebalanced in (sec)": self.monotonic() - start}
 
         return data
diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
new file mode 100644
index 0000000..42228e3
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+
+from ducktape.mark import parametrize
+from ducktape.mark.resource import cluster
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.zk.zookeeper import ZookeeperService
+from ignitetest.version import DEV_BRANCH, LATEST_2_7
+from ignitetest.tests.utils.ignite_test import IgniteTest
+
+from jinja2 import Template
+
+class DiscoveryTest(IgniteTest):
+    NUM_NODES = 7
+
+    CONFIG_TEMPLATE = """
+        <property name="clientMode" value="{{ client_mode | lower }}"/>
+        {% if zookeeper_settings %}
+        {% with zk = zookeeper_settings %}
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi">
+                <property name="zkConnectionString" value="{{ zk.connection_string }}"/>
+                <property name="sessionTimeout" value="{{ zk.session_timeout or 3000 }}"/>
+                <property name="zkRootPath" value="{{ zk.root_path or '/apacheIgnite' }}"/>
+                <property name="joinTimeout" value="{{ zk.join_timeout or 10000 }}"/>
+            </bean>
+        </property>
+        {% endwith %}
+        {% endif %}
+    """
+
+    def __init__(self, test_context):
+        super(DiscoveryTest, self).__init__(test_context=test_context)
+        self.zk = None
+        self.servers = None
+
+    @staticmethod
+    def properties(client_mode="false", zookeeper_settings=None):
+        return Template(DiscoveryTest.CONFIG_TEMPLATE) \
+            .render(client_mode=client_mode, zookeeper_settings=zookeeper_settings)
+
+    def setUp(self):
+        pass
+
+    def teardown(self):
+        if self.zk:
+            self.zk.stop()
+
+        if self.servers:
+            self.servers.stop()
+
+    @cluster(num_nodes=NUM_NODES)
+    @parametrize(version=str(DEV_BRANCH))
+    @parametrize(version=str(LATEST_2_7))
+    def test_tcp(self, version):
+        return self.__basic_test__(version, False)
+
+    @cluster(num_nodes=NUM_NODES + 3)
+    @parametrize(version=str(DEV_BRANCH))
+    @parametrize(version=str(LATEST_2_7))
+    def test_zk(self, version):
+        return self.__basic_test__(version, True)
+
+    def __basic_test__(self, version, with_zk=False):
+        if with_zk:
+            self.zk = ZookeeperService(self.test_context, 3)
+            self.stage("Starting Zookeper quorum")
+            self.zk.start()
+            properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()})
+            self.stage("Zookeper quorum started")
+        else:
+            properties = self.properties()
+
+        self.servers = IgniteService(
+            self.test_context,
+            num_nodes=self.NUM_NODES,
+            properties=properties,
+            version=version)
+
+        self.stage("Starting ignite cluster")
+
+        start = self.monotonic()
+        self.servers.start()
+        data = {'Ignite cluster start time (s)': self.monotonic() - start }
+        self.stage("Topology is ready")
+
+        # Node failure detection
+        fail_node, survived_node = self.choose_random_node_to_kill(self.servers)
+        self.servers.stop_node(fail_node, clean_shutdown=False)
+
+        start = self.monotonic()
+        self.servers.await_event_on_node("Node FAILED", random.choice(survived_node), 60, True)
+
+        data['Failure of node detected in time (s)'] = self.monotonic() - start
+
+        return data
+
+    @staticmethod
+    def choose_random_node_to_kill(service):
+        idx = random.randint(0, len(service.nodes) - 1)
+
+        survive = [node for i, node in enumerate(service.nodes) if i != idx]
+        kill = service.nodes[idx]
+
+        return kill, survive
diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
index 1ec2132..a9c13d9 100644
--- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from ducktape.tests.test import Test
+from monotonic import monotonic
 
 
 class IgniteTest(Test):
@@ -22,3 +23,15 @@ class IgniteTest(Test):
 
     def stage(self, msg):
         self.logger.info("[TEST_STAGE] " + msg + "...")
+
+    @staticmethod
+    def monotonic():
+        """
+        monotonic() -> float
+
+        :return:
+            The value (in fractional seconds) of a monotonic clock, i.e. a clock that cannot go backwards.
+            The clock is not affected by system clock updates. The reference point of the returned value is undefined,
+            so that only the difference between the results of consecutive calls is valid.
+        """
+        return monotonic()
diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py
index e82ed2b..2eef514 100644
--- a/modules/ducktests/tests/setup.py
+++ b/modules/ducktests/tests/setup.py
@@ -52,6 +52,6 @@ setup(name="ignitetest",
       packages=find_packages(),
       include_package_data=True,
       install_requires=["ducktape==0.7.7", "requests==2.20.0"],
-      tests_require=["pytest", "mock"],
+      tests_require=["pytest", "mock", "monotonic"],
       cmdclass={'test': PyTest}
 )