You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2020/07/23 12:02:14 UTC
[ignite] branch ignite-ducktape updated: Add jmx client and some
basic discovery stuff fo IgniteClusterNode. (#8070)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new 57aa778 Add jmx client and some basic discovery stuff fo IgniteClusterNode. (#8070)
57aa778 is described below
commit 57aa7780f34167f17b5365b5a77d5478d0d5daef
Author: Ivan Daschinskiy <iv...@gmail.com>
AuthorDate: Thu Jul 23 15:02:00 2020 +0300
Add jmx client and some basic discovery stuff fo IgniteClusterNode. (#8070)
---
modules/ducktests/tests/docker/Dockerfile | 7 ++
.../ducktests/tests/ignitetest/services/ignite.py | 9 +-
.../tests/ignitetest/services/ignite_app.py | 15 +--
.../tests/ignitetest/services/ignite_spark_app.py | 4 +-
.../{ignite_spark_app.py => utils/decorators.py} | 27 ++---
.../ignitetest/services/utils/ignite_aware.py | 11 +-
.../ignitetest/services/utils/ignite_aware_app.py | 5 +-
.../tests/ignitetest/services/utils/jmx_utils.py | 124 +++++++++++++++++++++
.../tests/ignitetest/services/zk/zookeeper.py | 14 +--
.../tests/ignitetest/tests/discovery_test.py | 19 +++-
10 files changed, 190 insertions(+), 45 deletions(-)
diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile
index 0baefda..fedea54 100644
--- a/modules/ducktests/tests/docker/Dockerfile
+++ b/modules/ducktests/tests/docker/Dockerfile
@@ -83,6 +83,13 @@ ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a"
RUN apt-get install fuse
RUN cd /opt && git clone -q https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2
+#Install jmxterm
+ARG JMXTERM_NAME="jmxterm"
+ARG JMXTERM_VERSION="1.0.1"
+ARG JMXTERM_ARTIFACT="$JMXTERM_NAME-$JMXTERM_VERSION-uber.jar"
+RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMXTERM_VERSION/$JMXTERM_ARTIFACT \
+ && mv $JMXTERM_ARTIFACT $JMXTERM_NAME.jar
+
# Set up the ducker user.
RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
USER ducker
diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py
index a2855e3..e68d0bb 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite.py
@@ -17,7 +17,6 @@ import os.path
import signal
from ducktape.cluster.remoteaccount import RemoteCommandError
-from ducktape.services.service import Service
from ducktape.utils.util import wait_until
from ignitetest.services.utils.ignite_aware import IgniteAwareService
@@ -39,15 +38,15 @@ class IgniteService(IgniteAwareService):
}
def __init__(self, context, num_nodes, version=DEV_BRANCH, properties=""):
- IgniteAwareService.__init__(self, context, num_nodes, version, properties)
+ super(IgniteService, self).__init__(context, num_nodes, version, properties)
def start(self, timeout_sec=180):
- Service.start(self)
+ super(IgniteService, self).start()
self.logger.info("Waiting for Ignite(s) to start...")
for node in self.nodes:
- self.await_node_stated(node, timeout_sec)
+ self.await_node_started(node, timeout_sec)
def start_cmd(self, node):
jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file "
@@ -64,7 +63,7 @@ class IgniteService(IgniteAwareService):
IgniteService.STDOUT_STDERR_CAPTURE)
return cmd
- def await_node_stated(self, node, timeout_sec):
+ def await_node_started(self, node, timeout_sec):
self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True)
if len(self.pids(node)) == 0:
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py
index 87cb8e1..237436d 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_app.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py
@@ -24,15 +24,8 @@ The Ignite application service allows to perform custom logic writen on java.
class IgniteApplicationService(IgniteAwareApplicationService):
- def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60):
- IgniteAwareApplicationService.__init__(
- self, context, java_class_name, version, properties, params, timeout_sec,
- service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteApplicationService")
-
- def start(self):
- Service.start(self)
+ service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService"
- self.logger.info("Waiting for Ignite Application (%s) to start..." % self.java_class_name)
-
- self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True)
- self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True)
+ def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60):
+ super(IgniteApplicationService, self).__init__(context, java_class_name, version, properties, params,
+ timeout_sec, self.service_java_class_name)
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
index dffe7ac..c2f0496 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
@@ -22,8 +22,8 @@ from ignitetest.version import DEV_BRANCH
class SparkIgniteApplicationService(IgniteAwareApplicationService):
def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60):
- IgniteAwareApplicationService.__init__(
- self, context, java_class_name, version, properties, params, timeout_sec)
+ super(SparkIgniteApplicationService, self).__init__(context, java_class_name, version, properties, params,
+ timeout_sec)
def env(self):
return IgniteAwareApplicationService.env(self) + \
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py
similarity index 52%
copy from modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
copy to modules/ducktests/tests/ignitetest/services/utils/decorators.py
index dffe7ac..788ddba 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py
@@ -12,20 +12,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import functools
+from threading import RLock
-"""
-The Ignite-Spark application service.
-"""
-from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService
-from ignitetest.version import DEV_BRANCH
+def memoize(func):
+ cache = func.cache = {}
+ lock = RLock()
-class SparkIgniteApplicationService(IgniteAwareApplicationService):
- def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60):
- IgniteAwareApplicationService.__init__(
- self, context, java_class_name, version, properties, params, timeout_sec)
+ @functools.wraps(func)
+ def memoized_func(*args, **kwargs):
+ key = str(args) + str(kwargs)
+ if key not in cache:
+ with lock:
+ if key not in cache:
+ cache[key] = func(*args, **kwargs)
+ return cache[key]
- def env(self):
- return IgniteAwareApplicationService.env(self) + \
- "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \
- "rocketmq\"; "
+ return memoized_func
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index dd82feb..106493c 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -12,14 +12,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import os
from abc import abstractmethod
from ducktape.services.background_thread import BackgroundThreadService
+from ducktape.utils.util import wait_until
from ignitetest.services.utils.ignite_config import IgniteConfig
from ignitetest.services.utils.ignite_path import IgnitePath
+from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin
"""
The base class to build services aware of Ignite.
@@ -41,7 +42,7 @@ class IgniteAwareService(BackgroundThreadService):
}
def __init__(self, context, num_nodes, version, properties):
- BackgroundThreadService.__init__(self, context, num_nodes)
+ super(IgniteAwareService, self).__init__(context, num_nodes)
self.log_level = "DEBUG"
self.config = IgniteConfig()
@@ -55,7 +56,11 @@ class IgniteAwareService(BackgroundThreadService):
def start_node(self, node):
self.init_persistent(node)
- BackgroundThreadService.start_node(self, node)
+ super(IgniteAwareService, self).start_node(node)
+
+ wait_until(lambda: len(self.pids(node)) > 0, timeout_sec=10)
+
+ ignite_jmx_mixin(node, self.pids(node))
def init_persistent(self, node):
node.account.mkdirs(self.PERSISTENT_ROOT)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
index 916d155..7b37633 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
@@ -26,7 +26,7 @@ The base class to build Ignite aware application written on java.
class IgniteAwareApplicationService(IgniteAwareService):
def __init__(self, context, java_class_name, version, properties, params, timeout_sec,
service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"):
- IgniteAwareService.__init__(self, context, 1, version, properties)
+ super(IgniteAwareApplicationService, self).__init__(context, 1, version, properties)
self.servicejava_class_name = service_java_class_name
self.java_class_name = java_class_name
@@ -35,10 +35,11 @@ class IgniteAwareApplicationService(IgniteAwareService):
self.params = params
def start(self):
- Service.start(self)
+ super(IgniteAwareApplicationService, self).start()
self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name)
+ self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True)
self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True)
def start_cmd(self, node):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
new file mode 100644
index 0000000..6843c94
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py
@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+from ignitetest.services.utils.decorators import memoize
+
+
+def ignite_jmx_mixin(node, pids):
+ setattr(node, 'pids', pids)
+ base_cls = node.__class__
+ base_cls_name = node.__class__.__name__
+ node.__class__ = type(base_cls_name, (base_cls, IgniteJmxMixin), {})
+
+
+class JmxMBean(object):
+ def __init__(self, client, name):
+ self.client = client
+ self.name = name
+
+ def __getattr__(self, attr):
+ return self.client.mbean_attribute(self.name, attr)
+
+
+class JmxClient(object):
+ jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n'
+
+ def __init__(self, node):
+ self.node = node
+ self.pid = node.pids[0]
+
+ @memoize
+ def find_mbean(self, pattern, domain='org.apache'):
+ cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \
+ % (self.pid, domain, self.jmx_util_cmd, pattern)
+
+ name = next(self.run_cmd(cmd)).strip()
+
+ return JmxMBean(self, name)
+
+ def mbean_attribute(self, mbean, attr):
+ cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \
+ % (self.pid, mbean, attr, self.jmx_util_cmd, attr)
+
+ return iter(s.strip() for s in self.run_cmd(cmd))
+
+ def run_cmd(self, cmd):
+ return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str)
+
+
+class DiscoveryInfo(object):
+ def __init__(self, coordinator, local_raw):
+ self._local_raw = local_raw
+ self._coordinator = coordinator
+
+ @property
+ def id(self):
+ return self.__find__("id=([^\\s]+),")
+
+ @property
+ def coordinator(self):
+ return self._coordinator
+
+ @property
+ def consistent_id(self):
+ return self.__find__("consistentId=([^\\s]+),")
+
+ @property
+ def is_client(self):
+ return self.__find__("isClient=([^\\s]+),") == "true"
+
+ @property
+ def order(self):
+ return int(self.__find__("order=(\\d+),"))
+
+ @property
+ def int_order(self):
+ val = self.__find__("intOrder=(\\d+),")
+ return int(val) if val else -1
+
+ def __find__(self, pattern):
+ res = re.search(pattern, self._local_raw)
+ return res.group(1) if res else None
+
+
+class IgniteJmxMixin(object):
+ @memoize
+ def jmx_client(self):
+ # noinspection PyTypeChecker
+ return JmxClient(self)
+
+ @memoize
+ def id(self):
+ return next(self.kernal_mbean().LocalNodeId).strip()
+
+ def discovery_info(self):
+ disco_mbean = self.disco_mbean()
+ crd = next(disco_mbean.Coordinator).strip()
+ local = next(disco_mbean.LocalNodeFormatted).strip()
+
+ return DiscoveryInfo(crd, local)
+
+ def kernal_mbean(self):
+ return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal')
+
+ @memoize
+ def disco_mbean(self):
+ disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip()
+
+ if 'ZookeeperDiscoverySpi' in disco_spi:
+ return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi')
+ else:
+ return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi')
diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
index 7767ce9..d32cbbc 100644
--- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
+++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
@@ -12,10 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import os.path
-from ducktape.cluster.remoteaccount import RemoteCommandError
from ducktape.services.service import Service
@@ -99,13 +97,13 @@ class ZookeeperService(Service):
err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname
)
+ @staticmethod
+ def java_class_name():
+ """ The class name of the Zookeeper quorum peers. """
+ return "org.apache.zookeeper.server.quorum.QuorumPeerMain"
+
def pids(self, node):
- try:
- cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'"
- pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
- return pid_arr
- except (RemoteCommandError, ValueError) as e:
- return []
+ return node.account.java_pids(self.java_class_name())
def alive(self, node):
return len(self.pids(node)) > 0
diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
index 42228e3..dd25205 100644
--- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import random
from ducktape.mark import parametrize
@@ -25,6 +24,7 @@ from ignitetest.tests.utils.ignite_test import IgniteTest
from jinja2 import Template
+
class DiscoveryTest(IgniteTest):
NUM_NODES = 7
@@ -101,6 +101,23 @@ class DiscoveryTest(IgniteTest):
# Node failure detection
fail_node, survived_node = self.choose_random_node_to_kill(self.servers)
+
+ data["nodes"] = [node.id() for node in self.servers.nodes]
+
+ disco_infos = []
+ for node in self.servers.nodes:
+ disco_info = node.discovery_info()
+ disco_infos.append({
+ "id": disco_info.id,
+ "consistent_id": disco_info.consistent_id,
+ "coordinator": disco_info.coordinator,
+ "order": disco_info.order,
+ "int_order": disco_info.int_order,
+ "is_client": disco_info.is_client
+ })
+
+ data["node_disco_info"] = disco_infos
+
self.servers.stop_node(fail_node, clean_shutdown=False)
start = self.monotonic()