You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by GitBox <gi...@apache.org> on 2020/08/10 13:59:50 UTC
[GitHub] [ignite] timoninmaxim commented on a change in pull request #8127: Ignite ducktape control sh
timoninmaxim commented on a change in pull request #8127:
URL: https://github.com/apache/ignite/pull/8127#discussion_r467887216
##########
File path: bin/include/build-classpath.sh
##########
@@ -47,14 +47,13 @@ includeToClassPath() {
for file in $1/*
do
- if [[ -z "${EXCLUDE_MODULES}" ]] || [[ ${EXCLUDE_MODULES} != *"`basename $file`"* ]]; then
- echo "$file included"
+ if [[ -z "${EXCLUDE_MODULES:-}" ]] || [[ ${EXCLUDE_MODULES:-} != *"`basename $file`"* ]]; then
Review comment:
do we really need EXCLUDE_MODULES?
##########
File path: modules/ducktests/tests/ignitetest/services/utils/control_utility.py
##########
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains control utility wrapper.
+"""
+import random
+import re
+from collections import namedtuple
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+
+
+class ControlUtility:
+ """
+ Control utility (control.sh) wrapper.
+ """
+ BASE_COMMAND = "control.sh"
+
+ def __init__(self, cluster, text_context):
+ self._cluster = cluster
+ self.logger = text_context.logger
+
+ def baseline(self):
+ """
+ :return Baseline nodes.
+ """
+ return self.cluster_state().baseline
+
+ def cluster_state(self):
+ """
+ :return: Cluster state.
+ """
+ output = self.__run("--baseline")
+
+ return self.__parse_cluster_state(output)
+
+ def set_baseline(self, baseline):
+ """
+ :param baseline: Baseline nodes or topology version to set as baseline.
+ """
+ if isinstance(baseline, int):
+ result = self.__run("--baseline version %d --yes" % baseline)
+ else:
+ result = self.__run("--baseline set %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in baseline]))
+
+ return self.__parse_cluster_state(result)
+
+ def add_to_baseline(self, nodes):
+ """
+ :param nodes: Nodes that should be added to baseline.
+ """
+ result = self.__run("--baseline add %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in nodes]))
+
+ return self.__parse_cluster_state(result)
+
+ def remove_from_baseline(self, nodes):
+ """
+ :param nodes: Nodes that should be removed to baseline.
+ """
+ result = self.__run("--baseline remove %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in nodes]))
+
+ return self.__parse_cluster_state(result)
+
+ def disable_baseline_auto_adjust(self):
+ """
+ Disable baseline auto adjust.
+ """
+ return self.__run("--baseline auto_adjust disable --yes")
+
+ def enable_baseline_auto_adjust(self, timeout=None):
+ """
+ Enable baseline auto adjust.
+ :param timeout: Auto adjust timeout in millis.
+ """
+ timeout_str = "timeout %d" % timeout if timeout else ""
+ return self.__run("--baseline auto_adjust enable %s --yes" % timeout_str)
+
+ def activate(self):
+ """
+ Activate cluster.
+ """
+ return self.__run("--activate --yes")
+
+ def deactivate(self):
+ """
+ Deactivate cluster.
+ """
+ return self.__run("--deactivate --yes")
+
+ @staticmethod
+ def __parse_cluster_state(output):
+ state_pattern = re.compile("Cluster state: ([^\\s]+)")
+ topology_pattern = re.compile("Current topology version: (\\d+)")
+ baseline_pattern = re.compile("Consistent(Id|ID)=([^\\s]+),\\sS(tate|TATE)=([^\\s]+),?(\\sOrder=(\\d+))?")
+
+ match = state_pattern.search(output)
+ state = match.group(1) if match else None
Review comment:
What do you think, is it better to use named group instead of indices?
https://docs.python.org/3/howto/regex.html#non-capturing-and-named-groups
##########
File path: modules/ducktests/tests/ignitetest/tests/control_utility_test.py
##########
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains control.sh utility tests.
+"""
+from ducktape.mark import parametrize
+from ducktape.mark.resource import cluster
+from ducktape.utils.util import wait_until
+from jinja2 import Template
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.utils.control_utility import ControlUtility, ControlUtilityError
+from ignitetest.tests.utils.ignite_test import IgniteTest
+from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0
+
+
+# pylint: disable=W0223
+class BaselineTests(IgniteTest):
+ """
+ Tests baseline command
+ """
+ NUM_NODES = 3
+
+ CONFIG_TEMPLATE = """
+ {% if version > "2.9.0" %}
+ <property name="clusterStateOnStart" value="INACTIVE"/>
+ {% else %}
+ <property name="activeOnStart" value="false"/>
+ {% endif %}
+ <property name="dataStorageConfiguration">
+ <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+ <property name="defaultDataRegionConfiguration">
+ <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+ <property name="persistenceEnabled" value="true"/>
+ <property name="maxSize" value="#{100L * 1024 * 1024}"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ """
+
+ @staticmethod
+ def properties(version):
+ """
+ Render properties for ignite node configuration.
+ """
+ return Template(BaselineTests.CONFIG_TEMPLATE) \
+ .render(version=version)
+
+ def __init__(self, test_context):
+ super(BaselineTests, self).__init__(test_context)
+ self.servers = None
+
+ @cluster(num_nodes=NUM_NODES)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_8))
+ @parametrize(version=str(LATEST_2_7))
+ def test_baseline_set(self, version):
+ """
+ Test baseline set.
+ """
+ blt_size = self.NUM_NODES - 2
+ self.servers = self.__start_ignite_nodes(version, blt_size)
+
+ control_utility = ControlUtility(self.servers, self.test_context)
+ control_utility.activate()
+
+ # Check baseline of activated cluster.
+ baseline = control_utility.baseline()
+ self.__check_baseline_size(baseline, blt_size)
+ self.__check_nodes_in_baseline(self.servers.nodes, baseline)
+
+ # Set baseline using list of conststent ids.
+ new_node = self.__start_ignite_nodes(version, 1)
+ control_utility.set_baseline(self.servers.nodes + new_node.nodes)
+ blt_size += 1
+
+ baseline = control_utility.baseline()
+ self.__check_baseline_size(baseline, blt_size)
+ self.__check_nodes_in_baseline(new_node.nodes, baseline)
+
+ # Set baseline using topology version.
+ new_node = self.__start_ignite_nodes(version, 1)
+ _, version, _ = control_utility.cluster_state()
+ control_utility.set_baseline(version)
+ blt_size += 1
+
+ baseline = control_utility.baseline()
+ self.__check_baseline_size(baseline, blt_size)
+ self.__check_nodes_in_baseline(new_node.nodes, baseline)
+
+ @cluster(num_nodes=NUM_NODES)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_8))
+ @parametrize(version=str(LATEST_2_7))
+ def test_baseline_add_remove(self, version):
+ """
+ Test add and remove nodes from baseline.
+ """
+ blt_size = self.NUM_NODES - 1
+ self.servers = self.__start_ignite_nodes(version, blt_size)
+
+ control_utility = ControlUtility(self.servers, self.test_context)
+
+ control_utility.activate()
+
+ # Add node to baseline.
+ new_node = self.__start_ignite_nodes(version, 1)
+ control_utility.add_to_baseline(new_node.nodes)
+ blt_size += 1
+
+ baseline = control_utility.baseline()
+ self.__check_baseline_size(baseline, blt_size)
+ self.__check_nodes_in_baseline(new_node.nodes, baseline)
+
+ # Expected failure (remove of online node is not allowed).
+ try:
+ control_utility.remove_from_baseline(new_node.nodes)
+
+ assert False, "Remove of online node from baseline should fail!"
+ except ControlUtilityError:
+ pass
+
+ # Remove of offline node from baseline.
+ new_node.stop()
+
+ self.servers.await_event("Node left topology", timeout_sec=30, from_the_beginning=True)
+
+ control_utility.remove_from_baseline(new_node.nodes)
+ blt_size -= 1
+
+ baseline = control_utility.baseline()
+ self.__check_baseline_size(baseline, blt_size)
+ self.__check_nodes_not_in_baseline(new_node.nodes, baseline)
+
+ @cluster(num_nodes=NUM_NODES)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_8))
+ @parametrize(version=str(LATEST_2_7))
+ def test_activate_deactivate(self, version):
+ """
+ Test activate and deactivate cluster.
+ """
+ self.servers = self.__start_ignite_nodes(version, self.NUM_NODES)
+
+ control_utility = ControlUtility(self.servers, self.test_context)
+
+ control_utility.activate()
+
+ state, _, _ = control_utility.cluster_state()
+
+ assert state.lower() == 'active', 'Unexpected state %s' % state
+
+ control_utility.deactivate()
+
+ state, _, _ = control_utility.cluster_state()
+
+ assert state.lower() == 'inactive', 'Unexpected state %s' % state
+
+ @cluster(num_nodes=NUM_NODES)
+ @parametrize(version=str(DEV_BRANCH))
+ @parametrize(version=str(LATEST_2_8))
+ def test_baseline_autoadjust(self, version):
+ """
+ Test activate and deactivate cluster.
+ """
+ if version < V_2_8_0:
Review comment:
This condition is not part of test itself, let's make decorator from that. Otherwise we should not used it at all, as every test should be marked this way.
##########
File path: modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
##########
@@ -156,10 +164,9 @@ def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backof
:param backoff_sec: Number of seconds to back off between each failure to meet the condition
before checking again.
"""
- assert len(self.nodes) == 1
Review comment:
Was there a reason to have the assert?
##########
File path: modules/ducktests/tests/ignitetest/services/utils/control_utility.py
##########
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains control utility wrapper.
+"""
+import random
+import re
+from collections import namedtuple
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+
+
+class ControlUtility:
+ """
+ Control utility (control.sh) wrapper.
+ """
+ BASE_COMMAND = "control.sh"
+
+ def __init__(self, cluster, text_context):
+ self._cluster = cluster
+ self.logger = text_context.logger
+
+ def baseline(self):
+ """
+ :return Baseline nodes.
+ """
+ return self.cluster_state().baseline
+
+ def cluster_state(self):
+ """
+ :return: Cluster state.
+ """
+ output = self.__run("--baseline")
+
+ return self.__parse_cluster_state(output)
+
+ def set_baseline(self, baseline):
+ """
+ :param baseline: Baseline nodes or topology version to set as baseline.
+ """
+ if isinstance(baseline, int):
+ result = self.__run("--baseline version %d --yes" % baseline)
+ else:
+ result = self.__run("--baseline set %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in baseline]))
+
+ return self.__parse_cluster_state(result)
+
+ def add_to_baseline(self, nodes):
+ """
+ :param nodes: Nodes that should be added to baseline.
+ """
+ result = self.__run("--baseline add %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in nodes]))
+
+ return self.__parse_cluster_state(result)
+
+ def remove_from_baseline(self, nodes):
+ """
+ :param nodes: Nodes that should be removed to baseline.
+ """
+ result = self.__run("--baseline remove %s --yes" %
+ ",".join([node.account.externally_routable_ip for node in nodes]))
+
+ return self.__parse_cluster_state(result)
+
+ def disable_baseline_auto_adjust(self):
+ """
+ Disable baseline auto adjust.
+ """
+ return self.__run("--baseline auto_adjust disable --yes")
+
+ def enable_baseline_auto_adjust(self, timeout=None):
+ """
+ Enable baseline auto adjust.
+ :param timeout: Auto adjust timeout in millis.
+ """
+ timeout_str = "timeout %d" % timeout if timeout else ""
+ return self.__run("--baseline auto_adjust enable %s --yes" % timeout_str)
+
+ def activate(self):
+ """
+ Activate cluster.
+ """
+ return self.__run("--activate --yes")
+
+ def deactivate(self):
+ """
+ Deactivate cluster.
+ """
+ return self.__run("--deactivate --yes")
+
+ @staticmethod
+ def __parse_cluster_state(output):
+ state_pattern = re.compile("Cluster state: ([^\\s]+)")
+ topology_pattern = re.compile("Current topology version: (\\d+)")
+ baseline_pattern = re.compile("Consistent(Id|ID)=([^\\s]+),\\sS(tate|TATE)=([^\\s]+),?(\\sOrder=(\\d+))?")
+
+ match = state_pattern.search(output)
+ state = match.group(1) if match else None
+
+ match = topology_pattern.search(output)
+ topology = int(match.group(1)) if match else None
+
+ baseline = [BaselineNode(consistent_id=m[1], state=m[3], order=int(m[5]) if m[5] else None)
+ for m in baseline_pattern.findall(output)]
+
+ return ClusterState(state=state, topology_version=topology, baseline=baseline)
+
+ def __run(self, cmd):
+ node = random.choice(self.__alives())
+
+ self.logger.debug("Run command %s on node %s", cmd, node.name)
+
+ raw_output = node.account.ssh_capture(self.__form_cmd(node, cmd), allow_fail=True)
+ code, output = self.__parse_output(raw_output)
+
+ self.logger.debug("Output of command %s on node %s, exited with code %d, is %s", cmd, node.name, code, output)
+
+ if code != 0:
+ raise ControlUtilityError(node.account, cmd, code, output)
+
+ return output
+
+ def __form_cmd(self, node, cmd):
+ return self._cluster.path.script("%s --host %s %s" % (self.BASE_COMMAND, node.account.externally_routable_ip,
+ cmd))
+
+ @staticmethod
+ def __parse_output(raw_output):
+ exit_code = raw_output.channel_file.channel.recv_exit_status()
+ output = "".join(raw_output)
+
+ pattern = re.compile("Command \\[[^\\s]*\\] finished with code: (\\d+)")
+ match = pattern.search(output)
+
+ if match:
+ return int(match.group(1)), output
Review comment:
Is it ok, that it's possible to receive `exit_code !=0` but in the same time text output will have code equals to 0?
##########
File path: modules/ducktests/tests/ignitetest/services/utils/control_utility.py
##########
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains control utility wrapper.
+"""
+import random
+import re
+from collections import namedtuple
+
+from ducktape.cluster.remoteaccount import RemoteCommandError
+
+
+class ControlUtility:
+ """
+ Control utility (control.sh) wrapper.
+ """
+ BASE_COMMAND = "control.sh"
+
+ def __init__(self, cluster, text_context):
+ self._cluster = cluster
+ self.logger = text_context.logger
+
+ def baseline(self):
+ """
+ :return Baseline nodes.
+ """
+ return self.cluster_state().baseline
+
+ def cluster_state(self):
+ """
+ :return: Cluster state.
+ """
+ output = self.__run("--baseline")
+
+ return self.__parse_cluster_state(output)
Review comment:
Use 'result' instead of 'output', as it's used in other methods. Or vice versa
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org