You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2021/04/14 09:37:50 UTC
[ignite] branch ignite-ducktape updated: IGNITE-14503 Ability to
specify project (fork) via the Version (#8996)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new a98a4b3 IGNITE-14503 Ability to specify project (fork) via the Version (#8996)
a98a4b3 is described below
commit a98a4b30d8355a6a7de6b0477b19810786f46fa1
Author: Anton Vinogradov <av...@apache.org>
AuthorDate: Wed Apr 14 12:37:27 2021 +0300
IGNITE-14503 Ability to specify project (fork) via the Version (#8996)
---
.../tests/checks/utils/check_dev_version.py | 51 -----------
.../tests/checks/utils/check_parametrized.py | 34 ++++++-
.../ducktests/tests/checks/utils/check_version.py | 101 +++++++++++++++++++++
.../ducktests/tests/ignitetest/services/spark.py | 8 +-
.../ignitetest/services/utils/ignite_aware.py | 8 +-
.../tests/ignitetest/services/utils/ignite_spec.py | 26 +++---
.../tests/ignitetest/services/utils/path.py | 26 ++----
.../tests/ignitetest/services/zk/zookeeper.py | 8 +-
.../ignitetest/tests/cellular_affinity_test.py | 3 +-
modules/ducktests/tests/ignitetest/utils/_mark.py | 14 ++-
.../ducktests/tests/ignitetest/utils/version.py | 37 ++++++--
11 files changed, 203 insertions(+), 113 deletions(-)
diff --git a/modules/ducktests/tests/checks/utils/check_dev_version.py b/modules/ducktests/tests/checks/utils/check_dev_version.py
deleted file mode 100644
index d991556..0000000
--- a/modules/ducktests/tests/checks/utils/check_dev_version.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Checks DEV_BRANCH version.
-"""
-
-from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST
-from ignitetest import __version__
-
-
-# pylint: disable=no-self-use
-class CheckDevVersion:
- """
- Checks developer version.
- """
- def check_dev_version(self):
- """"
- Check developer version.
- """
- dev = IgniteVersion('dev')
-
- assert DEV_BRANCH == dev
- assert DEV_BRANCH.version == dev.version
-
- index = __version__.find('-')
-
- if index > 0:
- ver = IgniteVersion(__version__[:index])
-
- assert dev > ver
- assert dev.version > ver.version
-
- assert dev.is_dev
-
- assert str(dev) == 'dev'
-
- assert dev > LATEST
- assert dev.version > LATEST.version
diff --git a/modules/ducktests/tests/checks/utils/check_parametrized.py b/modules/ducktests/tests/checks/utils/check_parametrized.py
index dd87907a..a108543 100644
--- a/modules/ducktests/tests/checks/utils/check_parametrized.py
+++ b/modules/ducktests/tests/checks/utils/check_parametrized.py
@@ -17,14 +17,14 @@
Checks custom parametrizers.
"""
-import itertools
from unittest.mock import Mock
+import itertools
import pytest
from ducktape.mark import parametrized, parametrize, matrix, ignore
from ducktape.mark.mark_expander import MarkedFunctionExpander
-from ignitetest.utils import ignite_versions, ignore_if
+from ignitetest.utils import ignite_versions, ignore_if
from ignitetest.utils._mark import IgniteVersionParametrize
from ignitetest.utils.version import IgniteVersion, V_2_8_0, V_2_8_1, V_2_7_6, DEV_BRANCH
@@ -138,6 +138,36 @@ class CheckIgniteVersions:
self._check_injection(context_list, versions=versions, global_args=global_args, pairs=True)
+ @pytest.mark.parametrize(
+ ['versions', 'global_args', 'result'],
+ [pytest.param(['2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'dev', 'ignite-dev', 'fork-dev'],
+ {'project': 'superfork'},
+ ['superfork-2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'superfork-dev', 'ignite-dev', 'fork-dev']),
+ pytest.param(['2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'dev', 'ignite-dev', 'fork-dev'],
+ {}, # project: ignite (default)
+ ['ignite-2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'ignite-dev', 'ignite-dev', 'fork-dev']),
+ pytest.param(['10.4.42', '0.6.53', 'fork-me'], # ignored
+ {'project': 'superfork',
+ 'ignite_versions': ['2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'dev', 'ignite-dev', 'fork-dev']},
+ ['superfork-2.9.9', 'ignite-2.9.9', 'fork-2.9.9', 'superfork-dev', 'ignite-dev', 'fork-dev'])])
+ def check_project_injection(self, versions, global_args, result): # pylint: disable=R0201
+ """
+ Checks joining project to the version.
+ """
+
+ @ignite_versions(*versions, version_prefix='ver')
+ def function(ver):
+ return IgniteVersion(ver)
+
+ context_list = expand_function(func=function, sess_ctx=mock_session_ctx(global_args=global_args))
+
+ check_versions = list(map(IgniteVersion, result))
+
+ assert len(check_versions) == len(context_list)
+
+ for i, ctx in enumerate(reversed(context_list)):
+ assert ctx.function() == check_versions[i]
+
def check_with_others_marks(self): # pylint: disable=R0201
"""
Checks that ignite version parametrization works with others correctly.
diff --git a/modules/ducktests/tests/checks/utils/check_version.py b/modules/ducktests/tests/checks/utils/check_version.py
new file mode 100644
index 0000000..05cbf28
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/check_version.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks Version.
+"""
+
+from ignitetest import __version__
+from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST
+
+
+def check_dev_version():
+ """"
+ Check developer version.
+ """
+ dev = IgniteVersion('dev')
+ ignite_dev = IgniteVersion('ignite-dev')
+ fork_dev = IgniteVersion('fork-dev')
+
+ assert DEV_BRANCH.is_dev
+ assert dev.is_dev
+ assert ignite_dev.is_dev
+ assert fork_dev.is_dev
+
+ assert DEV_BRANCH == dev == ignite_dev
+
+ # pylint: disable=W0511
+ # todo solve comparability issues and uncomment the following
+ # with pytest.raises(Exception):
+ # assert DEV_BRANCH != fork_dev # incomparable
+
+ assert DEV_BRANCH.version == dev.version == ignite_dev.version == fork_dev.version
+ assert DEV_BRANCH.project == dev.project == ignite_dev.project == "ignite"
+ assert fork_dev.project == "fork"
+
+ index = __version__.find('-')
+
+ if index > 0:
+ ver = IgniteVersion(__version__[:index])
+
+ assert dev > ver
+ assert ignite_dev > ver
+ assert dev.version > ver.version
+ assert ignite_dev.version > ver.version
+
+ assert dev > LATEST
+ assert dev.version > LATEST.version
+ assert ignite_dev > LATEST
+ assert ignite_dev.version > LATEST.version
+ assert DEV_BRANCH > LATEST
+ assert DEV_BRANCH.version > LATEST.version
+
+ # pylint: disable=W0511
+ # todo solve comparability issues and uncomment the following
+ # with pytest.raises(Exception):
+ # assert fork_dev != LATEST # incomparable
+
+ assert fork_dev.version != LATEST.version
+
+ assert str(dev) == str(ignite_dev) == str(DEV_BRANCH) == 'ignite-dev'
+ assert str(fork_dev) == 'fork-dev'
+
+
+def check_numeric_version():
+ """
+ Checks numeric version.
+ """
+ v_2_99_1 = IgniteVersion('2.99.1')
+ ignite_v_2_99_1 = IgniteVersion('ignite-2.99.1')
+ fork_v_2_99_1 = IgniteVersion('fork-2.99.1')
+
+ assert not v_2_99_1.is_dev
+ assert not ignite_v_2_99_1.is_dev
+ assert not fork_v_2_99_1.is_dev
+ assert not LATEST.is_dev
+
+ assert v_2_99_1 == ignite_v_2_99_1
+
+ # pylint: disable=W0511
+ # todo solve comparability issues and uncomment the following
+ # with pytest.raises(Exception):
+ # assert v_2_99_1 != fork_v_2_99_1 # incomparable
+
+ assert v_2_99_1.version == ignite_v_2_99_1.version == fork_v_2_99_1.version == [2, 99, 1]
+ assert v_2_99_1.project == ignite_v_2_99_1.project == LATEST.project == "ignite"
+ assert fork_v_2_99_1.project == "fork"
+
+ assert str(v_2_99_1) == str(ignite_v_2_99_1) == 'ignite-2.99.1'
+ assert str(fork_v_2_99_1) == 'fork-2.99.1'
diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py
index 83cfdfb..41be184 100644
--- a/modules/ducktests/tests/ignitetest/services/spark.py
+++ b/modules/ducktests/tests/ignitetest/services/spark.py
@@ -45,12 +45,8 @@ class SparkService(DucktestsService, PathAware):
self.init_logs_attribute()
@property
- def project(self):
- return "spark"
-
- @property
- def version(self):
- return self._version
+ def product(self):
+ return "%s-%s" % ("spark", self._version)
@property
def globals(self):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index 69308a4..5d5dc07 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -81,12 +81,8 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, metaclass=ABC
self.start_ignite = kwargs.get("start_ignite", True)
@property
- def version(self):
- return self.config.version
-
- @property
- def project(self):
- return self.spec.project
+ def product(self):
+ return str(self.config.version)
@property
def globals(self):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
index 35c28a3..76df89d 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
@@ -61,9 +61,10 @@ class IgniteSpec(metaclass=ABCMeta):
"""
This class is a basic Spec
"""
+
# pylint: disable=R0913
- def __init__(self, path_aware, config, project, jvm_opts=None, full_jvm_opts=None):
- self.project = project
+ def __init__(self, path_aware, config, jvm_opts=None, full_jvm_opts=None):
+ self.config = config
self.path_aware = path_aware
self.envs = {}
@@ -76,8 +77,6 @@ class IgniteSpec(metaclass=ABCMeta):
self.jvm_opts = create_jvm_settings(opts=jvm_opts,
gc_dump_path=os.path.join(path_aware.log_dir, "ignite_gc.log"),
oom_path=os.path.join(path_aware.log_dir, "ignite_out_of_mem.hprof"))
- self.config = config
- self.version = config.version
@property
def config_template(self):
@@ -88,22 +87,21 @@ class IgniteSpec(metaclass=ABCMeta):
return IgniteClientConfigTemplate()
return IgniteServerConfigTemplate()
- def __home(self, version=None, project=None):
+ def __home(self, product=None):
"""
Get home directory for current spec.
"""
- project = project if project else self.project
- version = version if version else self.version
- return get_home_dir(self.path_aware.install_root, project, version)
+ product = product if product else str(self.config.version)
+ return get_home_dir(self.path_aware.install_root, product)
def _module(self, name):
"""
Get module path for current spec.
"""
if name == "ducktests":
- return get_module_path(self.__home(DEV_BRANCH, project="ignite"), name, DEV_BRANCH)
+ return get_module_path(self.__home(str(DEV_BRANCH)), name, DEV_BRANCH.is_dev)
- return get_module_path(self.__home(self.version), name, self.version)
+ return get_module_path(self.__home(), name, self.config.version.is_dev)
@abstractmethod
def command(self, node):
@@ -173,7 +171,8 @@ class ApacheIgniteNodeSpec(IgniteNodeSpec):
Implementation IgniteNodeSpec for Apache Ignite project
"""
def __init__(self, context, modules, **kwargs):
- super().__init__(project=context.globals.get("project", "ignite"), **kwargs)
+ super().__init__(**kwargs)
+ self.context = context
libs = (modules or [])
libs.append("log4j")
@@ -198,7 +197,7 @@ class ApacheIgniteApplicationSpec(IgniteApplicationSpec):
"""
# pylint: disable=too-many-arguments
def __init__(self, context, modules, main_java_class, java_class_name, params, start_ignite, **kwargs):
- super().__init__(project=context.globals.get("project", "ignite"), **kwargs)
+ super().__init__(**kwargs)
self.context = context
libs = modules or []
@@ -231,8 +230,7 @@ class ApacheIgniteApplicationSpec(IgniteApplicationSpec):
]
def __jackson(self):
- version = self.version
- if not version.is_dev:
+ if not self.config.version.is_dev:
aws = self._module("aws")
return self.context.cluster.nodes[0].account.ssh_capture(
"ls -d %s/* | grep jackson | tr '\n' ':' | sed 's/.$//'" % aws)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/path.py b/modules/ducktests/tests/ignitetest/services/utils/path.py
index 9ecd805..3ccae1b 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/path.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/path.py
@@ -24,18 +24,18 @@ from ignitetest.services.utils.config_template import IgniteLoggerConfigTemplate
from ignitetest.utils.version import DEV_BRANCH
-def get_home_dir(install_root, project, version):
+def get_home_dir(install_root, product):
"""
- Get path to binary release (home) directory depending on version.
+ Get path to binary release (home) directory.
"""
- return os.path.join(install_root, f"{project}-{version}")
+ return os.path.join(install_root, product)
-def get_module_path(project_dir, module_name, version):
+def get_module_path(project_dir, module_name, is_dev):
"""
Get absolute path to the specified module.
"""
- if version.is_dev:
+ if is_dev:
module_path = os.path.join("modules", module_name, "target")
else:
module_path = os.path.join("libs", "optional", "ignite-%s" % module_name)
@@ -107,16 +107,9 @@ class PathAware:
@property
@abstractmethod
- def project(self):
+ def product(self):
"""
- :return: project name, for example 'zookeeper' for Apache Zookeeper.
- """
-
- @property
- @abstractmethod
- def version(self):
- """
- :return: version of project.
+ :return: Represents product (folder name), typically project/fork name with version.
"""
@property
@@ -131,7 +124,7 @@ class PathAware:
"""
:return: path to binary release (home) directory
"""
- return get_home_dir(self.install_root, self.project, self.version)
+ return get_home_dir(self.install_root, self.product)
@property
def temp_dir(self):
@@ -196,8 +189,7 @@ class IgnitePathAware(PathAware, metaclass=ABCMeta):
"""
:return: path to the certificate directory.
"""
- return os.path.join(get_home_dir(self.install_root, self.project, DEV_BRANCH),
- "modules", "ducktests", "tests", "certs")
+ return os.path.join(get_home_dir(self.install_root, str(DEV_BRANCH)), "modules", "ducktests", "tests", "certs")
def script(self, script_name):
"""
diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
index cd76991..6758994 100644
--- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
+++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py
@@ -63,8 +63,8 @@ class ZookeeperService(DucktestsService, PathAware):
self.init_logs_attribute()
@property
- def version(self):
- return self.settings.version
+ def product(self):
+ return "%s-%s" % ("zookeeper", self.settings.version)
@property
def globals(self):
@@ -78,10 +78,6 @@ class ZookeeperService(DucktestsService, PathAware):
def config_file(self):
return os.path.join(self.persistent_root, "zookeeper.properties")
- @property
- def project(self):
- return "zookeeper"
-
def start(self, **kwargs):
super().start(**kwargs)
self.logger.info("Waiting for Zookeeper quorum...")
diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py
index 38f8279..ae6850e 100644
--- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py
@@ -29,7 +29,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration,
from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster, from_zookeeper_cluster, \
TcpDiscoverySpi
from ignitetest.services.zk.zookeeper import ZookeeperSettings, ZookeeperService
-from ignitetest.utils import ignite_versions, cluster, ignore_if
+from ignitetest.utils import ignite_versions, cluster
from ignitetest.utils.enum import constructible
from ignitetest.utils.ignite_test import IgniteTest
from ignitetest.utils.version import DEV_BRANCH, IgniteVersion, LATEST
@@ -104,7 +104,6 @@ class CellularAffinity(IgniteTest):
cacheName=CellularAffinity.CACHE_NAME)
@cluster(num_nodes=NODES_PER_CELL * 3 + 1)
- @ignore_if(lambda version, globals: version < DEV_BRANCH)
@ignite_versions(str(DEV_BRANCH))
def test_distribution(self, ignite_version):
"""
diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py
index 69bb685..28998be 100644
--- a/modules/ducktests/tests/ignitetest/utils/_mark.py
+++ b/modules/ducktests/tests/ignitetest/utils/_mark.py
@@ -17,8 +17,8 @@
Module contains useful test decorators.
"""
-from collections.abc import Iterable
import copy
+from collections.abc import Iterable
from ducktape.cluster.cluster_spec import ClusterSpec
from ducktape.mark._mark import Ignore, Mark, _inject
@@ -73,6 +73,8 @@ class IgniteVersionParametrize(Mark):
raise AssertionError("Expected string or iterable as parameter in ignite_versions, "
"%s of type %s passed" % (ver, type(ver)))
+ self.versions = self._inject_global_project(self.versions, seed_context.globals.get("project"))
+
new_context_list = []
if context_list:
for ctx in context_list:
@@ -87,6 +89,16 @@ class IgniteVersionParametrize(Mark):
return new_context_list
+ @staticmethod
+ def _inject_global_project(version, project):
+ if isinstance(version, (list, tuple)):
+ return list(map(lambda v: IgniteVersionParametrize._inject_global_project(v, project), version))
+
+ if (version.lower() == "dev" or version[0].isdigit()) and project:
+ version = "%s-%s" % (project, version)
+
+ return version
+
def _prepare_new_ctx(self, version, seed_context, ctx=None):
injected_args = dict(ctx.injected_args) if ctx and ctx.injected_args else {}
diff --git a/modules/ducktests/tests/ignitetest/utils/version.py b/modules/ducktests/tests/ignitetest/utils/version.py
index deaa776..cbb2f4d 100644
--- a/modules/ducktests/tests/ignitetest/utils/version.py
+++ b/modules/ducktests/tests/ignitetest/utils/version.py
@@ -16,7 +16,7 @@
"""
Module contains ignite version utility class.
"""
-
+import re
from distutils.version import LooseVersion
from ignitetest import __version__
@@ -34,18 +34,39 @@ class IgniteVersion(LooseVersion):
v28 = IgniteVersion("2.8.1")
assert v28 > v27 # assertion passes!
"""
- def __init__(self, version_string):
- self.is_dev = (version_string.lower() == __version__.lower()) or (version_string.lower() == "dev")
+
+ DEV_VERSION = "dev"
+ DEFAULT_PROJECT = "ignite"
+
+ def __init__(self, vstring=None):
+ if vstring == self.DEV_VERSION:
+ self.project = self.DEFAULT_PROJECT
+ version = vstring
+ else:
+ match = re.match(r'([a-zA-Z]*)-*([\d(dev)]+.*)', vstring)
+ self.project = self.DEFAULT_PROJECT if not match.group(1) else match.group(1)
+ version = match.group(2)
+
+ self.is_dev = (version.lower() == __version__.lower()) or version == self.DEV_VERSION
+
if self.is_dev:
- version_string = __version__
+ version = __version__ # we may also parse pom file to gain correct version (in future)
- super().__init__(version_string)
+ super().__init__(version)
def __str__(self):
- if self.is_dev:
- return "dev"
+ return "%s-%s" % (self.project, "dev" if self.is_dev else super().__str__())
+
+ def _cmp(self, other):
+ if isinstance(other, str):
+ other = IgniteVersion(other)
+
+ # pylint: disable=W0511
+ # todo solve comparability issues and uncomment the following
+ # if self.project != other.project:
+ # raise Exception("Incomperable versons v1=%s, v2=%s because of different projects" % (self, other))
- return super().__str__()
+ return super()._cmp(other)
def __repr__(self):
return "IgniteVersion ('%s')" % str(self)