You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2021/02/19 13:28:15 UTC
[ignite] branch ignite-ducktape updated: IGNITE-13492: Added
snapshot test to ducktests (#8575)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new 8fa155c IGNITE-13492: Added snapshot test to ducktests (#8575)
8fa155c is described below
commit 8fa155c0c7f1c9de1be5c82c4a6e1423713dc3de
Author: Sergei Ryzhov <s....@gmail.com>
AuthorDate: Fri Feb 19 16:27:45 2021 +0300
IGNITE-13492: Added snapshot test to ducktests (#8575)
---
modules/ducktests/pom.xml | 6 +
.../tests/snapshot_test/DataLoaderApplication.java | 68 +++++++++++
.../ignitetest/services/utils/control_utility.py | 73 +++++++++++-
.../ignitetest/services/utils/ignite_aware.py | 13 +++
.../utils/ignite_configuration/__init__.py | 1 +
.../tests/ignitetest/services/utils/path.py | 14 +++
.../services/utils/templates/ignite.xml.j2 | 6 +
.../tests/ignitetest/tests/snapshot_test.py | 125 +++++++++++++++++++++
.../tests/ignitetest/tests/suites/slow_suite.yml | 3 +
scripts/build-module.sh | 1 +
10 files changed, 305 insertions(+), 5 deletions(-)
diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml
index 8f46d90..0889443 100644
--- a/modules/ducktests/pom.xml
+++ b/modules/ducktests/pom.xml
@@ -49,6 +49,12 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-control-utility</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-spark</artifactId>
<version>${project.version}</version>
<exclusions>
diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java
new file mode 100644
index 0000000..b562411
--- /dev/null
+++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.snapshot_test;
+
+import java.util.Collections;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading data to cache.
+ */
+public class DataLoaderApplication extends IgniteAwareApplication {
+ /** {@inheritDoc} */
+ @Override public void run(JsonNode jNode) {
+ String cacheName = jNode.get("cacheName").asText();
+
+ long start = jNode.get("start").asLong();
+
+ long interval = jNode.get("interval").asLong();
+
+ int valSize = jNode.get("valueSizeKb").asInt() * 1024;
+
+ markInitialized();
+
+ QueryEntity qryEntity = new QueryEntity()
+ .setKeyFieldName("id")
+ .setKeyType(Long.class.getName())
+ .setTableName("TEST_TABLE")
+ .setValueType(byte[].class.getName())
+ .addQueryField("id", Long.class.getName(), null)
+ .setIndexes(Collections.singletonList(new QueryIndex("id")));
+
+ CacheConfiguration<Long, byte[]> cacheCfg = new CacheConfiguration<>(cacheName);
+ cacheCfg.setCacheMode(CacheMode.REPLICATED);
+ cacheCfg.setQueryEntities(Collections.singletonList(qryEntity));
+
+ ignite.getOrCreateCache(cacheCfg);
+
+ byte[] data = new byte[valSize];
+
+ try (IgniteDataStreamer<Long, byte[]> dataStreamer = ignite.dataStreamer(cacheName)) {
+ for (long i = start; i < start + interval; i++)
+ dataStreamer.addData(i, data);
+ }
+
+ markFinished();
+ }
+}
diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
index 9f04b96..f46a620 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
@@ -19,12 +19,15 @@ This module contains control utility wrapper.
import os
import random
import re
+import socket
import time
+from datetime import datetime, timedelta
from typing import NamedTuple
from ducktape.cluster.remoteaccount import RemoteCommandError
from ignitetest.services.utils.ssl.ssl_factory import DEFAULT_PASSWORD, DEFAULT_TRUSTSTORE, DEFAULT_ADMIN_KEYSTORE
+from ignitetest.services.utils.jmx_utils import JmxClient
class ControlUtility:
@@ -154,6 +157,63 @@ class ControlUtility:
res = self.__parse_tx_list(output)
return res if res else output
+ def validate_indexes(self):
+ """
+ Validate indexes.
+ """
+ data = self.__run("--cache validate_indexes")
+
+ assert ('no issues found.' in data), data
+
+ def idle_verify(self):
+ """
+ Idle verify.
+ """
+ data = self.__run("--cache idle_verify")
+
+ assert ('idle_verify check has finished, no conflicts have been found.' in data), data
+
+ def idle_verify_dump(self, node=None):
+ """
+ Idle verify dump.
+ :param node: Node on which the command will be executed and the dump file will be located.
+ """
+ data = self.__run("--cache idle_verify --dump", node=node)
+
+ assert ('VisorIdleVerifyDumpTask successfully' in data), data
+
+ return re.search(r'/.*.txt', data).group(0)
+
+ def snapshot_create(self, snapshot_name: str, timeout_sec: int = 60):
+ """
+ Create snapshot.
+ :param snapshot_name: Name of Snapshot.
+ :param timeout_sec: Timeout to await snapshot to complete.
+ """
+ res = self.__run(f"--snapshot create {snapshot_name}")
+
+ assert "Command [SNAPSHOT] finished with code: 0" in res
+
+ delta_time = datetime.now() + timedelta(seconds=timeout_sec)
+
+ while datetime.now() < delta_time:
+ for node in self._cluster.nodes:
+ mbean = JmxClient(node).find_mbean('.*name=snapshot')
+
+ if snapshot_name != next(mbean.LastSnapshotName):
+ continue
+
+ start_time = int(next(mbean.LastSnapshotStartTime))
+ end_time = int(next(mbean.LastSnapshotEndTime))
+ err_msg = next(mbean.LastSnapshotErrorMessage)
+
+ if (start_time < end_time) and (err_msg == ''):
+ assert snapshot_name == next(mbean.LastSnapshotName)
+ return
+
+ raise TimeoutError(f'Failed to wait for the snapshot operation to complete: '
+ f'snapshot_name={snapshot_name} in {timeout_sec} seconds.')
+
@staticmethod
def __tx_command(**kwargs):
tokens = ["--tx"]
@@ -275,12 +335,15 @@ class ControlUtility:
return ClusterState(state=state, topology_version=topology, baseline=baseline)
- def __run(self, cmd):
- node = random.choice(self.__alives())
+ def __run(self, cmd, node=None):
+ if node is None:
+ node = random.choice(self.__alives())
self.logger.debug(f"Run command {cmd} on node {node.name}")
- raw_output = node.account.ssh_capture(self.__form_cmd(node, cmd), allow_fail=True)
+ node_ip = socket.gethostbyname(node.account.hostname)
+
+ raw_output = node.account.ssh_capture(self.__form_cmd(node_ip, cmd), allow_fail=True)
code, output = self.__parse_output(raw_output)
self.logger.debug(f"Output of command {cmd} on node {node.name}, exited with code {code}, is {output}")
@@ -290,13 +353,13 @@ class ControlUtility:
return output
- def __form_cmd(self, node, cmd):
+ def __form_cmd(self, node_ip, cmd):
ssl = ""
if hasattr(self, 'key_store_path'):
ssl = f" --keystore {self.key_store_path} --keystore-password {self.key_store_password} " \
f"--truststore {self.trust_store_path} --truststore-password {self.trust_store_password}"
- return self._cluster.script(f"{self.BASE_COMMAND} --host {node.account.externally_routable_ip} {cmd} {ssl}")
+ return self._cluster.script(f"{self.BASE_COMMAND} --host {node_ip} {cmd} {ssl}")
@staticmethod
def __parse_output(raw_output):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index 352b92f..fa4c5c4 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -446,3 +446,16 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, metaclass=ABC
cmd = "grep -E '%s' %s | sed -r 's/%s/\\1/'" % (regexp, node.log_file, regexp)
return IgniteAwareService.exec_command(node, cmd).strip().lower()
+
+ def restore_from_snapshot(self, snapshot_name: str):
+ """
+ Restore from snapshot.
+ :param snapshot_name: Name of Snapshot.
+ """
+ snapshot_db = os.path.join(self.snapshots_dir, snapshot_name, "db")
+
+ for node in self.nodes:
+ assert len(self.pids(node)) == 0
+
+ node.account.ssh(f'rm -rf {self.database_dir}', allow_fail=False)
+ node.account.ssh(f'cp -r {snapshot_db} {self.work_dir}', allow_fail=False)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
index 7799288..7d1c907 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
@@ -45,6 +45,7 @@ class IgniteConfiguration(NamedTuple):
local_host: str = None
ssl_context_factory: SslContextFactory = None
connector_configuration: ConnectorConfiguration = None
+ metric_exporter: str = None
class IgniteClientConfiguration(IgniteConfiguration):
diff --git a/modules/ducktests/tests/ignitetest/services/utils/path.py b/modules/ducktests/tests/ignitetest/services/utils/path.py
index 7747568..9419078 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/path.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/path.py
@@ -167,6 +167,20 @@ class IgnitePathAware(PathAware, metaclass=ABCMeta):
return os.path.join(self.persistent_root, "ignite-log4j.xml")
@property
+ def database_dir(self):
+ """
+ :return: path to database directory
+ """
+ return os.path.join(self.work_dir, "db")
+
+ @property
+ def snapshots_dir(self):
+ """
+ :return: path to snapshots directory
+ """
+ return os.path.join(self.work_dir, "snapshots")
+
+ @property
def certificate_dir(self):
"""
:return: path to the certificate directory.
diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2
index 8bbff0f..96eef5b 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2
+++ b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2
@@ -42,6 +42,12 @@
<property name="failureDetectionTimeout" value="{{ config.failure_detection_timeout }}"/>
<property name="systemWorkerBlockedTimeout" value="{{ config.sys_worker_blocked_timeout }}"/>
+ {% if config.metric_exporter %}
+ <property name="metricExporterSpi">
+ <bean class="{{ config.metric_exporter }}"/>
+ </property>
+ {% endif %}
+
{{ misc_utils.cluster_state(config.cluster_state, config.version) }}
{{ communication.communication_spi(config.communication_spi) }}
diff --git a/modules/ducktests/tests/ignitetest/tests/snapshot_test.py b/modules/ducktests/tests/ignitetest/tests/snapshot_test.py
new file mode 100644
index 0000000..edb8012
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/tests/snapshot_test.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module contains snapshot test.
+"""
+
+from ducktape.mark.resource import cluster
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.ignite_app import IgniteApplicationService
+from ignitetest.services.utils.control_utility import ControlUtility
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration
+from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster
+from ignitetest.utils import ignite_versions
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import IgniteVersion, LATEST_2_9
+
+
+# pylint: disable=W0223
+class SnapshotTest(IgniteTest):
+ """
+ Test Snapshot.
+ """
+ SNAPSHOT_NAME = "test_snapshot"
+
+ CACHE_NAME = "TEST_CACHE"
+
+ @cluster(num_nodes=4)
+ @ignite_versions(str(LATEST_2_9))
+ def snapshot_test(self, ignite_version):
+ """
+ Basic snapshot test.
+ """
+ version = IgniteVersion(ignite_version)
+
+ ignite_config = IgniteConfiguration(
+ version=version,
+ data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)),
+ metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi'
+ )
+
+ service = IgniteService(self.test_context, ignite_config, num_nodes=len(self.test_context.cluster) - 1)
+ service.start()
+
+ control_utility = ControlUtility(service)
+ control_utility.activate()
+
+ client_config = IgniteConfiguration(
+ client_mode=True,
+ version=version,
+ discovery_spi=from_ignite_cluster(service)
+ )
+
+ loader = IgniteApplicationService(
+ self.test_context,
+ client_config,
+ java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication",
+ params={
+ "start": 0,
+ "cacheName": self.CACHE_NAME,
+ "interval": 500_000,
+ "valueSizeKb": 1
+ }
+ )
+
+ loader.run()
+ loader.free()
+
+ control_utility.validate_indexes()
+ control_utility.idle_verify()
+ node = service.nodes[0]
+
+ dump_1 = control_utility.idle_verify_dump(node)
+
+ control_utility.snapshot_create(self.SNAPSHOT_NAME)
+
+ loader = IgniteApplicationService(
+ self.test_context,
+ client_config,
+ java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication",
+ params={
+ "start": 500_000,
+ "cacheName": self.CACHE_NAME,
+ "interval": 100_000,
+ "valueSizeKb": 1
+ }
+ )
+
+ loader.start(clean=False)
+ loader.wait()
+
+ dump_2 = control_utility.idle_verify_dump(node)
+
+ diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True)
+ assert len(diff) != 0
+
+ service.stop()
+
+ service.restore_from_snapshot(self.SNAPSHOT_NAME)
+
+ service.start(clean=False)
+
+ control_utility.activate()
+
+ control_utility.validate_indexes()
+ control_utility.idle_verify()
+
+ dump_3 = control_utility.idle_verify_dump(node)
+
+ diff = node.account.ssh_output(f'diff {dump_1} {dump_3}', allow_fail=True)
+ assert len(diff) == 0, diff
diff --git a/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml
index 6f066d5..863f9a4 100644
--- a/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml
+++ b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml
@@ -15,3 +15,6 @@
discovery:
- ../discovery_test.py
+
+snapshot:
+ - ../snapshot_test.py
diff --git a/scripts/build-module.sh b/scripts/build-module.sh
index 541ac8b..b3260b8 100755
--- a/scripts/build-module.sh
+++ b/scripts/build-module.sh
@@ -22,4 +22,5 @@
# Usage: ./scripts/build-module.sh ducktests
#
+mvn clean -Pall-java,all-scala
mvn package -pl :ignite-$1 -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true -am
\ No newline at end of file