You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2023/10/18 11:43:32 UTC
[ignite] branch master updated: IGNITE-20470 [ducktests] Test for the cache dump (#10998)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 43c6432879f IGNITE-20470 [ducktests] Test for the cache dump (#10998)
43c6432879f is described below
commit 43c6432879f737055f4ac3bba674ce7fb6250626
Author: Sergey Korotkov <se...@gmail.com>
AuthorDate: Wed Oct 18 18:43:17 2023 +0700
IGNITE-20470 [ducktests] Test for the cache dump (#10998)
---
.../internal/ducktest/tests/dump/DumpUtility.java | 51 +++++++++
.../ignitetest/services/utils/control_utility.py | 15 ++-
.../ignitetest/services/utils/dump_utility.py | 48 ++++++++
.../ignitetest/services/utils/ignite_aware.py | 8 ++
.../ducktests/tests/ignitetest/tests/dump_test.py | 123 +++++++++++++++++++++
5 files changed, 243 insertions(+), 2 deletions(-)
diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dump/DumpUtility.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dump/DumpUtility.java
new file mode 100644
index 00000000000..3c6983f7b1b
--- /dev/null
+++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dump/DumpUtility.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.dump;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Application to control the cache dump operations (just create for now).
+ */
+public class DumpUtility extends IgniteAwareApplication {
+ /** {@inheritDoc} */
+ @Override public void run(JsonNode jsonNode) {
+ String cmd = jsonNode.get("cmd").asText();
+ String dumpName = jsonNode.get("dumpName").asText();
+
+ markInitialized();
+
+ switch (cmd) {
+ case "create":
+ long startTime = System.nanoTime();
+
+ ignite.snapshot().createDump(dumpName, null).get();
+
+ long resultTime = System.nanoTime() - startTime;
+
+ recordResult("DUMP_CREATE_TIME_MS", resultTime / 1_000_000);
+
+ break;
+ default:
+ throw new RuntimeException("Wrong cmd parameter for the dump control utility: '" + cmd + "'");
+ }
+
+ markFinished();
+ }
+}
diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
index 1b1dd997cfd..d95f97b0a13 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py
@@ -220,6 +220,17 @@ class ControlUtility:
raise TimeoutError(f'Failed to wait for the snapshot operation to complete: '
f'snapshot_name={snapshot_name} in {timeout_sec} seconds.')
+ def snapshot_check(self, snapshot_name: str):
+ """
+ Check snapshot.
+ :param snapshot_name: Name of snapshot.
+ """
+ res = self.__run(f"--snapshot check {snapshot_name}")
+
+ assert "The check procedure has finished, no conflicts have been found." in res
+
+ return res
+
def start_performance_statistics(self):
"""
Start performance statistics collecting in the cluster.
@@ -425,10 +436,10 @@ class ControlUtility:
auth = f" --user {self.username} --password {self.password} "
return "%s %s" % \
- (envs_to_exports(self.__envs()),
+ (envs_to_exports(self.envs()),
self._cluster.script(f"{self.BASE_COMMAND} --host {node_ip} {cmd} {ssl} {auth}"))
- def __envs(self):
+ def envs(self):
"""
:return: environment set.
"""
diff --git a/modules/ducktests/tests/ignitetest/services/utils/dump_utility.py b/modules/ducktests/tests/ignitetest/services/utils/dump_utility.py
new file mode 100644
index 00000000000..65d32919edb
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/utils/dump_utility.py
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ignitetest.services.ignite_app import IgniteApplicationService
+
+
+class DumpUtility:
+ """
+ Control the cache dump operations.
+ """
+ def __init__(self, test_context, cluster):
+ self.cluster = cluster
+
+ self.app = IgniteApplicationService(
+ test_context,
+ cluster.config._replace(client_mode=True),
+ java_class_name="org.apache.ignite.internal.ducktest.tests.dump.DumpUtility"
+ )
+
+ def create(self, dump_name):
+ """
+ Create cache dump.
+ :param dump_name: Name of the dump.
+ """
+ self.app.params = {
+ "cmd": "create",
+ "dumpName": dump_name
+ }
+
+ self.app.start(clean=False)
+
+ self.app.wait()
+
+ dump_create_time_ms = self.app.extract_result("DUMP_CREATE_TIME_MS")
+
+ return int(dump_create_time_ms) if dump_create_time_ms != "" else None
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index ef5d05b4582..dbbe5787411 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -581,6 +581,14 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMix
"""
return [node for node in self.nodes if self.alive(node)]
+ @staticmethod
+ def get_file_size(node, file):
+ out = IgniteAwareService.exec_command(node, f'du -s --block-size=1 {file}')
+
+ data = out.split("\t")
+
+ return int(data[0])
+
def node_failed_event_pattern(failed_node_id=None):
"""Failed node pattern in log."""
diff --git a/modules/ducktests/tests/ignitetest/tests/dump_test.py b/modules/ducktests/tests/ignitetest/tests/dump_test.py
new file mode 100644
index 00000000000..db58d506444
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/tests/dump_test.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module contains cache dump tests.
+"""
+import os
+import re
+from ducktape.mark import defaults
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.utils.control_utility import ControlUtility
+from ignitetest.services.utils.dump_utility import DumpUtility
+from ignitetest.services.utils.ignite_aware import IgniteAwareService
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration
+from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster
+from ignitetest.tests.util import DataGenerationParams, preload_data
+from ignitetest.utils import cluster, ignite_versions
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import IgniteVersion, DEV_BRANCH
+
+DUMP_NAME = "test_dump"
+CACHE_NAME = "test-cache"
+
+
+class DumpTest(IgniteTest):
+ """
+ Test cache dump.
+ """
+
+ @cluster(num_nodes=5)
+ @ignite_versions(str(DEV_BRANCH))
+ @defaults(nodes=[1, 3], backups=[1], cache_count=[1], entry_count=[50_000], entry_size=[1024], preloaders=[1])
+ def dump_test(self, ignite_version, nodes, backups, cache_count, entry_count, entry_size, preloaders):
+ """
+ Basic dump test.
+ """
+ data_gen_params = DataGenerationParams(backups=backups, cache_count=cache_count, entry_count=entry_count,
+ entry_size=entry_size, preloaders=preloaders)
+
+ ignite_config = IgniteConfiguration(
+ version=IgniteVersion(ignite_version),
+ data_storage=DataStorageConfiguration(
+ checkpoint_frequency=5000,
+ default=DataRegionConfiguration(
+ persistence_enabled=True,
+ max_size=data_gen_params.data_region_max_size
+ )
+ ),
+ metric_exporters={'org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi'}
+ )
+
+ ignite = IgniteService(self.test_context, ignite_config, num_nodes=nodes)
+ ignite.start()
+
+ control_utility = ControlUtility(ignite)
+ control_utility.activate()
+
+ preload_data(
+ self.test_context,
+ ignite.config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignite)),
+ data_gen_params=data_gen_params)
+
+ result = self.get_data_region_size(ignite)
+
+ result.update(self.create_dump(ignite))
+
+ result.update(self.check_dump(ignite))
+
+ return result
+
+ def create_dump(self, ignite):
+ dump_utility = DumpUtility(self.test_context, ignite)
+
+ dump_create_time_ms = dump_utility.create(DUMP_NAME)
+
+ dump_size = {}
+ for node in ignite.nodes:
+ dump_size[node.consistent_id] = IgniteAwareService.get_file_size(
+ node, os.path.join(ignite.snapshots_dir, DUMP_NAME))
+
+ return {
+ "dump_create_time_ms": dump_create_time_ms,
+ "dump_size": dump_size
+ }
+
+ @staticmethod
+ def check_dump(ignite):
+ control_utility = ControlUtility(ignite)
+
+ output = control_utility.snapshot_check(DUMP_NAME)
+
+ pattern = re.compile("Execution time: (?P<time_ms>\\d+) ms")
+ match = pattern.search(output)
+
+ return {
+ "dump_check_time_ms": int(match.group("time_ms")) if match else None
+ }
+
+ @staticmethod
+ def get_data_region_size(ignite):
+ data_region_size = {}
+
+ for node in ignite.nodes:
+ mbean = node.jmx_client().find_mbean('.*group=io.*name="dataregion.default"')
+ data_region_size[node.consistent_id] = int(next(mbean.TotalUsedSize))
+
+ return {
+ "data_region_size": data_region_size
+ }