You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2021/03/30 11:45:19 UTC
[ignite] branch ignite-ducktape updated: IGNITE-14390 Add rebalance
statistic to test result data (#8945)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new 235b25e IGNITE-14390 Add rebalance statistic to test result data (#8945)
235b25e is described below
commit 235b25e99e3bac3e9e69b4f66d5c430512c44b95
Author: Dmitriy Sorokin <sb...@gmail.com>
AuthorDate: Tue Mar 30 14:45:00 2021 +0300
IGNITE-14390 Add rebalance statistic to test result data (#8945)
---
.../tests/ignitetest/tests/rebalance/__init__.py | 82 +++++++++++-----------
.../ignitetest/tests/rebalance/in_memory_test.py | 17 ++---
2 files changed, 46 insertions(+), 53 deletions(-)
diff --git a/modules/ducktests/tests/ignitetest/tests/rebalance/__init__.py b/modules/ducktests/tests/ignitetest/tests/rebalance/__init__.py
index 9ec1b96..92cfb6e 100644
--- a/modules/ducktests/tests/ignitetest/tests/rebalance/__init__.py
+++ b/modules/ducktests/tests/ignitetest/tests/rebalance/__init__.py
@@ -17,7 +17,6 @@
This package contains rebalance tests.
"""
-from datetime import datetime
from typing import NamedTuple
# pylint: disable=W0622
@@ -136,23 +135,15 @@ def get_rebalance_metrics(node, cache_group):
:return: RebalanceMetrics instance.
"""
mbean = node.jmx_client().find_mbean('.*group=cacheGroups.*name="%s"' % cache_group)
- start_time = to_datetime(int(next(mbean.RebalancingStartTime)))
- end_time = to_datetime(int(next(mbean.RebalancingEndTime)))
+ start_time = int(next(mbean.RebalancingStartTime))
+ end_time = int(next(mbean.RebalancingEndTime))
return RebalanceMetrics(
received_bytes=int(next(mbean.RebalancingReceivedBytes)),
start_time=start_time,
end_time=end_time,
- duration=(end_time - start_time).total_seconds() if start_time and end_time else 0)
-
-
-def to_datetime(timestamp):
- """
- Converts timestamp in millicesonds to datetime.
- :param timestamp: Timestamp in milliseconds.
- :return: datetime constructed from timestamp or None if ts == -1.
- """
- return None if timestamp == -1 else datetime.fromtimestamp(timestamp / 1000.0)
+ duration=(end_time - start_time) if start_time != -1 and end_time != -1 else 0,
+ node=node.name)
class RebalanceMetrics(NamedTuple):
@@ -160,39 +151,46 @@ class RebalanceMetrics(NamedTuple):
Rebalance metrics
"""
received_bytes: int = 0
- start_time: datetime = None
- end_time: datetime = None
- duration: float = 0
+ start_time: int = 0
+ end_time: int = 0
+ duration: int = 0
+ node: str = None
def aggregate_rebalance_stats(nodes, cache_count):
"""
- Aggregates rebalance stats for specified nodes and cache count:
- received_bytes -> sum(all of received_bytes)
- start_time -> min(all of start_time)
- end_time -> max(all of end_time)
- duration -> sum(all of duration)
+ Aggregates rebalance stats for specified nodes and cache count.
:param nodes: Nodes list.
:param cache_count: Cache count.
- :return: RebalanceMetrics instance with aggregated values.
+ :return: Aggregated rebalance stats dictionary.
"""
- received_bytes = 0
- start_time = None
- end_time = None
- duration = 0
-
- for node in nodes:
- for cache_idx in range(cache_count):
- metrics = get_rebalance_metrics(node, "test-cache-%d" % (cache_idx + 1))
- received_bytes += metrics.received_bytes
- if metrics.start_time is not None:
- start_time = min(t for t in [start_time, metrics.start_time] if t is not None)
- if metrics.end_time is not None:
- end_time = max(t for t in [end_time, metrics.end_time] if t is not None)
- duration += metrics.duration
-
- return RebalanceMetrics(
- received_bytes=received_bytes,
- start_time=start_time,
- end_time=end_time,
- duration=duration)
+ def __stats(cache_idx):
+ cache_name = "test-cache-%d" % (cache_idx + 1)
+
+ stats = {
+ "cache": cache_name,
+ "start_time": {},
+ "end_time": {},
+ "duration": {},
+ "received_bytes": {}
+ }
+
+ metrics = list(map(lambda node: get_rebalance_metrics(node, cache_name), nodes))
+
+ def __key(tup):
+ return tup[1]
+
+ stats["start_time"]["min"] = min(map(lambda item: (item.node, item.start_time), metrics), key=__key)
+ stats["start_time"]["max"] = max(map(lambda item: (item.node, item.start_time), metrics), key=__key)
+ stats["end_time"]["min"] = min(map(lambda item: (item.node, item.end_time), metrics), key=__key)
+ stats["end_time"]["max"] = max(map(lambda item: (item.node, item.end_time), metrics), key=__key)
+ stats["duration"]["min"] = min(map(lambda item: (item.node, item.duration), metrics), key=__key)
+ stats["duration"]["max"] = max(map(lambda item: (item.node, item.duration), metrics), key=__key)
+ stats["duration"]["sum"] = sum(map(lambda item: item.duration, metrics))
+ stats["received_bytes"]["min"] = min(map(lambda item: (item.node, item.received_bytes), metrics), key=__key)
+ stats["received_bytes"]["max"] = max(map(lambda item: (item.node, item.received_bytes), metrics), key=__key)
+ stats["received_bytes"]["sum"] = sum(map(lambda item: item.received_bytes, metrics))
+
+ return stats
+
+ return list(map(__stats, range(cache_count)))
diff --git a/modules/ducktests/tests/ignitetest/tests/rebalance/in_memory_test.py b/modules/ducktests/tests/ignitetest/tests/rebalance/in_memory_test.py
index 74b7921..92799ea 100644
--- a/modules/ducktests/tests/ignitetest/tests/rebalance/in_memory_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/rebalance/in_memory_test.py
@@ -132,22 +132,17 @@ class RebalanceInMemoryTest(IgniteTest):
num_nodes=1)
ignite.start()
- start_node, start_time = await_rebalance_start(ignite)
+ start_node, _ = await_rebalance_start(ignite)
- end_time = await_rebalance_complete(ignite, start_node, cache_count)
+ await_rebalance_complete(ignite, start_node, cache_count)
rebalance_nodes = ignite.nodes[:-1] if trigger_event else ignite.nodes
stats = aggregate_rebalance_stats(rebalance_nodes, cache_count)
- def speed(dur):
- return (int(stats.received_bytes / dur * 1000 / 1024 / 1024) / 1000.0) if dur else None
-
return {
- "Rebalanced in (sec)": (end_time - start_time).total_seconds(),
- "Rebalance nodes": len(rebalance_nodes),
- "Rebalance speed (Total, MiB/sec)": speed((stats.end_time - stats.start_time).total_seconds()),
- "Rebalance speed (Average per node, MiB/sec)": speed(stats.duration),
- "Preloaded in (sec)": preload_time,
- "Preloaded data size (MiB)": round(cache_count * entry_count * entry_size / (1024 * 1024), 3)
+ "rebalance_nodes": len(rebalance_nodes),
+ "rebalance_stats": stats,
+ "preload_time": int(preload_time * 1000),
+ "preloaded_bytes": cache_count * entry_count * entry_size
}