You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by ji...@apache.org on 2021/06/23 10:42:38 UTC
[incubator-pegasus] 01/05: refactor: remove the useless class
table_stats (#720)
This is an automated email from the ASF dual-hosted git repository.
jiashuo pushed a commit to branch v2.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit a26835d049137b87deb271978adfdff219b09b4a
Author: zhao liwei <zl...@163.com>
AuthorDate: Fri Apr 16 10:25:43 2021 +0800
refactor: remove the useless class table_stats (#720)
---
rdsn | 2 +-
src/server/info_collector.cpp | 10 +--
src/server/info_collector.h | 102 ++++++++++------------
src/server/table_stats.h | 195 ------------------------------------------
src/shell/command_helper.h | 63 ++++++++++++--
5 files changed, 108 insertions(+), 264 deletions(-)
diff --git a/rdsn b/rdsn
index 7ccb58e..4be1128 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit 7ccb58ed9c6319f1a8ccb5fe28276a8a1014379a
+Subproject commit 4be112891938ccea16bee2805a24861b2557726e
diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp
index a4f4d76..42be060 100644
--- a/src/server/info_collector.cpp
+++ b/src/server/info_collector.cpp
@@ -149,16 +149,16 @@ void info_collector::on_app_stat()
return;
}
- table_stats all_stats("_all_");
+ row_data all_stats("_all_");
for (const auto &app_rows : all_rows) {
// get statistics data for app
- table_stats app_stats(app_rows.first);
+ row_data app_stats(app_rows.first);
for (auto partition_row : app_rows.second) {
app_stats.aggregate(partition_row);
}
- get_app_counters(app_stats.app_name)->set(app_stats);
+ get_app_counters(app_stats.row_name)->set(app_stats);
// get row data statistics for all of the apps
- all_stats.merge(app_stats);
+ all_stats.aggregate(app_stats);
// hotspot_partition_calculator is used for detecting hotspots
auto hotspot_partition_calculator =
@@ -166,7 +166,7 @@ void info_collector::on_app_stat()
hotspot_partition_calculator->data_aggregate(app_rows.second);
hotspot_partition_calculator->data_analyse();
}
- get_app_counters(all_stats.app_name)->set(all_stats);
+ get_app_counters(all_stats.row_name)->set(all_stats);
ddebug_f("stat apps succeed, app_count = {}, total_read_qps = {}, total_write_qps = {}",
all_rows.size(),
diff --git a/src/server/info_collector.h b/src/server/info_collector.h
index 65312a0..5f09aa4 100644
--- a/src/server/info_collector.h
+++ b/src/server/info_collector.h
@@ -33,7 +33,6 @@
#include <event2/bufferevent.h>
#include "../shell/commands.h"
-#include "table_stats.h"
namespace pegasus {
namespace server {
@@ -51,65 +50,58 @@ public:
return std::abs(total) < 1e-6 ? 0 : hit / total * 1e6;
}
- void set(const table_stats &row_stats)
+ void set(const row_data &row_stats)
{
- get_qps->set(row_stats.total_get_qps);
- multi_get_qps->set(row_stats.total_multi_get_qps);
- put_qps->set(row_stats.total_put_qps);
- multi_put_qps->set(row_stats.total_multi_put_qps);
- remove_qps->set(row_stats.total_remove_qps);
- multi_remove_qps->set(row_stats.total_multi_remove_qps);
- incr_qps->set(row_stats.total_incr_qps);
- check_and_set_qps->set(row_stats.total_check_and_set_qps);
- check_and_mutate_qps->set(row_stats.total_check_and_mutate_qps);
- scan_qps->set(row_stats.total_scan_qps);
- duplicate_qps->set(row_stats.total_duplicate_qps);
- dup_shipped_ops->set(row_stats.total_dup_shipped_ops);
- dup_failed_shipping_ops->set(row_stats.total_dup_failed_shipping_ops);
- recent_read_cu->set(row_stats.total_recent_read_cu);
- recent_write_cu->set(row_stats.total_recent_write_cu);
- recent_expire_count->set(row_stats.total_recent_expire_count);
- recent_filter_count->set(row_stats.total_recent_filter_count);
- recent_abnormal_count->set(row_stats.total_recent_abnormal_count);
- recent_write_throttling_delay_count->set(
- row_stats.total_recent_write_throttling_delay_count);
+ get_qps->set(row_stats.get_qps);
+ multi_get_qps->set(row_stats.multi_get_qps);
+ put_qps->set(row_stats.put_qps);
+ multi_put_qps->set(row_stats.multi_put_qps);
+ remove_qps->set(row_stats.remove_qps);
+ multi_remove_qps->set(row_stats.multi_remove_qps);
+ incr_qps->set(row_stats.incr_qps);
+ check_and_set_qps->set(row_stats.check_and_set_qps);
+ check_and_mutate_qps->set(row_stats.check_and_mutate_qps);
+ scan_qps->set(row_stats.scan_qps);
+ duplicate_qps->set(row_stats.duplicate_qps);
+ dup_shipped_ops->set(row_stats.dup_shipped_ops);
+ dup_failed_shipping_ops->set(row_stats.dup_failed_shipping_ops);
+ recent_read_cu->set(row_stats.recent_read_cu);
+ recent_write_cu->set(row_stats.recent_write_cu);
+ recent_expire_count->set(row_stats.recent_expire_count);
+ recent_filter_count->set(row_stats.recent_filter_count);
+ recent_abnormal_count->set(row_stats.recent_abnormal_count);
+ recent_write_throttling_delay_count->set(row_stats.recent_write_throttling_delay_count);
recent_write_throttling_reject_count->set(
- row_stats.total_recent_write_throttling_reject_count);
- recent_read_throttling_delay_count->set(
- row_stats.total_recent_read_throttling_delay_count);
- recent_read_throttling_reject_count->set(
- row_stats.total_recent_read_throttling_reject_count);
- storage_mb->set(row_stats.total_storage_mb);
- storage_count->set(row_stats.total_storage_count);
- rdb_block_cache_hit_rate->set(
- convert_to_1M_ratio(row_stats.total_rdb_block_cache_hit_count,
- row_stats.total_rdb_block_cache_total_count));
+ row_stats.recent_write_throttling_reject_count);
+ recent_read_throttling_delay_count->set(row_stats.recent_read_throttling_delay_count);
+ recent_read_throttling_reject_count->set(row_stats.recent_read_throttling_reject_count);
+ storage_mb->set(row_stats.storage_mb);
+ storage_count->set(row_stats.storage_count);
+ rdb_block_cache_hit_rate->set(convert_to_1M_ratio(
+ row_stats.rdb_block_cache_hit_count, row_stats.rdb_block_cache_total_count));
rdb_index_and_filter_blocks_mem_usage->set(
- row_stats.total_rdb_index_and_filter_blocks_mem_usage);
- rdb_memtable_mem_usage->set(row_stats.total_rdb_memtable_mem_usage);
- rdb_estimate_num_keys->set(row_stats.total_rdb_estimate_num_keys);
- rdb_bf_seek_negatives_rate->set(convert_to_1M_ratio(
- row_stats.total_rdb_bf_seek_negatives, row_stats.total_rdb_bf_seek_total));
- rdb_bf_point_negatives_rate->set(
- convert_to_1M_ratio(row_stats.total_rdb_bf_point_negatives,
- row_stats.total_rdb_bf_point_negatives +
- row_stats.total_rdb_bf_point_positive_total));
- rdb_bf_point_false_positive_rate->set(
- convert_to_1M_ratio(row_stats.total_rdb_bf_point_positive_total -
- row_stats.total_rdb_bf_point_positive_true,
- (row_stats.total_rdb_bf_point_positive_total -
- row_stats.total_rdb_bf_point_positive_true) +
- row_stats.total_rdb_bf_point_negatives));
+ row_stats.rdb_index_and_filter_blocks_mem_usage);
+ rdb_memtable_mem_usage->set(row_stats.rdb_memtable_mem_usage);
+ rdb_estimate_num_keys->set(row_stats.rdb_estimate_num_keys);
+ rdb_bf_seek_negatives_rate->set(
+ convert_to_1M_ratio(row_stats.rdb_bf_seek_negatives, row_stats.rdb_bf_seek_total));
+ rdb_bf_point_negatives_rate->set(convert_to_1M_ratio(
+ row_stats.rdb_bf_point_negatives,
+ row_stats.rdb_bf_point_negatives + row_stats.rdb_bf_point_positive_total));
+ rdb_bf_point_false_positive_rate->set(convert_to_1M_ratio(
+ row_stats.rdb_bf_point_positive_total - row_stats.rdb_bf_point_positive_true,
+ (row_stats.rdb_bf_point_positive_total - row_stats.rdb_bf_point_positive_true) +
+ row_stats.rdb_bf_point_negatives));
read_qps->set(row_stats.get_total_read_qps());
write_qps->set(row_stats.get_total_write_qps());
- backup_request_qps->set(row_stats.total_backup_request_qps);
- get_bytes->set(row_stats.total_get_bytes);
- multi_get_bytes->set(row_stats.total_multi_get_bytes);
- scan_bytes->set(row_stats.total_scan_bytes);
- put_bytes->set(row_stats.total_put_bytes);
- multi_put_bytes->set(row_stats.total_multi_put_bytes);
- check_and_set_bytes->set(row_stats.total_check_and_set_bytes);
- check_and_mutate_bytes->set(row_stats.total_check_and_mutate_bytes);
+ backup_request_qps->set(row_stats.backup_request_qps);
+ get_bytes->set(row_stats.get_bytes);
+ multi_get_bytes->set(row_stats.multi_get_bytes);
+ scan_bytes->set(row_stats.scan_bytes);
+ put_bytes->set(row_stats.put_bytes);
+ multi_put_bytes->set(row_stats.multi_put_bytes);
+ check_and_set_bytes->set(row_stats.check_and_set_bytes);
+ check_and_mutate_bytes->set(row_stats.check_and_mutate_bytes);
read_bytes->set(row_stats.get_total_read_bytes());
write_bytes->set(row_stats.get_total_write_bytes());
}
diff --git a/src/server/table_stats.h b/src/server/table_stats.h
deleted file mode 100644
index c3bafcd..0000000
--- a/src/server/table_stats.h
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-struct table_stats
-{
- table_stats(const std::string &app_name) : app_name(app_name) {}
-
- double get_total_read_qps() const
- {
- return total_get_qps + total_multi_get_qps + total_scan_qps;
- }
-
- double get_total_write_qps() const
- {
- return total_put_qps + total_multi_put_qps + total_remove_qps + total_multi_remove_qps +
- total_incr_qps + total_check_and_set_qps + total_check_and_mutate_qps +
- total_duplicate_qps;
- }
-
- double get_total_read_bytes() const
- {
- return total_get_bytes + total_multi_get_bytes + total_scan_bytes;
- }
-
- double get_total_write_bytes() const
- {
- return total_put_bytes + total_multi_put_bytes + total_check_and_set_bytes +
- total_check_and_mutate_bytes;
- }
-
- void aggregate(const row_data &row)
- {
- total_get_qps += row.get_qps;
- total_multi_get_qps += row.multi_get_qps;
- total_put_qps += row.put_qps;
- total_multi_put_qps += row.multi_put_qps;
- total_remove_qps += row.remove_qps;
- total_multi_remove_qps += row.multi_remove_qps;
- total_incr_qps += row.incr_qps;
- total_check_and_set_qps += row.check_and_set_qps;
- total_check_and_mutate_qps += row.check_and_mutate_qps;
- total_scan_qps += row.scan_qps;
- total_duplicate_qps += row.duplicate_qps;
- total_dup_shipped_ops += row.dup_shipped_ops;
- total_dup_failed_shipping_ops += row.dup_failed_shipping_ops;
- total_recent_read_cu += row.recent_read_cu;
- total_recent_write_cu += row.recent_write_cu;
- total_recent_expire_count += row.recent_expire_count;
- total_recent_filter_count += row.recent_filter_count;
- total_recent_abnormal_count += row.recent_abnormal_count;
- total_recent_write_throttling_delay_count += row.recent_write_throttling_delay_count;
- total_recent_write_throttling_reject_count += row.recent_write_throttling_reject_count;
- total_recent_read_throttling_delay_count += row.recent_read_throttling_delay_count;
- total_recent_read_throttling_reject_count += row.recent_read_throttling_reject_count;
- total_storage_mb += row.storage_mb;
- total_storage_count += row.storage_count;
- total_rdb_block_cache_hit_count += row.rdb_block_cache_hit_count;
- total_rdb_block_cache_total_count += row.rdb_block_cache_total_count;
- total_rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage;
- total_rdb_memtable_mem_usage += row.rdb_memtable_mem_usage;
- total_rdb_estimate_num_keys += row.rdb_estimate_num_keys;
- total_rdb_bf_seek_negatives += row.rdb_bf_seek_negatives;
- total_rdb_bf_seek_total += row.rdb_bf_seek_total;
- total_rdb_bf_point_positive_true += row.rdb_bf_point_positive_true;
- total_rdb_bf_point_positive_total += row.rdb_bf_point_positive_total;
- total_rdb_bf_point_negatives += row.rdb_bf_point_negatives;
- total_backup_request_qps += row.backup_request_qps;
- total_get_bytes += row.get_bytes;
- total_multi_get_bytes += row.multi_get_bytes;
- total_scan_bytes += row.scan_bytes;
- total_put_bytes += row.put_bytes;
- total_multi_put_bytes += row.multi_put_bytes;
- total_check_and_set_bytes += row.check_and_set_bytes;
- total_check_and_mutate_bytes += row.check_and_mutate_bytes;
- }
-
- void merge(const table_stats &row_stats)
- {
- total_get_qps += row_stats.total_get_qps;
- total_multi_get_qps += row_stats.total_multi_get_qps;
- total_put_qps += row_stats.total_put_qps;
- total_multi_put_qps += row_stats.total_multi_put_qps;
- total_remove_qps += row_stats.total_remove_qps;
- total_multi_remove_qps += row_stats.total_multi_remove_qps;
- total_incr_qps += row_stats.total_incr_qps;
- total_check_and_set_qps += row_stats.total_check_and_set_qps;
- total_check_and_mutate_qps += row_stats.total_check_and_mutate_qps;
- total_scan_qps += row_stats.total_scan_qps;
- total_duplicate_qps += row_stats.total_duplicate_qps;
- total_dup_shipped_ops += row_stats.total_dup_shipped_ops;
- total_dup_failed_shipping_ops += row_stats.total_dup_failed_shipping_ops;
- total_recent_read_cu += row_stats.total_recent_read_cu;
- total_recent_write_cu += row_stats.total_recent_write_cu;
- total_recent_expire_count += row_stats.total_recent_expire_count;
- total_recent_filter_count += row_stats.total_recent_filter_count;
- total_recent_abnormal_count += row_stats.total_recent_abnormal_count;
- total_recent_write_throttling_delay_count +=
- row_stats.total_recent_write_throttling_delay_count;
- total_recent_write_throttling_reject_count +=
- row_stats.total_recent_write_throttling_reject_count;
- total_recent_read_throttling_delay_count +=
- row_stats.total_recent_read_throttling_delay_count;
- total_recent_read_throttling_reject_count +=
- row_stats.total_recent_read_throttling_reject_count;
- total_storage_mb += row_stats.total_storage_mb;
- total_storage_count += row_stats.total_storage_count;
- total_rdb_block_cache_hit_count += row_stats.total_rdb_block_cache_hit_count;
- total_rdb_block_cache_total_count += row_stats.total_rdb_block_cache_total_count;
- total_rdb_index_and_filter_blocks_mem_usage +=
- row_stats.total_rdb_index_and_filter_blocks_mem_usage;
- total_rdb_memtable_mem_usage += row_stats.total_rdb_memtable_mem_usage;
- total_rdb_estimate_num_keys += row_stats.total_rdb_estimate_num_keys;
- total_rdb_bf_seek_negatives += row_stats.total_rdb_bf_seek_negatives;
- total_rdb_bf_seek_total += row_stats.total_rdb_bf_seek_total;
- total_rdb_bf_point_positive_true += row_stats.total_rdb_bf_point_positive_true;
- total_rdb_bf_point_positive_total += row_stats.total_rdb_bf_point_positive_total;
- total_rdb_bf_point_negatives += row_stats.total_rdb_bf_point_negatives;
- total_backup_request_qps += row_stats.total_backup_request_qps;
- total_get_bytes += row_stats.total_get_bytes;
- total_multi_get_bytes += row_stats.total_multi_get_bytes;
- total_scan_bytes += row_stats.total_scan_bytes;
- total_put_bytes += row_stats.total_put_bytes;
- total_multi_put_bytes += row_stats.total_multi_put_bytes;
- total_check_and_set_bytes += row_stats.total_check_and_set_bytes;
- total_check_and_mutate_bytes += row_stats.total_check_and_mutate_bytes;
- }
-
- std::string app_name;
- double total_get_qps = 0;
- double total_multi_get_qps = 0;
- double total_put_qps = 0;
- double total_multi_put_qps = 0;
- double total_remove_qps = 0;
- double total_multi_remove_qps = 0;
- double total_incr_qps = 0;
- double total_check_and_set_qps = 0;
- double total_check_and_mutate_qps = 0;
- double total_scan_qps = 0;
- double total_duplicate_qps = 0;
- double total_dup_shipped_ops = 0;
- double total_dup_failed_shipping_ops = 0;
- double total_recent_read_cu = 0;
- double total_recent_write_cu = 0;
- double total_recent_expire_count = 0;
- double total_recent_filter_count = 0;
- double total_recent_abnormal_count = 0;
- double total_recent_write_throttling_delay_count = 0;
- double total_recent_write_throttling_reject_count = 0;
- double total_recent_read_throttling_delay_count = 0;
- double total_recent_read_throttling_reject_count = 0;
- double total_storage_mb = 0;
- double total_storage_count = 0;
- double total_rdb_block_cache_hit_count = 0;
- double total_rdb_block_cache_total_count = 0;
- double total_rdb_index_and_filter_blocks_mem_usage = 0;
- double total_rdb_memtable_mem_usage = 0;
- double total_rdb_estimate_num_keys = 0;
- double total_rdb_bf_seek_negatives = 0;
- double total_rdb_bf_seek_total = 0;
- double total_rdb_bf_point_positive_true = 0;
- double total_rdb_bf_point_positive_total = 0;
- double total_rdb_bf_point_negatives = 0;
- double total_backup_request_qps = 0;
- double total_get_bytes = 0;
- double total_multi_get_bytes = 0;
- double total_scan_bytes = 0;
- double total_put_bytes = 0;
- double total_multi_put_bytes = 0;
- double total_check_and_set_bytes = 0;
- double total_check_and_mutate_bytes = 0;
- double max_total_qps = 0;
- double min_total_qps = INT_MAX;
- double max_total_cu = 0;
- double min_total_cu = INT_MAX;
- std::string max_qps_partition_id;
- std::string max_cu_partition_id;
-};
diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h
index 0b9f959..4d46895 100644
--- a/src/shell/command_helper.h
+++ b/src/shell/command_helper.h
@@ -555,14 +555,8 @@ inline bool parse_app_perf_counter_name(const std::string &name,
struct row_data
{
- double get_total_qps() const
- {
- return get_qps + multi_get_qps + scan_qps + put_qps + multi_put_qps + remove_qps +
- multi_remove_qps + incr_qps + check_and_set_qps + check_and_mutate_qps +
- duplicate_qps;
- }
-
- double get_total_cu() const { return recent_read_cu + recent_write_cu; }
+ row_data() = default;
+ explicit row_data(const std::string &row_name) : row_name(row_name) {}
double get_total_read_qps() const { return get_qps + multi_get_qps + scan_qps; }
@@ -572,6 +566,59 @@ struct row_data
check_and_mutate_qps;
}
+ double get_total_read_bytes() const { return get_bytes + multi_get_bytes + scan_bytes; }
+
+ double get_total_write_bytes() const
+ {
+ return put_bytes + multi_put_bytes + check_and_set_bytes + check_and_mutate_bytes;
+ }
+
+ void aggregate(const row_data &row)
+ {
+ get_qps += row.get_qps;
+ multi_get_qps += row.multi_get_qps;
+ put_qps += row.put_qps;
+ multi_put_qps += row.multi_put_qps;
+ remove_qps += row.remove_qps;
+ multi_remove_qps += row.multi_remove_qps;
+ incr_qps += row.incr_qps;
+ check_and_set_qps += row.check_and_set_qps;
+ check_and_mutate_qps += row.check_and_mutate_qps;
+ scan_qps += row.scan_qps;
+ duplicate_qps += row.duplicate_qps;
+ dup_shipped_ops += row.dup_shipped_ops;
+ dup_failed_shipping_ops += row.dup_failed_shipping_ops;
+ recent_read_cu += row.recent_read_cu;
+ recent_write_cu += row.recent_write_cu;
+ recent_expire_count += row.recent_expire_count;
+ recent_filter_count += row.recent_filter_count;
+ recent_abnormal_count += row.recent_abnormal_count;
+ recent_write_throttling_delay_count += row.recent_write_throttling_delay_count;
+ recent_write_throttling_reject_count += row.recent_write_throttling_reject_count;
+ recent_read_throttling_delay_count += row.recent_read_throttling_delay_count;
+ recent_read_throttling_reject_count += row.recent_read_throttling_reject_count;
+ storage_mb += row.storage_mb;
+ storage_count += row.storage_count;
+ rdb_block_cache_hit_count += row.rdb_block_cache_hit_count;
+ rdb_block_cache_total_count += row.rdb_block_cache_total_count;
+ rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage;
+ rdb_memtable_mem_usage += row.rdb_memtable_mem_usage;
+ rdb_estimate_num_keys += row.rdb_estimate_num_keys;
+ rdb_bf_seek_negatives += row.rdb_bf_seek_negatives;
+ rdb_bf_seek_total += row.rdb_bf_seek_total;
+ rdb_bf_point_positive_true += row.rdb_bf_point_positive_true;
+ rdb_bf_point_positive_total += row.rdb_bf_point_positive_total;
+ rdb_bf_point_negatives += row.rdb_bf_point_negatives;
+ backup_request_qps += row.backup_request_qps;
+ get_bytes += row.get_bytes;
+ multi_get_bytes += row.multi_get_bytes;
+ scan_bytes += row.scan_bytes;
+ put_bytes += row.put_bytes;
+ multi_put_bytes += row.multi_put_bytes;
+ check_and_set_bytes += row.check_and_set_bytes;
+ check_and_mutate_bytes += row.check_and_mutate_bytes;
+ }
+
std::string row_name;
int32_t app_id = 0;
int32_t partition_count = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org