[doris] branch master updated: [enhancement](BE)add metric for too many version (#14735)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 58bc254529 [enhancement](BE)add metric for too many version (#14735)
58bc254529 is described below

commit 58bc2545290c9c1f56c67c1e79924cfab4101dcd
Author: AlexYue <yj...@gmail.com>
AuthorDate: Mon Dec 5 11:37:14 2022 +0800

    [enhancement](BE)add metric for too many version (#14735)
    
    * add one funciton to get if exceeds version limit
    
    add bvar to indicate version exceed
    
    * resolve
    
    * remove unnecessary header file
---
 be/src/olap/delta_writer.cpp |  2 +-
 be/src/olap/push_handler.cpp |  2 +-
 be/src/olap/tablet.cpp       | 14 ++++++++++++++
 be/src/olap/tablet.h         |  1 +
 4 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index d75e0b63d4..b0f2dc6f30 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -111,7 +111,7 @@ Status DeltaWriter::init() {
     }
 
     // check tablet version number
-    if (_tablet->version_count() > config::max_tablet_version_num - 100) {
+    if (_tablet->exceed_version_limit(config::max_tablet_version_num - 100)) {
         //trigger compaction
         StorageEngine::instance()->submit_compaction_task(_tablet,
                                                           CompactionType::CUMULATIVE_COMPACTION);
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index e6b858015a..e43092a359 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -127,7 +127,7 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
     }
 
     // check if version number exceed limit
-    if (tablet->version_count() > config::max_tablet_version_num) {
+    if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
         LOG(WARNING) << "failed to push data. version count: " << tablet->version_count()
                      << ", exceed limit: " << config::max_tablet_version_num
                      << ". tablet: " << tablet->full_name();
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index d741203f1c..05d42ff607 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -17,6 +17,8 @@
 
 #include "olap/tablet.h"
 
+#include <bvar/reducer.h>
+#include <bvar/window.h>
 #include <ctype.h>
 #include <fmt/core.h>
 #include <glog/logging.h>
@@ -75,6 +77,10 @@ using std::vector;
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_bytes, MetricUnit::BYTES);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_finish_count, MetricUnit::OPERATIONS);
 
+bvar::Adder<uint64_t> exceed_version_limit_counter;
+bvar::Window<bvar::Adder<uint64_t>> xceed_version_limit_counter_minute(
+        &exceed_version_limit_counter, 60);
+
 TabletSharedPtr Tablet::create_tablet_from_meta(TabletMetaSharedPtr tablet_meta,
                                                 DataDir* data_dir) {
     return std::make_shared<Tablet>(tablet_meta, data_dir);
@@ -675,6 +681,14 @@ Status Tablet::check_version_integrity(const Version& version, bool quiet) {
     return capture_consistent_versions(version, nullptr, quiet);
 }
 
+bool Tablet::exceed_version_limit(int32_t limit) const {
+    if (_tablet_meta->version_count() > limit) {
+        exceed_version_limit_counter << 1;
+        return true;
+    }
+    return false;
+}
+
 // If any rowset contains the specific version, it means the version already exist
 bool Tablet::check_version_exist(const Version& version) const {
     for (auto& it : _rs_version_map) {
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 0990de435b..c64803a62e 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -93,6 +93,7 @@ public:
 
     size_t num_rows();
     int version_count() const;
+    bool exceed_version_limit(int32_t limit) const;
     Version max_version() const;
     Version max_version_unlocked() const;
     CumulativeCompactionPolicy* cumulative_compaction_policy();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org