You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2026/06/09 18:04:53 UTC

(impala) branch master updated (cf03d9376 -> 498d24d98)

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


    from cf03d9376 IMPALA-12696: Update HS2 definitions (TCLIService.thrift)
     new aac753cb3 IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric
     new 498d24d98 IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/util/memory-metrics.cc | 62 +++++++++++++++++++++++++---------------
 be/src/util/memory-metrics.h  | 66 +++++++++++++++++++++++--------------------
 be/src/util/metrics-test.cc   |  4 ---
 common/thrift/metrics.json    | 18 ++----------
 4 files changed, 79 insertions(+), 71 deletions(-)


(impala) 01/02: IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit aac753cb3208effb6f41f916b53a17a05ed967c6
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue May 26 11:30:22 2026 -0700

    IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric
    
    In IMPALA-14874, we introduced several tcmalloc metrics
    including tcmalloc.current-total-thread-cache-bytes and
    tcmalloc.thread-cache-free-bytes. These are actually the
    same underlying metric, so this removes tcmalloc.thread-cache-free-bytes.
    
    Testing:
     - Ran core tests
    
    Change-Id: I7ff431b00c10fe9ff99bfd6c909739d8a97fcf3a
    Reviewed-on: http://gerrit.cloudera.org:8080/24399
    Reviewed-by: Michael Smith <mi...@cloudera.com>
    Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
    Tested-by: Joe McDonnell <jo...@cloudera.com>
---
 be/src/util/memory-metrics.cc |  5 -----
 be/src/util/memory-metrics.h  |  3 ---
 be/src/util/metrics-test.cc   |  4 ----
 common/thrift/metrics.json    | 14 +-------------
 4 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc
index 9cc0bb3e0..021209599 100644
--- a/be/src/util/memory-metrics.cc
+++ b/be/src/util/memory-metrics.cc
@@ -59,7 +59,6 @@ TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
 TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
 TcmallocMetric* TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = nullptr;
 TcmallocMetric* TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::THREAD_CACHE_FREE_BYTES = nullptr;
 TcmallocMetric::PhysicalBytesMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
 
 SanitizerMallocMetric* SanitizerMallocMetric::BYTES_ALLOCATED = nullptr;
@@ -146,10 +145,6 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
       tcmalloc_metrics, "tcmalloc.transfer-cache-free-bytes",
       "tcmalloc.transfer_cache_free_bytes");
 
-  TcmallocMetric::THREAD_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
-      tcmalloc_metrics, "tcmalloc.thread-cache-free-bytes",
-      "tcmalloc.thread_cache_free_bytes");
-
   TcmallocMetric::PHYSICAL_BYTES_RESERVED =
       tcmalloc_metrics->RegisterMetric(new TcmallocMetric::PhysicalBytesMetric(
           MetricDefs::Get("tcmalloc.physical-bytes-reserved")));
diff --git a/be/src/util/memory-metrics.h b/be/src/util/memory-metrics.h
index 19930e46a..7c068606c 100644
--- a/be/src/util/memory-metrics.h
+++ b/be/src/util/memory-metrics.h
@@ -118,9 +118,6 @@ class TcmallocMetric : public IntGauge {
   /// Number of free bytes waiting to be transferred between central and thread caches.
   static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
 
-  /// Number of free bytes in thread caches.
-  static TcmallocMetric* THREAD_CACHE_FREE_BYTES;
-
   /// Derived metric computing the amount of physical memory (in bytes) used by the
   /// process, including that actually in use and free bytes reserved by tcmalloc. Does not
   /// include the tcmalloc metadata.
diff --git a/be/src/util/metrics-test.cc b/be/src/util/metrics-test.cc
index b8f6ef587..4c8eb6938 100644
--- a/be/src/util/metrics-test.cc
+++ b/be/src/util/metrics-test.cc
@@ -287,10 +287,6 @@ TEST_F(MetricsTest, MemMetric) {
   IntGauge* transfer_cache_free_bytes =
       metrics.FindMetricForTesting<IntGauge>("tcmalloc.transfer-cache-free-bytes");
   ASSERT_TRUE(transfer_cache_free_bytes != NULL);
-
-  IntGauge* thread_cache_free_bytes =
-      metrics.FindMetricForTesting<IntGauge>("tcmalloc.thread-cache-free-bytes");
-  ASSERT_TRUE(thread_cache_free_bytes != NULL);
 #endif
 }
 
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 51761f752..381392222 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2609,7 +2609,7 @@
     "key": "tcmalloc.total-bytes-reserved"
   },
   {
-    "description": "Number of bytes used across all thread caches.",
+    "description": "Number of bytes used across all thread caches. They always count towards virtual memory usage, and unless the underlying memory is swapped out by the OS, they also count towards physical memory usage.",
     "contexts": [
       "STATESTORE",
       "CATALOGSERVER",
@@ -2644,18 +2644,6 @@
     "kind": "GAUGE",
     "key": "tcmalloc.transfer-cache-free-bytes"
   },
-  {
-    "description": "Number of free bytes in thread caches. They always count towards virtual memory usage, and unless the underlying memory is swapped out by the OS, they also count towards physical memory usage.",
-    "contexts": [
-      "STATESTORE",
-      "CATALOGSERVER",
-      "IMPALAD"
-    ],
-    "label": "TCMalloc Thread Cache Free Bytes",
-    "units": "BYTES",
-    "kind": "GAUGE",
-    "key": "tcmalloc.thread-cache-free-bytes"
-  },
   {
     "description": "Bytes allocated from the sanitizer malloc (Sanitizer debug builds only)",
     "contexts": [


(impala) 02/02: IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 498d24d982affcc92ebaedf700be2c3156d8c1bc
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue May 26 12:59:06 2026 -0700

    IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata
    
    Currently, our gperftools tcmalloc metrics like
    tcmalloc.total-bytes-reserved do not include the malloc metadata.
    Back when the tcmalloc metrics were added to Impala, gperftools
    did not expose metrics that would include the malloc metadata.
    Since then, gperftools has added the generic.total_physical_bytes
    metric, which includes the malloc metadata. Malloc metadata can be
    a couple hundred MBs on busy systems, so this is not a major
    change, but it seems like a slight improvement to our metrics.
    
    This changes tcmalloc.physical-bytes-reserved to use the
    generic.total_physical_bytes metric. It changes
    tcmalloc.total-bytes-reserved into a derived metric of
    tcmalloc.physical-bytes-reserved + tcmalloc.pageheap-unmapped-bytes.
    This brings our metrics in line with the text description from
    gperftools used for /memz.
    
    Testing:
     - Compared the metrics to the /memz page
    
    Change-Id: I5b89193ed0b488dce1498ba630fec91611d7c4fd
    Reviewed-on: http://gerrit.cloudera.org:8080/24400
    Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/util/memory-metrics.cc | 59 +++++++++++++++++++++++++++-------------
 be/src/util/memory-metrics.h  | 63 ++++++++++++++++++++++++-------------------
 common/thrift/metrics.json    |  4 +--
 3 files changed, 79 insertions(+), 47 deletions(-)

diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc
index 021209599..4b665c9a5 100644
--- a/be/src/util/memory-metrics.cc
+++ b/be/src/util/memory-metrics.cc
@@ -54,12 +54,12 @@ StringProperty* AggregateMemoryMetrics::THP_KHUGEPAGED_DEFRAG = nullptr;
 
 TcmallocMetric* TcmallocMetric::BYTES_IN_USE = nullptr;
 TcmallocMetric* TcmallocMetric::PAGEHEAP_FREE_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::TOTAL_BYTES_RESERVED = nullptr;
-TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
 TcmallocMetric* TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = nullptr;
 TcmallocMetric* TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = nullptr;
-TcmallocMetric::PhysicalBytesMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
+TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
+TcmallocMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
+TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
+TcmallocMetric::TotalBytesReservedMetric* TcmallocMetric::TOTAL_BYTES_RESERVED = nullptr;
 
 SanitizerMallocMetric* SanitizerMallocMetric::BYTES_ALLOCATED = nullptr;
 
@@ -120,34 +120,57 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
   MetricGroup* tcmalloc_metrics = metrics->GetOrCreateChildGroup("tcmalloc");
   // We rely on TCMalloc for our global memory metrics, so skip setting them up
   // if we're not using TCMalloc.
+
+  /// These metrics line up with the text description used for /memz:
+  /// MALLOC:     3388499040 ( 3231.5 MiB) Bytes in use by application (#1)
+  /// MALLOC: +            0 (    0.0 MiB) Bytes in page heap freelist (#2)
+  /// MALLOC: +     98309992 (   93.8 MiB) Bytes in central cache freelist (#3)
+  /// MALLOC: +      4746496 (    4.5 MiB) Bytes in transfer cache freelist (#4)
+  /// MALLOC: +    177297208 (  169.1 MiB) Bytes in thread cache freelists (#5)
+  /// MALLOC: +     14942208 (   14.2 MiB) Bytes in malloc metadata
+  /// MALLOC:   ------------
+  /// MALLOC: =   3683794944 ( 3513.1 MiB) Actual memory used (physical + swap) (#6)
+  /// MALLOC: +    149757952 (  142.8 MiB) Bytes released to OS (aka unmapped) (#7)
+  /// MALLOC:   ------------
+  /// MALLOC: =   3833552896 ( 3656.0 MiB) Virtual address space used (#8)
+
+  /// #1
   TcmallocMetric::BYTES_IN_USE = TcmallocMetric::CreateAndRegister(
       tcmalloc_metrics, "tcmalloc.bytes-in-use", "generic.current_allocated_bytes");
 
-  TcmallocMetric::TOTAL_BYTES_RESERVED = TcmallocMetric::CreateAndRegister(
-      tcmalloc_metrics, "tcmalloc.total-bytes-reserved", "generic.heap_size");
-
+  /// #2
   TcmallocMetric::PAGEHEAP_FREE_BYTES = TcmallocMetric::CreateAndRegister(
       tcmalloc_metrics, "tcmalloc.pageheap-free-bytes", "tcmalloc.pageheap_free_bytes");
 
-  TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES =
-      TcmallocMetric::CreateAndRegister(tcmalloc_metrics,
-          "tcmalloc.pageheap-unmapped-bytes", "tcmalloc.pageheap_unmapped_bytes");
-
-  TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = TcmallocMetric::CreateAndRegister(
-      tcmalloc_metrics, "tcmalloc.current-total-thread-cache-bytes",
-      "tcmalloc.current_total_thread_cache_bytes");
-
+  /// #3
   TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
       tcmalloc_metrics, "tcmalloc.central-cache-free-bytes",
       "tcmalloc.central_cache_free_bytes");
 
+  /// #4
   TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
       tcmalloc_metrics, "tcmalloc.transfer-cache-free-bytes",
       "tcmalloc.transfer_cache_free_bytes");
 
-  TcmallocMetric::PHYSICAL_BYTES_RESERVED =
-      tcmalloc_metrics->RegisterMetric(new TcmallocMetric::PhysicalBytesMetric(
-          MetricDefs::Get("tcmalloc.physical-bytes-reserved")));
+  /// #5
+  TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = TcmallocMetric::CreateAndRegister(
+      tcmalloc_metrics, "tcmalloc.current-total-thread-cache-bytes",
+      "tcmalloc.current_total_thread_cache_bytes");
+
+  /// #6
+  TcmallocMetric::PHYSICAL_BYTES_RESERVED = TcmallocMetric::CreateAndRegister(
+      tcmalloc_metrics, "tcmalloc.physical-bytes-reserved",
+      "generic.total_physical_bytes");
+
+  /// #7
+  TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES =
+      TcmallocMetric::CreateAndRegister(tcmalloc_metrics,
+          "tcmalloc.pageheap-unmapped-bytes", "tcmalloc.pageheap_unmapped_bytes");
+
+  /// #8
+  TcmallocMetric::TOTAL_BYTES_RESERVED =
+      tcmalloc_metrics->RegisterMetric(new TcmallocMetric::TotalBytesReservedMetric(
+          MetricDefs::Get("tcmalloc.total-bytes-reserved")));
 
   used_metrics.push_back(TcmallocMetric::PHYSICAL_BYTES_RESERVED);
 #endif
diff --git a/be/src/util/memory-metrics.h b/be/src/util/memory-metrics.h
index 7c068606c..4c9c3d0fc 100644
--- a/be/src/util/memory-metrics.h
+++ b/be/src/util/memory-metrics.h
@@ -89,48 +89,57 @@ class AggregateMemoryMetrics {
 };
 
 /// Specialised metric which exposes numeric properties from tcmalloc.
+/// These metrics line up with the text description used for /memz:
+/// MALLOC:     3388499040 ( 3231.5 MiB) Bytes in use by application (#1)
+/// MALLOC: +            0 (    0.0 MiB) Bytes in page heap freelist (#2)
+/// MALLOC: +     98309992 (   93.8 MiB) Bytes in central cache freelist (#3)
+/// MALLOC: +      4746496 (    4.5 MiB) Bytes in transfer cache freelist (#4)
+/// MALLOC: +    177297208 (  169.1 MiB) Bytes in thread cache freelists (#5)
+/// MALLOC: +     14942208 (   14.2 MiB) Bytes in malloc metadata
+/// MALLOC:   ------------
+/// MALLOC: =   3683794944 ( 3513.1 MiB) Actual memory used (physical + swap) (#6)
+/// MALLOC: +    149757952 (  142.8 MiB) Bytes released to OS (aka unmapped) (#7)
+/// MALLOC:   ------------
+/// MALLOC: =   3833552896 ( 3656.0 MiB) Virtual address space used (#8)
 class TcmallocMetric : public IntGauge {
  public:
-  /// Number of bytes allocated by tcmalloc, currently used by the application.
+  /// #1: Number of bytes allocated by tcmalloc, currently used by the application.
   static TcmallocMetric* BYTES_IN_USE;
 
-  /// Number of bytes of system memory reserved by tcmalloc, including that in use by the
-  /// application. Does not include what tcmalloc accounts for as 'malloc metadata' in
-  /// /memz. That is, this is memory reserved by tcmalloc that the application can use.
-  /// Includes unmapped virtual memory.
-  static TcmallocMetric* TOTAL_BYTES_RESERVED;
-
-  /// Number of bytes reserved and still mapped by tcmalloc that are not allocated to the
-  /// application.
+  /// #2: Number of bytes reserved and still mapped by tcmalloc that are not allocated
+  /// to the application.
   static TcmallocMetric* PAGEHEAP_FREE_BYTES;
 
-  /// Number of bytes once reserved by tcmalloc, but released back to the operating system
-  /// so that their use incurs a pagefault. Contributes to the total amount of virtual
-  /// address space used, but not to the physical memory usage.
-  static TcmallocMetric* PAGEHEAP_UNMAPPED_BYTES;
+  /// #3: Number of free bytes in the central cache assigned to size classes.
+  static TcmallocMetric* CENTRAL_CACHE_FREE_BYTES;
+
+  /// #4: Number of free bytes waiting to be transferred between central and thread
+  /// caches.
+  static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
 
-  /// Number of bytes used across all thread caches.
+  /// #5: Number of bytes used across all thread caches.
   static TcmallocMetric* CURRENT_TOTAL_THREAD_CACHE_BYTES;
 
-  /// Number of free bytes in the central cache assigned to size classes.
-  static TcmallocMetric* CENTRAL_CACHE_FREE_BYTES;
+  /// #6: Number of physical bytes in use (including any tcmalloc caches and metadata)
+  static TcmallocMetric* PHYSICAL_BYTES_RESERVED;
 
-  /// Number of free bytes waiting to be transferred between central and thread caches.
-  static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
+  /// #7: Number of bytes once reserved by tcmalloc, but released back to the operating
+  /// system so that their use incurs a pagefault. Contributes to the total amount of
+  /// virtual address space used, but not to the physical memory usage.
+  static TcmallocMetric* PAGEHEAP_UNMAPPED_BYTES;
 
-  /// Derived metric computing the amount of physical memory (in bytes) used by the
-  /// process, including that actually in use and free bytes reserved by tcmalloc. Does not
-  /// include the tcmalloc metadata.
-  class PhysicalBytesMetric : public IntGauge {
-   public:
-    PhysicalBytesMetric(const TMetricDef& def) : IntGauge(def, 0) { }
+  /// #8: Derived metric computing the amount of virtual memory (in bytes) used by the
+  /// process, including all different types of tcmalloc memory including unmapped
+  /// virtual memory.
+  class TotalBytesReservedMetric : public IntGauge {
+  public:
+    TotalBytesReservedMetric(const TMetricDef& def) : IntGauge(def, 0) { }
 
     virtual int64_t GetValue() override {
-      return TOTAL_BYTES_RESERVED->GetValue() - PAGEHEAP_UNMAPPED_BYTES->GetValue();
+      return PHYSICAL_BYTES_RESERVED->GetValue() + PAGEHEAP_UNMAPPED_BYTES->GetValue();
     }
   };
-
-  static PhysicalBytesMetric* PHYSICAL_BYTES_RESERVED;
+  static TotalBytesReservedMetric* TOTAL_BYTES_RESERVED;
 
   static TcmallocMetric* CreateAndRegister(MetricGroup* metrics, const std::string& key,
       const std::string& tcmalloc_var);
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 381392222..d72768e38 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2585,7 +2585,7 @@
     "key": "tcmalloc.pageheap-unmapped-bytes"
   },
   {
-    "description": "Derived metric computing the amount of physical memory (in bytes) used by the process, including that actually in use and free bytes reserved by tcmalloc. Does not include the tcmalloc metadata.",
+    "description": "Number of bytes of physical memory in use by the process, including tcmalloc caches and metadata.",
     "contexts": [
       "STATESTORE",
       "CATALOGSERVER",
@@ -2597,7 +2597,7 @@
     "key": "tcmalloc.physical-bytes-reserved"
   },
   {
-    "description": "Bytes of system memory reserved by TCMalloc.",
+    "description": "Derived metric computing the amount of system memory reserved by TCMalloc. This includes all tcmalloc caches, metadata, and unmapped virtual memory.",
     "contexts": [
       "STATESTORE",
       "CATALOGSERVER",