You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2026/06/09 18:04:53 UTC
(impala) branch master updated (cf03d9376 -> 498d24d98)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from cf03d9376 IMPALA-12696: Update HS2 definitions (TCLIService.thrift)
new aac753cb3 IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric
new 498d24d98 IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/util/memory-metrics.cc | 62 +++++++++++++++++++++++++---------------
be/src/util/memory-metrics.h | 66 +++++++++++++++++++++++--------------------
be/src/util/metrics-test.cc | 4 ---
common/thrift/metrics.json | 18 ++----------
4 files changed, 79 insertions(+), 71 deletions(-)
(impala) 01/02: IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit aac753cb3208effb6f41f916b53a17a05ed967c6
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue May 26 11:30:22 2026 -0700
IMPALA-15021: Remove duplicate tcmalloc.thread_cache_free_bytes metric
In IMPALA-14874, we introduced several tcmalloc metrics
including tcmalloc.current-total-thread-cache-bytes and
tcmalloc.thread-cache-free-bytes. These are actually the
same underlying metric, so this removes tcmalloc.thread-cache-free-bytes.
Testing:
- Ran core tests
Change-Id: I7ff431b00c10fe9ff99bfd6c909739d8a97fcf3a
Reviewed-on: http://gerrit.cloudera.org:8080/24399
Reviewed-by: Michael Smith <mi...@cloudera.com>
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
be/src/util/memory-metrics.cc | 5 -----
be/src/util/memory-metrics.h | 3 ---
be/src/util/metrics-test.cc | 4 ----
common/thrift/metrics.json | 14 +-------------
4 files changed, 1 insertion(+), 25 deletions(-)
diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc
index 9cc0bb3e0..021209599 100644
--- a/be/src/util/memory-metrics.cc
+++ b/be/src/util/memory-metrics.cc
@@ -59,7 +59,6 @@ TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
TcmallocMetric* TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = nullptr;
TcmallocMetric* TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::THREAD_CACHE_FREE_BYTES = nullptr;
TcmallocMetric::PhysicalBytesMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
SanitizerMallocMetric* SanitizerMallocMetric::BYTES_ALLOCATED = nullptr;
@@ -146,10 +145,6 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
tcmalloc_metrics, "tcmalloc.transfer-cache-free-bytes",
"tcmalloc.transfer_cache_free_bytes");
- TcmallocMetric::THREAD_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
- tcmalloc_metrics, "tcmalloc.thread-cache-free-bytes",
- "tcmalloc.thread_cache_free_bytes");
-
TcmallocMetric::PHYSICAL_BYTES_RESERVED =
tcmalloc_metrics->RegisterMetric(new TcmallocMetric::PhysicalBytesMetric(
MetricDefs::Get("tcmalloc.physical-bytes-reserved")));
diff --git a/be/src/util/memory-metrics.h b/be/src/util/memory-metrics.h
index 19930e46a..7c068606c 100644
--- a/be/src/util/memory-metrics.h
+++ b/be/src/util/memory-metrics.h
@@ -118,9 +118,6 @@ class TcmallocMetric : public IntGauge {
/// Number of free bytes waiting to be transferred between central and thread caches.
static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
- /// Number of free bytes in thread caches.
- static TcmallocMetric* THREAD_CACHE_FREE_BYTES;
-
/// Derived metric computing the amount of physical memory (in bytes) used by the
/// process, including that actually in use and free bytes reserved by tcmalloc. Does not
/// include the tcmalloc metadata.
diff --git a/be/src/util/metrics-test.cc b/be/src/util/metrics-test.cc
index b8f6ef587..4c8eb6938 100644
--- a/be/src/util/metrics-test.cc
+++ b/be/src/util/metrics-test.cc
@@ -287,10 +287,6 @@ TEST_F(MetricsTest, MemMetric) {
IntGauge* transfer_cache_free_bytes =
metrics.FindMetricForTesting<IntGauge>("tcmalloc.transfer-cache-free-bytes");
ASSERT_TRUE(transfer_cache_free_bytes != NULL);
-
- IntGauge* thread_cache_free_bytes =
- metrics.FindMetricForTesting<IntGauge>("tcmalloc.thread-cache-free-bytes");
- ASSERT_TRUE(thread_cache_free_bytes != NULL);
#endif
}
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 51761f752..381392222 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2609,7 +2609,7 @@
"key": "tcmalloc.total-bytes-reserved"
},
{
- "description": "Number of bytes used across all thread caches.",
+ "description": "Number of bytes used across all thread caches. They always count towards virtual memory usage, and unless the underlying memory is swapped out by the OS, they also count towards physical memory usage.",
"contexts": [
"STATESTORE",
"CATALOGSERVER",
@@ -2644,18 +2644,6 @@
"kind": "GAUGE",
"key": "tcmalloc.transfer-cache-free-bytes"
},
- {
- "description": "Number of free bytes in thread caches. They always count towards virtual memory usage, and unless the underlying memory is swapped out by the OS, they also count towards physical memory usage.",
- "contexts": [
- "STATESTORE",
- "CATALOGSERVER",
- "IMPALAD"
- ],
- "label": "TCMalloc Thread Cache Free Bytes",
- "units": "BYTES",
- "kind": "GAUGE",
- "key": "tcmalloc.thread-cache-free-bytes"
- },
{
"description": "Bytes allocated from the sanitizer malloc (Sanitizer debug builds only)",
"contexts": [
(impala) 02/02: IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 498d24d982affcc92ebaedf700be2c3156d8c1bc
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue May 26 12:59:06 2026 -0700
IMPALA-15022: Rework tcmalloc metrics to include tcmalloc metadata
Currently, our gperftools tcmalloc metrics like
tcmalloc.total-bytes-reserved do not include the malloc metadata.
Back when the tcmalloc metrics were added to Impala, gperftools
did not expose metrics that would include the malloc metadata.
Since then, gperftools has added the generic.total_physical_bytes
metric, which includes the malloc metadata. Malloc metadata can be
a couple hundred MBs on busy systems, so this is not a major
change, but it seems like a slight improvement to our metrics.
This changes tcmalloc.physical-bytes-reserved to use the
generic.total_physical_bytes metric. It changes
tcmalloc.total-bytes-reserved into a derived metric of
tcmalloc.physical-bytes-reserved + tcmalloc.pageheap-unmapped-bytes.
This brings our metrics in line with the text description from
gperftools used for /memz.
Testing:
- Compared the metrics to the /memz page
Change-Id: I5b89193ed0b488dce1498ba630fec91611d7c4fd
Reviewed-on: http://gerrit.cloudera.org:8080/24400
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/memory-metrics.cc | 59 +++++++++++++++++++++++++++-------------
be/src/util/memory-metrics.h | 63 ++++++++++++++++++++++++-------------------
common/thrift/metrics.json | 4 +--
3 files changed, 79 insertions(+), 47 deletions(-)
diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc
index 021209599..4b665c9a5 100644
--- a/be/src/util/memory-metrics.cc
+++ b/be/src/util/memory-metrics.cc
@@ -54,12 +54,12 @@ StringProperty* AggregateMemoryMetrics::THP_KHUGEPAGED_DEFRAG = nullptr;
TcmallocMetric* TcmallocMetric::BYTES_IN_USE = nullptr;
TcmallocMetric* TcmallocMetric::PAGEHEAP_FREE_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::TOTAL_BYTES_RESERVED = nullptr;
-TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
-TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
TcmallocMetric* TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = nullptr;
TcmallocMetric* TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = nullptr;
-TcmallocMetric::PhysicalBytesMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
+TcmallocMetric* TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = nullptr;
+TcmallocMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
+TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
+TcmallocMetric::TotalBytesReservedMetric* TcmallocMetric::TOTAL_BYTES_RESERVED = nullptr;
SanitizerMallocMetric* SanitizerMallocMetric::BYTES_ALLOCATED = nullptr;
@@ -120,34 +120,57 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
MetricGroup* tcmalloc_metrics = metrics->GetOrCreateChildGroup("tcmalloc");
// We rely on TCMalloc for our global memory metrics, so skip setting them up
// if we're not using TCMalloc.
+
+ /// These metrics line up with the text description used for /memz:
+ /// MALLOC: 3388499040 ( 3231.5 MiB) Bytes in use by application (#1)
+ /// MALLOC: + 0 ( 0.0 MiB) Bytes in page heap freelist (#2)
+ /// MALLOC: + 98309992 ( 93.8 MiB) Bytes in central cache freelist (#3)
+ /// MALLOC: + 4746496 ( 4.5 MiB) Bytes in transfer cache freelist (#4)
+ /// MALLOC: + 177297208 ( 169.1 MiB) Bytes in thread cache freelists (#5)
+ /// MALLOC: + 14942208 ( 14.2 MiB) Bytes in malloc metadata
+ /// MALLOC: ------------
+ /// MALLOC: = 3683794944 ( 3513.1 MiB) Actual memory used (physical + swap) (#6)
+ /// MALLOC: + 149757952 ( 142.8 MiB) Bytes released to OS (aka unmapped) (#7)
+ /// MALLOC: ------------
+ /// MALLOC: = 3833552896 ( 3656.0 MiB) Virtual address space used (#8)
+
+ /// #1
TcmallocMetric::BYTES_IN_USE = TcmallocMetric::CreateAndRegister(
tcmalloc_metrics, "tcmalloc.bytes-in-use", "generic.current_allocated_bytes");
- TcmallocMetric::TOTAL_BYTES_RESERVED = TcmallocMetric::CreateAndRegister(
- tcmalloc_metrics, "tcmalloc.total-bytes-reserved", "generic.heap_size");
-
+ /// #2
TcmallocMetric::PAGEHEAP_FREE_BYTES = TcmallocMetric::CreateAndRegister(
tcmalloc_metrics, "tcmalloc.pageheap-free-bytes", "tcmalloc.pageheap_free_bytes");
- TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES =
- TcmallocMetric::CreateAndRegister(tcmalloc_metrics,
- "tcmalloc.pageheap-unmapped-bytes", "tcmalloc.pageheap_unmapped_bytes");
-
- TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = TcmallocMetric::CreateAndRegister(
- tcmalloc_metrics, "tcmalloc.current-total-thread-cache-bytes",
- "tcmalloc.current_total_thread_cache_bytes");
-
+ /// #3
TcmallocMetric::CENTRAL_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
tcmalloc_metrics, "tcmalloc.central-cache-free-bytes",
"tcmalloc.central_cache_free_bytes");
+ /// #4
TcmallocMetric::TRANSFER_CACHE_FREE_BYTES = TcmallocMetric::CreateAndRegister(
tcmalloc_metrics, "tcmalloc.transfer-cache-free-bytes",
"tcmalloc.transfer_cache_free_bytes");
- TcmallocMetric::PHYSICAL_BYTES_RESERVED =
- tcmalloc_metrics->RegisterMetric(new TcmallocMetric::PhysicalBytesMetric(
- MetricDefs::Get("tcmalloc.physical-bytes-reserved")));
+ /// #5
+ TcmallocMetric::CURRENT_TOTAL_THREAD_CACHE_BYTES = TcmallocMetric::CreateAndRegister(
+ tcmalloc_metrics, "tcmalloc.current-total-thread-cache-bytes",
+ "tcmalloc.current_total_thread_cache_bytes");
+
+ /// #6
+ TcmallocMetric::PHYSICAL_BYTES_RESERVED = TcmallocMetric::CreateAndRegister(
+ tcmalloc_metrics, "tcmalloc.physical-bytes-reserved",
+ "generic.total_physical_bytes");
+
+ /// #7
+ TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES =
+ TcmallocMetric::CreateAndRegister(tcmalloc_metrics,
+ "tcmalloc.pageheap-unmapped-bytes", "tcmalloc.pageheap_unmapped_bytes");
+
+ /// #8
+ TcmallocMetric::TOTAL_BYTES_RESERVED =
+ tcmalloc_metrics->RegisterMetric(new TcmallocMetric::TotalBytesReservedMetric(
+ MetricDefs::Get("tcmalloc.total-bytes-reserved")));
used_metrics.push_back(TcmallocMetric::PHYSICAL_BYTES_RESERVED);
#endif
diff --git a/be/src/util/memory-metrics.h b/be/src/util/memory-metrics.h
index 7c068606c..4c9c3d0fc 100644
--- a/be/src/util/memory-metrics.h
+++ b/be/src/util/memory-metrics.h
@@ -89,48 +89,57 @@ class AggregateMemoryMetrics {
};
/// Specialised metric which exposes numeric properties from tcmalloc.
+/// These metrics line up with the text description used for /memz:
+/// MALLOC: 3388499040 ( 3231.5 MiB) Bytes in use by application (#1)
+/// MALLOC: + 0 ( 0.0 MiB) Bytes in page heap freelist (#2)
+/// MALLOC: + 98309992 ( 93.8 MiB) Bytes in central cache freelist (#3)
+/// MALLOC: + 4746496 ( 4.5 MiB) Bytes in transfer cache freelist (#4)
+/// MALLOC: + 177297208 ( 169.1 MiB) Bytes in thread cache freelists (#5)
+/// MALLOC: + 14942208 ( 14.2 MiB) Bytes in malloc metadata
+/// MALLOC: ------------
+/// MALLOC: = 3683794944 ( 3513.1 MiB) Actual memory used (physical + swap) (#6)
+/// MALLOC: + 149757952 ( 142.8 MiB) Bytes released to OS (aka unmapped) (#7)
+/// MALLOC: ------------
+/// MALLOC: = 3833552896 ( 3656.0 MiB) Virtual address space used (#8)
class TcmallocMetric : public IntGauge {
public:
- /// Number of bytes allocated by tcmalloc, currently used by the application.
+ /// #1: Number of bytes allocated by tcmalloc, currently used by the application.
static TcmallocMetric* BYTES_IN_USE;
- /// Number of bytes of system memory reserved by tcmalloc, including that in use by the
- /// application. Does not include what tcmalloc accounts for as 'malloc metadata' in
- /// /memz. That is, this is memory reserved by tcmalloc that the application can use.
- /// Includes unmapped virtual memory.
- static TcmallocMetric* TOTAL_BYTES_RESERVED;
-
- /// Number of bytes reserved and still mapped by tcmalloc that are not allocated to the
- /// application.
+ /// #2: Number of bytes reserved and still mapped by tcmalloc that are not allocated
+ /// to the application.
static TcmallocMetric* PAGEHEAP_FREE_BYTES;
- /// Number of bytes once reserved by tcmalloc, but released back to the operating system
- /// so that their use incurs a pagefault. Contributes to the total amount of virtual
- /// address space used, but not to the physical memory usage.
- static TcmallocMetric* PAGEHEAP_UNMAPPED_BYTES;
+ /// #3: Number of free bytes in the central cache assigned to size classes.
+ static TcmallocMetric* CENTRAL_CACHE_FREE_BYTES;
+
+ /// #4: Number of free bytes waiting to be transferred between central and thread
+ /// caches.
+ static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
- /// Number of bytes used across all thread caches.
+ /// #5: Number of bytes used across all thread caches.
static TcmallocMetric* CURRENT_TOTAL_THREAD_CACHE_BYTES;
- /// Number of free bytes in the central cache assigned to size classes.
- static TcmallocMetric* CENTRAL_CACHE_FREE_BYTES;
+ /// #6: Number of physical bytes in use (including any tcmalloc caches and metadata)
+ static TcmallocMetric* PHYSICAL_BYTES_RESERVED;
- /// Number of free bytes waiting to be transferred between central and thread caches.
- static TcmallocMetric* TRANSFER_CACHE_FREE_BYTES;
+ /// #7: Number of bytes once reserved by tcmalloc, but released back to the operating
+ /// system so that their use incurs a pagefault. Contributes to the total amount of
+ /// virtual address space used, but not to the physical memory usage.
+ static TcmallocMetric* PAGEHEAP_UNMAPPED_BYTES;
- /// Derived metric computing the amount of physical memory (in bytes) used by the
- /// process, including that actually in use and free bytes reserved by tcmalloc. Does not
- /// include the tcmalloc metadata.
- class PhysicalBytesMetric : public IntGauge {
- public:
- PhysicalBytesMetric(const TMetricDef& def) : IntGauge(def, 0) { }
+ /// #8: Derived metric computing the amount of virtual memory (in bytes) used by the
+ /// process, including all different types of tcmalloc memory including unmapped
+ /// virtual memory.
+ class TotalBytesReservedMetric : public IntGauge {
+ public:
+ TotalBytesReservedMetric(const TMetricDef& def) : IntGauge(def, 0) { }
virtual int64_t GetValue() override {
- return TOTAL_BYTES_RESERVED->GetValue() - PAGEHEAP_UNMAPPED_BYTES->GetValue();
+ return PHYSICAL_BYTES_RESERVED->GetValue() + PAGEHEAP_UNMAPPED_BYTES->GetValue();
}
};
-
- static PhysicalBytesMetric* PHYSICAL_BYTES_RESERVED;
+ static TotalBytesReservedMetric* TOTAL_BYTES_RESERVED;
static TcmallocMetric* CreateAndRegister(MetricGroup* metrics, const std::string& key,
const std::string& tcmalloc_var);
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 381392222..d72768e38 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2585,7 +2585,7 @@
"key": "tcmalloc.pageheap-unmapped-bytes"
},
{
- "description": "Derived metric computing the amount of physical memory (in bytes) used by the process, including that actually in use and free bytes reserved by tcmalloc. Does not include the tcmalloc metadata.",
+ "description": "Number of bytes of physical memory in use by the process, including tcmalloc caches and metadata.",
"contexts": [
"STATESTORE",
"CATALOGSERVER",
@@ -2597,7 +2597,7 @@
"key": "tcmalloc.physical-bytes-reserved"
},
{
- "description": "Bytes of system memory reserved by TCMalloc.",
+ "description": "Derived metric computing the amount of system memory reserved by TCMalloc. This includes all tcmalloc caches, metadata, and unmapped virtual memory.",
"contexts": [
"STATESTORE",
"CATALOGSERVER",