You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/03 14:14:50 UTC
(doris) branch branch-4.1 updated: branch-4.1: [opt](memory) lazy-allocate PrefetchBuffer backing buffer to reduce peak memory (#62005)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new b918e156a04 branch-4.1: [opt](memory) lazy-allocate PrefetchBuffer backing buffer to reduce peak memory (#62005)
b918e156a04 is described below
commit b918e156a04d91c89b3a76f41d0bce65856ff408
Author: hui lai <la...@selectdb.com>
AuthorDate: Fri Apr 3 22:14:43 2026 +0800
branch-4.1: [opt](memory) lazy-allocate PrefetchBuffer backing buffer to reduce peak memory (#62005)
pick https://github.com/apache/doris/pull/61482
---
be/src/io/fs/buffered_reader.cpp | 7 +++++++
be/src/io/fs/buffered_reader.h | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index 7c215696375..cc5a983b1cd 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -449,6 +449,13 @@ void PrefetchBuffer::prefetch_buffer() {
_prefetched.notify_all();
}
+ // Lazy-allocate the backing buffer on first actual prefetch, avoiding the cost of
+ // pre-allocating memory for readers that are initialized but never read (e.g. when
+ // many file readers are created concurrently for a TVF scan over many small S3 files).
+ if (!_buf) {
+ _buf = std::make_unique<char[]>(_size);
+ }
+
int read_range_index = search_read_range(_offset);
size_t buf_size;
if (read_range_index == -1) {
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index e9c696941eb..34b59d2e19a 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -430,7 +430,7 @@ struct PrefetchBuffer : std::enable_shared_from_this<PrefetchBuffer>, public Pro
_whole_buffer_size(whole_buffer_size),
_reader(reader),
_io_ctx(io_ctx),
- _buf(new char[buffer_size]),
+ _buf(nullptr),
_sync_profile(std::move(sync_profile)) {}
PrefetchBuffer(PrefetchBuffer&& other)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org