You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/05/09 15:38:14 UTC
(tsfile) 01/02: optimize get_timeseries_metadata() to eliminate N+1 I/O
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch colin_try
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit bbcc010da103511b6c08f2f6bd6b247c9089ac0d
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Sat May 9 20:00:54 2026 +0800
optimize get_timeseries_metadata() to eliminate N+1 I/O
- Add get_device_timeseries_meta_by_offset() that accepts pre-resolved
offsets, skipping redundant load_device_index_entry() tree search and
reusing the deserialized measurement index node via get_all_leaf()
instead of re-reading the same bytes through
load_all_measurement_index_entry()
- Add get_all_device_entries() to collect device IDs with their
(start_offset, end_offset) in a single index tree traversal
- Rewrite get_timeseries_metadata() to use the above two methods
- Remove redundant PageArena::init() call in get_timeseries_metadata_impl
---
cpp/src/file/tsfile_io_reader.cc | 64 ++++++++++++++++++++++++++
cpp/src/file/tsfile_io_reader.h | 5 +++
cpp/src/reader/tsfile_reader.cc | 97 ++++++++++++++++++++++++++++++++++++----
3 files changed, 158 insertions(+), 8 deletions(-)
diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc
index 351ab8620..fdf85b357 100644
--- a/cpp/src/file/tsfile_io_reader.cc
+++ b/cpp/src/file/tsfile_io_reader.cc
@@ -202,6 +202,70 @@ int TsFileIOReader::get_device_timeseries_meta_without_chunk_meta(
return ret;
}
+int TsFileIOReader::get_device_timeseries_meta_by_offset(
+ int64_t start_offset, int64_t end_offset,
+ std::vector<ITimeseriesIndex*>& timeseries_indexs, PageArena& pa) {
+ int ret = E_OK;
+ load_tsfile_meta_if_necessary();
+
+ std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t>>
+ meta_index_entry_list;
+ bool is_aligned = false;
+ TimeseriesIndex* time_timeseries_index = nullptr;
+
+ ASSERT(start_offset < end_offset);
+ const int32_t read_size = end_offset - start_offset;
+ int32_t ret_read_len = 0;
+ char* data_buf = (char*)pa.alloc(read_size);
+ void* m_idx_node_buf = pa.alloc(sizeof(MetaIndexNode));
+ if (IS_NULL(data_buf) || IS_NULL(m_idx_node_buf)) {
+ return E_OOM;
+ }
+ auto* top_node_ptr = new (m_idx_node_buf) MetaIndexNode(&pa);
+ auto top_node = std::shared_ptr<MetaIndexNode>(top_node_ptr,
+ MetaIndexNode::self_deleter);
+ if (RET_FAIL(read_file_->read(start_offset, data_buf, read_size,
+ ret_read_len))) {
+ return ret;
+ }
+ if (RET_FAIL(top_node->deserialize_from(data_buf, read_size))) {
+ return ret;
+ }
+
+ is_aligned = is_aligned_device(top_node);
+ if (is_aligned) {
+ if (RET_FAIL(
+ get_time_column_metadata(top_node, time_timeseries_index, pa))) {
+ return ret;
+ }
+ }
+
+ get_all_leaf(top_node, meta_index_entry_list);
+
+ if (RET_FAIL(do_load_all_timeseries_index(meta_index_entry_list, pa,
+ timeseries_indexs))) {
+ return ret;
+ }
+
+ if (is_aligned && time_timeseries_index != nullptr) {
+ for (size_t i = 0; i < timeseries_indexs.size(); i++) {
+ void* buf = pa.alloc(sizeof(AlignedTimeseriesIndex));
+ if (IS_NULL(buf)) {
+ return E_OOM;
+ }
+ auto* aligned_ts_idx = new (buf) AlignedTimeseriesIndex;
+ aligned_ts_idx->time_ts_idx_ = time_timeseries_index;
+ aligned_ts_idx->value_ts_idx_ =
+ dynamic_cast<TimeseriesIndex*>(timeseries_indexs[i]);
+ if (aligned_ts_idx->value_ts_idx_ == nullptr) {
+ return E_TYPE_NOT_MATCH;
+ }
+ timeseries_indexs[i] = aligned_ts_idx;
+ }
+ }
+ return ret;
+}
+
bool TsFileIOReader::filter_stasify(ITimeseriesIndex* ts_index,
Filter* time_filter) {
ASSERT(ts_index->get_statistic() != nullptr);
diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h
index 1624e8bee..506aa7f47 100644
--- a/cpp/src/file/tsfile_io_reader.h
+++ b/cpp/src/file/tsfile_io_reader.h
@@ -91,6 +91,11 @@ class TsFileIOReader {
std::vector<ITimeseriesIndex*>& timeseries_indexs,
common::PageArena& pa);
+ int get_device_timeseries_meta_by_offset(
+ int64_t start_offset, int64_t end_offset,
+ std::vector<ITimeseriesIndex*>& timeseries_indexs,
+ common::PageArena& pa);
+
private:
FORCE_INLINE int64_t file_size() const { return read_file_->file_size(); }
diff --git a/cpp/src/reader/tsfile_reader.cc b/cpp/src/reader/tsfile_reader.cc
index 196189f0f..f98852671 100644
--- a/cpp/src/reader/tsfile_reader.cc
+++ b/cpp/src/reader/tsfile_reader.cc
@@ -25,6 +25,69 @@
using namespace common;
using namespace storage;
+namespace {
+
+struct DeviceMetaEntry {
+ std::shared_ptr<IDeviceID> device_id;
+ int64_t start_offset;
+ int64_t end_offset;
+};
+
+int get_all_device_entries(std::vector<DeviceMetaEntry>& entries,
+ std::shared_ptr<MetaIndexNode> index_node,
+ ReadFile* read_file, PageArena& pa) {
+ int ret = E_OK;
+ if (index_node == nullptr) {
+ return ret;
+ }
+ if (index_node->node_type_ == LEAF_DEVICE) {
+ for (size_t i = 0; i < index_node->children_.size(); i++) {
+ DeviceMetaEntry entry;
+ entry.device_id = index_node->children_[i]->get_device_id();
+ entry.start_offset = index_node->children_[i]->get_offset();
+ entry.end_offset =
+ (i + 1 < index_node->children_.size())
+ ? index_node->children_[i + 1]->get_offset()
+ : index_node->end_offset_;
+ entries.push_back(entry);
+ }
+ } else {
+ for (size_t idx = 0; idx < index_node->children_.size(); idx++) {
+ auto meta_index_entry = index_node->children_[idx];
+ int start_offset = meta_index_entry->get_offset();
+ int end_offset = index_node->end_offset_;
+ if (idx + 1 < index_node->children_.size()) {
+ end_offset = index_node->children_[idx + 1]->get_offset();
+ }
+ ASSERT(end_offset - start_offset > 0);
+ const int32_t read_size = (int32_t)end_offset - start_offset;
+ int32_t ret_read_len = 0;
+ char* data_buf = (char*)pa.alloc(read_size);
+ void* m_idx_node_buf = pa.alloc(sizeof(MetaIndexNode));
+ if (IS_NULL(data_buf) || IS_NULL(m_idx_node_buf)) {
+ return E_OOM;
+ }
+ auto* top_node_ptr = new (m_idx_node_buf) MetaIndexNode(&pa);
+ auto top_node = std::shared_ptr<MetaIndexNode>(
+ top_node_ptr, [](MetaIndexNode* ptr) {
+ if (ptr) {
+ ptr->~MetaIndexNode();
+ }
+ });
+ if (RET_FAIL(read_file->read(start_offset, data_buf, read_size,
+ ret_read_len))) {
+ } else if (RET_FAIL(top_node->device_deserialize_from(
+ data_buf, read_size))) {
+ } else {
+ ret = get_all_device_entries(entries, top_node, read_file, pa);
+ }
+ }
+ }
+ return ret;
+}
+
+} // namespace
+
namespace storage {
TsFileReader::TsFileReader()
: read_file_(nullptr),
@@ -348,8 +411,6 @@ int TsFileReader::get_timeseries_metadata_impl(
std::vector<std::shared_ptr<ITimeseriesIndex>>& result) {
int ret = E_OK;
std::vector<ITimeseriesIndex*> timeseries_indexs;
- tsfile_reader_meta_pa_.init(512, MOD_TSFILE_READER);
- // Pointers are owned by tsfile_reader_meta_pa_; shared_ptr must not delete
auto noop_deleter = [](ITimeseriesIndex*) {};
if (RET_FAIL(
tsfile_executor_->get_tsfile_io_reader()
@@ -378,13 +439,33 @@ DeviceTimeseriesMetadataMap TsFileReader::get_timeseries_metadata(
}
DeviceTimeseriesMetadataMap TsFileReader::get_timeseries_metadata() {
- // Collect metadata for all devices present in the file
DeviceTimeseriesMetadataMap result;
- auto device_ids = get_all_device_ids();
- for (const auto& device_id : device_ids) {
- std::vector<std::shared_ptr<ITimeseriesIndex>> list;
- if (get_timeseries_metadata_impl(device_id, list) == E_OK) {
- result.insert(std::make_pair(device_id, std::move(list)));
+ TsFileMeta* tsfile_meta = tsfile_executor_->get_tsfile_meta();
+ if (tsfile_meta == nullptr) {
+ return result;
+ }
+
+ PageArena pa;
+ pa.init(512, MOD_TSFILE_READER);
+ std::vector<DeviceMetaEntry> entries;
+ for (auto& table_entry : tsfile_meta->table_metadata_index_node_map_) {
+ get_all_device_entries(entries, table_entry.second, read_file_, pa);
+ }
+
+ auto noop_deleter = [](ITimeseriesIndex*) {};
+ for (auto& device_entry : entries) {
+ std::vector<ITimeseriesIndex*> raw_ts_indexes;
+ if (tsfile_executor_->get_tsfile_io_reader()
+ ->get_device_timeseries_meta_by_offset(
+ device_entry.start_offset, device_entry.end_offset,
+ raw_ts_indexes, tsfile_reader_meta_pa_) == E_OK) {
+ std::vector<std::shared_ptr<ITimeseriesIndex>> list;
+ for (auto ts_idx : raw_ts_indexes) {
+ list.emplace_back(
+ std::shared_ptr<ITimeseriesIndex>(ts_idx, noop_deleter));
+ }
+ result.insert(
+ std::make_pair(device_entry.device_id, std::move(list)));
}
}
return result;