You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/12/16 06:38:02 UTC
(doris) branch branch-4.0 updated: branch-4.0 [Feature](Segment) Support external meta V3 with common format (#59042)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 21fa64b8581 branch-4.0 [Feature](Segment) Support external meta V3 with common format (#59042)
21fa64b8581 is described below
commit 21fa64b85818749a99639e9fd0f7b7efd2a13c43
Author: lihangyu <li...@selectdb.com>
AuthorDate: Tue Dec 16 14:37:54 2025 +0800
branch-4.0 [Feature](Segment) Support external meta V3 with common format (#59042)
backport https://github.com/apache/doris/pull/57702
https://github.com/apache/doris/pull/58461
https://github.com/apache/doris/pull/58589
https://github.com/apache/doris/pull/58681
---
be/src/cloud/pb_convert.cpp | 4 +
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 3 +-
.../rowset/segment_v2/column_meta_accessor.cpp | 218 +++++++
.../olap/rowset/segment_v2/column_meta_accessor.h | 93 +++
be/src/olap/rowset/segment_v2/column_reader.cpp | 28 +-
be/src/olap/rowset/segment_v2/column_reader.h | 18 +-
.../olap/rowset/segment_v2/column_reader_cache.cpp | 182 +++---
.../olap/rowset/segment_v2/column_reader_cache.h | 37 +-
.../rowset/segment_v2/external_col_meta_util.cpp | 199 ++++++
.../rowset/segment_v2/external_col_meta_util.h | 91 +++
be/src/olap/rowset/segment_v2/segment.cpp | 108 +---
be/src/olap/rowset/segment_v2/segment.h | 25 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 33 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 2 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 12 +-
.../segment_v2/variant/variant_column_reader.cpp | 688 ++++++++++++++------
.../segment_v2/variant/variant_column_reader.h | 122 +++-
.../variant/variant_column_writer_impl.cpp | 8 +-
.../variant/variant_column_writer_impl.h | 1 +
.../segment_v2/variant/variant_ext_meta_writer.cpp | 199 ++++++
.../segment_v2/variant/variant_ext_meta_writer.h | 99 +++
.../variant/variant_external_meta_reader.cpp | 238 +++++++
.../variant/variant_external_meta_reader.h | 95 +++
.../rowset/segment_v2/vertical_segment_writer.cpp | 14 +-
be/src/olap/tablet_manager.cpp | 3 +-
be/src/olap/tablet_meta.cpp | 30 +-
be/src/olap/tablet_meta.h | 7 +-
be/src/olap/tablet_schema.cpp | 9 +
be/src/olap/tablet_schema.h | 12 +
be/src/vec/common/schema_util.cpp | 25 +-
be/test/olap/date_bloom_filter_test.cpp | 19 +-
be/test/olap/delete_bitmap_calculator_test.cpp | 13 +
.../segment_v2/column_meta_accessor_test.cpp | 705 +++++++++++++++++++++
.../rowset/segment_v2/column_reader_cache_test.cpp | 47 +-
.../segment_v2/external_col_meta_util_test.cpp | 663 +++++++++++++++++++
be/test/olap/rowset/segment_v2/mock/mock_segment.h | 6 -
.../variant_column_writer_reader_test.cpp | 272 ++++++--
be/test/vec/common/schema_util_rowset_test.cpp | 130 +++-
.../main/java/org/apache/doris/common/Config.java | 293 ++++-----
.../src/main/java/org/apache/doris/DorisFE.java | 7 +-
.../org/apache/doris/alter/CloudRollupJobV2.java | 4 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../apache/doris/alter/SchemaChangeHandler.java | 14 +-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 3 +-
.../cloud/common/util/CloudPropertyAnalyzer.java | 1 -
.../cloud/datasource/CloudInternalCatalog.java | 15 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 7 +-
.../trees/plans/commands/info/CreateTableInfo.java | 14 +
.../java/org/apache/doris/qe/SessionVariable.java | 16 +-
.../org/apache/doris/task/CreateReplicaTask.java | 1 -
gensrc/proto/olap_file.proto | 6 +
gensrc/proto/segment_v2.proto | 29 +-
gensrc/thrift/AgentService.thrift | 4 +-
.../ext_meta/test_storage_format_v2_1.out | 21 +
.../test_variant_external_meta_concurrent.out | 29 +
.../test_variant_external_meta_edge_cases.out | 174 +++++
.../test_variant_external_meta_integration.out | 120 ++++
.../test_variant_external_meta_with_sparse.out | 197 ++++++
.../data/variant_p0/query_subcolumns.out | 211 ++++++
.../query_p0/system/test_table_properties.groovy | 9 +-
...test_show_create_table_and_views_nereids.groovy | 3 +-
.../load.groovy | 14 +
.../ext_meta/test_storage_format_v2_1.groovy | 98 +++
.../test_variant_external_meta_concurrent.groovy | 307 +++++++++
.../test_variant_external_meta_edge_cases.groovy | 296 +++++++++
.../test_variant_external_meta_integration.groovy | 355 +++++++++++
.../test_variant_external_meta_with_sparse.groovy | 251 ++++++++
...est_variant_compaction_with_sparse_limit.groovy | 6 +-
.../suites/variant_p0/query_subcolumns.groovy | 94 +++
69 files changed, 6358 insertions(+), 701 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 11d1e5a7c92..4cdccb09321 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -356,6 +356,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_column_meta_used(in.is_external_segment_column_meta_used());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
@@ -392,6 +393,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_column_meta_used(in.is_external_segment_column_meta_used());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
@@ -441,6 +443,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_column_meta_used(in.is_external_segment_column_meta_used());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
@@ -478,6 +481,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_column_meta_used(in.is_external_segment_column_meta_used());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
index 2b63b5ae5e7..bd8e2cf85f2 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
@@ -212,7 +212,8 @@ Status BinaryPrefixPageDecoder::seek_at_or_after_value(const void* value, bool*
_cur_pos++;
auto st = _read_next_value();
if (st.is<ErrorCode::END_OF_FILE>()) {
- return Status::Error<ErrorCode::ENTRY_NOT_FOUND>("all value small than the value");
+ return Status::Error<ErrorCode::ENTRY_NOT_FOUND, false>(
+ "all value small than the value");
}
if (!st.ok()) {
return st;
diff --git a/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp b/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp
new file mode 100644
index 00000000000..7982199d6a0
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
+
+#include <cstdint>
+#include <string>
+
+#include "io/io_common.h"
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+#include "olap/rowset/segment_v2/segment.h" // SEGMENT_FOOTER_VERSION_V3_EXT_COL_META
+
+namespace doris::segment_v2 {
+
+// Abstract base implementation shared by both V2 and V3 layouts.
+class ColumnMetaAccessor::Impl {
+public:
+ virtual ~Impl() = default;
+ // Build uid -> col_id mapping based on the provided footer.
+ virtual Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const = 0;
+ // Lookup ColumnMetaPB by logical ordinal id using the provided footer.
+ virtual Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const = 0;
+ // Traverse all ColumnMetaPBs using the provided footer.
+ virtual Status traverse_metas(
+ const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const = 0;
+};
+
+// V2: inline layout using only footer.columns().
+class ColumnMetaAccessorV2 : public ColumnMetaAccessor::Impl {
+public:
+ explicit ColumnMetaAccessorV2(io::FileReaderSPtr file_reader)
+ : _file_reader(std::move(file_reader)) {}
+
+ Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const override {
+ uid_to_colid->clear();
+ uint32_t ordinal = 0;
+ for (const auto& column_meta : footer.columns()) {
+ // 跳过 variant 子列（unique_id == -1）
+ if (column_meta.unique_id() == -1) {
+ ordinal++;
+ continue;
+ }
+ uid_to_colid->try_emplace(column_meta.unique_id(), ordinal++);
+ }
+ return Status::OK();
+ }
+
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const override {
+ if (footer.columns_size() == 0 ||
+ column_ordinal_id >= static_cast<uint32_t>(footer.columns_size())) {
+ return Status::Corruption(
+ "no inline column meta available for column_id={} (columns_size={})",
+ column_ordinal_id, footer.columns_size());
+ }
+ *out = footer.columns(static_cast<int>(column_ordinal_id));
+ return Status::OK();
+ }
+
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const override {
+ if (footer.columns_size() == 0) {
+ return Status::Corruption("no column meta found in footer (inline V2)");
+ }
+ for (const auto& column : footer.columns()) {
+ visitor(column);
+ }
+ return Status::OK();
+ }
+
+private:
+ io::FileReaderSPtr _file_reader;
+};
+
+// V3: use external Column Meta Region + column_meta_entries layout.
+class ColumnMetaAccessorV3 : public ColumnMetaAccessor::Impl {
+public:
+ ColumnMetaAccessorV3(io::FileReaderSPtr file_reader,
+ const ExternalColMetaUtil::ExternalMetaPointers& ptrs)
+ : _file_reader(std::move(file_reader)), _ptrs(ptrs) {}
+
+ Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const override {
+ uid_to_colid->clear();
+ return ExternalColMetaUtil::parse_uid_to_colid_map(footer, _ptrs, uid_to_colid);
+ }
+
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const override {
+ // Prefer external Column Meta Region first.
+ if (column_ordinal_id < _ptrs.num_columns) {
+ return ExternalColMetaUtil::read_col_meta(_file_reader, footer, _ptrs,
+ column_ordinal_id, out);
+ }
+ return Status::Corruption(
+ "no column meta available for column_id={} (inline/external missing)",
+ column_ordinal_id);
+ }
+
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const override {
+ const uint64_t region_size = (_ptrs.region_end > _ptrs.region_start)
+ ? (_ptrs.region_end - _ptrs.region_start)
+ : 0;
+ if (region_size == 0) {
+ return Status::Corruption("invalid external meta region size");
+ }
+
+ // Read entire meta region once to reduce random I/O.
+ std::string region_buf;
+ region_buf.resize(static_cast<size_t>(region_size));
+ size_t br = 0;
+ io::IOContext io_ctx {.is_index_data = true};
+ RETURN_IF_ERROR(_file_reader->read_at(_ptrs.region_start, Slice(region_buf), &br, &io_ctx));
+ if (br != region_size) {
+ return Status::Corruption("short read on meta region");
+ }
+
+ if (footer.column_meta_entries_size() != _ptrs.num_columns) {
+ return Status::Corruption("column_meta_entries size mismatch");
+ }
+
+ uint64_t offset = _ptrs.region_start;
+ for (uint32_t i = 0; i < _ptrs.num_columns; ++i) {
+ const auto& entry = footer.column_meta_entries(static_cast<int>(i));
+ const auto sz = static_cast<uint64_t>(entry.length());
+ const uint64_t pos = offset;
+ if (!ExternalColMetaUtil::is_valid_meta_slice(pos, sz, _ptrs)) {
+ return Status::Corruption("external meta entry out of region bounds");
+ }
+ const uint64_t rel = pos - _ptrs.region_start;
+ ColumnMetaPB meta;
+ if (!meta.ParseFromArray(region_buf.data() + rel, static_cast<int>(sz))) {
+ return Status::Corruption("failed parse ColumnMetaPB from region");
+ }
+ visitor(meta);
+ offset += sz;
+ }
+ return Status::OK();
+ }
+
+private:
+ io::FileReaderSPtr _file_reader;
+ ExternalColMetaUtil::ExternalMetaPointers _ptrs;
+};
+
+ColumnMetaAccessor::ColumnMetaAccessor() = default;
+ColumnMetaAccessor::~ColumnMetaAccessor() = default;
+
+Status ColumnMetaAccessor::init(const SegmentFooterPB& footer, io::FileReaderSPtr file_reader) {
+ // First check footer version to see if external Column Meta Region might exist,
+ // then try to parse external layout; if parsing fails, fall back to V2.
+ if (footer.version() >= SEGMENT_FOOTER_VERSION_V3_EXT_COL_META) {
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs));
+ _impl = std::make_unique<ColumnMetaAccessorV3>(std::move(file_reader), ptrs);
+ } else {
+ _impl = std::make_unique<ColumnMetaAccessorV2>(std::move(file_reader));
+ }
+ // Build inner uid -> column ordinal id mapping once during init.
+ _column_uid_to_column_ordinal.clear();
+ RETURN_IF_ERROR(_impl->build_uid_to_colid_map(footer, &_column_uid_to_column_ordinal));
+ return Status::OK();
+}
+
+Status ColumnMetaAccessor::get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const {
+ return _impl->get_column_meta_by_column_ordinal_id(footer, column_ordinal_id, out);
+}
+
+Status ColumnMetaAccessor::get_column_meta_by_uid(const SegmentFooterPB& footer, int32_t column_uid,
+ ColumnMetaPB* out) const {
+ auto it = _column_uid_to_column_ordinal.find(column_uid);
+ if (it == _column_uid_to_column_ordinal.end()) {
+ *out = ColumnMetaPB();
+ return Status::Error<ErrorCode::NOT_FOUND, false>(
+ "column not found in segment meta, col_uid={}", column_uid);
+ }
+ return _impl->get_column_meta_by_column_ordinal_id(footer, static_cast<uint32_t>(it->second),
+ out);
+}
+
+bool ColumnMetaAccessor::has_column_uid(int32_t column_uid) const {
+ return _column_uid_to_column_ordinal.contains(column_uid);
+}
+
+Status ColumnMetaAccessor::traverse_metas(
+ const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const {
+ return _impl->traverse_metas(footer, visitor);
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/column_meta_accessor.h b/be/src/olap/rowset/segment_v2/column_meta_accessor.h
new file mode 100644
index 00000000000..a99f8f314dc
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/column_meta_accessor.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <unordered_map>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+
+namespace doris::segment_v2 {
+
+// Centralized accessor for column metadata layout of a Segment.
+// It selects V2 (inline footer.columns) or V3 (external Column Meta Region)
+// implementation internally and exposes a unified API to callers, hiding
+// the physical layout (inline vs external CMO).
+class ColumnMetaAccessor {
+public:
+ class Impl;
+ ColumnMetaAccessor();
+ ~ColumnMetaAccessor();
+ ColumnMetaAccessor(ColumnMetaAccessor&&) noexcept;
+ ColumnMetaAccessor& operator=(ColumnMetaAccessor&&) noexcept;
+ ColumnMetaAccessor(const ColumnMetaAccessor&) = delete;
+ ColumnMetaAccessor& operator=(const ColumnMetaAccessor&) = delete;
+ // Initialize accessor and choose V2 / V3 implementation based on footer.
+ //
+ // The caller is responsible for ensuring that `footer` outlives this accessor.
+ Status init(const SegmentFooterPB& footer, io::FileReaderSPtr file_reader);
+
+ // Get ColumnMetaPB for the given column ordinal id using the provided footer.
+ //
+ // Semantics:
+ // - In legacy V2 segments, column_ordinal_id is the index into footer.columns().
+ // - In V3 segments with external CMO, column_ordinal_id indexes the external
+ // Column Meta Region in column order; when external meta is missing or
+ // we fall back to inline columns, the same logical ordinal is reused.
+ //
+ // Error convention:
+ // - If the ordinal is out of range or the corresponding meta is missing/corrupted,
+ // returns ErrorCode::CORRUPTION.
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const;
+
+ // Get ColumnMetaPB for the given column unique id using the provided footer.
+ // column_uid is TabletColumn.unique_id / ColumnMetaPB.unique_id.
+ //
+ // Mapping scope:
+ // - Only top-level physical columns (including Variant root columns) are indexed.
+ // - Variant extracted subcolumns (unique_id == -1) and path subcolumns are excluded.
+ //
+ // Error convention:
+ // - If the uid does not exist in this segment (no physical column for this uid),
+ // returns ErrorCode::NOT_FOUND.
+ Status get_column_meta_by_uid(const SegmentFooterPB& footer, int32_t column_uid,
+ ColumnMetaPB* out) const;
+
+ // Fast existence check for a given column unique id.
+ //
+ // Only checks the in-memory uid -> ordinal map; does not trigger any I/O and
+ // does not parse ColumnMetaPB. Semantics are consistent with
+ // get_column_meta_by_uid(...) in terms of uid presence (NOT_FOUND vs OK).
+ bool has_column_uid(int32_t column_uid) const;
+
+ // Traverse all ColumnMetaPBs using the provided footer.
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const;
+
+private:
+ std::unique_ptr<Impl> _impl;
+ // map column unique id ---> column ordinal id (footer ordinal in legacy mode, or external CMO column id)
+ std::unordered_map<int32_t, size_t> _column_uid_to_column_ordinal;
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 6ccddbf3a54..b10caf2d3b6 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -27,15 +27,11 @@
#include <utility>
#include "common/compiler_util.h" // IWYU pragma: keep
-#include "common/exception.h"
#include "common/status.h"
#include "io/fs/file_reader.h"
-#include "io/fs/file_system.h"
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
-#include "olap/comparison_predicate.h"
#include "olap/decimal12.h"
-#include "olap/inverted_index_parser.h"
#include "olap/iterators.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/ann_index/ann_index_reader.h"
@@ -43,6 +39,7 @@
#include "olap/rowset/segment_v2/binary_plain_page.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
#include "olap/rowset/segment_v2/bloom_filter_index_reader.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo
#include "olap/rowset/segment_v2/index_file_reader.h"
#include "olap/rowset/segment_v2/index_reader.h"
@@ -233,28 +230,6 @@ bool ColumnReader::is_compaction_reader_type(ReaderType type) {
type == ReaderType::READER_FULL_COMPACTION;
}
-Status ColumnReader::create_variant(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader) {
- std::unique_ptr<VariantColumnReader> reader_local(new VariantColumnReader());
- RETURN_IF_ERROR(reader_local->init(opts, footer, column_id, num_rows, file_reader));
- *reader = std::move(reader_local);
- return Status::OK();
-}
-
-Status ColumnReader::create(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader) {
- // create normal column reader or variant subcolumn reader with extracted columns info in footer
- if ((FieldType)footer.columns(column_id).type() != FieldType::OLAP_FIELD_TYPE_VARIANT) {
- return ColumnReader::create(opts, footer.columns(column_id), num_rows, file_reader, reader);
- }
- // create variant column reader with extracted columns info in footer with hierarchical data info
- return create_variant(opts, footer, column_id, num_rows, file_reader, reader);
-}
-
Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader) {
@@ -308,6 +283,7 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
_meta_children_column_type = (FieldType)meta.children_columns(0).type();
}
+ _data_type = vectorized::DataTypeFactory::instance().create_data_type(meta);
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index c274a2d7430..d2d0f8e8c63 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -70,7 +70,6 @@ struct Slice;
struct StringRef;
namespace segment_v2 {
-
class EncodingInfo;
class ColumnIterator;
class BloomFilterIndexReader;
@@ -81,6 +80,7 @@ class PageDecoder;
class RowRanges;
class ZoneMapIndexReader;
class IndexIterator;
+class ColumnMetaAccessor;
struct ColumnReaderOptions {
// whether verify checksum when read page
@@ -133,10 +133,7 @@ public:
static Status create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
- static Status create(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader);
+
static Status create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
@@ -149,10 +146,7 @@ public:
static Status create_agg_state(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
- static Status create_variant(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader);
+
enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING, ALL_DICT_ENCODING };
static bool is_compaction_reader_type(ReaderType type);
@@ -228,11 +222,15 @@ public:
void disable_index_meta_cache() { _use_index_page_cache = false; }
+ vectorized::DataTypePtr get_vec_data_type() { return _data_type; }
+
virtual FieldType get_meta_type() { return _meta_type; }
int64_t get_metadata_size() const override;
private:
+ friend class VariantColumnReader;
+
ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows,
io::FileReaderSPtr file_reader);
Status init(const ColumnMetaPB* meta);
@@ -283,6 +281,8 @@ private:
DictEncodingType _dict_encoding_type;
+ vectorized::DataTypePtr _data_type;
+
TypeInfoPtr _type_info =
TypeInfoPtr(nullptr,
nullptr); // initialized in init(), may changed by subclasses.
diff --git a/be/src/olap/rowset/segment_v2/column_reader_cache.cpp b/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
index 62719459ecf..dbffd10caaa 100644
--- a/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
@@ -17,8 +17,10 @@
#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
+#
bvar::Adder<int64_t> g_segment_column_reader_cache_count("segment_column_cache_count");
bvar::Adder<int64_t> g_segment_column_cache_hit_count("segment_column_cache_hit_count");
@@ -29,7 +31,16 @@ namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
-ColumnReaderCache::ColumnReaderCache(Segment* segment) : _segment(segment) {}
+ColumnReaderCache::ColumnReaderCache(
+ ColumnMetaAccessor* accessor, TabletSchemaSPtr tablet_schema,
+ io::FileReaderSPtr file_reader, uint64_t num_rows,
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)>
+ get_footer_cb)
+ : _accessor(accessor),
+ _tablet_schema(std::move(tablet_schema)),
+ _file_reader(std::move(file_reader)),
+ _num_rows(num_rows),
+ _get_footer_cb(std::move(get_footer_cb)) {}
ColumnReaderCache::~ColumnReaderCache() {
g_segment_column_reader_cache_count << -_cache_map.size();
@@ -49,20 +60,8 @@ std::shared_ptr<ColumnReader> ColumnReaderCache::_lookup(const ColumnReaderCache
return it->second->reader;
}
-Status ColumnReaderCache::_insert(const ColumnReaderCacheKey& key, const ColumnReaderOptions& opts,
- const SegmentFooterPB& footer, int32_t column_id,
- const io::FileReaderSPtr& file_reader, size_t num_rows,
- std::shared_ptr<ColumnReader>* column_reader) {
- std::lock_guard<std::mutex> lock(_cache_mutex);
- // If already exists, and move it to the front
- if (_cache_map.find(key) != _cache_map.end()) {
- g_segment_column_cache_hit_count << 1;
- auto it = _cache_map[key];
- DCHECK_EQ(it->key.first, key.first);
- _lru_list.splice(_lru_list.begin(), _lru_list, it);
- *column_reader = it->reader;
- return Status::OK();
- }
+void ColumnReaderCache::_insert_locked_nocheck(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& reader) {
// If capacity exceeded, remove least recently used (tail)
if (_cache_map.size() >= config::max_segment_partial_column_cache_size) {
g_segment_column_reader_cache_count << -1;
@@ -72,21 +71,16 @@ Status ColumnReaderCache::_insert(const ColumnReaderCacheKey& key, const ColumnR
_cache_map.erase(last_it->key);
_lru_list.pop_back();
}
- std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(ColumnReader::create(opts, footer, column_id, num_rows, file_reader, &reader));
- // Insert new node at the front
- std::shared_ptr<ColumnReader> reader_ptr = std::move(reader);
g_segment_column_reader_cache_count << 1;
- DCHECK(key.first >= 0) << " col_uid: " << key.first
- << " relative_path: " << key.second.get_path();
- _lru_list.push_front(CacheNode {key, reader_ptr, std::chrono::steady_clock::now()});
+ _lru_list.push_front(CacheNode {
+ .key = key, .reader = reader, .last_access = std::chrono::steady_clock::now()});
_cache_map[key] = _lru_list.begin();
- *column_reader = reader_ptr;
- VLOG_DEBUG << "insert cache: " << key.first << " " << key.second.get_path()
- << ", type: " << (int)reader_ptr->get_meta_type()
- << ", cache_size: " << _cache_map.size() << ", list_size: " << _lru_list.size()
- << ", cache_map: " << _cache_map.size() << ", lru_list: " << _lru_list.size();
- return Status::OK();
+}
+
+void ColumnReaderCache::_insert_direct(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& column_reader) {
+ std::lock_guard<std::mutex> lock(_cache_mutex);
+ _insert_locked_nocheck(key, column_reader);
}
std::map<int32_t, std::shared_ptr<ColumnReader>> ColumnReaderCache::get_available_readers(
@@ -109,80 +103,98 @@ Status ColumnReaderCache::get_column_reader(int32_t col_uid,
*column_reader = cached;
return Status::OK();
}
- auto it = _segment->_column_uid_to_footer_ordinal.find(col_uid);
- if (it == _segment->_column_uid_to_footer_ordinal.end()) {
- // no such column in this segment, return nullptr
- *column_reader = nullptr;
- return Status::Error<ErrorCode::NOT_FOUND, false>("column not found in segment, col_uid={}",
- col_uid);
- }
+ // Load footer once under cache mutex (not thread-safe otherwise)
std::shared_ptr<SegmentFooterPB> footer_pb_shared;
{
std::lock_guard<std::mutex> lock(_cache_mutex);
- // keep the lock until the footer is loaded, since _get_segment_footer is not thread safe
- RETURN_IF_ERROR(_segment->_get_segment_footer(footer_pb_shared, stats));
+ RETURN_IF_ERROR(_get_footer_cb(footer_pb_shared, stats));
+ }
+
+ // Lookup column meta by uid via ColumnMetaAccessor. If not initialized or not found, return NOT_FOUND.
+ ColumnMetaPB meta;
+ Status st_meta = _accessor->get_column_meta_by_uid(*footer_pb_shared, col_uid, &meta);
+ if (st_meta.is<ErrorCode::NOT_FOUND>()) {
+ *column_reader = nullptr;
+ return st_meta;
+ }
+ RETURN_IF_ERROR(st_meta);
+
+ ColumnReaderOptions opts {.kept_in_memory = _tablet_schema->is_in_memory(),
+ .be_exec_version = _be_exec_version,
+ .tablet_schema = _tablet_schema};
+
+ std::shared_ptr<ColumnReader> reader;
+ if ((FieldType)meta.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) {
+ // Variant root columns require VariantColumnReader, which encapsulates
+ // subcolumn layout, sparse columns and external meta.
+ std::unique_ptr<VariantColumnReader> variant_reader(new VariantColumnReader());
+ RETURN_IF_ERROR(variant_reader->init(opts, _accessor, footer_pb_shared, col_uid, _num_rows,
+ _file_reader));
+ reader.reset(variant_reader.release());
+ VLOG_DEBUG << "insert cache (variant): uid=" << col_uid << " col_id=" << meta.column_id();
+ } else {
+ // For non-variant columns, we can create reader directly from ColumnMetaPB.
+ VLOG_DEBUG << "insert cache: uid=" << col_uid << " col_id=" << meta.column_id();
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta, _num_rows, _file_reader, &reader));
}
- // lazy create column reader from footer
- const auto& col_footer_pb = footer_pb_shared->columns(static_cast<int>(it->second));
- ColumnReaderOptions opts {
- .kept_in_memory = _segment->tablet_schema()->is_in_memory(),
- .be_exec_version = _be_exec_version,
- .tablet_schema = _segment->tablet_schema(),
- };
- VLOG_DEBUG << "insert cache: " << col_uid << " "
- << ""
- << ", type: " << (int)col_footer_pb.type() << ", footer_ordinal: " << it->second;
- return _insert({col_uid, {}}, opts, *footer_pb_shared, static_cast<int>(it->second),
- _segment->_file_reader, _segment->num_rows(), column_reader);
+
+ _insert_direct({col_uid, {}}, reader);
+ *column_reader = std::move(reader);
+ return Status::OK();
}
-Status ColumnReaderCache::get_path_column_reader(uint32_t col_uid,
+Status ColumnReaderCache::get_path_column_reader(int32_t col_uid,
vectorized::PathInData relative_path,
std::shared_ptr<ColumnReader>* column_reader,
OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint) {
- // Attempt to find in cache
+ // Attempt to find in cache first
if (auto cached = _lookup({col_uid, relative_path})) {
*column_reader = cached;
return Status::OK();
}
- const SubcolumnColumnMetaInfo::Node* node = node_hint;
- std::shared_ptr<ColumnReader> variant_column_reader;
- if (node == nullptr) {
- RETURN_IF_ERROR(get_column_reader(col_uid, &variant_column_reader, stats));
- node = variant_column_reader
- ? static_cast<VariantColumnReader*>(variant_column_reader.get())
- ->get_subcolumn_meta_by_path(relative_path)
- : nullptr;
+
+ if (!_accessor->has_column_uid(col_uid)) {
+ *column_reader = nullptr;
+ return Status::Error<ErrorCode::NOT_FOUND, false>("column not found in segment, col_uid={}",
+ col_uid);
}
- if (node != nullptr) {
- // lazy create column reader from footer
- DCHECK_GE(node->data.footer_ordinal, 0);
- std::shared_ptr<SegmentFooterPB> footer_pb_shared;
- {
- std::lock_guard<std::mutex> lock(_cache_mutex);
- // keep the lock until the footer is loaded, since _get_segment_footer is not thread safe
- RETURN_IF_ERROR(_segment->_get_segment_footer(footer_pb_shared, stats));
- }
- const auto& col_footer_pb = footer_pb_shared->columns(node->data.footer_ordinal);
- ColumnReaderOptions opts {
- .kept_in_memory = _segment->tablet_schema()->is_in_memory(),
- .be_exec_version = _be_exec_version,
- .tablet_schema = _segment->tablet_schema(),
- };
- VLOG_DEBUG << "insert cache: " << col_uid << " "
- << ""
- << ", type: " << (int)col_footer_pb.type()
- << ", footer_ordinal: " << node->data.footer_ordinal;
- RETURN_IF_ERROR(_insert({col_uid, node->path}, opts, *footer_pb_shared,
- node->data.footer_ordinal, _segment->_file_reader,
- _segment->num_rows(), column_reader));
+
+ // Load footer once under cache mutex (not thread-safe otherwise)
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ {
+ std::lock_guard<std::mutex> lock(_cache_mutex);
+ RETURN_IF_ERROR(_get_footer_cb(footer_pb_shared, stats));
+ }
+
+ // Ensure variant root reader is available in cache.
+ ColumnReaderOptions opts {.kept_in_memory = _tablet_schema->is_in_memory(),
+ .be_exec_version = _be_exec_version,
+ .tablet_schema = _tablet_schema};
+ std::shared_ptr<ColumnReader> variant_column_reader;
+ RETURN_IF_ERROR(get_column_reader(col_uid, &variant_column_reader, stats));
+
+ if (relative_path.empty()) {
+ *column_reader = std::move(variant_column_reader);
return Status::OK();
}
- // no such column in this segment, return nullptr
- *column_reader = nullptr;
- return Status::Error<ErrorCode::NOT_FOUND, false>("column not found in segment, col_uid={}",
- col_uid);
+
+ // Delegate path-level reader creation to VariantColumnReader, which hides
+ // inline vs external meta details.
+ std::shared_ptr<ColumnReader> path_reader;
+ auto* vreader = static_cast<VariantColumnReader*>(variant_column_reader.get());
+ Status st = vreader->create_path_reader(relative_path, opts, _accessor, *footer_pb_shared,
+ _file_reader, _num_rows, &path_reader);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ *column_reader = nullptr;
+ return st;
+ }
+ RETURN_IF_ERROR(st);
+
+ // Cache and return
+ _insert_direct({col_uid, relative_path}, path_reader);
+ *column_reader = std::move(path_reader);
+ return Status::OK();
}
#include "common/compile_check_end.h"
diff --git a/be/src/olap/rowset/segment_v2/column_reader_cache.h b/be/src/olap/rowset/segment_v2/column_reader_cache.h
index ce4060a5f26..8efd829c3c1 100644
--- a/be/src/olap/rowset/segment_v2/column_reader_cache.h
+++ b/be/src/olap/rowset/segment_v2/column_reader_cache.h
@@ -23,8 +23,7 @@
namespace doris::segment_v2 {
class ColumnReaderCache;
-class Segment;
-using SegmentSPtr = std::shared_ptr<Segment>;
+class ColumnMetaAccessor;
// Key: pair of column uid and variant path (empty for normal column)
using ColumnReaderCacheKey = std::pair<int32_t, vectorized::PathInData>;
@@ -40,7 +39,14 @@ struct CacheNode {
// Limit the column reader count to config::max_segment_partial_column_cache_size
class ColumnReaderCache {
public:
- explicit ColumnReaderCache(Segment* segment);
+ // Main constructor used in production: cache is bound to a specific segment's
+ // ColumnMetaAccessor, TabletSchema, file reader and row count, plus a footer
+ // getter callback (Segment::_get_segment_footer).
+ ColumnReaderCache(
+ ColumnMetaAccessor* accessor, TabletSchemaSPtr tablet_schema,
+ io::FileReaderSPtr file_reader, uint64_t num_rows,
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)>
+ get_footer_cb);
virtual ~ColumnReaderCache();
// Get all available readers
// if include_subcolumns is true, return all available readers, including subcolumn readers
@@ -52,7 +58,7 @@ public:
OlapReaderStatistics* stats);
// Get column reader by column unique id and path(leaf node of variant's subcolumn)
- virtual Status get_path_column_reader(uint32_t col_uid, vectorized::PathInData relative_path,
+ virtual Status get_path_column_reader(int32_t col_uid, vectorized::PathInData relative_path,
std::shared_ptr<ColumnReader>* column_reader,
OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint = nullptr);
@@ -60,11 +66,13 @@ public:
private:
// Lookup function remains similar
std::shared_ptr<ColumnReader> _lookup(const ColumnReaderCacheKey& key);
- // Insert and create column reader if not exists
- Status _insert(const ColumnReaderCacheKey& key, const ColumnReaderOptions& opts,
- const SegmentFooterPB& footer, int32_t column_id,
- const io::FileReaderSPtr& file_reader, size_t num_rows,
- std::shared_ptr<ColumnReader>* column_reader);
+ // Insert helper: assumes _cache_mutex already locked, will evict if needed
+ void _insert_locked_nocheck(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& reader);
+
+ // Insert an already-created reader directly into cache
+ void _insert_direct(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& column_reader);
// keep _lru_list and _cache_map thread safe
std::mutex _cache_mutex;
// Doubly-linked list to maintain LRU order
@@ -72,8 +80,15 @@ private:
// Map from key to list iterator for O(1) access
std::unordered_map<ColumnReaderCacheKey, std::list<CacheNode>::iterator> _cache_map;
int _be_exec_version = BeExecVersionManager::get_newest_version();
- // ATTN: Do not use share_ptr, reference will be recycled
- Segment* _segment;
+
+ // Non-owning pointer to the segment's ColumnMetaAccessor.
+ ColumnMetaAccessor* _accessor = nullptr;
+ // Segment-level context needed to construct ColumnReader.
+ TabletSchemaSPtr _tablet_schema;
+ io::FileReaderSPtr _file_reader;
+ uint64_t _num_rows = 0;
+ // Callback to get footer, usually bound to Segment::_get_segment_footer.
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)> _get_footer_cb;
};
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp b/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp
new file mode 100644
index 00000000000..315713a44b9
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "io/io_common.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+
+namespace doris::segment_v2 {
+
+Status ExternalColMetaUtil::parse_external_meta_pointers(
+ const SegmentFooterPB& footer, ExternalColMetaUtil::ExternalMetaPointers* out) {
+ out->region_start = footer.col_meta_region_start();
+ out->num_columns = static_cast<uint32_t>(footer.column_meta_entries_size());
+
+ // Validate pointers and basic consistency to ensure external meta is actually present.
+ if (out->region_start == 0) {
+ return Status::Corruption("external meta region_start is 0");
+ }
+ if (out->num_columns == 0) {
+ return Status::Corruption("external meta num_columns is 0");
+ }
+
+ // Compute region_end = region_start + sum(length_i) with overflow guards.
+ uint64_t region_size = 0;
+ for (int i = 0; i < footer.column_meta_entries_size(); ++i) {
+ const auto& entry = footer.column_meta_entries(i);
+ const auto len = static_cast<uint64_t>(entry.length());
+ if (len == 0) {
+ return Status::Corruption("column_meta_entries[{}] has zero length", i);
+ }
+ if (region_size + len < region_size) {
+ return Status::Corruption("region_size overflow at column_meta_entries[{}]", i);
+ }
+ region_size += len;
+ }
+
+ out->region_end = out->region_start + region_size;
+ if (out->region_end <= out->region_start) {
+ return Status::Corruption("invalid region_end: start={}, end={}", out->region_start,
+ out->region_end);
+ }
+ return Status::OK();
+}
+
+Status ExternalColMetaUtil::parse_uid_to_colid_map(
+ const SegmentFooterPB& footer, const ExternalColMetaUtil::ExternalMetaPointers& ptrs,
+ std::unordered_map<int32_t, size_t>* out) {
+ out->clear();
+ out->reserve(ptrs.num_columns);
+
+ for (int i = 0; i < footer.column_meta_entries_size(); ++i) {
+ const auto& entry = footer.column_meta_entries(i);
+ // Skip subcolumns (unique_id == -1) for top-level map
+ if (entry.unique_id() != -1) {
+ (*out)[entry.unique_id()] = static_cast<size_t>(i);
+ }
+ }
+ return Status::OK();
+}
+
+bool ExternalColMetaUtil::is_valid_meta_slice(uint64_t pos, uint64_t size,
+ const ExternalColMetaUtil::ExternalMetaPointers& p) {
+ if (size == 0) {
+ return false;
+ }
+ if (pos < p.region_start) {
+ return false;
+ }
+ if (pos + size < pos) {
+ return false;
+ } // overflow guard
+ if (pos + size > p.region_end) {
+ return false;
+ }
+ return true;
+}
+
+Status ExternalColMetaUtil::read_col_meta(const io::FileReaderSPtr& file_reader,
+ const SegmentFooterPB& footer,
+ const ExternalColMetaUtil::ExternalMetaPointers& p,
+ uint32_t col_id, ColumnMetaPB* out_meta) {
+ if (col_id >= p.num_columns) {
+ return Status::Corruption("col_id {} out of range {}", col_id, p.num_columns);
+ }
+ if (footer.column_meta_entries_size() != p.num_columns) {
+ return Status::Corruption("column_meta_entries size mismatch: entries={}, num_columns={}",
+ footer.column_meta_entries_size(), p.num_columns);
+ }
+ // compute meta offset via prefix-sum of entry lengths.
+ // TODO: prepare position and size in advance, and then read the meta.
+ uint64_t pos = p.region_start;
+ for (uint32_t i = 0; i < col_id; ++i) {
+ pos += static_cast<uint64_t>(footer.column_meta_entries(i).length());
+ }
+ const auto size = static_cast<uint64_t>(footer.column_meta_entries(col_id).length());
+ if (!ExternalColMetaUtil::is_valid_meta_slice(pos, size, p)) {
+ return Status::Corruption(
+ "invalid ColumnMetaPB bounds: col_id={}, pos={}, size={}, region_start={}, "
+ "region_end={}",
+ col_id, pos, size, p.region_start, p.region_end);
+ }
+ if (size > static_cast<uint64_t>(std::numeric_limits<size_t>::max())) {
+ return Status::Corruption("ColumnMetaPB size too large to allocate: {}", size);
+ }
+ std::string buf;
+ buf.resize(static_cast<size_t>(size));
+ size_t meta_read = 0;
+ io::IOContext io_ctx {.is_index_data = true};
+ RETURN_IF_ERROR(file_reader->read_at(pos, Slice(buf.data(), buf.size()), &meta_read, &io_ctx));
+ if (meta_read != size) {
+ return Status::Corruption("short read ColumnMetaPB: expect={}, actual={}", size, meta_read);
+ }
+ if (!out_meta->ParseFromArray(buf.data(), static_cast<int>(buf.size()))) {
+ return Status::Corruption("failed to parse ColumnMetaPB, col_id={}", col_id);
+ }
+ return Status::OK();
+}
+
+Status ExternalColMetaUtil::write_external_column_meta(
+ io::FileWriter* file_writer, SegmentFooterPB* footer, CompressionTypePB compression_type,
+ const std::function<Status(const std::vector<Slice>&)>& write_cb) {
+ if (footer->columns_size() == 0) {
+ return Status::OK();
+ }
+
+ std::vector<ColumnMetaPB> all_metas;
+ // 0) Externalize variant subcolumns first, then proceed with per-column meta region.
+ {
+ auto variant_ext_meta_agg =
+ std::make_unique<VariantExtMetaWriter>(file_writer, compression_type);
+ RETURN_IF_ERROR(variant_ext_meta_agg->externalize_from_footer(footer, &all_metas));
+ }
+
+ // Use the reorganized metas (including subcolumns)
+ const uint64_t meta_region_start = file_writer->bytes_appended();
+
+ footer->clear_column_meta_entries();
+
+ // 1) write contiguous meta blobs in col_id order and record lengths in footer.
+ for (const auto& meta : all_metas) {
+ std::string meta_bytes;
+ if (!meta.SerializeToString(&meta_bytes)) {
+ return Status::InternalError("failed to serialize column meta, unique_id={}",
+ meta.unique_id());
+ }
+ Slice s(meta_bytes);
+ RETURN_IF_ERROR(write_cb({s}));
+ auto* entry = footer->add_column_meta_entries();
+ entry->set_unique_id(static_cast<int32_t>(meta.unique_id()));
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ // 2) write pointers via proto fields
+ footer->set_col_meta_region_start(meta_region_start);
+
+ // 3) clear inline columns to enable true on-demand meta loading
+ // Note: footer->columns has already been pruned to only Top Level Columns in externalize_from_footer
+ // But for full externalization, we might want to clear them all or keep only necessary info?
+ // The original logic was footer->clear_columns().
+ // If we clear columns, the Reader needs to know how to reconstruct the schema.
+ // Currently, SegmentFooterPB.columns is used as the schema source if present.
+ // If we clear it, Reader must rely on External Meta.
+ // However, the Reader typically reads footer first. If columns is empty, it assumes V3 and reads external.
+ // So yes, we should clear it.
+ // But wait, in externalize_from_footer we carefully put Top Level columns back into footer->columns.
+ // Why? Because in previous logic, we might want to keep roots in footer?
+ // The previous logic: "replace Footer.columns with only the kept top-level columns".
+ // BUT then `write_external_column_meta` calls `footer->clear_columns()` at the end!
+ // So `footer->columns` will be empty anyway.
+ // The only reason to reconstruct `footer->columns` in `externalize_from_footer` is if `write_external_column_meta` logic depended on it.
+ // In my updated `write_external_column_meta`, I iterate over `all_metas` which is returned by `externalize_from_footer`.
+ // So I don't strictly need `footer->columns` to be correct in between.
+ // However, strictly following protocol: `externalize_from_footer` modifies footer to reflect "logical" columns (Top Level).
+ // And then `write_external_column_meta` finalizes it by clearing them and setting pointers.
+
+ footer->clear_columns();
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/external_col_meta_util.h b/be/src/olap/rowset/segment_v2/external_col_meta_util.h
new file mode 100644
index 00000000000..6a0e245c2f9
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/external_col_meta_util.h
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+#include <vector>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_writer.h"
+#include "util/slice.h"
+
+namespace doris::segment_v2 {
+// -----------------------------------------------------------------------------
+// External Column Meta layout and footer pointers
+// -----------------------------------------------------------------------------
+//
+// Segment tail:
+//
+// ┌──────────────────────────────────────────────────────────────────────┐
+// │ ... data pages ... │
+// │ │
+// │ Column Meta Region (variable-size, per top-level column, col_id asc) │
+// │ [Meta[col0]] [Meta[col1]] ... [Meta[colN-1]] │
+// │ │
+// │ Footer (SegmentFooterPB) │
+// │ fields: │
+// │ - col_meta_region_start : uint64 │
+// │ - column_meta_entries : repeated ColumnMetaEntryPB │
+// │ - unique_id : int32 │
+// │ - length : uint32 │
+// │ - num_columns : uint32 │
+// └──────────────────────────────────────────────────────────────────────┘
+//
+
+// Wrapper class to expose static methods for external column meta utilities.
+class ExternalColMetaUtil {
+public:
+ struct ExternalMetaPointers {
+ uint64_t region_start = 0;
+ // region_end is one-past-the-end file offset of the Column Meta Region.
+ uint64_t region_end = 0;
+ uint32_t num_columns = 0;
+ };
+
+ // Parse three external meta pointers from Footer.
+ // Returns Status::OK() on success, or Status::Corruption() with detailed error message.
+ static Status parse_external_meta_pointers(const SegmentFooterPB& footer,
+ ExternalMetaPointers* out);
+
+ // Parse uid->col_id mapping from footer.column_meta_entries.
+ // Returns Status::OK() on success, or Status::Corruption() with detailed error message.
+ static Status parse_uid_to_colid_map(const SegmentFooterPB& footer,
+ const ExternalMetaPointers& ptrs,
+ std::unordered_map<int32_t, size_t>* out);
+
+ static bool is_valid_meta_slice(uint64_t pos, uint64_t size, const ExternalMetaPointers& p);
+
+ // Read one ColumnMetaPB from external Column Meta Region; use INDEX cache lane because
+ // it's small metadata. Column offsets are derived from footer.column_meta_lengths via
+ // prefix sums.
+ static Status read_col_meta(const io::FileReaderSPtr& file_reader,
+ const SegmentFooterPB& footer, const ExternalMetaPointers& p,
+ uint32_t col_id, ColumnMetaPB* out_meta);
+
+ // Write external ColumnMetaPB contiguous region + footer pointers + column_meta_entries.
+ // write_cb: a callback that appends slices to the underlying file (e.g., SegmentWriter::_write_raw_data).
+ static Status write_external_column_meta(
+ io::FileWriter* file_writer, SegmentFooterPB* footer,
+ CompressionTypePB compression_type,
+ const std::function<Status(const std::vector<Slice>&)>& write_cb);
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index a417d9dcd6a..9b51a01d165 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -42,6 +42,7 @@
#include "olap/olap_common.h"
#include "olap/primary_key_index.h"
#include "olap/rowset/rowset_reader_context.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
#include "olap/rowset/segment_v2/empty_segment_iterator.h"
@@ -260,8 +261,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
}
if (read_options.col_id_to_predicates.contains(column_id) &&
can_apply_predicate_safely(column_id, *schema,
- read_options.target_cast_type_for_variants,
- read_options.io_ctx.reader_type)) {
+ read_options.target_cast_type_for_variants, read_options)) {
bool matched = true;
RETURN_IF_ERROR(reader->match_condition(entry.second.get(), &matched));
if (!matched) {
@@ -293,7 +293,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
DCHECK(reader != nullptr);
if (can_apply_predicate_safely(runtime_predicate->column_id(), *schema,
read_options.target_cast_type_for_variants,
- read_options.io_ctx.reader_type)) {
+ read_options)) {
bool matched = true;
RETURN_IF_ERROR(reader->match_condition(&and_predicate, &matched));
if (!matched) {
@@ -597,7 +597,7 @@ Status Segment::healthy_status() {
// Return the storage datatype of related column to field.
vectorized::DataTypePtr Segment::get_data_type_of(const TabletColumn& column,
- bool read_flat_leaves) {
+ const StorageReadOptions& read_options) {
const vectorized::PathInDataPtr path = column.path_info_ptr();
// none variant column
@@ -607,71 +607,26 @@ vectorized::DataTypePtr Segment::get_data_type_of(const TabletColumn& column,
// Path exists, proceed with variant logic.
vectorized::PathInData relative_path = path->copy_pop_front();
- int32_t unique_id = column.unique_id() > 0 ? column.unique_id() : column.parent_unique_id();
+ int32_t unique_id = column.unique_id() >= 0 ? column.unique_id() : column.parent_unique_id();
- // Find the reader for the base variant column.
- if (!_column_uid_to_footer_ordinal.contains(unique_id)) {
+ // If this uid does not exist in segment meta, fallback to schema type.
+ if (!_column_meta_accessor->has_column_uid(unique_id)) {
return vectorized::DataTypeFactory::instance().create_data_type(column);
}
- std::shared_ptr<ColumnReader> reader;
- // get the parent variant column reader
+ std::shared_ptr<ColumnReader> v_reader;
+
+ // Get the parent variant column reader
OlapReaderStatistics stats;
// If status is not ok, it will throw exception(data corruption)
- THROW_IF_ERROR(get_column_reader(unique_id, &reader, &stats));
- DCHECK(reader != nullptr);
- const auto* variant_reader = static_cast<const VariantColumnReader*>(reader.get());
-
- // Find the specific node within the variant structure using the relative path.
- const auto* node = variant_reader->get_subcolumn_meta_by_path(relative_path);
-
- if (relative_path.get_path().find(BeConsts::SPARSE_COLUMN_PATH) != std::string::npos) {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
-
- // Case 1: Node not found for the given path within the variant reader.
- // If relative_path is empty, it means the original path pointed to the root
- // of the variant column itself. We should return the Variant type.
- // If node is nullptr, it means the path is not exist in the variant sub columns.
- if (node == nullptr || relative_path.empty()) {
- // nested subcolumn is not exist in the sparse column
- if (column.is_nested_subcolumn()) {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
-
- // when the path is in the sparse column or exceeded the limit, return the variant type.
- if (variant_reader->exist_in_sparse_column(relative_path) ||
- variant_reader->is_exceeded_sparse_column_limit()) {
- return column.is_nullable() ? vectorized::make_nullable(
- std::make_shared<vectorized::DataTypeVariant>(
- column.variant_max_subcolumns_count()))
- : std::make_shared<vectorized::DataTypeVariant>(
- column.variant_max_subcolumns_count());
- }
- // now, path is not in this segment, return the default type from column.
- else {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
- }
-
- bool exist_in_sparse = variant_reader->exist_in_sparse_column(relative_path);
- bool is_physical_leaf = node->children.empty();
-
- if (is_physical_leaf && column.is_nested_subcolumn()) {
- return node->data.file_column_type;
- }
-
- // Condition to return the specific underlying type of the node:
- // 1. We are reading flat leaves (ignoring hierarchy).
- // 2. OR It's a leaf in the physical column structure AND it doesn't *also* exist
- // in the sparse column (meaning it's purely a materialized leaf).
- if (read_flat_leaves || (is_physical_leaf && !exist_in_sparse &&
- !variant_reader->is_exceeded_sparse_column_limit())) {
- return node->data.file_column_type;
- }
-
- // not the compaction read, return the default type from column.
- return vectorized::DataTypeFactory::instance().create_data_type(column);
+ THROW_IF_ERROR(get_column_reader(unique_id, &v_reader, &stats));
+ DCHECK(v_reader != nullptr);
+ auto* variant_reader = static_cast<VariantColumnReader*>(v_reader.get());
+ // Delegate type inference for variant paths to VariantColumnReader.
+ vectorized::DataTypePtr type;
+ THROW_IF_ERROR(variant_reader->infer_data_type_for_path(&type, column, read_options,
+ _column_reader_cache.get()));
+ return type;
}
Status Segment::_create_column_meta_once(OlapReaderStatistics* stats) {
@@ -684,17 +639,14 @@ Status Segment::_create_column_meta_once(OlapReaderStatistics* stats) {
}
Status Segment::_create_column_meta(const SegmentFooterPB& footer) {
- // unique_id -> idx in footer.columns()
- uint32_t ordinal = 0;
- for (const auto& column_meta : footer.columns()) {
- // no need to create column reader for variant's subcolumn
- if (column_meta.unique_id() == -1) {
- ordinal++;
- continue;
- }
- _column_uid_to_footer_ordinal.try_emplace(column_meta.unique_id(), ordinal++);
- }
- _column_reader_cache = std::make_unique<ColumnReaderCache>(this);
+ // Initialize column meta accessor which internally maintains uid -> column_ordinal mapping.
+ _column_meta_accessor = std::make_unique<ColumnMetaAccessor>();
+ RETURN_IF_ERROR(_column_meta_accessor->init(footer, _file_reader));
+ _column_reader_cache = std::make_unique<ColumnReaderCache>(
+ _column_meta_accessor.get(), _tablet_schema, _file_reader, _num_rows,
+ [this](std::shared_ptr<SegmentFooterPB>& footer_pb, OlapReaderStatistics* stats) {
+ return _get_segment_footer(footer_pb, stats);
+ });
return Status::OK();
}
@@ -739,11 +691,13 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
// For compability reason unique_id may less than 0 for variant extracted column
int32_t unique_id = tablet_column.unique_id() >= 0 ? tablet_column.unique_id()
: tablet_column.parent_unique_id();
- // init default iterator
- if (!_column_uid_to_footer_ordinal.contains(unique_id)) {
+
+ // If column meta for this uid is not found in this segment, use default iterator.
+ if (!_column_meta_accessor->has_column_uid(unique_id)) {
RETURN_IF_ERROR(new_default_iterator(tablet_column, iter));
return Status::OK();
}
+
// init iterator by unique id
std::shared_ptr<ColumnReader> reader;
RETURN_IF_ERROR(get_column_reader(unique_id, &reader, opt->stats));
@@ -992,7 +946,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto
slot->col_unique_id(),
assert_cast<const vectorized::DataTypeVariant&>(*remove_nullable(slot->type()))
.variant_max_subcolumns_count());
- auto storage_type = get_data_type_of(column, false);
+ auto storage_type = get_data_type_of(column, storage_read_options);
vectorized::MutableColumnPtr file_storage_column = storage_type->create_column();
DCHECK(storage_type != nullptr);
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index 337b22f2bd4..e67d0ce86a4 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -70,6 +70,7 @@ class InvertedIndexIterator;
class IndexFileReader;
class IndexIterator;
class ColumnReaderCache;
+class ColumnMetaAccessor;
using SegmentSharedPtr = std::shared_ptr<Segment>;
// A Segment is used to represent a segment in memory format. When segment is
@@ -166,21 +167,21 @@ public:
// another method `get_metadata_size` not include the column reader, only the segment object itself.
int64_t meta_mem_usage() const { return _meta_mem_usage; }
- // Get the inner file column's data type
- // ignore_chidren set to false will treat field as variant
- // when it contains children with field paths.
- // nullptr will returned if storage type does not contains such column
- std::shared_ptr<const vectorized::IDataType> get_data_type_of(const TabletColumn& column,
- bool read_flat_leaves);
+ // Get the inner file column's data type.
+ // When `read_options` is provided, the decision (e.g. flat-leaf vs hierarchical) can depend
+ // on the reader type and tablet schema; when it is nullptr, we treat it as a query reader.
+ // nullptr will be returned if storage type does not contain such column.
+ std::shared_ptr<const vectorized::IDataType> get_data_type_of(
+ const TabletColumn& column, const StorageReadOptions& read_options);
- // If column in segment is the same type in schema, then it is safe to apply predicate
+ // If column in segment is the same type in schema, then it is safe to apply predicate.
bool can_apply_predicate_safely(
int cid, const Schema& schema,
const std::map<std::string, vectorized::DataTypePtr>& target_cast_type_for_variants,
- ReaderType read_type) {
+ const StorageReadOptions& read_options) {
const doris::Field* col = schema.column(cid);
vectorized::DataTypePtr storage_column_type =
- get_data_type_of(col->get_desc(), read_type != ReaderType::READER_QUERY);
+ get_data_type_of(col->get_desc(), read_options);
if (storage_column_type == nullptr || col->type() != FieldType::OLAP_FIELD_TYPE_VARIANT ||
!target_cast_type_for_variants.contains(col->name())) {
// Default column iterator or not variant column
@@ -203,6 +204,8 @@ public:
Status get_column_reader(int32_t col_uid, std::shared_ptr<ColumnReader>* column_reader,
OlapReaderStatistics* stats);
+ Status traverse_column_meta_pbs(const std::function<void(const ColumnMetaPB&)>& visitor);
+
private:
DISALLOW_COPY_AND_ASSIGN(Segment);
Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema,
@@ -258,8 +261,8 @@ private:
// Limited cache for column readers
std::unique_ptr<ColumnReaderCache> _column_reader_cache;
- // map column unique id ---> it's footer ordinal
- std::unordered_map<int32_t, size_t> _column_uid_to_footer_ordinal;
+ // Centralized accessor for column metadata layout and uid->column_ordinal mapping.
+ std::unique_ptr<ColumnMetaAccessor> _column_meta_accessor;
// Init from ColumnMetaPB in SegmentFooterPB
// map column unique id ---> it's inner data type
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index b0efd2f2284..43659b368d1 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -301,8 +301,7 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
for (const auto& predicate : opts.column_predicates) {
if (!_segment->can_apply_predicate_safely(predicate->column_id(), *_schema,
- _opts.target_cast_type_for_variants,
- _opts.io_ctx.reader_type)) {
+ _opts.target_cast_type_for_variants, _opts)) {
continue;
}
_col_predicates.emplace_back(predicate);
@@ -331,8 +330,7 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
for (int i = 0; i < _schema->columns().size(); ++i) {
const Field* col = _schema->column(i);
if (col) {
- auto storage_type = _segment->get_data_type_of(
- col->get_desc(), _opts.io_ctx.reader_type != ReaderType::READER_QUERY);
+ auto storage_type = _segment->get_data_type_of(col->get_desc(), _opts);
if (storage_type == nullptr) {
storage_type = vectorized::DataTypeFactory::instance().create_data_type(
col->get_desc(), col->is_nullable());
@@ -806,9 +804,8 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row
if (_opts.io_ctx.reader_type == ReaderType::READER_QUERY) {
RowRanges dict_row_ranges = RowRanges::create_single(num_rows());
for (auto cid : cids) {
- if (!_segment->can_apply_predicate_safely(cid, *_schema,
- _opts.target_cast_type_for_variants,
- _opts.io_ctx.reader_type)) {
+ if (!_segment->can_apply_predicate_safely(
+ cid, *_schema, _opts.target_cast_type_for_variants, _opts)) {
continue;
}
DCHECK(_opts.col_id_to_predicates.count(cid) > 0);
@@ -839,8 +836,7 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row
for (auto& cid : cids) {
DCHECK(_opts.col_id_to_predicates.count(cid) > 0);
if (!_segment->can_apply_predicate_safely(cid, *_schema,
- _opts.target_cast_type_for_variants,
- _opts.io_ctx.reader_type)) {
+ _opts.target_cast_type_for_variants, _opts)) {
continue;
}
// get row ranges by bf index of this column,
@@ -869,8 +865,7 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row
for (const auto& cid : cids) {
DCHECK(_opts.col_id_to_predicates.count(cid) > 0);
if (!_segment->can_apply_predicate_safely(cid, *_schema,
- _opts.target_cast_type_for_variants,
- _opts.io_ctx.reader_type)) {
+ _opts.target_cast_type_for_variants, _opts)) {
continue;
}
// do not check zonemap if predicate does not support zonemap
@@ -903,7 +898,7 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row
_opts.topn_filter_target_node_id);
if (_segment->can_apply_predicate_safely(runtime_predicate->column_id(), *_schema,
_opts.target_cast_type_for_variants,
- _opts.io_ctx.reader_type)) {
+ _opts)) {
AndBlockColumnPredicate and_predicate;
and_predicate.add_column_predicate(
SingleColumnBlockPredicate::create_unique(runtime_predicate.get()));
@@ -1360,6 +1355,8 @@ Status SegmentIterator::_init_index_iterators() {
// We use this column to locate the metadata for the inverted index, which requires a unique_id and path.
const auto& column = _opts.tablet_schema->column(cid);
std::vector<const TabletIndex*> inverted_indexs;
+ // Keep shared_ptr alive to prevent use-after-free when accessing raw pointers
+ TabletIndexes inverted_indexs_holder;
// If the column is an extracted column, we need to find the sub-column in the parent column reader.
std::shared_ptr<ColumnReader> column_reader;
if (column.is_extracted_column()) {
@@ -1368,12 +1365,18 @@ Status SegmentIterator::_init_index_iterators() {
column_reader == nullptr) {
continue;
}
- inverted_indexs = assert_cast<VariantColumnReader*>(column_reader.get())
- ->find_subcolumn_tablet_indexes(column.suffix_path());
+ inverted_indexs_holder =
+ assert_cast<VariantColumnReader*>(column_reader.get())
+ ->find_subcolumn_tablet_indexes(column,
+ _storage_name_and_type[cid].second);
+ // Extract raw pointers from shared_ptr for iteration
+ for (const auto& index_ptr : inverted_indexs_holder) {
+ inverted_indexs.push_back(index_ptr.get());
+ }
}
// If the column is not an extracted column, we can directly get the inverted index metadata from the tablet schema.
else {
- inverted_indexs = {_segment->_tablet_schema->inverted_indexs(column)};
+ inverted_indexs = _segment->_tablet_schema->inverted_indexs(column);
}
for (const auto& inverted_index : inverted_indexs) {
RETURN_IF_ERROR(_segment->new_index_iterator(column, inverted_index, _opts,
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index c9b1b0029c6..94aa87adc79 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -254,7 +254,7 @@ private:
continue;
}
vectorized::DataTypePtr storage_type =
- _segment->get_data_type_of(_schema->column(cid)->get_desc(), false);
+ _segment->get_data_type_of(_schema->column(cid)->get_desc(), _opts);
if (storage_type && !storage_type->equals(*block->get_by_position(block_cid).type)) {
// Do additional cast
vectorized::MutableColumnPtr tmp = storage_type->create_column();
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 95a3076f912..3ee0ca28f53 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -46,10 +46,12 @@
#include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "olap/rowset/segment_creator.h"
#include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
#include "olap/rowset/segment_v2/index_file_writer.h"
#include "olap/rowset/segment_v2/index_writer.h"
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/page_pointer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
#include "olap/rowset/segment_v2/variant_stats_calculator.h"
#include "olap/segment_loader.h"
#include "olap/short_key_index.h"
@@ -72,7 +74,6 @@
#include "vec/jsonb/serialize.h"
#include "vec/olap/olap_data_convertor.h"
#include "vec/runtime/vdatetime_value.h"
-
namespace doris {
namespace segment_v2 {
#include "common/compile_check_begin.h"
@@ -1120,6 +1121,15 @@ Status SegmentWriter::_write_primary_key_index() {
Status SegmentWriter::_write_footer() {
_footer.set_num_rows(_row_count);
+ // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
+ if (_tablet_schema->is_external_segment_column_meta_used()) {
+ _footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+ VLOG_DEBUG << "use external column meta";
+ // External ColumnMetaPB writing (optional)
+ RETURN_IF_ERROR(ExternalColMetaUtil::write_external_column_meta(
+ _file_writer, &_footer, _opts.compression_type,
+ [this](const std::vector<Slice>& slices) { return _write_raw_data(slices); }));
+ }
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4)
std::string footer_buf;
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
index 83d09b5f474..1e411dec498 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
@@ -26,16 +26,20 @@
#include "common/config.h"
#include "common/status.h"
#include "io/fs/file_reader.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/hierarchical_data_iterator.h"
#include "olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h"
#include "olap/rowset/segment_v2/variant/sparse_column_merge_iterator.h"
#include "olap/tablet_schema.h"
+#include "util/slice.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
#include "vec/columns/column_variant.h"
#include "vec/common/assert_cast.h"
#include "vec/common/schema_util.h"
@@ -48,6 +52,7 @@ namespace doris::segment_v2 {
const SubcolumnColumnMetaInfo::Node* VariantColumnReader::get_subcolumn_meta_by_path(
const vectorized::PathInData& relative_path) const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
const auto* node = _subcolumns_meta_info->find_leaf(relative_path);
if (node) {
return node;
@@ -203,6 +208,7 @@ UnifiedSparseColumnReader::select_reader_and_cache_key(const std::string& relati
bool VariantColumnReader::exist_in_sparse_column(
const vectorized::PathInData& relative_path) const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
// Check if path exist in sparse column
bool existed_in_sparse_column =
!_statistics->sparse_column_non_null_size.empty() &&
@@ -218,6 +224,11 @@ bool VariantColumnReader::exist_in_sparse_column(
}
bool VariantColumnReader::is_exceeded_sparse_column_limit() const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
+ return _is_exceeded_sparse_column_limit_unlocked();
+}
+
+bool VariantColumnReader::_is_exceeded_sparse_column_limit_unlocked() const {
bool exceeded_sparse_column_limit = !_statistics->sparse_column_non_null_size.empty() &&
_statistics->sparse_column_non_null_size.size() >=
_variant_sparse_column_statistics_size;
@@ -235,6 +246,7 @@ bool VariantColumnReader::is_exceeded_sparse_column_limit() const {
}
int64_t VariantColumnReader::get_metadata_size() const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
int64_t size = ColumnReader::get_metadata_size();
if (_statistics) {
for (const auto& [path, _] : _statistics->subcolumns_non_null_size) {
@@ -258,7 +270,16 @@ Status VariantColumnReader::_create_hierarchical_reader(ColumnIteratorUPtr* read
const SubcolumnColumnMetaInfo::Node* root,
ColumnReaderCache* column_reader_cache,
OlapReaderStatistics* stats) {
+ // make sure external meta is loaded otherwise can't find any meta data for extracted columns
+ // TODO(lhy): this will load all external meta if not loaded, and memory will be consumed.
+ RETURN_IF_ERROR(load_external_meta_once());
+
stats->variant_subtree_hierarchical_iter_count++;
+ // After external meta is loaded, protect reads from `_statistics` and
+ // `_subcolumns_meta_info` against concurrent writers.
+ // english only in comments
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
+
// Node contains column with children columns or has correspoding sparse columns
// Create reader with hirachical data.
std::unique_ptr<SubstreamIterator> sparse_iter;
@@ -269,6 +290,9 @@ Status VariantColumnReader::_create_hierarchical_reader(ColumnIteratorUPtr* read
sparse_iter = std::make_unique<SubstreamIterator>(
vectorized::ColumnVariant::create_sparse_column_fn(), std::move(iter), nullptr);
}
+ if (node == nullptr) {
+ node = _subcolumns_meta_info->find_exact(path);
+ }
// If read the full path of variant read in MERGE_ROOT, otherwise READ_DIRECT
HierarchicalDataIterator::ReadType read_type =
(path == root->path) ? HierarchicalDataIterator::ReadType::MERGE_ROOT
@@ -296,6 +320,7 @@ Status VariantColumnReader::_create_sparse_merge_reader(ColumnIteratorUPtr* iter
SparseColumnCacheSPtr sparse_column_cache,
ColumnReaderCache* column_reader_cache,
std::optional<uint32_t> bucket_index) {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
// Get subcolumns path set from tablet schema
const auto& path_set_info = opts->tablet_schema->path_set_info(target_col.parent_unique_id());
@@ -345,6 +370,7 @@ Status VariantColumnReader::_create_sparse_merge_reader(ColumnIteratorUPtr* iter
Status VariantColumnReader::_new_default_iter_with_same_nested(
ColumnIteratorUPtr* iterator, const TabletColumn& tablet_column,
const StorageReadOptions* opt, ColumnReaderCache* column_reader_cache) {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
auto relative_path = tablet_column.path_info_ptr()->copy_pop_front();
// We find node that represents the same Nested type as path.
const auto* parent = _subcolumns_meta_info->find_best_match(relative_path);
@@ -400,11 +426,23 @@ Result<SparseColumnCacheSPtr> VariantColumnReader::_get_shared_column_cache(
return sparse_column_cache_ptr->at(path);
}
-Status VariantColumnReader::_new_iterator_with_flat_leaves(
- ColumnIteratorUPtr* iterator, const TabletColumn& target_col,
- const StorageReadOptions* opts, bool exceeded_sparse_column_limit,
- bool existed_in_sparse_column, ColumnReaderCache* column_reader_cache,
- PathToSparseColumnCache* sparse_column_cache_ptr) {
+vectorized::DataTypePtr create_variant_type(const TabletColumn& target_col) {
+ return target_col.is_nullable()
+ ? vectorized::make_nullable(std::make_shared<vectorized::DataTypeVariant>(
+ target_col.variant_max_subcolumns_count()))
+ : std::make_shared<vectorized::DataTypeVariant>(
+ target_col.variant_max_subcolumns_count());
+}
+
+Status VariantColumnReader::_build_read_plan_flat_leaves(
+ ReadPlan* plan, const TabletColumn& target_col, const StorageReadOptions* opts,
+ ColumnReaderCache* column_reader_cache, PathToSparseColumnCache* sparse_column_cache_ptr) {
+ // make sure external meta is loaded otherwise can't find any meta data for extracted columns
+ // TODO(lhy): this will load all external meta if not loaded, and memory will be consumed.
+ RETURN_IF_ERROR(load_external_meta_once());
+
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
+
DCHECK(opts != nullptr);
auto relative_path = target_col.path_info_ptr()->copy_pop_front();
// compaction need to read flat leaves nodes data to prevent from amplification
@@ -416,12 +454,12 @@ Status VariantColumnReader::_new_iterator_with_flat_leaves(
// Case 1: single sparse column path
if (rel == SPARSE_COLUMN_PATH && !_sparse_reader.has_buckets() &&
_sparse_reader.single() != nullptr) {
- // read sparse column and filter extracted columns in subcolumn_path_map
- SparseColumnCacheSPtr sparse_column_cache = DORIS_TRY(_get_shared_column_cache(
- sparse_column_cache_ptr, SPARSE_COLUMN_PATH, _sparse_reader.single()));
- // get subcolumns in sparse path set which will be merged into sparse column
- RETURN_IF_ERROR(_create_sparse_merge_reader(iterator, opts, target_col,
- sparse_column_cache, column_reader_cache));
+ plan->kind = ReadKind::SPARSE_MERGE;
+ plan->type = vectorized::DataTypeFactory::instance().create_data_type(target_col);
+ plan->relative_path = relative_path;
+ plan->sparse_column_reader = _sparse_reader.single();
+ plan->sparse_cache_key = SPARSE_COLUMN_PATH;
+ plan->bucket_index.reset();
return Status::OK();
}
// Case 2: bucketized sparse column path: __DORIS_VARIANT_SPARSE__.b{i}
@@ -434,76 +472,145 @@ Status VariantColumnReader::_new_iterator_with_flat_leaves(
if (bucket_index >= buckets.size() || !buckets[bucket_index]) {
return Status::NotFound("bucket sparse column reader not found: {}", rel);
}
- std::string cache_key =
+ plan->kind = ReadKind::SPARSE_MERGE;
+ plan->type = vectorized::DataTypeFactory::instance().create_data_type(target_col);
+ plan->relative_path = relative_path;
+ plan->sparse_column_reader = buckets[bucket_index];
+ plan->sparse_cache_key =
std::string(SPARSE_COLUMN_PATH) + ".b" + std::to_string(bucket_index);
- SparseColumnCacheSPtr sparse_column_cache = DORIS_TRY(_get_shared_column_cache(
- sparse_column_cache_ptr, cache_key, buckets[bucket_index]));
- RETURN_IF_ERROR(_create_sparse_merge_reader(iterator, opts, target_col,
- sparse_column_cache, column_reader_cache,
- bucket_index));
+ plan->bucket_index = bucket_index;
return Status::OK();
}
if (target_col.is_nested_subcolumn()) {
- // using the sibling of the nested column to fill the target nested column
- RETURN_IF_ERROR(_new_default_iter_with_same_nested(iterator, target_col, opts,
- column_reader_cache));
+ plan->kind = ReadKind::DEFAULT_NESTED;
+ plan->type = vectorized::DataTypeFactory::instance().create_data_type(target_col);
+ plan->relative_path = relative_path;
return Status::OK();
}
// If the path is typed, it means the path is not a sparse column, so we can't read the sparse column
// even if the sparse column size is reached limit
+ bool existed_in_sparse_column =
+ !_statistics->sparse_column_non_null_size.empty() &&
+ _statistics->sparse_column_non_null_size.contains(relative_path.get_path());
+ bool exceeded_sparse_column_limit = is_exceeded_sparse_column_limit();
if (existed_in_sparse_column || exceeded_sparse_column_limit) {
// Sparse column exists or reached sparse size limit, read sparse column
auto [reader, cache_key] =
_sparse_reader.select_reader_and_cache_key(relative_path.get_path());
DCHECK(reader != nullptr);
- SparseColumnCacheSPtr sparse_column_cache =
- DORIS_TRY(_get_shared_column_cache(sparse_column_cache_ptr, cache_key, reader));
- DCHECK(opts);
- *iterator = std::make_unique<SparseColumnExtractIterator>(
- relative_path.get_path(), std::move(sparse_column_cache), opts);
+ plan->kind = ReadKind::SPARSE_EXTRACT;
+ plan->type = create_variant_type(target_col);
+ plan->relative_path = relative_path;
+ plan->sparse_column_reader = std::move(reader);
+ plan->sparse_cache_key = std::move(cache_key);
+ plan->bucket_index.reset();
return Status::OK();
}
VLOG_DEBUG << "new_default_iter: " << target_col.path_info_ptr()->get_path();
- RETURN_IF_ERROR(Segment::new_default_iterator(target_col, iterator));
+ plan->kind = ReadKind::DEFAULT_FILL;
+ plan->type = vectorized::DataTypeFactory::instance().create_data_type(target_col);
+ plan->relative_path = relative_path;
return Status::OK();
}
if (relative_path.empty()) {
// root path, use VariantRootColumnIterator
- *iterator = std::make_unique<VariantRootColumnIterator>(
- std::make_unique<FileColumnIterator>(_root_column_reader));
+ plan->kind = ReadKind::ROOT_FLAT;
+ plan->type = create_variant_type(target_col);
+ plan->relative_path = relative_path;
return Status::OK();
}
VLOG_DEBUG << "new iterator: " << target_col.path_info_ptr()->get_path();
std::shared_ptr<ColumnReader> column_reader;
RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
target_col.parent_unique_id(), node->path, &column_reader, opts->stats, node));
- RETURN_IF_ERROR(column_reader->new_iterator(iterator, nullptr));
+ plan->kind = ReadKind::LEAF;
+ plan->type = column_reader->get_vec_data_type();
+ plan->relative_path = relative_path;
+ plan->leaf_column_reader = std::move(column_reader);
return Status::OK();
}
-Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
- const TabletColumn* target_col,
- const StorageReadOptions* opt) {
- // return new_iterator(iterator, target_col, opt, nullptr);
- return Status::NotSupported("Not implemented");
+bool VariantColumnReader::has_prefix_path(const vectorized::PathInData& relative_path) const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
+ return _has_prefix_path_unlocked(relative_path);
}
-Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
- const TabletColumn* target_col,
- const StorageReadOptions* opt,
- ColumnReaderCache* column_reader_cache,
- PathToSparseColumnCache* sparse_column_cache_ptr) {
+bool VariantColumnReader::_has_prefix_path_unlocked(
+ const vectorized::PathInData& relative_path) const {
+ if (relative_path.empty()) {
+ return true;
+ }
+ const std::string path = relative_path.get_path();
+ const std::string dot_prefix = relative_path.get_path() + ".";
+
+ // 1) exact node exists and has children.
+ if (const auto* node = _subcolumns_meta_info->find_exact(relative_path)) {
+ if (!node->children.empty()) {
+ return true;
+ }
+ }
+
+ // 2) Check sparse column stats: use lower_bound to test the `p.` prefix range
+ // example sparse columns path: a.b.c, a.b.e, access prefix: a.b.
+ // then we must read the sparse columns
+ if (_statistics && !_statistics->sparse_column_non_null_size.empty()) {
+ auto it = _statistics->sparse_column_non_null_size.lower_bound(dot_prefix);
+ if (it != _statistics->sparse_column_non_null_size.end() &&
+ it->first.starts_with(dot_prefix)) {
+ return true;
+ }
+ }
+
+ // 3) Check external meta store (if available).
+ if (_ext_meta_reader && _ext_meta_reader->available()) {
+ bool has = false;
+ // Pass strict prefix `p.` to avoid false positives like `a.b` matching `a.bc`.
+ if (_ext_meta_reader->has_prefix(dot_prefix, &has).ok() && has) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+Status VariantColumnReader::_build_read_plan(ReadPlan* plan, const TabletColumn& target_col,
+ const StorageReadOptions* opt,
+ ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr) {
+ // root column use unique id, leaf column use parent_unique_id
int32_t col_uid =
- target_col->unique_id() >= 0 ? target_col->unique_id() : target_col->parent_unique_id();
+ target_col.unique_id() >= 0 ? target_col.unique_id() : target_col.parent_unique_id();
// root column use unique id, leaf column use parent_unique_id
- auto relative_path = target_col->path_info_ptr()->copy_pop_front();
+ auto relative_path = target_col.path_info_ptr()->copy_pop_front();
+
+ // If the variant column has extracted columns and is a compaction reader, then read flat leaves
+ // Otherwise read hierarchical data, since the variant subcolumns are flattened in
+ // schema_util::get_compaction_schema. For checksum reader, we need to read flat leaves to
+ // get the correct data if has extracted columns.
+ auto need_read_flat_leaves = [](const StorageReadOptions* opts) {
+ return opts != nullptr && opts->tablet_schema != nullptr &&
+ std::ranges::any_of(
+ opts->tablet_schema->columns(),
+ [](const auto& column) { return column->is_extracted_column(); }) &&
+ (is_compaction_reader_type(opts->io_ctx.reader_type) ||
+ opts->io_ctx.reader_type == ReaderType::READER_CHECKSUM);
+ };
+
+ // Flat-leaf compaction/checksum mode: delegate to dedicated planner which handles locking
+ // and external meta loading internally.
+ // english only in comments
+ if (need_read_flat_leaves(opt)) {
+ return _build_read_plan_flat_leaves(plan, target_col, opt, column_reader_cache,
+ sparse_column_cache_ptr);
+ }
+
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
const auto* root = _subcolumns_meta_info->get_root();
- const auto* node = target_col->has_path_info()
- ? _subcolumns_meta_info->find_exact(relative_path)
- : nullptr;
+ const auto* node =
+ target_col.has_path_info() ? _subcolumns_meta_info->find_exact(relative_path) : nullptr;
// try rebuild path with hierarchical
// example path(['a.b']) -> path(['a', 'b'])
@@ -515,8 +622,7 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
// Check if path exist in sparse column
bool existed_in_sparse_column =
!_statistics->sparse_column_non_null_size.empty() &&
- _statistics->sparse_column_non_null_size.find(relative_path.get_path()) !=
- _statistics->sparse_column_non_null_size.end();
+ _statistics->sparse_column_non_null_size.contains(relative_path.get_path());
DBUG_EXECUTE_IF("exist_in_sparse_column_must_be_false", {
if (existed_in_sparse_column) {
@@ -528,42 +634,20 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
// Otherwise the prefix is not exist and the sparse column size is reached limit
// which means the path maybe exist in sparse_column
- bool exceeded_sparse_column_limit = is_exceeded_sparse_column_limit();
-
- // If the variant column has extracted columns and is a compaction reader, then read flat leaves
- // Otherwise read hierarchical data, since the variant subcolumns are flattened in schema_util::VariantCompactionUtil::get_extended_compaction_schema
- // when config::enable_vertical_compact_variant_subcolumns is true
- // For checksum reader, we need to read flat leaves to get the correct data if has extracted columns
- auto need_read_flat_leaves = [](const StorageReadOptions* opts) {
- return opts != nullptr && opts->tablet_schema != nullptr &&
- std::ranges::any_of(
- opts->tablet_schema->columns(),
- [](const auto& column) { return column->is_extracted_column(); }) &&
- (is_compaction_reader_type(opts->io_ctx.reader_type) ||
- opts->io_ctx.reader_type == ReaderType::READER_CHECKSUM);
- };
-
- if (need_read_flat_leaves(opt)) {
- // original path, compaction with wide schema
- return _new_iterator_with_flat_leaves(
- iterator, *target_col, opt, exceeded_sparse_column_limit, existed_in_sparse_column,
- column_reader_cache, sparse_column_cache_ptr);
- }
+ bool exceeded_sparse_column_limit = _is_exceeded_sparse_column_limit_unlocked();
// Check if path is prefix, example sparse columns path: a.b.c, a.b.e, access prefix: a.b.
- // then we must read the sparse columns
- const std::string& prefix = relative_path.get_path() + ".";
- bool prefix_existed_in_sparse_column =
- !_statistics->sparse_column_non_null_size.empty() &&
- (_statistics->sparse_column_non_null_size.lower_bound(prefix) !=
- _statistics->sparse_column_non_null_size.end()) &&
- _statistics->sparse_column_non_null_size.lower_bound(prefix)->first.starts_with(prefix);
- // if prefix exists in sparse column, read sparse column with hierarchical reader
- if (prefix_existed_in_sparse_column || exceeded_sparse_column_limit) {
+ // Or access root path
+ if (_has_prefix_path_unlocked(relative_path)) {
// Example {"b" : {"c":456,"e":7.111}}
- // b.c is sparse column, b.e is subcolumn, so b is both the prefix of sparse column and subcolumn
- return _create_hierarchical_reader(iterator, col_uid, relative_path, node, root,
- column_reader_cache, opt->stats);
+ // b.c is sparse column, b.e is subcolumn, so b is both the prefix of sparse column and
+ // subcolumn
+ plan->kind = ReadKind::HIERARCHICAL;
+ plan->type = create_variant_type(target_col);
+ plan->relative_path = relative_path;
+ plan->node = node;
+ plan->root = root;
+ return Status::OK();
}
// if path exists in sparse column, read sparse column with extract reader
@@ -574,58 +658,260 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
// Then we should use hierarchical reader to read b
auto [reader, cache_key] =
_sparse_reader.select_reader_and_cache_key(relative_path.get_path());
- SparseColumnCacheSPtr sparse_column_cache =
- DORIS_TRY(_get_shared_column_cache(sparse_column_cache_ptr, cache_key, reader));
- DCHECK(opt);
- // Sparse column exists or reached sparse size limit, read sparse column
- *iterator = std::make_unique<SparseColumnExtractIterator>(
- relative_path.get_path(), std::move(sparse_column_cache), opt);
- opt->stats->variant_subtree_sparse_iter_count++;
+ DCHECK(reader);
+ plan->kind = ReadKind::SPARSE_EXTRACT;
+ plan->type = create_variant_type(target_col);
+ plan->relative_path = relative_path;
+ plan->sparse_column_reader = std::move(reader);
+ plan->sparse_cache_key = std::move(cache_key);
+ plan->bucket_index.reset();
return Status::OK();
}
+ // read leaf node
if (node != nullptr) {
- // relative_path means the root node, should always use HierarchicalDataIterator
- if (node->is_leaf_node() && !relative_path.empty()) {
- // Node contains column without any child sub columns and no corresponding sparse columns
- // Direct read extracted columns
- const auto* leaf_node = _subcolumns_meta_info->find_leaf(relative_path);
+ DCHECK(node->is_leaf_node());
+ // Node contains column without any child sub columns and no corresponding sparse columns
+ // Direct read extracted columns
+ const auto* leaf_node = _subcolumns_meta_info->find_leaf(relative_path);
+ std::shared_ptr<ColumnReader> leaf_column_reader;
+ RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
+ col_uid, leaf_node->path, &leaf_column_reader, opt->stats, leaf_node));
+ plan->kind = ReadKind::LEAF;
+ plan->type = leaf_column_reader->get_vec_data_type();
+ plan->relative_path = relative_path;
+ plan->leaf_column_reader = std::move(leaf_column_reader);
+ } else {
+ if (_ext_meta_reader && _ext_meta_reader->available()) {
+ // Get path reader from external meta
std::shared_ptr<ColumnReader> leaf_column_reader;
- RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
- col_uid, leaf_node->path, &leaf_column_reader, opt->stats, leaf_node));
- RETURN_IF_ERROR(leaf_column_reader->new_iterator(iterator, nullptr));
- opt->stats->variant_subtree_leaf_iter_count++;
- } else {
- RETURN_IF_ERROR(_create_hierarchical_reader(iterator, col_uid, relative_path, node,
- root, column_reader_cache, opt->stats));
+ Status st = column_reader_cache->get_path_column_reader(
+ col_uid, relative_path, &leaf_column_reader, opt->stats, nullptr);
+ DCHECK(!_has_prefix_path_unlocked(relative_path));
+ if (st.ok()) {
+ // Try external meta fallback: build a leaf reader on demand from externalized meta
+ plan->kind = ReadKind::LEAF;
+ plan->type = leaf_column_reader->get_vec_data_type();
+ plan->relative_path = relative_path;
+ plan->leaf_column_reader = std::move(leaf_column_reader);
+ return Status::OK();
+ }
+ if (!st.is<ErrorCode::NOT_FOUND>()) {
+ return st;
+ }
+ // not found, need continue
}
- } else {
- // Sparse column not exists and not reached stats limit, then the target path is not exist, get a default iterator
- RETURN_IF_ERROR(Segment::new_default_iterator(*target_col, iterator));
- opt->stats->variant_subtree_default_iter_count++;
+ if (exceeded_sparse_column_limit) {
+ // maybe exist prefix path in sparse column
+ plan->kind = ReadKind::HIERARCHICAL;
+ plan->type = create_variant_type(target_col);
+ plan->relative_path = relative_path;
+ plan->node = node;
+ plan->root = root;
+ return Status::OK();
+ }
+ // Sparse column not exists and not reached stats limit, then the target path is not
+ // exist, get a default iterator
+ plan->kind = ReadKind::DEFAULT_FILL;
+ plan->type = vectorized::DataTypeFactory::instance().create_data_type(target_col);
+ plan->relative_path = relative_path;
}
return Status::OK();
}
-Status VariantColumnReader::init(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- io::FileReaderSPtr file_reader) {
+Status VariantColumnReader::_create_iterator_from_plan(
+ ColumnIteratorUPtr* iterator, const ReadPlan& plan, const TabletColumn& target_col,
+ const StorageReadOptions* opt, ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr) {
+ switch (plan.kind) {
+ case ReadKind::ROOT_FLAT: {
+ *iterator = std::make_unique<VariantRootColumnIterator>(
+ std::make_unique<FileColumnIterator>(_root_column_reader));
+ return Status::OK();
+ }
+ case ReadKind::HIERARCHICAL: {
+ int32_t col_uid = target_col.unique_id() >= 0 ? target_col.unique_id()
+ : target_col.parent_unique_id();
+ RETURN_IF_ERROR(_create_hierarchical_reader(iterator, col_uid, plan.relative_path,
+ plan.node, plan.root, column_reader_cache,
+ opt->stats));
+ return Status::OK();
+ }
+ case ReadKind::LEAF: {
+ DCHECK(plan.leaf_column_reader != nullptr);
+ RETURN_IF_ERROR(plan.leaf_column_reader->new_iterator(iterator, nullptr));
+ if (opt && opt->stats) {
+ opt->stats->variant_subtree_leaf_iter_count++;
+ }
+ return Status::OK();
+ }
+ case ReadKind::SPARSE_EXTRACT: {
+ DCHECK(plan.sparse_column_reader != nullptr);
+ SparseColumnCacheSPtr sparse_column_cache = DORIS_TRY(_get_shared_column_cache(
+ sparse_column_cache_ptr, plan.sparse_cache_key, plan.sparse_column_reader));
+ *iterator = std::make_unique<SparseColumnExtractIterator>(
+ plan.relative_path.get_path(), std::move(sparse_column_cache), opt);
+ if (opt && opt->stats) {
+ opt->stats->variant_subtree_sparse_iter_count++;
+ }
+ return Status::OK();
+ }
+ case ReadKind::SPARSE_MERGE: {
+ DCHECK(plan.sparse_column_reader != nullptr);
+ SparseColumnCacheSPtr sparse_column_cache = DORIS_TRY(_get_shared_column_cache(
+ sparse_column_cache_ptr, plan.sparse_cache_key, plan.sparse_column_reader));
+ RETURN_IF_ERROR(_create_sparse_merge_reader(iterator, opt, target_col, sparse_column_cache,
+ column_reader_cache, plan.bucket_index));
+ return Status::OK();
+ }
+ case ReadKind::DEFAULT_NESTED: {
+ RETURN_IF_ERROR(
+ _new_default_iter_with_same_nested(iterator, target_col, opt, column_reader_cache));
+ return Status::OK();
+ }
+ case ReadKind::DEFAULT_FILL: {
+ RETURN_IF_ERROR(Segment::new_default_iterator(target_col, iterator));
+ if (opt && opt->stats) {
+ opt->stats->variant_subtree_default_iter_count++;
+ }
+ return Status::OK();
+ }
+ }
+ return Status::InternalError("Unknown ReadKind for VariantColumnReader");
+}
+
+Status VariantColumnReader::_new_iterator_with_flat_leaves(
+ ColumnIteratorUPtr* iterator, vectorized::DataTypePtr* type, const TabletColumn& target_col,
+ const StorageReadOptions* opts, bool /*exceeded_sparse_column_limit*/,
+ bool /*existed_in_sparse_column*/, ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr) {
+ ReadPlan plan;
+ RETURN_IF_ERROR(_build_read_plan_flat_leaves(&plan, target_col, opts, column_reader_cache,
+ sparse_column_cache_ptr));
+ *type = plan.type;
+ return _create_iterator_from_plan(iterator, plan, target_col, opts, column_reader_cache,
+ sparse_column_cache_ptr);
+}
+
+Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
+ const TabletColumn* target_col,
+ const StorageReadOptions* opt) {
+ // return new_iterator(iterator, target_col, opt, nullptr);
+ return Status::NotSupported("Not implemented");
+}
+
+Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
+ const TabletColumn* target_col,
+ const StorageReadOptions* opt,
+ ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr) {
+ ReadPlan plan;
+ RETURN_IF_ERROR(_build_read_plan(&plan, *target_col, opt, column_reader_cache,
+ sparse_column_cache_ptr));
+ // Caller of this overload does not need the storage type; only iterator is used.
+ return _create_iterator_from_plan(iterator, plan, *target_col, opt, column_reader_cache,
+ sparse_column_cache_ptr);
+}
+
+Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const std::shared_ptr<SegmentFooterPB>& footer, int32_t column_uid,
+ uint64_t num_rows, io::FileReaderSPtr file_reader) {
// init sub columns
_subcolumns_meta_info = std::make_unique<SubcolumnColumnMetaInfo>();
_statistics = std::make_unique<VariantStatistics>();
- const ColumnMetaPB& self_column_pb = footer.columns(column_id);
+
+ // Prefer external root ColumnMetaPB via ColumnMetaAccessor when available.
+ ColumnMetaPB self_column_pb;
+ {
+ // Locate root column meta by unique id; this hides inline vs external CMO layout.
+ RETURN_IF_ERROR(accessor->get_column_meta_by_uid(*footer, column_uid, &self_column_pb));
+ // root column
+ // root subcolumn is ColumnVariant::MostCommonType which is jsonb
+ vectorized::DataTypePtr root_type =
+ self_column_pb.is_nullable()
+ ? make_nullable(
+ std::make_unique<vectorized::ColumnVariant::MostCommonType>())
+ : std::make_unique<vectorized::ColumnVariant::MostCommonType>();
+ int32_t root_footer_ordinal = -1;
+ if (self_column_pb.has_column_id()) {
+ // Narrow explicitly to avoid implicit narrowing warnings.
+ root_footer_ordinal = static_cast<int32_t>(self_column_pb.column_id());
+ }
+ _subcolumns_meta_info->create_root(SubcolumnMeta {
+ .file_column_type = root_type,
+ // Use column_id from meta as footer ordinal when inline footer is available.
+ .footer_ordinal = root_footer_ordinal});
+ RETURN_IF_ERROR(ColumnReader::create(opts, self_column_pb, num_rows, file_reader,
+ &_root_column_reader));
+ }
+
+ _data_type = vectorized::DataTypeFactory::instance().create_data_type(self_column_pb);
+ _root_unique_id = self_column_pb.unique_id();
const auto& parent_index = opts.tablet_schema->inverted_indexs(self_column_pb.unique_id());
// record variant_sparse_column_statistics_size from parent column
_variant_sparse_column_statistics_size =
opts.tablet_schema->column_by_uid(self_column_pb.unique_id())
.variant_max_sparse_column_statistics_size();
-
+ _tablet_schema = opts.tablet_schema;
// collect bucketized sparse readers for this variant column
std::map<int, std::shared_ptr<ColumnReader>> tmp_bucket_readers;
std::map<std::string, int64_t> aggregated_bucket_stats;
- for (int32_t ordinal = 0; ordinal < footer.columns_size(); ++ordinal) {
- const ColumnMetaPB& column_pb = footer.columns(ordinal);
+ // helper to handle sparse meta (single or bucket) from a ColumnMetaPB
+ auto handle_sparse_meta = [&](const ColumnMetaPB& col, bool* handled) -> Status {
+ *handled = false;
+ if (!col.has_column_path_info()) {
+ return Status::OK();
+ }
+ vectorized::PathInData path;
+ path.from_protobuf(col.column_path_info());
+ auto relative = path.copy_pop_front();
+ if (relative.empty()) {
+ return Status::OK();
+ }
+ std::string rel_str = relative.get_path();
+ if (rel_str == SPARSE_COLUMN_PATH) {
+ DCHECK(col.has_variant_statistics()) << col.DebugString();
+ const auto& variant_stats = col.variant_statistics();
+ for (const auto& [subpath, size] : variant_stats.sparse_column_non_null_size()) {
+ _statistics->sparse_column_non_null_size.emplace(subpath, size);
+ }
+ std::shared_ptr<ColumnReader> single_reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, col, footer->num_rows(), file_reader,
+ &single_reader));
+ _sparse_reader.set_single(std::move(single_reader));
+ *handled = true;
+ return Status::OK();
+ }
+ std::string bucket_prefix = std::string(SPARSE_COLUMN_PATH) + ".b";
+ if (rel_str.starts_with(bucket_prefix)) {
+ int idx = atoi(rel_str.substr(bucket_prefix.size()).c_str());
+ DCHECK(col.has_variant_statistics()) << col.DebugString();
+ const auto& variant_stats = col.variant_statistics();
+ for (const auto& [subpath, size] : variant_stats.sparse_column_non_null_size()) {
+ aggregated_bucket_stats[subpath] += size;
+ }
+ std::shared_ptr<ColumnReader> reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, col, num_rows, file_reader, &reader));
+ tmp_bucket_readers[idx] = reader;
+ *handled = true;
+ return Status::OK();
+ }
+ return Status::OK();
+ };
+
+ // First try initialize sparse from root's embedded children_columns (new segments)
+ if (self_column_pb.children_columns_size() > 0) {
+ for (int i = 0; i < self_column_pb.children_columns_size(); ++i) {
+ const ColumnMetaPB& child_pb = self_column_pb.children_columns(i);
+ bool handled = false;
+ RETURN_IF_ERROR(handle_sparse_meta(child_pb, &handled));
+ }
+ }
+
+ // init from inline columns meta
+ for (int32_t ordinal = 0; ordinal < footer->columns_size(); ++ordinal) {
+ const ColumnMetaPB& column_pb = footer->columns(ordinal);
// Find all columns belonging to the current variant column
// 1. not the variant column
if (!column_pb.has_column_path_info()) {
@@ -650,78 +936,30 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, const SegmentF
// init sparse column readers
auto relative_sparse = path.copy_pop_front();
auto rel_str = relative_sparse.get_path();
- if (rel_str == SPARSE_COLUMN_PATH) {
- DCHECK(column_pb.has_variant_statistics()) << column_pb.DebugString();
- const auto& variant_stats = column_pb.variant_statistics();
- for (const auto& [subpath, size] : variant_stats.sparse_column_non_null_size()) {
- _statistics->sparse_column_non_null_size.emplace(subpath, size);
- }
- std::shared_ptr<ColumnReader> single_reader;
- RETURN_IF_ERROR(ColumnReader::create(opts, column_pb, footer.num_rows(), file_reader,
- &single_reader));
- _sparse_reader.set_single(std::move(single_reader));
- continue;
- } else {
- // detect bucket: __DORIS_VARIANT_SPARSE__.b{idx}
- std::string bucket_prefix = std::string(SPARSE_COLUMN_PATH) + ".b";
- if (rel_str.rfind(bucket_prefix, 0) == 0) {
- int idx = atoi(rel_str.substr(bucket_prefix.size()).c_str());
- DCHECK(column_pb.has_variant_statistics()) << column_pb.DebugString();
- const auto& variant_stats = column_pb.variant_statistics();
- for (const auto& [subpath, size] : variant_stats.sparse_column_non_null_size()) {
- aggregated_bucket_stats[subpath] += size;
- }
- std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(
- ColumnReader::create(opts, column_pb, num_rows, file_reader, &reader));
- tmp_bucket_readers[idx] = reader;
+ {
+ bool handled = false;
+ RETURN_IF_ERROR(handle_sparse_meta(column_pb, &handled));
+ if (handled) {
continue;
}
}
// init subcolumns
auto relative_path = path.copy_pop_front();
- auto get_data_type_fn = [&]() {
- // root subcolumn is ColumnVariant::MostCommonType which is jsonb
- if (relative_path.empty()) {
- return self_column_pb.is_nullable()
- ? make_nullable(std::make_unique<
- vectorized::ColumnVariant::MostCommonType>())
- : std::make_unique<vectorized::ColumnVariant::MostCommonType>();
- }
- return vectorized::DataTypeFactory::instance().create_data_type(column_pb);
- };
- // init subcolumns
- if (_subcolumns_meta_info->get_root() == nullptr) {
- _subcolumns_meta_info->create_root(SubcolumnMeta {});
- }
if (relative_path.empty()) {
- // root column
- _subcolumns_meta_info->get_mutable_root()->modify_to_scalar(
- SubcolumnMeta {get_data_type_fn(), ordinal});
- RETURN_IF_ERROR(ColumnReader::create(opts, column_pb, num_rows, file_reader,
- &_root_column_reader));
- } else {
- // check the root is already a leaf node
- if (column_pb.has_none_null_size()) {
- _statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
- column_pb.none_null_size());
- }
- _subcolumns_meta_info->add(relative_path, SubcolumnMeta {get_data_type_fn(), ordinal});
- TabletSchema::SubColumnInfo sub_column_info;
- // if subcolumn has index, add index to _variant_subcolumns_indexes
- if (vectorized::schema_util::generate_sub_column_info(
- *opts.tablet_schema, self_column_pb.unique_id(), relative_path.get_path(),
- &sub_column_info) &&
- !sub_column_info.indexes.empty()) {
- _variant_subcolumns_indexes[path.get_path()] = std::move(sub_column_info.indexes);
- }
- // if parent column has index, add index to _variant_subcolumns_indexes
- else if (!parent_index.empty()) {
- vectorized::schema_util::inherit_index(
- parent_index, _variant_subcolumns_indexes[path.get_path()], column_pb);
- }
+ continue;
+ }
+ // check the root is already a leaf node
+ if (column_pb.has_none_null_size()) {
+ _statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
+ column_pb.none_null_size());
}
+ _subcolumns_meta_info->add(
+ relative_path,
+ SubcolumnMeta {
+ .file_column_type =
+ vectorized::DataTypeFactory::instance().create_data_type(column_pb),
+ .footer_ordinal = ordinal});
}
// finalize bucket readers if any
@@ -741,24 +979,97 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, const SegmentF
_statistics->sparse_column_non_null_size.emplace(path, size);
}
}
+ _segment_file_reader = file_reader;
+ _num_rows = num_rows;
+ // try build external meta readers (optional)
+ _ext_meta_reader = std::make_unique<VariantExternalMetaReader>();
+ RETURN_IF_ERROR(_ext_meta_reader->init_from_footer(footer, file_reader, _root_unique_id));
return Status::OK();
}
+Status VariantColumnReader::create_reader_from_external_meta(const std::string& path,
+ const ColumnReaderOptions& opts,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows,
+ std::shared_ptr<ColumnReader>* out) {
+ if (!_ext_meta_reader || !_ext_meta_reader->available()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("no external variant meta");
+ }
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(_ext_meta_reader->lookup_meta_by_path(path, &meta));
+ return ColumnReader::create(opts, meta, num_rows, file_reader, out);
+}
-std::vector<const TabletIndex*> VariantColumnReader::find_subcolumn_tablet_indexes(
- const std::string& path) {
- auto it = _variant_subcolumns_indexes.find(path);
- std::vector<const TabletIndex*> indexes;
- if (it != _variant_subcolumns_indexes.end()) {
- for (const auto& index : it->second) {
- indexes.push_back(index.get());
- }
+Status VariantColumnReader::create_path_reader(const vectorized::PathInData& relative_path,
+ const ColumnReaderOptions& opts,
+ ColumnMetaAccessor* accessor,
+ const SegmentFooterPB& footer,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows,
+ std::shared_ptr<ColumnReader>* out) {
+ // 1) Try inline subcolumn meta if available (footer_ordinal >= 0)
+ const auto* node = get_subcolumn_meta_by_path(relative_path);
+ if (node != nullptr && node->data.footer_ordinal >= 0) {
+ // leaf node, get the column meta by footer ordinal
+ const int32_t column_ordinal = node->data.footer_ordinal;
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(
+ accessor->get_column_meta_by_column_ordinal_id(footer, column_ordinal, &meta));
+ return ColumnReader::create(opts, meta, num_rows, file_reader, out);
+ }
+
+ // 2) Try external meta layout (if available)
+ Status st = create_reader_from_external_meta(relative_path.get_path(), opts, file_reader,
+ num_rows, out);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ *out = nullptr;
+ return st;
+ }
+ return st;
+}
+
+Status VariantColumnReader::load_external_meta_once() {
+ if (!_ext_meta_reader || !_ext_meta_reader->available()) {
+ return Status::OK();
+ }
+ // Ensure only one writer can populate `_subcolumns_meta_info` / `_statistics`
+ // while readers of these structures hold shared locks.
+ // english only in comments
+ std::unique_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
+ return _ext_meta_reader->load_all_once(_subcolumns_meta_info.get(), _statistics.get());
+}
+
+TabletIndexes VariantColumnReader::find_subcolumn_tablet_indexes(
+ const TabletColumn& column, const vectorized::DataTypePtr& data_type) {
+ TabletSchema::SubColumnInfo sub_column_info;
+ const auto& parent_index = _tablet_schema->inverted_indexs(column.parent_unique_id());
+ auto relative_path = column.path_info_ptr()->copy_pop_front();
+ // if subcolumn has index, add index to _variant_subcolumns_indexes
+ if (vectorized::schema_util::generate_sub_column_info(
+ *_tablet_schema, column.parent_unique_id(), relative_path.get_path(),
+ &sub_column_info) &&
+ !sub_column_info.indexes.empty()) {
+ }
+ // if parent column has index, add index to _variant_subcolumns_indexes
+ else if (!parent_index.empty() &&
+ data_type->get_primitive_type() != PrimitiveType::TYPE_VARIANT &&
+ data_type->get_primitive_type() != PrimitiveType::TYPE_MAP /*SPARSE COLUMN*/) {
+ // type in column maynot be real type, so use data_type to get the real type
+ TabletColumn target_column = vectorized::schema_util::get_column_by_type(
+ data_type, column.name(),
+ {.unique_id = -1,
+ .parent_unique_id = column.parent_unique_id(),
+ .path_info = *column.path_info_ptr()});
+ vectorized::schema_util::inherit_index(parent_index, sub_column_info.indexes,
+ target_column);
}
- return indexes;
+ // Return shared_ptr directly to maintain object lifetime
+ return sub_column_info.indexes;
}
void VariantColumnReader::get_subcolumns_types(
std::unordered_map<vectorized::PathInData, vectorized::DataTypes,
vectorized::PathInData::Hash>* subcolumns_types) const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
for (const auto& subcolumn_reader : *_subcolumns_meta_info) {
auto& path_types = (*subcolumns_types)[subcolumn_reader->path];
path_types.push_back(subcolumn_reader->data.file_column_type);
@@ -766,6 +1077,7 @@ void VariantColumnReader::get_subcolumns_types(
}
void VariantColumnReader::get_typed_paths(std::unordered_set<std::string>* typed_paths) const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
for (const auto& entry : *_subcolumns_meta_info) {
if (entry->path.get_is_typed()) {
typed_paths->insert(entry->path.get_path());
@@ -776,6 +1088,7 @@ void VariantColumnReader::get_typed_paths(std::unordered_set<std::string>* typed
void VariantColumnReader::get_nested_paths(
std::unordered_set<vectorized::PathInData, vectorized::PathInData::Hash>* nested_paths)
const {
+ std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
for (const auto& entry : *_subcolumns_meta_info) {
if (entry->path.has_nested_part()) {
nested_paths->insert(entry->path);
@@ -783,6 +1096,17 @@ void VariantColumnReader::get_nested_paths(
}
}
+Status VariantColumnReader::infer_data_type_for_path(vectorized::DataTypePtr* type,
+ const TabletColumn& column,
+ const StorageReadOptions& opts,
+ ColumnReaderCache* column_reader_cache) {
+ DCHECK(column.has_path_info());
+ ReadPlan plan;
+ RETURN_IF_ERROR(_build_read_plan(&plan, column, &opts, column_reader_cache, nullptr));
+ *type = plan.type;
+ return Status::OK();
+}
+
Status VariantRootColumnIterator::_process_root_column(
vectorized::MutableColumnPtr& dst, vectorized::MutableColumnPtr& root_column,
const vectorized::DataTypePtr& most_common_type) {
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
index f53b78ca723..3143fd94d81 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
@@ -22,14 +22,18 @@
#include <map>
#include <memory>
+#include <shared_mutex>
#include <string>
#include <unordered_map>
#include <vector>
#include "olap/rowset/segment_v2/column_reader.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
#include "olap/rowset/segment_v2/page_handle.h"
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
#include "olap/rowset/segment_v2/variant/variant_statistics.h"
#include "olap/tablet_schema.h"
+#include "util/once.h"
#include "vec/columns/column_variant.h"
#include "vec/columns/subcolumn_tree.h"
#include "vec/json/path_in_data.h"
@@ -38,6 +42,7 @@ namespace doris {
class TabletIndex;
class StorageReadOptions;
+class TabletSchema;
namespace segment_v2 {
@@ -200,7 +205,8 @@ class VariantColumnReader : public ColumnReader {
public:
VariantColumnReader() = default;
- Status init(const ColumnReaderOptions& opts, const SegmentFooterPB& footer, uint32_t column_id,
+ Status init(const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const std::shared_ptr<SegmentFooterPB>& footer, int32_t column_uid,
uint64_t num_rows, io::FileReaderSPtr file_reader);
Status new_iterator(ColumnIteratorUPtr* iterator, const TabletColumn* col,
@@ -221,7 +227,9 @@ public:
int64_t get_metadata_size() const override;
- std::vector<const TabletIndex*> find_subcolumn_tablet_indexes(const std::string&);
+ // Return shared_ptr to ensure the lifetime of TabletIndex objects
+ TabletIndexes find_subcolumn_tablet_indexes(const TabletColumn& target_column,
+ const vectorized::DataTypePtr& data_type);
bool exist_in_sparse_column(const vectorized::PathInData& path) const;
@@ -231,24 +239,113 @@ public:
return _subcolumns_meta_info.get();
}
+ // Get the types of all subcolumns in the variant column.
void get_subcolumns_types(
std::unordered_map<vectorized::PathInData, vectorized::DataTypes,
vectorized::PathInData::Hash>* subcolumns_types) const;
+ // Get the typed paths in the variant column.
void get_typed_paths(std::unordered_set<std::string>* typed_paths) const;
+ // Get the nested paths in the variant column.
void get_nested_paths(std::unordered_set<vectorized::PathInData, vectorized::PathInData::Hash>*
nested_paths) const;
+ // Infer the storage data type for a variant subcolumn using full StorageReadOptions
+ // (reader type, tablet schema, etc). This shares the same decision logic as
+ // `_build_read_plan`, but does not create any iterator.
+ Status infer_data_type_for_path(vectorized::DataTypePtr* type, const TabletColumn& column,
+ const StorageReadOptions& opts,
+ ColumnReaderCache* column_reader_cache);
+
+ // Create a ColumnReader for a sub-column identified by `relative_path`.
+ // This method will first try inline footer.columns via footer_ordinal and then
+ // fall back to external meta if available. Callers do not need to care about
+ // the underlying layout (inline vs external).
+ Status create_path_reader(const vectorized::PathInData& relative_path,
+ const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const SegmentFooterPB& footer, const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows, std::shared_ptr<ColumnReader>* out);
+
+ // Try create a ColumnReader from externalized meta (path -> ColumnMetaPB bytes) if present.
+ // Only used internally by create_path_reader. External callers should not rely
+ // on external meta details directly.
+ Status create_reader_from_external_meta(const std::string& path,
+ const ColumnReaderOptions& opts,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows, std::shared_ptr<ColumnReader>* out);
+
+ // Ensure external meta is loaded only once across concurrent callers.
+ Status load_external_meta_once();
+
+ // Determine whether `path` is a strict prefix of any existing subcolumn path.
+ // Consider three sources:
+ // 1) Extracted subcolumns in `_subcolumns_meta_info`
+ // 2) Sparse column statistics in `_statistics->sparse_column_non_null_size`
+ // 3) Externalized metas via `_ext_meta_reader`
+ bool has_prefix_path(const vectorized::PathInData& relative_path) const;
+
private:
+ // Internal unlocked helpers. Caller must hold `_subcolumns_meta_mutex` when using them.
+ // english only in comments
+ bool _is_exceeded_sparse_column_limit_unlocked() const;
+ bool _has_prefix_path_unlocked(const vectorized::PathInData& relative_path) const;
+
+ // Describe how a variant sub-path should be read. This is a logical plan only and
+ // does not create any concrete ColumnIterator.
+ enum class ReadKind {
+ ROOT_FLAT, // root variant using `VariantRootColumnIterator`
+ HIERARCHICAL, // hierarchical merge (root + subcolumns + sparse)
+ LEAF, // direct leaf reader
+ SPARSE_EXTRACT, // extract single path from sparse column
+ SPARSE_MERGE, // merge subcolumns into sparse column
+ DEFAULT_NESTED, // fill nested subcolumn using sibling nested column
+ DEFAULT_FILL // default iterator when path not exist
+ };
+
+ struct ReadPlan {
+ ReadKind kind {ReadKind::DEFAULT_FILL};
+ vectorized::DataTypePtr type;
+
+ // path & meta context
+ vectorized::PathInData relative_path;
+ const SubcolumnColumnMetaInfo::Node* node = nullptr;
+ const SubcolumnColumnMetaInfo::Node* root = nullptr;
+
+ // readers for LEAF / sparse cases
+ std::shared_ptr<ColumnReader> leaf_column_reader;
+ std::shared_ptr<ColumnReader> sparse_column_reader;
+
+ // sparse extras
+ std::string sparse_cache_key;
+ std::optional<uint32_t> bucket_index;
+ };
+
+ // Build read plan for flat-leaf (compaction/checksum) mode. Only decides the
+ // resulting type and how to read, without creating iterators.
+ Status _build_read_plan_flat_leaves(ReadPlan* plan, const TabletColumn& col,
+ const StorageReadOptions* opts,
+ ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr);
+
+ // Build read plan for the general hierarchical reading mode.
+ Status _build_read_plan(ReadPlan* plan, const TabletColumn& target_col,
+ const StorageReadOptions* opt, ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr);
+
+ // Materialize a concrete ColumnIterator according to the previously built plan.
+ Status _create_iterator_from_plan(ColumnIteratorUPtr* iterator, const ReadPlan& plan,
+ const TabletColumn& target_col, const StorageReadOptions* opt,
+ ColumnReaderCache* column_reader_cache,
+ PathToSparseColumnCache* sparse_column_cache_ptr);
// init for compaction read
Status _new_default_iter_with_same_nested(ColumnIteratorUPtr* iterator, const TabletColumn& col,
const StorageReadOptions* opt,
ColumnReaderCache* column_reader_cache);
Status _new_iterator_with_flat_leaves(
- ColumnIteratorUPtr* iterator, const TabletColumn& col, const StorageReadOptions* opts,
- bool exceeded_sparse_column_limit, bool existed_in_sparse_column,
- ColumnReaderCache* column_reader_cache,
+ ColumnIteratorUPtr* iterator, vectorized::DataTypePtr* type, const TabletColumn& col,
+ const StorageReadOptions* opts, bool exceeded_sparse_column_limit,
+ bool existed_in_sparse_column, ColumnReaderCache* column_reader_cache,
PathToSparseColumnCache* sparse_column_cache_ptr = nullptr);
Status _create_hierarchical_reader(ColumnIteratorUPtr* reader, int32_t col_uid,
@@ -269,16 +366,27 @@ private:
PathToSparseColumnCache* sparse_column_cache_ptr, const std::string& path,
std::shared_ptr<ColumnReader> sparse_column_reader);
+ // Protect `_subcolumns_meta_info` and `_statistics` when loading external meta.
+ // english only in comments
+ mutable std::shared_mutex _subcolumns_meta_mutex;
std::unique_ptr<SubcolumnColumnMetaInfo> _subcolumns_meta_info;
// Sparse column readers (single or bucketized)
UnifiedSparseColumnReader _sparse_reader;
std::shared_ptr<ColumnReader> _root_column_reader;
std::unique_ptr<VariantStatistics> _statistics;
- // key: subcolumn path, value: subcolumn indexes
- std::unordered_map<std::string, TabletIndexes> _variant_subcolumns_indexes;
+ std::shared_ptr<TabletSchema> _tablet_schema;
// variant_sparse_column_statistics_size
size_t _variant_sparse_column_statistics_size =
BeConsts::DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATS_SIZE;
+
+ // Externalized meta reader (optional)
+ std::unique_ptr<VariantExternalMetaReader> _ext_meta_reader;
+
+ io::FileReaderSPtr _segment_file_reader;
+ uint64_t _num_rows {0};
+ uint32_t _root_unique_id {0};
+
+ // call-once guard moved into VariantExternalMetaReader
};
class VariantRootColumnIterator : public ColumnIterator {
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
index 2e9320f714e..155180411e7 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
@@ -31,9 +31,12 @@
#include "olap/rowset/rowset_fwd.h"
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/column_writer.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
#include "olap/segment_loader.h"
#include "olap/tablet_schema.h"
+#include "olap/types.h"
#include "util/simd/bits.h"
+#include "util/slice.h"
#include "vec/columns/column.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_variant.h"
@@ -456,6 +459,9 @@ Status VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnVariant* p
};
_subcolumns_indexes.resize(ptr->get_subcolumns().size());
// convert sub column data from engine format to storage layer format
+ // NOTE: We only keep up to variant_max_subcolumns_count as extracted columns; others are externalized.
+ // uint32_t extracted = 0;
+ // uint32_t extract_limit = _tablet_column->variant_max_subcolumns_count();
for (const auto& entry :
vectorized::schema_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
const auto& least_common_type = entry->data.get_least_common_type();
@@ -469,7 +475,7 @@ Status VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnVariant* p
}
CHECK(entry->data.is_finalized());
- // create subcolumn writer
+ // create subcolumn writer if under limit; otherwise externalize ColumnMetaPB via IndexedColumn
int current_column_id = column_id++;
TabletColumn tablet_column;
int64_t none_null_value_size = entry->data.get_non_null_value_size();
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
index 12e553adda8..c2aa7bd9a05 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
@@ -21,6 +21,7 @@
#include "common/status.h"
#include "olap/rowset/segment_v2/column_writer.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
#include "olap/rowset/segment_v2/variant/variant_statistics.h"
#include "olap/tablet_schema.h"
#include "vec/columns/column.h"
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp
new file mode 100644
index 00000000000..b31a9ceff20
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/types.h"
+#include "vec/common/schema_util.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+
+Status VariantExtMetaWriter::_ensure_inited(Writers* w) {
+ if (w->inited) {
+ return Status::OK();
+ }
+
+ // key writer: VARCHAR, value index ON, ordinal index ON
+ IndexedColumnWriterOptions dict_opts;
+ dict_opts.write_value_index = true;
+ dict_opts.write_ordinal_index = true;
+ dict_opts.encoding = PREFIX_ENCODING;
+ dict_opts.compression = _comp;
+ const TypeInfo* dict_type = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
+ w->key_writer = std::make_unique<IndexedColumnWriter>(dict_opts, dict_type, _fw);
+ RETURN_IF_ERROR(w->key_writer->init());
+
+ w->inited = true;
+ return Status::OK();
+}
+
+Status VariantExtMetaWriter::add(int32_t root_uid, const Slice& key) {
+ auto& w = _writers_by_uid[root_uid];
+ RETURN_IF_ERROR(_ensure_inited(&w));
+
+ // Only store the relative path string as key. The concrete column_id in
+ // Column Meta Region will be derived on the reader side using:
+ // col_id = root_column_id + 1 + key_ordinal
+ // based on the agreed layout of ColumnMetaEntryPB in SegmentFooterPB.
+ RETURN_IF_ERROR(w.key_writer->add(&key));
+ ++w.count;
+ return Status::OK();
+}
+
+Status VariantExtMetaWriter::flush_to_footer(SegmentFooterPB* footer) {
+ for (auto& [uid, w] : _writers_by_uid) {
+ if (!w.inited || w.count == 0) {
+ continue;
+ }
+ doris::segment_v2::IndexedColumnMetaPB key_meta;
+ RETURN_IF_ERROR(w.key_writer->finish(&key_meta));
+
+ // keys
+ std::string k = std::string("variant_meta_keys.") + std::to_string(uid);
+ std::string v;
+ key_meta.AppendToString(&v);
+ auto* p1 = footer->add_file_meta_datas();
+ p1->set_key(k);
+ p1->set_value(v);
+ }
+ _writers_by_uid.clear();
+ return Status::OK();
+}
+
+// Helper struct to manage column classification
+struct ColumnClassification {
+ std::vector<ColumnMetaPB> top_level_columns;
+ std::unordered_map<int32_t, std::vector<std::pair<std::string, ColumnMetaPB>>> pending_subcols;
+ std::unordered_map<int32_t, std::vector<ColumnMetaPB>> pending_sparse;
+ size_t externalized_count = 0;
+};
+
+// Separate function to classify columns
+static void classify_columns(const SegmentFooterPB& footer, ColumnClassification& classification) {
+ classification.top_level_columns.reserve(footer.columns_size());
+
+ for (int i = 0; i < footer.columns_size(); ++i) {
+ const ColumnMetaPB& col = footer.columns(i);
+ bool is_subcol = false;
+ if (col.has_column_path_info()) {
+ vectorized::PathInData full_path;
+ full_path.from_protobuf(col.column_path_info());
+ vectorized::PathInData rel = full_path.copy_pop_front();
+ if (!rel.empty()) {
+ is_subcol = true;
+ std::string rel_path = rel.get_path();
+ int32_t root_uid = col.column_path_info().parrent_column_unique_id();
+
+ if (rel_path.find("__DORIS_VARIANT_SPARSE__") != std::string::npos) {
+ classification.pending_sparse[root_uid].emplace_back(col);
+ } else {
+ classification.pending_subcols[root_uid].emplace_back(rel_path, col);
+ classification.externalized_count++;
+ }
+ }
+ }
+
+ if (!is_subcol) {
+ classification.top_level_columns.emplace_back(col);
+ }
+ }
+}
+
+Status VariantExtMetaWriter::externalize_from_footer(SegmentFooterPB* footer,
+ std::vector<ColumnMetaPB>* out_metas) {
+ // Variant meta pre-processing and reorganization:
+ // 1. Identify Variant Root columns and their subcolumns.
+ // 2. Collect sparse subcolumns to embed in Root.
+ // 3. Collect non-sparse subcolumns to externalize to Meta Region.
+ // 4. Rebuild the column list (out_metas) ensuring subcolumns follow their Root.
+ // 5. Record mapping (Path -> Index in out_metas) for externalized subcolumns.
+
+ ColumnClassification classification;
+ classify_columns(*footer, classification);
+
+ // Rebuild out_metas
+ out_metas->clear();
+ out_metas->reserve(footer->columns_size()); // Rough estimate
+
+ std::vector<ColumnMetaPB> kept_footer_columns;
+ kept_footer_columns.reserve(footer->columns_size());
+
+ for (auto& col : classification.top_level_columns) {
+ // Add Top Level Column
+ out_metas->emplace_back(col);
+ // Keep it in footer columns as well
+ kept_footer_columns.emplace_back(col);
+ ColumnMetaPB* current_root_meta = &out_metas->back();
+ ColumnMetaPB* footer_root_meta = &kept_footer_columns.back();
+
+ // Check if it is a Variant Root
+ if (col.type() == int(FieldType::OLAP_FIELD_TYPE_VARIANT)) {
+ int32_t root_uid = col.unique_id();
+
+ // 1. Embed sparse subcolumns
+ if (auto it = classification.pending_sparse.find(root_uid);
+ it != classification.pending_sparse.end()) {
+ for (const auto& sparse_meta : it->second) {
+ current_root_meta->add_children_columns()->CopyFrom(sparse_meta);
+ footer_root_meta->add_children_columns()->CopyFrom(sparse_meta);
+ }
+ }
+
+ // 2. Append non-sparse subcolumns to out_metas and record path-only keys
+ if (auto it = classification.pending_subcols.find(root_uid);
+ it != classification.pending_subcols.end()) {
+ // Sort by path for consistent index order
+ std::sort(it->second.begin(), it->second.end(),
+ [](const auto& a, const auto& b) { return a.first < b.first; });
+
+ for (auto& [path, meta] : it->second) {
+ // Append to out_metas
+ out_metas->emplace_back(meta);
+ // Mark as subcolumn placeholder for footer logic (unique_id = -1)
+ out_metas->back().set_unique_id(-1);
+
+ // Record path key for this subcolumn. Column id will be derived
+ // by reader as (root_col_id + 1 + ordinal).
+ RETURN_IF_ERROR(add(root_uid, Slice(path)));
+ }
+ }
+ }
+ }
+
+ // Write the index to footer
+ RETURN_IF_ERROR(flush_to_footer(footer));
+
+ // Update footer.columns to only contain Top Level Columns (with embedded sparse cols)
+ footer->clear_columns();
+ for (const auto& c : kept_footer_columns) {
+ footer->add_columns()->CopyFrom(c);
+ }
+
+ VLOG_DEBUG << "VariantExtMetaWriter::externalize_from_footer, externalized subcolumns: "
+ << classification.externalized_count
+ << ", kept columns: " << kept_footer_columns.size()
+ << ", total meta entries: " << out_metas->size();
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h
new file mode 100644
index 00000000000..e70429983dd
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/types.h"
+#include "util/slice.h"
+
+namespace doris::segment_v2 {
+
+// -----------------------------------------------------------------------------
+// Variant External Meta (path -> ColumnMetaPB bytes) storage format
+// -----------------------------------------------------------------------------
+// Purpose: externalize ColumnMetaPB of variant subcolumns as a mapping from
+// relative path to column meta entries in the external Column Meta Region,
+// instead of packing all into Footer.columns. This enables on-demand subcolumn
+// loading and fast path lookup.
+//
+// Physical layout (per variant root uid): one IndexedColumn of keys is written
+// - variant_meta_keys.<root_uid> : IndexedColumn(VARCHAR)
+// • value index: ON (fast existence/lookup by path)
+// • ordinal index: ON (enable full scan / load_all)
+// • encoding: PREFIX_ENCODING
+// • content: Key = Path (relative path string only)
+//
+// Footer carrier:
+// footer.file_meta_datas contains one entry per root uid:
+// - key = "variant_meta_keys.<root_uid>" , value = IndexedColumnMetaPB bytes
+//
+// Interop with external column meta (column_meta_entries):
+// - Variant root ColumnMetaPB is externalized via column_meta_entries (addressed by col_id)
+// - Variant subcolumn ColumnMetaPB:
+// • non-sparse: stored in Column Meta Region, addressed by a unique col_id (in column_meta_entries).
+// The mapping from Path -> col_id is stored in this class (VariantExtMetaWriter).
+// • sparse (including buckets): embedded into root's ColumnMetaPB.children_columns and
+// loaded together with root meta via column_meta_entries
+//
+class VariantExtMetaWriter {
+public:
+ VariantExtMetaWriter(io::FileWriter* fw, CompressionTypePB comp) : _fw(fw), _comp(comp) {}
+
+ // Add one path entry for the specified variant root uid.
+ // key: subcolumn path (VARCHAR)
+ // The concrete ColumnMetaPB location (column_id) is derived on read side
+ // via (root_column_id + 1 + ordinal) based on the agreed Column Meta Region
+ // layout.
+ Status add(int32_t root_uid, const Slice& key);
+
+ // Finish writers and append their metas into footer.file_meta_datas.
+ Status flush_to_footer(SegmentFooterPB* footer);
+
+ // Scan footer.columns, find variant extracted subcolumns, externalize them into
+ // ext meta index and remove them from footer.columns.
+ // This method reorganizes the columns:
+ // 1. Separates Top Level Columns (kept in footer) and Variant Subcolumns.
+ // 2. Records Subcolumns' indices (relative to the combined list) into the Variant Index.
+ // 3. Populates out_metas with the combined list of all columns (Top Level + Subcolumns),
+ // ordered such that Subcolumns immediately follow their Root Variant.
+ Status externalize_from_footer(SegmentFooterPB* footer, std::vector<ColumnMetaPB>* out_metas);
+
+ bool empty() const { return _writers_by_uid.empty(); }
+
+private:
+ struct Writers {
+ std::unique_ptr<IndexedColumnWriter> key_writer; // value index
+ size_t count = 0;
+ bool inited = false;
+ };
+
+ Status _ensure_inited(Writers* w);
+
+ io::FileWriter* _fw;
+ CompressionTypePB _comp;
+ std::unordered_map<int32_t, Writers> _writers_by_uid;
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp
new file mode 100644
index 00000000000..c068a5423a7
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
+
+#include <gen_cpp/segment_v2.pb.h>
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
+#include "vec/columns/column_string.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+
+#include "common/compile_check_begin.h"
+
+Status VariantExternalMetaReader::_find_key_meta(const SegmentFooterPB& footer, int32_t root_uid,
+ const MetadataPairPB** keys_meta_pair) const {
+ *keys_meta_pair = nullptr;
+ // prefer suffixed pairs
+ std::string suffix = "." + std::to_string(root_uid);
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == std::string("variant_meta_keys") + suffix) {
+ *keys_meta_pair = &m;
+ return Status::OK();
+ }
+ }
+ // fallback: legacy single-variant footer
+ if (!*keys_meta_pair) {
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == "variant_meta_keys") {
+ *keys_meta_pair = &m;
+ return Status::OK();
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::init_from_footer(std::shared_ptr<const SegmentFooterPB> footer,
+ const io::FileReaderSPtr& file_reader,
+ int32_t root_uid) {
+ _footer = std::move(footer);
+ _file_reader = file_reader;
+
+ const MetadataPairPB* keys_meta_pair = nullptr;
+ RETURN_IF_ERROR(_find_key_meta(*_footer, root_uid, &keys_meta_pair));
+ // If keys meta is absent, external meta is considered unavailable.
+ if (!keys_meta_pair) {
+ return Status::OK();
+ }
+
+ // Parse external Column Meta Region pointers and uid->col_id map.
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_external_meta_pointers(*_footer, &_meta_ptrs));
+ std::unordered_map<int32_t, size_t> uid2colid;
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_uid_to_colid_map(*_footer, _meta_ptrs, &uid2colid));
+ auto it = uid2colid.find(root_uid);
+ if (it == uid2colid.end()) {
+ return Status::Corruption("root uid {} not found in external uid->col_id map", root_uid);
+ }
+ _root_col_id = static_cast<uint32_t>(it->second);
+
+ // Open the path-key indexed column.
+ doris::segment_v2::IndexedColumnMetaPB key_meta;
+ if (!key_meta.ParseFromArray(keys_meta_pair->value().data(),
+ static_cast<int>(keys_meta_pair->value().size()))) {
+ return Status::Corruption("bad variant_meta_keys meta");
+ }
+
+ _key_reader = std::make_unique<segment_v2::IndexedColumnReader>(file_reader, key_meta);
+ RETURN_IF_ERROR(_key_reader->load(true, false));
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::lookup_meta_by_path(const std::string& rel_path,
+ ColumnMetaPB* out_meta) const {
+ if (!available()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("no external variant meta");
+ }
+ // 1. Seek for Path
+ auto key_it = segment_v2::IndexedColumnIterator(_key_reader.get());
+ bool exact = false;
+ Status st = key_it.seek_at_or_after(&rel_path, &exact);
+ if (st.is<ErrorCode::ENTRY_NOT_FOUND>()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("variant meta key not found");
+ }
+ if (!st.ok()) {
+ return st;
+ }
+
+ // Record ordinal immediately after seek; get_current_ordinal() is only valid
+ // while iterator is in "seeked" state (before calling next_batch()).
+ auto ordinal = static_cast<uint32_t>(key_it.get_current_ordinal());
+
+ // 2. Read key and verify exact match (keys store pure relative path).
+ size_t n = 1;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n != 1) {
+ return Status::Corruption("variant meta key read failed");
+ }
+ auto* s = assert_cast<vectorized::ColumnString*>(dst.get());
+ auto ref = s->get_data_at(0);
+
+ if (ref.size != rel_path.size() || memcmp(ref.data, rel_path.data(), ref.size) != 0) {
+ // seek_at_or_after found something greater, but path doesn't match
+ return Status::Error<ErrorCode::NOT_FOUND, false>("variant meta key not found");
+ }
+
+ // 3. Derive column id from root_col_id and key ordinal, then read ColumnMetaPB.
+ auto col_id = _root_col_id + 1 + ordinal;
+ return ExternalColMetaUtil::read_col_meta(_file_reader, *_footer, _meta_ptrs, col_id, out_meta);
+}
+
+Status VariantExternalMetaReader::load_all(SubcolumnColumnMetaInfo* out_meta_tree,
+ VariantStatistics* out_stats) {
+ DCHECK(available());
+ auto key_it = segment_v2::IndexedColumnIterator(_key_reader.get());
+ RETURN_IF_ERROR(key_it.seek_to_ordinal(0));
+ auto total = static_cast<size_t>(_key_reader->num_values());
+ size_t built = 0;
+
+ while (built < total) {
+ size_t n = total - built;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n == 0) {
+ break;
+ }
+ for (size_t i = 0; i < n; ++i) {
+ // Derive column id for this key by its global ordinal.
+ auto ordinal = static_cast<uint32_t>(built + i);
+ auto col_id = _root_col_id + 1 + ordinal;
+
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(ExternalColMetaUtil::read_col_meta(_file_reader, *_footer, _meta_ptrs,
+ col_id, &meta));
+
+ if (!meta.has_column_path_info()) {
+ continue;
+ }
+ vectorized::PathInData full_path;
+ full_path.from_protobuf(meta.column_path_info());
+ auto relative_path = full_path.copy_pop_front();
+ if (relative_path.empty()) {
+ continue; // skip root
+ }
+ if (out_meta_tree->find_leaf(relative_path)) {
+ continue; // already exists
+ }
+ if (meta.has_none_null_size() && out_stats != nullptr) {
+ out_stats->subcolumns_non_null_size.emplace(relative_path.get_path(),
+ meta.none_null_size());
+ }
+ auto file_type = vectorized::DataTypeFactory::instance().create_data_type(meta);
+ out_meta_tree->add(relative_path,
+ SubcolumnMeta {.file_column_type = file_type, .footer_ordinal = -1});
+ }
+ built += n;
+ }
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::has_prefix(const std::string& prefix, bool* out) const {
+ // english only in comments
+ DCHECK(out != nullptr);
+ DCHECK(available());
+ *out = false;
+ // Empty prefix means everything matches; guard for safety
+ if (prefix.empty()) {
+ *out = true;
+ return Status::OK();
+ }
+
+ segment_v2::IndexedColumnIterator key_it(_key_reader.get());
+ bool exact = false;
+ Status st = key_it.seek_at_or_after(&prefix, &exact);
+ if (st.is<ErrorCode::ENTRY_NOT_FOUND>()) {
+ *out = false;
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
+ size_t n = 1;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n == 0) {
+ *out = false;
+ return Status::OK();
+ }
+ auto* s = assert_cast<vectorized::ColumnString*>(dst.get());
+ auto ref = s->get_data_at(0);
+ std::string_view key_sv(ref.data, ref.size);
+
+ // starts_with check. Key is Path + ID.
+ // If prefix is just path part, it matches.
+ // If prefix contains ID part (unlikely use case), it also matches.
+ *out = key_sv.size() >= prefix.size() && key_sv.starts_with(prefix);
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::load_all_once(SubcolumnColumnMetaInfo* out_meta_tree,
+ VariantStatistics* out_stats) {
+ DCHECK(available());
+ return _load_once_call.call([&]() -> Status {
+ if (_loaded) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(load_all(out_meta_tree, out_stats));
+ _loaded = true;
+ return Status::OK();
+ });
+}
+
+#include "common/compile_check_end.h"
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h
new file mode 100644
index 00000000000..bea5187324c
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/segment_v2.pb.h>
+
+#include <memory>
+#include <string>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
+#include "olap/rowset/segment_v2/stream_reader.h"
+#include "olap/rowset/segment_v2/variant/variant_statistics.h"
+#include "util/once.h"
+
+namespace doris::segment_v2 {
+
+#include "common/compile_check_begin.h"
+
+// Encapsulates reading of externalized Variant subcolumn metas.
+// It discovers key/value indexed-columns from SegmentFooterPB, supports:
+// - availability check
+// - lookup ColumnMetaPB by relative path
+// - one-time bulk load of all metas into SubcolumnColumnMetaInfo and VariantStatistics
+class VariantExternalMetaReader {
+public:
+ VariantExternalMetaReader() = default;
+
+ // Initialize by locating and opening ext meta key/value indexed columns.
+ // root_uid is used to find suffixed keys like: variant_meta_keys.<root_uid>
+ //
+ // NOTE: footer is kept as shared_ptr to ensure its lifetime is at least as
+ // long as this reader; callers should pass in the same footer shared_ptr
+ // they own (e.g. from Segment::_get_segment_footer).
+ Status init_from_footer(std::shared_ptr<const SegmentFooterPB> footer,
+ const io::FileReaderSPtr& file_reader, int32_t root_uid);
+
+ bool available() const { return _key_reader != nullptr; }
+
+ // Lookup a single ColumnMetaPB by relative path. Returns NOT_FOUND if missing/unavailable.
+ Status lookup_meta_by_path(const std::string& rel_path, ColumnMetaPB* out_meta) const;
+
+ // Check whether there exists any key in external meta that starts with `prefix`.
+ // This performs a lower_bound (seek_at_or_after) on the sorted key column
+ // and verifies the first key is prefixed by `prefix`.
+ Status has_prefix(const std::string& prefix, bool* out) const;
+
+ // Ensure external metas are loaded exactly once and merged into provided structures.
+ Status load_all_once(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics* out_stats);
+
+ // Load and merge all external metas without call-once guard.
+ Status load_all(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics* out_stats);
+
+private:
+ // helpers
+ Status _find_key_meta(const SegmentFooterPB& footer, int32_t root_uid,
+ const MetadataPairPB** keys_meta_pair) const;
+
+ std::unique_ptr<IndexedColumnReader> _key_reader;
+
+ // Shared footer to access Column Meta Region info (column_meta_entries).
+ // Using shared_ptr prevents dangling pointer when the original owner releases it.
+ std::shared_ptr<const SegmentFooterPB> _footer;
+ ExternalColMetaUtil::ExternalMetaPointers _meta_ptrs;
+ io::FileReaderSPtr _file_reader;
+
+ // Global column_id of the variant root in external Column Meta Region.
+ // For its non-sparse subcolumns, their column_id is derived as:
+ // root_col_id + 1 + key_ordinal
+ uint32_t _root_col_id = 0;
+
+ // call-once guard for bulk loading
+ DorisCallOnce<Status> _load_once_call;
+ bool _loaded = false;
+};
+
+#include "common/compile_check_end.h"
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 7320ce18371..8455f9eabf3 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -49,10 +49,12 @@
#include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "olap/rowset/segment_creator.h"
#include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
#include "olap/rowset/segment_v2/index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/page_pointer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
#include "olap/segment_loader.h"
#include "olap/short_key_index.h"
#include "olap/tablet_schema.h"
@@ -78,7 +80,6 @@
#include "vec/json/path_in_data.h"
#include "vec/jsonb/serialize.h"
#include "vec/olap/olap_data_convertor.h"
-
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
@@ -1320,6 +1321,17 @@ Status VerticalSegmentWriter::_write_primary_key_index() {
Status VerticalSegmentWriter::_write_footer() {
_footer.set_num_rows(_row_count);
+ // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
+
+ if (_tablet_schema->is_external_segment_column_meta_used()) {
+ _footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+ VLOG_DEBUG << "use external column meta";
+ // External ColumnMetaPB writing (optional)
+ RETURN_IF_ERROR(ExternalColMetaUtil::write_external_column_meta(
+ _file_writer, &_footer, _opts.compression_type,
+ [this](const std::vector<Slice>& slices) { return _write_raw_data(slices); }));
+ }
+
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4)
VLOG_DEBUG << "footer " << _footer.DebugString();
std::string footer_buf;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 80fb14c5e22..38a6a4c06dd 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1564,7 +1564,8 @@ Status TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& reque
(*tablet_meta)->set_preferred_rowset_type(_engine.default_rowset_type());
} else if (request.storage_format == TStorageFormat::V1) {
(*tablet_meta)->set_preferred_rowset_type(ALPHA_ROWSET);
- } else if (request.storage_format == TStorageFormat::V2) {
+ } else if (request.storage_format == TStorageFormat::V2 ||
+ request.storage_format == TStorageFormat::V3) {
(*tablet_meta)->set_preferred_rowset_type(BETA_ROWSET);
} else {
return Status::Error<CE_CMD_PARAMS_ERROR>("invalid TStorageFormat: {}",
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 8e1f4453fea..d5acbc0bb31 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -106,12 +106,16 @@ TabletMetaSharedPtr TabletMeta::create(
break;
}
}
+ // Decide storage format for this tablet. DEFAULT / not-set fall back to V2 on BE side.
+ TStorageFormat::type storage_format =
+ request.__isset.storage_format ? request.storage_format : TStorageFormat::V2;
return std::make_shared<TabletMeta>(
request.table_id, request.partition_id, request.tablet_id, request.replica_id,
request.tablet_schema.schema_hash, shard_id, request.tablet_schema, next_unique_id,
col_ordinal_to_unique_id, tablet_uid,
request.__isset.tablet_type ? request.tablet_type : TTabletType::TABLET_TYPE_DISK,
- request.compression_type, request.storage_policy_id,
+ request.__isset.compression_type ? request.compression_type : TCompressionType::LZ4F,
+ request.__isset.storage_policy_id ? request.storage_policy_id : -1,
request.__isset.enable_unique_key_merge_on_write
? request.enable_unique_key_merge_on_write
: false,
@@ -121,7 +125,7 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
request.time_series_compaction_level_threshold, inverted_index_file_storage_format,
- request.tde_algorithm);
+ request.tde_algorithm, storage_format);
}
TabletMeta::~TabletMeta() {
@@ -149,10 +153,12 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
int64_t time_series_compaction_empty_rowsets_threshold,
int64_t time_series_compaction_level_threshold,
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format,
- TEncryptionAlgorithm::type tde_algorithm)
+ TEncryptionAlgorithm::type tde_algorithm,
+ TStorageFormat::type storage_format)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
- _delete_bitmap(new DeleteBitmap(tablet_id)) {
+ _delete_bitmap(new DeleteBitmap(tablet_id)),
+ _storage_format(storage_format) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(table_id);
tablet_meta_pb.set_partition_id(partition_id);
@@ -390,6 +396,22 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
tablet_meta_pb.set_encryption_algorithm(EncryptionAlgorithmPB::PLAINTEXT);
}
+ // Initialize default external ColumnMeta usage according to storage format.
+ // V2: legacy behavior, inline ColumnMetaPB only.
+ // V3: V2 + external ColumnMetaPB (CMO) enabled by default.
+ switch (_storage_format) {
+ case TStorageFormat::V2:
+ case TStorageFormat::DEFAULT:
+ case TStorageFormat::V1:
+ break;
+ case TStorageFormat::V3:
+ schema->set_is_external_segment_column_meta_used(true);
+ _schema->set_external_segment_meta_used_default(true);
+ break;
+ default:
+ break;
+ }
+
init_from_pb(tablet_meta_pb);
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index dcefc309a44..8d4d6b69719 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -117,7 +117,8 @@ public:
int64_t time_series_compaction_level_threshold = 1,
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
TInvertedIndexFileStorageFormat::V2,
- TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT);
+ TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT,
+ TStorageFormat::type storage_format = TStorageFormat::V2);
// If need add a filed in TableMeta, filed init copy in copy construct function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -373,6 +374,10 @@ private:
EncryptionAlgorithmPB _encryption_algorithm = PLAINTEXT;
+ // Persisted storage format for this tablet (e.g. V2, V3). Used to derive
+ // schema-level defaults such as external ColumnMeta usage.
+ TStorageFormat::type _storage_format = TStorageFormat::V2;
+
mutable std::shared_mutex _meta_lock;
};
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 7d9336cf108..243f5d200b4 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1168,6 +1168,11 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
_row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
schema.row_store_column_unique_ids().end());
_enable_variant_flatten_nested = schema.enable_variant_flatten_nested();
+ if (schema.has_is_external_segment_column_meta_used()) {
+ _is_external_segment_column_meta_used = schema.is_external_segment_column_meta_used();
+ } else {
+ _is_external_segment_column_meta_used = false;
+ }
update_metadata_size();
}
@@ -1420,6 +1425,8 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign(
_row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end());
tablet_schema_pb->set_enable_variant_flatten_nested(_enable_variant_flatten_nested);
+ tablet_schema_pb->set_is_external_segment_column_meta_used(
+ _is_external_segment_column_meta_used);
}
size_t TabletSchema::row_size() const {
@@ -1781,6 +1788,8 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
if (a._storage_dict_page_size != b._storage_dict_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false;
if (a._enable_variant_flatten_nested != b._enable_variant_flatten_nested) return false;
+ if (a._is_external_segment_column_meta_used != b._is_external_segment_column_meta_used)
+ return false;
return true;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index b4666577c0d..2de26f26e62 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -677,6 +677,15 @@ public:
return 0;
}
+ // Whether new segments use externalized ColumnMetaPB layout (CMO) by default
+ bool is_external_segment_column_meta_used() const {
+ return _is_external_segment_column_meta_used;
+ }
+
+ void set_external_segment_meta_used_default(bool v) {
+ _is_external_segment_column_meta_used = v;
+ }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -755,6 +764,9 @@ private:
// value: indexes
using PatternToIndex = std::unordered_map<std::string, std::vector<TabletIndexPtr>>;
std::unordered_map<int32_t, PatternToIndex> _index_by_unique_id_with_pattern;
+
+ // Default behavior for new segments: use external ColumnMeta region + CMO table if true
+ bool _is_external_segment_column_meta_used = false;
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp
index 3d3bd0eafb4..d89a63c67c5 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -698,8 +698,10 @@ Status VariantCompactionUtil::aggregate_path_to_stats(
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* variant_column_reader =
- assert_cast<const segment_v2::VariantColumnReader*>(column_reader.get());
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting stats
+ RETURN_IF_ERROR(variant_column_reader->load_external_meta_once());
const auto* source_stats = variant_column_reader->get_stats();
CHECK(source_stats);
@@ -737,8 +739,10 @@ Status VariantCompactionUtil::aggregate_variant_extended_info(
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* variant_column_reader =
- assert_cast<const segment_v2::VariantColumnReader*>(column_reader.get());
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting stats
+ RETURN_IF_ERROR(variant_column_reader->load_external_meta_once());
const auto* source_stats = variant_column_reader->get_stats();
CHECK(source_stats);
@@ -1416,9 +1420,16 @@ TabletSchemaSPtr VariantCompactionUtil::calculate_variant_extended_schema(
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* subcolumn_meta_info =
- assert_cast<VariantColumnReader*>(column_reader.get())
- ->get_subcolumns_meta_info();
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting subcolumn meta info
+ st = variant_column_reader->load_external_meta_once();
+ if (!st.ok()) {
+ LOG(WARNING) << "Failed to load external meta for column: " << column->name()
+ << " error: " << st.to_string();
+ continue;
+ }
+ const auto* subcolumn_meta_info = variant_column_reader->get_subcolumns_meta_info();
for (const auto& entry : *subcolumn_meta_info) {
if (entry->path.empty()) {
continue;
diff --git a/be/test/olap/date_bloom_filter_test.cpp b/be/test/olap/date_bloom_filter_test.cpp
index d4eac77c943..6ef6eacb3e7 100644
--- a/be/test/olap/date_bloom_filter_test.cpp
+++ b/be/test/olap/date_bloom_filter_test.cpp
@@ -18,6 +18,7 @@
#include <gen_cpp/segment_v2.pb.h>
#include <gtest/gtest.h>
+#include "common/status.h"
#include "olap/comparison_predicate.h"
#include "olap/in_list_predicate.h"
#include "olap/rowset/beta_rowset.h"
@@ -144,15 +145,21 @@ TEST_F(DateBloomFilterTest, query_index_test) {
columns[0]->insert_many_fix_len_data(reinterpret_cast<const char*>(&olap_date_value), 1);
columns[1]->insert_many_fix_len_data(reinterpret_cast<const char*>(&olap_datetime_value), 1);
- EXPECT_TRUE(rowset_writer->add_block(&block).ok());
- EXPECT_TRUE(rowset_writer->flush().ok());
- EXPECT_TRUE(rowset_writer->build(rowset).ok());
- EXPECT_TRUE(_tablet->add_rowset(rowset).ok());
+ Status st;
+ st = rowset_writer->add_block(&block);
+ ASSERT_TRUE(st.ok()) << st; // add_block failed, no valid segment will be written
+ st = rowset_writer->flush();
+ ASSERT_TRUE(st.ok()) << st; // flush failed
+ st = rowset_writer->build(rowset);
+ ASSERT_TRUE(st.ok()) << st; // build rowset failed
+ st = _tablet->add_rowset(rowset);
+ ASSERT_TRUE(st.ok()) << st; // add_rowset failed
segment_v2::SegmentSharedPtr segment;
- EXPECT_TRUE(((BetaRowset*)rowset.get())->load_segment(0, nullptr, &segment).ok());
+ st = ((BetaRowset*)rowset.get())->load_segment(0, nullptr, &segment);
+ ASSERT_TRUE(st.ok()) << st; // load_segment failed, segment may be null
std::shared_ptr<SegmentFooterPB> footer_pb_shared;
- auto st = segment->_get_segment_footer(footer_pb_shared, nullptr);
+ st = segment->_get_segment_footer(footer_pb_shared, nullptr);
EXPECT_TRUE(st.ok());
st = segment->_create_column_meta(*footer_pb_shared);
EXPECT_TRUE(st.ok());
diff --git a/be/test/olap/delete_bitmap_calculator_test.cpp b/be/test/olap/delete_bitmap_calculator_test.cpp
index 406b8c53ad8..b51760fbd36 100644
--- a/be/test/olap/delete_bitmap_calculator_test.cpp
+++ b/be/test/olap/delete_bitmap_calculator_test.cpp
@@ -107,6 +107,19 @@ void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, siz
EXPECT_EQ(nrows, (*res)->num_rows());
}
+// Convenience overload that also returns the on-disk path of the built segment.
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir, std::string* out_path) {
+ std::string filename = fmt::format("{}_{}.dat", rowset_id.to_string(), segment_id);
+ std::string path = fmt::format("{}/{}", segment_dir, filename);
+ if (out_path != nullptr) {
+ *out_path = path;
+ }
+ build_segment(std::move(opts), std::move(build_schema), segment_id, std::move(query_schema),
+ nrows, std::move(generator), res, std::move(segment_dir));
+}
+
TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
KeysType keys_type = UNIQUE_KEYS) {
TabletSchemaSPtr res = std::make_shared<TabletSchema>();
diff --git a/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp b/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp
new file mode 100644
index 00000000000..b118968a589
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp
@@ -0,0 +1,705 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "io/fs/local_file_system.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_writer.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+
+using namespace doris;
+using namespace doris::segment_v2;
+
+namespace {
+
+constexpr std::string_view kTestDir = "./ut_dir/column_meta_accessor_test";
+
+// Helper to create a clean test directory
+std::string make_test_file_path(const std::string& file_name) {
+ auto fs = io::global_local_filesystem();
+ static_cast<void>(fs->delete_directory(kTestDir));
+ CHECK(fs->create_directory(kTestDir).ok());
+ return std::string(kTestDir) + "/" + file_name;
+}
+
+// Helper to write segment footer trailer (footer + metadata)
+Status append_footer_trailer(io::FileWriter* fw, SegmentFooterPB* footer) {
+ std::string footer_buf;
+ if (!footer->SerializeToString(&footer_buf)) {
+ return Status::InternalError("failed to serialize SegmentFooterPB");
+ }
+ faststring fixed_buf;
+ put_fixed32_le(&fixed_buf, static_cast<uint32_t>(footer_buf.size()));
+ uint32_t checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ put_fixed32_le(&fixed_buf, checksum);
+ fixed_buf.append("D0R1", 4);
+ std::vector<Slice> slices {Slice(footer_buf), Slice(fixed_buf)};
+ RETURN_IF_ERROR(fw->appendv(slices.data(), slices.size()));
+ return Status::OK();
+}
+
+// Helper to read footer from file
+Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* footer) {
+ const auto file_size = fr->size();
+ if (file_size < 12) {
+ return Status::Corruption("file too small");
+ }
+ uint8_t fixed_buf[12];
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12, Slice(fixed_buf, sizeof(fixed_buf)), &bytes_read));
+ if (bytes_read != sizeof(fixed_buf)) {
+ return Status::Corruption("short read footer trailer");
+ }
+ const uint32_t footer_length = decode_fixed32_le(fixed_buf);
+ if (file_size < 12 + footer_length) {
+ return Status::Corruption("bad footer length");
+ }
+ std::string footer_buf;
+ footer_buf.resize(footer_length);
+ bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12 - footer_length,
+ Slice(footer_buf.data(), footer_buf.size()), &bytes_read));
+ if (bytes_read != footer_length) {
+ return Status::Corruption("short read footer");
+ }
+ if (!footer->ParseFromArray(footer_buf.data(), static_cast<int>(footer_buf.size()))) {
+ return Status::Corruption("failed to parse footer");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+namespace doris {
+
+using Generator = std::function<void(size_t rid, int cid, RowCursorCell& cell)>;
+
+// Helper declarations are defined in tablet_schema_helper.{h,cpp} and
+// delete_bitmap_calculator_test.cpp.
+TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
+ KeysType keys_type = UNIQUE_KEYS);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir, std::string* out_path);
+
+} // namespace doris
+
+// Test V2 (inline) mode with valid column metadata
+TEST(ColumnMetaAccessorTest, V2_BasicInlineColumns) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+
+ // Add 3 columns with different types
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+ col0->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ auto* col1 = footer.add_columns();
+ col1->set_unique_id(20);
+ col1->set_column_id(1);
+ col1->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ auto* col2 = footer.add_columns();
+ col2->set_unique_id(30);
+ col2->set_column_id(2);
+ col2->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 10);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 1, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 20);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 2, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 30);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ // Test has_column_uid for existing and non-existing uids
+ EXPECT_TRUE(accessor.has_column_uid(10));
+ EXPECT_TRUE(accessor.has_column_uid(20));
+ EXPECT_TRUE(accessor.has_column_uid(30));
+ EXPECT_FALSE(accessor.has_column_uid(999));
+
+ // Test get_column_meta_by_uid
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 10, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 20, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 1);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 30, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 2);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ // Test traverse_metas
+ std::vector<int32_t> traversed_uids;
+ ASSERT_TRUE(accessor.traverse_metas(footer,
+ [&](const ColumnMetaPB& m) {
+ traversed_uids.push_back(m.unique_id());
+ })
+ .ok());
+ EXPECT_EQ(traversed_uids.size(), 3);
+ EXPECT_EQ(traversed_uids[0], 10);
+ EXPECT_EQ(traversed_uids[1], 20);
+ EXPECT_EQ(traversed_uids[2], 30);
+}
+
+// Test V2 mode with variant subcolumns (unique_id == -1 should be skipped)
+TEST(ColumnMetaAccessorTest, V2_SkipVariantSubcolumns) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+
+ // Regular column
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+
+ // Variant subcolumn (should be skipped in uid->colid map)
+ auto* col1 = footer.add_columns();
+ col1->set_unique_id(-1);
+ col1->set_column_id(1);
+
+ // Regular column
+ auto* col2 = footer.add_columns();
+ col2->set_unique_id(30);
+ col2->set_column_id(2);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Only regular columns should be accessible by uid; variant subcolumns (uid == -1) are skipped.
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 10, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 30, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 2);
+ Status st = accessor.get_column_meta_by_uid(footer, -1, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::NOT_FOUND>());
+
+ // has_column_uid should also reflect the same semantics
+ EXPECT_TRUE(accessor.has_column_uid(10));
+ EXPECT_TRUE(accessor.has_column_uid(30));
+ EXPECT_FALSE(accessor.has_column_uid(-1));
+}
+
+// Test V2 mode error cases
+TEST(ColumnMetaAccessorTest, V2_ErrorCases) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+
+ // Add one column
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id with invalid column id
+ ColumnMetaPB meta;
+ Status st = accessor.get_column_meta_by_column_ordinal_id(footer, 999, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+
+ // Test empty footer
+ SegmentFooterPB empty_footer;
+ empty_footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+ ColumnMetaAccessor empty_accessor;
+ ASSERT_TRUE(empty_accessor.init(empty_footer, nullptr).ok());
+
+ st = empty_accessor.get_column_meta_by_column_ordinal_id(empty_footer, 0, &meta);
+ EXPECT_FALSE(st.ok());
+
+ st = empty_accessor.traverse_metas(empty_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+}
+
+// Test V3 (external) mode with valid external meta region
+TEST(ColumnMetaAccessorTest, V3_ExternalMetaRegion) {
+ // 1. Create a V3 footer with external meta layout
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ // Prepare 2 columns
+ std::vector<ColumnMetaPB> columns;
+ ColumnMetaPB col0;
+ col0.set_unique_id(100);
+ col0.set_column_id(0);
+ col0.set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ col0.set_length(4);
+ columns.push_back(col0);
+
+ ColumnMetaPB col1;
+ col1.set_unique_id(200);
+ col1.set_column_id(1);
+ col1.set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+ col1.set_length(16);
+ columns.push_back(col1);
+
+ // 2. Write to file with external meta region
+ std::string file_path = make_test_file_path("v3_external_meta.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ // Write some dummy data before meta region
+ std::string dummy_data(128, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write each column meta and record entry
+ for (const auto& col : columns) {
+ std::string meta_bytes;
+ ASSERT_TRUE(col.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(col.unique_id());
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ // Write footer trailer
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ // 3. Read back and test accessor
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 100);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 1, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 200);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ // Test get_column_meta_by_uid
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(loaded_footer, 100, &meta).ok());
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(loaded_footer, 200, &meta).ok());
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ // Test traverse_metas
+ std::vector<int32_t> traversed_uids;
+ ASSERT_TRUE(accessor.traverse_metas(loaded_footer,
+ [&](const ColumnMetaPB& m) {
+ traversed_uids.push_back(m.unique_id());
+ })
+ .ok());
+ EXPECT_EQ(traversed_uids.size(), 2);
+ EXPECT_EQ(traversed_uids[0], 100);
+ EXPECT_EQ(traversed_uids[1], 200);
+}
+
+// Test V3 mode error cases
+TEST(ColumnMetaAccessorTest, V3_ErrorCases_InvalidColumnId) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ // Create minimal valid external meta
+ std::string file_path = make_test_file_path("v3_error_test.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ // Write some padding before meta region so that region_start is non-zero.
+ // ExternalColMetaUtil::parse_external_meta_pointers() treats region_start == 0
+ // as invalid layout.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ ColumnMetaPB col0;
+ col0.set_unique_id(10);
+ col0.set_column_id(0);
+ std::string meta_bytes;
+ ASSERT_TRUE(col0.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10);
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Test invalid column_id
+ ColumnMetaPB meta;
+ Status st = accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 999, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with empty external meta region
+TEST(ColumnMetaAccessorTest, V3_EmptyExternalMetaRegion) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+ footer.set_col_meta_region_start(100);
+ // No column_meta_entries added (empty)
+
+ ColumnMetaAccessor accessor;
+ // For an empty external meta region (num_columns == 0), the current
+ // implementation treats this as a corrupted layout and init() fails.
+ Status st = accessor.init(footer, nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with mismatched entries count
+TEST(ColumnMetaAccessorTest, V3_MismatchedEntriesCount) {
+ std::string file_path = make_test_file_path("v3_mismatch.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ // Write some padding before meta region so that region_start is non-zero.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write 2 column metas
+ for (int i = 0; i < 2; ++i) {
+ ColumnMetaPB col;
+ col.set_unique_id(10 + i);
+ col.set_column_id(i);
+ std::string meta_bytes;
+ ASSERT_TRUE(col.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10 + i);
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ // Initialize accessor with the original footer (entries count matches num_columns).
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Manually corrupt the footer by adding extra entry after init so that
+ // footer.column_meta_entries_size() != ptrs.num_columns, which should be
+ // detected by traverse_metas.
+ loaded_footer.add_column_meta_entries()->set_unique_id(999);
+
+ // traverse_metas should detect mismatch
+ Status st = accessor.traverse_metas(loaded_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with corrupted protobuf data
+TEST(ColumnMetaAccessorTest, V3_CorruptedProtobufData) {
+ std::string file_path = make_test_file_path("v3_corrupted.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ // Write some padding before meta region so that region_start is non-zero.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write garbage data instead of valid protobuf
+ std::string garbage(100, 'Z');
+ ASSERT_TRUE(fw->append(Slice(garbage)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10);
+ entry->set_length(100);
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // traverse_metas should fail to parse corrupted data
+ Status st = accessor.traverse_metas(loaded_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test version detection and automatic V2/V3 selection
+TEST(ColumnMetaAccessorTest, VersionDetection) {
+ // V1 footer should use V2 accessor (inline mode)
+ {
+ SegmentFooterPB footer;
+ footer.set_version(1);
+ auto* col = footer.add_columns();
+ col->set_unique_id(10);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 10);
+ }
+
+ // V2 footer should use V2 accessor
+ {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+ auto* col = footer.add_columns();
+ col->set_unique_id(20);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 20);
+ }
+
+ // V3 footer without valid external meta is treated as corrupted layout and
+ // init() should fail (no automatic fallback to V2 inline mode).
+ {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+ auto* col = footer.add_columns();
+ col->set_unique_id(30);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ Status st = accessor.init(footer, nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+ }
+}
+
+// Test large number of columns
+TEST(ColumnMetaAccessorTest, LargeNumberOfColumns) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+
+ const int num_columns = 1000;
+ for (int i = 0; i < num_columns; ++i) {
+ auto* col = footer.add_columns();
+ col->set_unique_id(i);
+ col->set_column_id(i);
+ col->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ }
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Spot check mapping via get_column_meta_by_uid
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 500, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 500);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 999, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 999);
+
+ // Test traverse_metas counts correctly
+ int count = 0;
+ ASSERT_TRUE(accessor.traverse_metas(footer, [&](const ColumnMetaPB&) { count++; }).ok());
+ EXPECT_EQ(count, num_columns);
+}
+
+// Compare footer size with and without external ColumnMetaPB region under high column count
+// using real segments written by SegmentWriter.
+// This test builds two real segments (inline V2 vs external V3 layout) on disk:
+// - schema has 10,00 INT columns
+// - each column has 1,000 rows of data
+// and verifies that the external layout significantly reduces the proto size of the footer.
+TEST(ColumnMetaAccessorTest, FooterSizeWithManyColumnsExternalVsInline) {
+ constexpr int kNumColumns = 1000;
+ constexpr int kNumRowsPerColumn = 100;
+
+ // Ensure test directory exists before building real segments.
+ {
+ auto fs = io::global_local_filesystem();
+ static_cast<void>(fs->delete_directory(kTestDir));
+ ASSERT_TRUE(fs->create_directory(kTestDir).ok());
+ }
+
+ // 1. Build common TabletSchema with many INT columns.
+ std::vector<TabletColumnPtr> columns;
+ columns.reserve(kNumColumns);
+ for (int i = 0; i < kNumColumns; ++i) {
+ auto col = std::make_shared<TabletColumn>();
+ col->set_unique_id(i);
+ col->set_name(fmt::format("c{}", i));
+ col->set_type(FieldType::OLAP_FIELD_TYPE_INT);
+ // Mark the first column as key so that SegmentWriter produces non-empty
+ // min/max encoded keys, which are asserted inside the shared build_segment helper.
+ col->set_is_key(i == 0);
+ col->set_is_nullable(true);
+ col->set_length(4);
+ // Set index_length for key columns so SegmentWriter can build key index properly.
+ col->set_index_length(4);
+ columns.emplace_back(std::move(col));
+ }
+
+ TabletSchemaSPtr inline_schema = create_schema(columns, UNIQUE_KEYS);
+ TabletSchemaSPtr external_schema = create_schema(columns, UNIQUE_KEYS);
+ // Enable external ColumnMetaPB for the second schema so that SegmentWriter
+ // produces a V3 footer with externalized column meta region.
+ external_schema->set_external_segment_meta_used_default(true);
+
+ // 2. Common SegmentWriter options and row generator.
+ SegmentWriterOptions opts;
+ opts.enable_unique_key_merge_on_write = false;
+
+ auto generator = [](size_t rid, int cid, RowCursorCell& cell) {
+ cell.set_not_null();
+ // deterministic int payload: value = rid * 10 + cid
+ *reinterpret_cast<int*>(cell.mutable_cell_ptr()) = static_cast<int>(rid * 10 + cid);
+ };
+
+ // 3. Build inline segment (V2 footer, inline ColumnMetaPB).
+ std::shared_ptr<Segment> inline_segment;
+ std::string inline_segment_path;
+ build_segment(opts, inline_schema,
+ /*segment_id=*/0, inline_schema, kNumRowsPerColumn, generator, &inline_segment,
+ std::string(kTestDir), &inline_segment_path);
+ ASSERT_NE(inline_segment, nullptr);
+
+ // 4. Build external segment (V3 footer, external ColumnMetaPB).
+ std::shared_ptr<Segment> external_segment;
+ std::string external_segment_path;
+ build_segment(opts, external_schema,
+ /*segment_id=*/1, external_schema, kNumRowsPerColumn, generator,
+ &external_segment, std::string(kTestDir), &external_segment_path);
+ ASSERT_NE(external_segment, nullptr);
+
+ // 5. Read footers back from real segment files and compare proto sizes.
+ auto fs = io::global_local_filesystem();
+ io::FileReaderSPtr fr_inline;
+ io::FileReaderSPtr fr_external;
+ io::FileReaderOptions reader_opts;
+
+ ASSERT_TRUE(fs->open_file(inline_segment_path, &fr_inline, &reader_opts).ok());
+ ASSERT_TRUE(fs->open_file(external_segment_path, &fr_external, &reader_opts).ok());
+
+ SegmentFooterPB inline_footer;
+ SegmentFooterPB external_footer;
+ ASSERT_TRUE(read_footer_from_file(fr_inline, &inline_footer).ok());
+ ASSERT_TRUE(read_footer_from_file(fr_external, &external_footer).ok());
+
+ const size_t inline_footer_size = inline_footer.ByteSizeLong();
+ const size_t external_footer_size = external_footer.ByteSizeLong();
+
+ std::cout << "Real segment footer size with " << kNumColumns
+ << " columns (inline vs external): inline=" << inline_footer_size
+ << " bytes, external=" << external_footer_size << " bytes" << std::endl;
+
+ // External layout should significantly reduce the proto size of the footer.
+ EXPECT_LT(external_footer_size, inline_footer_size / 10);
+}
+
+// Test concurrent access (thread safety not guaranteed by ColumnMetaAccessor itself,
+// but test that multiple sequential calls work correctly)
+TEST(ColumnMetaAccessorTest, MultipleSequentialAccesses) {
+ SegmentFooterPB footer;
+ footer.set_version(SEGMENT_FOOTER_VERSION_V2_BASELINE);
+
+ for (int i = 0; i < 10; ++i) {
+ auto* col = footer.add_columns();
+ col->set_unique_id(i);
+ col->set_column_id(i);
+ }
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Multiple calls should give consistent results
+ for (int round = 0; round < 100; ++round) {
+ for (int i = 0; i < 10; ++i) {
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, i, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), i);
+ // ensure uid lookup also works repeatedly
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, i, &meta).ok());
+ }
+ }
+}
diff --git a/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
index 7469ae4e635..75afa597302 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
@@ -24,12 +24,11 @@
#include <thread>
#include <vector>
-#include "agent/be_exec_version_manager.h"
#include "common/config.h"
-#include "gen_cpp/olap_file.pb.h"
#include "gen_cpp/segment_v2.pb.h"
#include "io/fs/file_reader.h"
#include "mock/mock_segment.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
@@ -65,11 +64,22 @@ protected:
// Create mock segment
_mock_segment = std::make_unique<StrictMock<MockSegment>>();
- // Create cache
- _cache = std::make_unique<ColumnReaderCache>(_mock_segment.get());
-
- // Set up basic mock expectations
+ // Set up basic mock expectations before constructing the cache, because
+ // ColumnReaderCache's ctor reads _mock_segment->num_rows().
setup_basic_mocks();
+
+ // Create cache bound to mock segment context.
+ // ColumnMetaAccessor will be initialized later in setup_segment_footer()
+ // once the test-specific footer has been constructed.
+ io::FileReaderSPtr file_reader; // nullptr is fine for inline-only tests
+ auto footer_cb = [this](std::shared_ptr<SegmentFooterPB>& footer_pb_shared,
+ OlapReaderStatistics* stats) -> Status {
+ // Delegate to MockSegment::_get_segment_footer
+ return _mock_segment->_get_segment_footer(footer_pb_shared, stats);
+ };
+ _cache = std::make_unique<ColumnReaderCache>(&_accessor, _mock_segment->tablet_schema(),
+ file_reader, _mock_segment->num_rows(),
+ footer_cb);
}
void TearDown() override {
@@ -108,9 +118,13 @@ protected:
*footer->add_columns() = col;
}
_mock_segment->set_footer(footer);
+ // Initialize ColumnMetaAccessor with the current footer; external meta is not
+ // used in these tests so we can pass a null file reader.
+ CHECK(_accessor.init(*footer, nullptr).ok());
}
std::unique_ptr<StrictMock<MockSegment>> _mock_segment;
+ ColumnMetaAccessor _accessor;
std::unique_ptr<ColumnReaderCache> _cache;
OlapReaderStatistics _stats;
};
@@ -353,17 +367,17 @@ TEST_F(ColumnReaderCacheTest, DifferentColumnTypes) {
FieldType::OLAP_FIELD_TYPE_DOUBLE, FieldType::OLAP_FIELD_TYPE_BOOL};
for (size_t i = 0; i < types.size(); ++i) {
- setup_column_uid_mapping(i + 1, 0);
+ setup_column_uid_mapping(static_cast<int32_t>(i + 1), 0);
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(types[i]));
- col_meta.set_unique_id(i + 1);
+ col_meta.set_unique_id(static_cast<int32_t>(i + 1));
col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
std::shared_ptr<ColumnReader> reader;
- Status status = _cache->get_column_reader(i + 1, &reader, &_stats);
+ Status status = _cache->get_column_reader(static_cast<int32_t>(i + 1), &reader, &_stats);
EXPECT_TRUE(status.ok());
EXPECT_NE(reader, nullptr);
}
@@ -395,20 +409,22 @@ TEST_F(ColumnReaderCacheTest, NodeHintParameter) {
vectorized::PathInData path("hint_field");
std::shared_ptr<ColumnReader> reader;
Status status = _cache->get_path_column_reader(1, path, &reader, &_stats, &node_hint);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(reader, nullptr);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_FOUND>());
+ EXPECT_EQ(reader, nullptr);
// Test cache hit
status = _cache->get_path_column_reader(1, path, &reader, &_stats, &node_hint);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(reader, nullptr);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_FOUND>());
+ EXPECT_EQ(reader, nullptr);
}
// Test cache performance under load
TEST_F(ColumnReaderCacheTest, PerformanceUnderLoad) {
const int num_columns = 100;
- // Set up many columns
+ // Set up many columns in a single footer so that ColumnMetaAccessor and
+ // ColumnReaderCache both see a consistent multi-column layout.
+ std::vector<ColumnMetaPB> all_columns;
for (int i = 1; i <= num_columns; ++i) {
setup_column_uid_mapping(i, 0);
@@ -417,8 +433,9 @@ TEST_F(ColumnReaderCacheTest, PerformanceUnderLoad) {
col_meta.set_unique_id(i);
col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
- setup_segment_footer({col_meta});
+ all_columns.push_back(col_meta);
}
+ setup_segment_footer(all_columns);
auto start_time = std::chrono::high_resolution_clock::now();
diff --git a/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp b/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp
new file mode 100644
index 00000000000..6b7a4c583c8
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp
@@ -0,0 +1,663 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "io/fs/local_file_system.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_writer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
+#include "olap/tablet_schema_helper.h"
+#include "olap/types.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+#include "vec/json/path_in_data.h"
+
+using namespace doris;
+using namespace doris::segment_v2;
+using namespace doris::vectorized;
+
+namespace {
+
+constexpr std::string_view kTestDir = "./ut_dir/external_col_meta_util_test";
+
+// Helper to create a clean local test directory and return absolute path + filename.
+std::string make_test_file_path(const std::string& file_name) {
+ auto fs = io::global_local_filesystem();
+ // english only in comments
+ static_cast<void>(fs->delete_directory(kTestDir));
+ CHECK(fs->create_directory(kTestDir).ok());
+ return std::string(kTestDir) + "/" + file_name;
+}
+
+// Helper to write a simple "segment-like" footer trailer. Current tests do not call
+// Segment::_parse_footer, so the exact magic value is not important as long as the
+// layout is self‑consistent.
+Status append_footer_trailer(io::FileWriter* fw, SegmentFooterPB* footer) {
+ std::string footer_buf;
+ if (!footer->SerializeToString(&footer_buf)) {
+ return Status::InternalError("failed to serialize SegmentFooterPB for test");
+ }
+ faststring fixed_buf;
+ // footer size (4 bytes)
+ put_fixed32_le(&fixed_buf, static_cast<uint32_t>(footer_buf.size()));
+ // footer checksum (4 bytes)
+ uint32_t checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ put_fixed32_le(&fixed_buf, checksum);
+ // magic number (4 bytes)
+ fixed_buf.append("D0R1", 4);
+ std::vector<Slice> slices {Slice(footer_buf), Slice(fixed_buf)};
+ RETURN_IF_ERROR(fw->appendv(slices.data(), slices.size()));
+ return Status::OK();
+}
+
+// Helper to read SegmentFooterPB back from a segment-like file and validate checksum.
+Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* footer) {
+ const auto file_size = fr->size();
+ if (file_size < 12) {
+ return Status::Corruption("test segment file too small: {}", file_size);
+ }
+ uint8_t fixed_buf[12];
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12, Slice(fixed_buf, sizeof(fixed_buf)), &bytes_read));
+ if (bytes_read != sizeof(fixed_buf)) {
+ return Status::Corruption("short read footer trailer, read={}, expect={}", bytes_read,
+ sizeof(fixed_buf));
+ }
+ const uint32_t footer_length = decode_fixed32_le(fixed_buf);
+ if (file_size < 12 + footer_length) {
+ return Status::Corruption("bad footer length in test segment: file_size={}, footer_len={}",
+ file_size, footer_length);
+ }
+ std::string footer_buf;
+ footer_buf.resize(footer_length);
+ bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12 - footer_length,
+ Slice(footer_buf.data(), footer_buf.size()), &bytes_read));
+ if (bytes_read != footer_length) {
+ return Status::Corruption("short read footer pb, read={}, expect={}", bytes_read,
+ footer_length);
+ }
+ const uint32_t expect_checksum = decode_fixed32_le(fixed_buf + 4);
+ const uint32_t actual_checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ if (actual_checksum != expect_checksum) {
+ return Status::Corruption("footer checksum mismatch, actual={}, expect={}", actual_checksum,
+ expect_checksum);
+ }
+ if (!footer->ParseFromArray(footer_buf.data(), static_cast<int>(footer_buf.size()))) {
+ return Status::Corruption("failed to parse SegmentFooterPB from test segment");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+namespace doris {
+
+using Generator = std::function<void(size_t rid, int cid, RowCursorCell& cell)>;
+
+// Helper declarations are defined in tablet_schema_helper.{h,cpp} and
+// delete_bitmap_calculator_test.cpp.
+TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
+ KeysType keys_type = UNIQUE_KEYS);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir, std::string* out_path);
+
+} // namespace doris
+
+// Basic unit test for ExternalColMetaUtil pointer parsing and uid‑>col_id map.
+TEST(ExternalColMetaUtilTest, PointersAndUidMapValidation) {
+ SegmentFooterPB footer;
+ footer.set_col_meta_region_start(100U);
+ auto* e0 = footer.add_column_meta_entries();
+ e0->set_unique_id(10);
+ e0->set_length(50);
+ auto* e1 = footer.add_column_meta_entries();
+ e1->set_unique_id(20);
+ e1->set_length(60);
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ EXPECT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ EXPECT_EQ(ptrs.region_start, 100U);
+ EXPECT_EQ(ptrs.num_columns, 2U);
+ EXPECT_EQ(ptrs.region_end, 210U);
+
+ std::unordered_map<int32_t, size_t> uid2colid;
+ EXPECT_TRUE(ExternalColMetaUtil::parse_uid_to_colid_map(footer, ptrs, &uid2colid).ok());
+ EXPECT_EQ(uid2colid.size(), 2U);
+ EXPECT_EQ(uid2colid[10], 0U);
+ EXPECT_EQ(uid2colid[20], 1U);
+
+ // Negative cases for pointers
+ footer.clear_column_meta_entries();
+ EXPECT_FALSE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ footer.add_column_meta_entries()->set_unique_id(10);
+ footer.add_column_meta_entries()->set_unique_id(20);
+ footer.mutable_column_meta_entries(0)->set_length(50);
+ footer.mutable_column_meta_entries(1)->set_length(60);
+ // zero length is illegal
+ footer.mutable_column_meta_entries(1)->set_length(0);
+ EXPECT_FALSE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ footer.mutable_column_meta_entries(1)->set_length(60);
+
+ // Slice validation
+ ExternalColMetaUtil::ExternalMetaPointers p;
+ p.region_start = 100;
+ p.region_end = 200;
+ EXPECT_TRUE(ExternalColMetaUtil::is_valid_meta_slice(100, 10, p));
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(100, 0, p)); // size == 0
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(50, 10, p)); // pos < region_start
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(195, 10, p)); // pos + size > region_end
+ // overflow guard: deliberately craft overflow
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(std::numeric_limits<uint64_t>::max() - 1,
+ 4, p));
+}
+
+// End‑to‑end test of VariantExtMetaWriter + VariantExternalMetaReader on a standalone file.
+// This does NOT involve full tablet/rowset plumbing, but exercises:
+// - externalize_from_footer(): classification of root/non‑sparse/sparse columns
+// - flush_to_footer(): writing IndexedColumn metas into footer.file_meta_datas
+// - init_from_footer()/lookup_meta_by_path(): reading back subcolumn ColumnMetaPB by path
+// - legacy unsuffixed keys: "variant_meta_keys" / "variant_meta_values"
+TEST(ExternalColMetaUtilTest, VariantExtMetaWriterAndReaderInteropAndCompatibility) {
+ // 1. Prepare a simple footer with:
+ // - one non‑variant column (kept as is)
+ // - one variant root (kept, path "v1")
+ // - one non‑sparse subcolumn "v1.key0" (externalized)
+ // - one sparse subcolumn "__DORIS_VARIANT_SPARSE__.key_sparse" (embedded into root)
+ SegmentFooterPB footer;
+ // non‑variant top level column
+ {
+ auto* col = footer.add_columns();
+ col->set_unique_id(0);
+ // use int(FieldType) to match production code
+ col->set_type(int(FieldType::OLAP_FIELD_TYPE_INT)); // simple scalar
+ }
+
+ const int32_t root_uid = 1;
+ // variant root column with path "v1"
+ ColumnMetaPB* root = footer.add_columns();
+ root->set_unique_id(root_uid);
+ root->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+ {
+ PathInData full_root_path("v1");
+ full_root_path.to_protobuf(root->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ }
+
+ // non‑sparse subcolumn: v1.key0
+ ColumnMetaPB* non_sparse = footer.add_columns();
+ non_sparse->set_unique_id(2);
+ non_sparse->set_type(int(FieldType::OLAP_FIELD_TYPE_INT));
+ {
+ PathInData full_path("v1.key0");
+ full_path.to_protobuf(non_sparse->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ non_sparse->set_none_null_size(123);
+ }
+
+ // sparse subcolumn: v1.__DORIS_VARIANT_SPARSE__
+ ColumnMetaPB* sparse = footer.add_columns();
+ sparse->set_unique_id(3);
+ sparse->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+ {
+ // For sparse column, the relative path (after pop_front) should contain
+ // "__DORIS_VARIANT_SPARSE__" so that VariantExtMetaWriter treats it as sparse.
+ PathInData full_path("v1.__DORIS_VARIANT_SPARSE__");
+ full_path.to_protobuf(sparse->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ // carry sparse statistics in VariantStatisticsPB
+ auto* stats = sparse->mutable_variant_statistics();
+ (*stats->mutable_sparse_column_non_null_size())["key_sparse"] = 10;
+ }
+
+ // 2. Externalize non‑sparse subcolumns into a dedicated file via VariantExtMetaWriter.
+ std::string file_path = make_test_file_path("variant_ext_meta.bin");
+ io::FileWriterPtr fw;
+ auto fs = io::global_local_filesystem();
+ Status st = fs->create_file(file_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ std::vector<ColumnMetaPB> out_metas;
+ {
+ VariantExtMetaWriter ext_writer(fw.get(), CompressionTypePB::LZ4);
+ st = ext_writer.externalize_from_footer(&footer, &out_metas);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ }
+
+ // Capture region start AFTER Variant Index is written
+ uint64_t meta_region_start = fw->bytes_appended();
+
+ // Manually write the Meta Region (simulating ExternalColMetaUtil::write_external_column_meta)
+ footer.clear_column_meta_entries();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ for (const auto& meta : out_metas) {
+ std::string meta_bytes;
+ meta.SerializeToString(&meta_bytes);
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(meta.unique_id());
+ entry->set_length(meta_bytes.size());
+ }
+
+ // After externalize_from_footer:
+ // - footer.columns() should contain:
+ // * non‑variant column uid=0
+ // * root variant column uid=1, whose children_columns includes the sparse meta
+ // - "v1.key0" should be removed from footer.columns() and only exist in external meta.
+ ASSERT_EQ(footer.columns_size(), 2);
+ const ColumnMetaPB& kept0 = footer.columns(0);
+ const ColumnMetaPB& kept1 = footer.columns(1);
+ EXPECT_EQ(kept0.unique_id(), 0);
+ EXPECT_EQ(kept1.unique_id(), root_uid);
+ // root should have embedded sparse children
+ ASSERT_EQ(kept1.children_columns_size(), 1);
+ const ColumnMetaPB& embedded_sparse = kept1.children_columns(0);
+ EXPECT_TRUE(embedded_sparse.has_column_path_info());
+ PathInData embedded_path;
+ embedded_path.from_protobuf(embedded_sparse.column_path_info());
+ // full path should keep root prefix
+ EXPECT_EQ(embedded_path.get_path(), "v1.__DORIS_VARIANT_SPARSE__");
+ ASSERT_TRUE(embedded_sparse.has_variant_statistics());
+ EXPECT_EQ(embedded_sparse.variant_statistics().sparse_column_non_null_size().at("key_sparse"),
+ 10);
+
+ // 3. Verify that ext meta keys metas were flushed into footer.file_meta_datas.
+ bool found_keys_meta = false;
+ std::string keys_meta_bytes;
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == "variant_meta_keys." + std::to_string(root_uid)) {
+ found_keys_meta = true;
+ keys_meta_bytes = m.value();
+ }
+ }
+ ASSERT_TRUE(found_keys_meta);
+ // variant_meta_values should NOT be present
+
+ // 4. Finish the "segment‑like" file by appending footer + trailer, then close.
+ st = append_footer_trailer(fw.get(), &footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(fw->close().ok());
+
+ // 5. Open a reader on the same file and build VariantExternalMetaReader from the footer.
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ st = fs->open_file(file_path, &fr, &opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ VariantExternalMetaReader reader_new;
+ auto footer_sp = std::make_shared<SegmentFooterPB>();
+ footer_sp->CopyFrom(footer);
+ st = reader_new.init_from_footer(footer_sp, fr, root_uid);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(reader_new.available());
+
+ // 6. Lookup meta by relative path and verify content.
+ ColumnMetaPB loaded_meta;
+ st = reader_new.lookup_meta_by_path("key0", &loaded_meta);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(loaded_meta.has_column_path_info());
+ PathInData loaded_path;
+ loaded_path.from_protobuf(loaded_meta.column_path_info());
+ // full path should still contain root prefix "v1.key0"
+ EXPECT_EQ(loaded_path.get_path(), "v1.key0");
+ EXPECT_EQ(loaded_meta.none_null_size(), 123);
+ EXPECT_EQ(loaded_meta.column_path_info().parrent_column_unique_id(), root_uid);
+
+ // 7. Legacy compatibility: construct a footer with unsuffixed keys:
+ // "variant_meta_keys"
+ SegmentFooterPB legacy_footer;
+ legacy_footer.CopyFrom(footer);
+ legacy_footer.clear_file_meta_datas();
+ {
+ auto* p1 = legacy_footer.add_file_meta_datas();
+ p1->set_key("variant_meta_keys");
+ p1->set_value(keys_meta_bytes);
+ }
+
+ VariantExternalMetaReader reader_legacy;
+ auto legacy_footer_sp = std::make_shared<SegmentFooterPB>();
+ legacy_footer_sp->CopyFrom(legacy_footer);
+ st = reader_legacy.init_from_footer(legacy_footer_sp, fr, root_uid);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(reader_legacy.available());
+
+ ColumnMetaPB loaded_meta_legacy;
+ st = reader_legacy.lookup_meta_by_path("key0", &loaded_meta_legacy);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ PathInData loaded_path_legacy;
+ loaded_path_legacy.from_protobuf(loaded_meta_legacy.column_path_info());
+ EXPECT_EQ(loaded_path_legacy.get_path(), "v1.key0");
+ EXPECT_EQ(loaded_meta_legacy.none_null_size(), 123);
+ EXPECT_EQ(loaded_meta_legacy.column_path_info().parrent_column_unique_id(), root_uid);
+
+ // 8. Bulk load all external metas once and verify subcolumn tree and statistics.
+ SubcolumnColumnMetaInfo meta_tree;
+ VariantStatistics stats;
+ st = reader_new.load_all_once(&meta_tree, &stats);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_FALSE(meta_tree.empty());
+ // Non-sparse subcolumn v1.key0 should be recorded as relative path "key0".
+ ASSERT_EQ(stats.subcolumns_non_null_size.size(), 1);
+ EXPECT_EQ(stats.subcolumns_non_null_size.at("key0"), 123);
+
+ // load_all_once should be idempotent.
+ st = reader_new.load_all_once(&meta_tree, &stats);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ // 9. has_prefix should detect existing keys and non-existing prefixes.
+ bool has = false;
+ st = reader_new.has_prefix("key", &has);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_TRUE(has);
+ has = false;
+ st = reader_new.has_prefix("non_existent_prefix", &has);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_FALSE(has);
+
+ has = false;
+ st = reader_new.has_prefix("", &has);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_TRUE(has);
+}
+
+// End-to-end test: use ExternalColMetaUtil to write external ColumnMeta region + CMO
+// into a V3 segment-like file, then read footer back and verify:
+// - footer.version() is V3
+// - external meta pointers / uid->col_id map are present and valid
+// - ColumnMetaPB can be read back through CMO by col_id
+TEST(ExternalColMetaUtilTest, WriteAndReadV3SegmentExternalMetaRoundTrip) {
+ // 1. Build a simple footer with two top-level columns so uid->col_id map is deterministic.
+ SegmentFooterPB footer;
+ {
+ auto* col0 = footer.add_columns();
+ col0->set_column_id(0);
+ col0->set_unique_id(10);
+ col0->set_type(int(FieldType::OLAP_FIELD_TYPE_INT));
+ col0->set_length(4);
+ col0->set_encoding(DEFAULT_ENCODING);
+ col0->set_compression(CompressionTypePB::LZ4);
+ col0->set_is_nullable(false);
+
+ auto* col1 = footer.add_columns();
+ col1->set_column_id(1);
+ col1->set_unique_id(20);
+ col1->set_type(int(FieldType::OLAP_FIELD_TYPE_BIGINT));
+ col1->set_length(8);
+ col1->set_encoding(DEFAULT_ENCODING);
+ col1->set_compression(CompressionTypePB::LZ4);
+ col1->set_is_nullable(true);
+ }
+
+ // Stamp V3 version like SegmentWriter::_write_footer does for external meta enabled tablets.
+ footer.set_version(segment_v2::SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ std::string file_path = make_test_file_path("v3_ext_col_meta_segment.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ Status st = fs->create_file(file_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ // Simulate real segment layout: write some dummy data pages before external meta region
+ // so that `col_meta_region_start` is non-zero (as in production segments).
+ {
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ }
+
+ // 2. Write external ColumnMeta region + CMO + uid->col_id mapping.
+ st = ExternalColMetaUtil::write_external_column_meta(
+ fw.get(), &footer, CompressionTypePB::LZ4,
+ [fw_raw = fw.get()](const std::vector<Slice>& slices) {
+ return fw_raw->appendv(slices.data(), slices.size());
+ });
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ // After write_external_column_meta, inline footer.columns() are cleared in V3 mode.
+ ASSERT_EQ(footer.columns_size(), 0);
+
+ // 3. Finish by appending footer + trailer and close writer.
+ st = append_footer_trailer(fw.get(), &footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(fw->close().ok());
+
+ // 4. Re-open file, parse footer back from on-disk trailer, and verify footer.version() is V3.
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ st = fs->open_file(file_path, &fr, &opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ SegmentFooterPB loaded_footer;
+ st = read_footer_from_file(fr, &loaded_footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(loaded_footer.version(), segment_v2::SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ // 5. Parse external meta pointers and uid->col_id mapping from the loaded footer.
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(loaded_footer, &ptrs).ok());
+ std::unordered_map<int32_t, size_t> uid2colid;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_uid_to_colid_map(loaded_footer, ptrs, &uid2colid).ok());
+ ASSERT_EQ(uid2colid.size(), 2U);
+ EXPECT_EQ(uid2colid[10], 0U);
+ EXPECT_EQ(uid2colid[20], 1U);
+
+ // 6. Read ColumnMetaPB back from external meta region and verify it matches the
+ // original unique_ids.
+ ColumnMetaPB meta0;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, /*col_id=*/0, &meta0);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta0.unique_id(), 10);
+ EXPECT_EQ(meta0.column_id(), 0);
+
+ ColumnMetaPB meta1;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, /*col_id=*/1, &meta1);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta1.unique_id(), 20);
+ EXPECT_EQ(meta1.column_id(), 1);
+}
+
+// Build a real Segment file with data rows, then verify that Segment::num_rows()
+// matches the written row count and that footer metadata is consistent with schema.
+TEST(ExternalColMetaUtilTest, BuildSegmentAndVerifyDataAndFooterMeta) {
+ // 1. Build a simple UNIQUE_KEYS tablet schema: one key column, one value column.
+ std::vector<TabletColumnPtr> columns;
+ columns.emplace_back(create_int_key(/*id=*/0));
+ columns.emplace_back(create_int_value(/*id=*/1));
+ TabletSchemaSPtr tablet_schema = create_schema(columns, UNIQUE_KEYS);
+ // Enable external ColumnMetaPB so that SegmentWriter produces a V3 footer with
+ // externalized column meta region + column_meta_entries.
+ tablet_schema->set_external_segment_meta_used_default(true);
+
+ // 2. Use existing build_segment helper (SegmentWriter-based) to write a segment file.
+ SegmentWriterOptions opts;
+ opts.enable_unique_key_merge_on_write = true;
+ const size_t nrows = 16;
+
+ auto generator = [](size_t rid, int cid, RowCursorCell& cell) {
+ cell.set_not_null();
+ // deterministic int payload: value = rid * 10 + cid
+ *reinterpret_cast<int*>(cell.mutable_cell_ptr()) = static_cast<int>(rid * 10 + cid);
+ };
+
+ std::shared_ptr<Segment> segment;
+ std::string segment_path;
+ build_segment(opts, tablet_schema,
+ /*segment_id=*/0, tablet_schema, nrows, generator, &segment,
+ std::string(kTestDir), &segment_path);
+
+ ASSERT_NE(segment, nullptr);
+
+ // 3. Verify basic segment data property (row count).
+ EXPECT_EQ(segment->num_rows(), nrows);
+
+ // 4. Verify external ColumnMetaPB region is actually written on disk and consistent with schema.
+ auto fs = io::global_local_filesystem();
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions reader_opts;
+ Status st = fs->open_file(segment_path, &fr, &reader_opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ SegmentFooterPB loaded_footer;
+ st = read_footer_from_file(fr, &loaded_footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(loaded_footer.version(), segment_v2::SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(loaded_footer, &ptrs).ok());
+ ASSERT_EQ(ptrs.num_columns, tablet_schema->num_columns());
+
+ // Read back per-column meta from external region and check uid/col_id mapping.
+ for (uint32_t cid = 0; cid < ptrs.num_columns; ++cid) {
+ ColumnMetaPB meta;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, cid, &meta);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta.unique_id(), tablet_schema->column(cid).unique_id());
+ EXPECT_EQ(meta.column_id(), cid);
+ }
+}
+
+// Validate argument checks in read_col_meta that do not trigger any IO.
+TEST(ExternalColMetaUtilTest, ReadColMetaArgumentValidation) {
+ SegmentFooterPB footer;
+ // Prepare one entry.
+ auto* e = footer.add_column_meta_entries();
+ e->set_unique_id(1);
+ e->set_length(16);
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ColumnMetaPB meta;
+ io::FileReaderSPtr dummy_reader; // not used on these failing paths
+
+ // 1) col_id out of range.
+ ptrs.num_columns = 1;
+ ptrs.region_start = 100;
+ ptrs.region_end = 200;
+ Status st = ExternalColMetaUtil::read_col_meta(dummy_reader, footer, ptrs,
+ /*col_id=*/1, &meta);
+ EXPECT_FALSE(st.ok());
+
+ // 2) footer.column_meta_entries_size() != ptrs.num_columns.
+ ptrs.num_columns = 2;
+ st = ExternalColMetaUtil::read_col_meta(dummy_reader, footer, ptrs,
+ /*col_id=*/0, &meta);
+ EXPECT_FALSE(st.ok());
+
+ // 3) Invalid meta slice (pos + size > region_end).
+ ptrs.num_columns = 1;
+ ptrs.region_start = 100;
+ ptrs.region_end = 105;
+ footer.mutable_column_meta_entries(0)->set_length(32);
+ st = ExternalColMetaUtil::read_col_meta(dummy_reader, footer, ptrs,
+ /*col_id=*/0, &meta);
+ EXPECT_FALSE(st.ok());
+}
+
+// Validate IO and parse error handling paths in read_col_meta.
+TEST(ExternalColMetaUtilTest, ReadColMetaIOAndParseErrors) {
+ auto fs = io::global_local_filesystem();
+
+ // 1) Short read: file smaller than declared ColumnMetaPB length.
+ std::string short_path = make_test_file_path("read_col_meta_short.bin");
+ io::FileWriterPtr fw;
+ Status st = fs->create_file(short_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ std::string small_payload = "abcd";
+ ASSERT_TRUE(fw->append(Slice(small_payload)).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr short_fr;
+ io::FileReaderOptions ropts;
+ st = fs->open_file(short_path, &short_fr, &ropts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ SegmentFooterPB footer;
+ auto* e = footer.add_column_meta_entries();
+ e->set_unique_id(1);
+ e->set_length(static_cast<uint32_t>(small_payload.size() * 2)); // request more than exists
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ptrs.region_start = 0;
+ ptrs.region_end = ptrs.region_start + e->length();
+ ptrs.num_columns = 1;
+
+ ColumnMetaPB meta;
+ st = ExternalColMetaUtil::read_col_meta(short_fr, footer, ptrs, /*col_id=*/0, &meta);
+ EXPECT_FALSE(st.ok()); // short read should fail
+
+ // 2) Parse error: enough bytes but not a valid ColumnMetaPB.
+ std::string parse_path = make_test_file_path("read_col_meta_parse.bin");
+ st = fs->create_file(parse_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ std::string bogus(32, '\xff');
+ ASSERT_TRUE(fw->append(Slice(bogus)).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr parse_fr;
+ st = fs->open_file(parse_path, &parse_fr, &ropts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ footer.clear_column_meta_entries();
+ e = footer.add_column_meta_entries();
+ e->set_unique_id(1);
+ e->set_length(static_cast<uint32_t>(bogus.size()));
+
+ ptrs.region_start = 0;
+ ptrs.region_end = ptrs.region_start + e->length();
+ ptrs.num_columns = 1;
+
+ st = ExternalColMetaUtil::read_col_meta(parse_fr, footer, ptrs, /*col_id=*/0, &meta);
+ EXPECT_FALSE(st.ok()); // ParseFromArray should fail on bogus data
+}
+
+// Ensure write_external_column_meta is a no-op when there are no columns.
+TEST(ExternalColMetaUtilTest, WriteExternalColumnMetaEmptyColumns) {
+ SegmentFooterPB footer;
+ ASSERT_EQ(footer.columns_size(), 0);
+
+ std::string file_path = make_test_file_path("empty_footer_ext_meta.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ Status st = fs->create_file(file_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ st = ExternalColMetaUtil::write_external_column_meta(
+ fw.get(), &footer, CompressionTypePB::LZ4,
+ [fw_raw = fw.get()](const std::vector<Slice>& slices) {
+ return fw_raw->appendv(slices.data(), slices.size());
+ });
+ EXPECT_TRUE(st.ok()) << st.to_json();
+ // No column meta entries or region start should be written for empty footer.
+ EXPECT_EQ(footer.columns_size(), 0);
+ EXPECT_EQ(footer.column_meta_entries_size(), 0);
+ EXPECT_EQ(footer.col_meta_region_start(), 0);
+ EXPECT_TRUE(fw->close().ok());
+}
diff --git a/be/test/olap/rowset/segment_v2/mock/mock_segment.h b/be/test/olap/rowset/segment_v2/mock/mock_segment.h
index f4421c37f7d..99bc363e010 100644
--- a/be/test/olap/rowset/segment_v2/mock/mock_segment.h
+++ b/be/test/olap/rowset/segment_v2/mock/mock_segment.h
@@ -52,16 +52,10 @@ public:
_tablet_schema->_cols.push_back(std::make_shared<TabletColumn>());
_tablet_schema->_cols.back()->set_unique_id(col_uid);
_tablet_schema->_field_uniqueid_to_index[col_uid] = footer_ordinal;
- _column_uid_to_footer_ordinal[col_uid] = footer_ordinal;
}
void set_footer(std::shared_ptr<SegmentFooterPB> footer) { _footer = footer; }
- // Access to internal data for testing
- const std::unordered_map<int32_t, size_t>& get_column_uid_mapping() const {
- return _column_uid_to_footer_ordinal;
- }
-
std::shared_ptr<SegmentFooterPB> get_footer() const { return _footer; }
std::shared_ptr<SegmentFooterPB> _footer;
diff --git a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
index 5907e2d6a31..fc5c3a913cc 100644
--- a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
@@ -15,7 +15,11 @@
// specific language governing permissions and limitations
// under the License.
+#include <atomic>
+#include <thread>
+
#include "gtest/gtest.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
#include "olap/rowset/segment_v2/variant/hierarchical_data_iterator.h"
@@ -64,13 +68,16 @@ class MockColumnReaderCache : public segment_v2::ColumnReaderCache {
public:
MockColumnReaderCache(const SegmentFooterPB& footer, const io::FileReaderSPtr& file_reader,
const std::shared_ptr<TabletSchema>& tablet_schema)
- : ColumnReaderCache(nullptr),
+ : ColumnReaderCache(nullptr, nullptr, nullptr, 0,
+ [](std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*) {
+ return Status::OK();
+ }),
_footer(footer),
_file_reader(file_reader),
_tablet_schema(tablet_schema) {}
Status get_path_column_reader(
- uint32_t col_uid, vectorized::PathInData relative_path,
+ int32_t col_uid, vectorized::PathInData relative_path,
std::shared_ptr<segment_v2::ColumnReader>* column_reader, OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint = nullptr) override {
DCHECK(node_hint != nullptr);
@@ -98,6 +105,30 @@ private:
const std::shared_ptr<TabletSchema>& _tablet_schema;
};
+// Helper to create a root VariantColumnReader using ColumnMetaAccessor, which
+// hides inline vs external column meta layout (V2 vs V3).
+static Status create_variant_root_reader(const SegmentFooterPB& footer,
+ const io::FileReaderSPtr& file_reader,
+ const TabletSchemaSPtr& tablet_schema,
+ std::shared_ptr<segment_v2::ColumnReader>* out) {
+ segment_v2::ColumnMetaAccessor accessor;
+ RETURN_IF_ERROR(accessor.init(footer, file_reader));
+
+ segment_v2::ColumnReaderOptions opts;
+ opts.kept_in_memory = false;
+ opts.be_exec_version = BeExecVersionManager::get_newest_version();
+ opts.tablet_schema = tablet_schema;
+
+ auto variant_reader = std::make_shared<segment_v2::VariantColumnReader>();
+ int32_t root_uid = tablet_schema->column(0).unique_id();
+ auto footer_sp = std::make_shared<SegmentFooterPB>();
+ footer_sp->CopyFrom(footer);
+ RETURN_IF_ERROR(variant_reader->init(opts, &accessor, footer_sp, root_uid, footer.num_rows(),
+ file_reader));
+ *out = std::move(variant_reader);
+ return Status::OK();
+}
+
class VariantColumnWriterReaderTest : public testing::Test {
public:
void SetUp() override {
@@ -210,6 +241,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -283,10 +318,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ // create root variant reader using ColumnMetaAccessor (supports inline/external meta)
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
@@ -348,7 +382,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
{
auto iter = assert_cast<HierarchicalDataIterator*>(it.get());
std::shared_ptr<ColumnReader> column_reader1;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader1);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader1);
EXPECT_TRUE(st.ok()) << st.msg();
std::cout << "hier:" << iter->get_current_ordinal() << std::endl;
// now we can find exist
@@ -757,6 +791,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -829,10 +867,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -953,6 +989,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1025,6 +1065,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1125,10 +1169,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -1176,6 +1218,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize)
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1267,6 +1313,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1358,6 +1408,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1451,6 +1505,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1544,6 +1602,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1636,6 +1698,10 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1717,10 +1783,8 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions reader_opts;
- reader_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> reader;
- st = ColumnReader::create(reader_opts, footer, 0, 1000, file_reader, &reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(reader.get());
EXPECT_TRUE(variant_column_reader != nullptr);
@@ -1767,6 +1831,10 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1860,10 +1928,8 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions reader_opts;
- reader_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> reader;
- st = ColumnReader::create(reader_opts, footer, 0, 1000, file_reader, &reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(reader.get());
EXPECT_TRUE(variant_column_reader != nullptr);
@@ -1912,6 +1978,10 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
@@ -2017,11 +2087,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2066,10 +2133,11 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
ColumnIteratorUPtr nested_column_iter;
MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
+ DataTypePtr nested_storage_type;
- st = variant_column_reader->_new_iterator_with_flat_leaves(&nested_column_iter, target_column,
- &storageReadOptions, false, false,
- &column_reader_cache);
+ st = variant_column_reader->_new_iterator_with_flat_leaves(
+ &nested_column_iter, &nested_storage_type, target_column, &storageReadOptions, false,
+ false, &column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
// check iter for read_by_rowids, next_batch
auto nested_iter = assert_cast<DefaultNestedColumnIterator*>(nested_column_iter.get());
@@ -2097,8 +2165,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
{
ColumnIteratorUPtr nested_column_iter11;
st = variant_column_reader->_new_iterator_with_flat_leaves(
- &nested_column_iter11, target_column, &storageReadOptions, false, false,
- &column_reader_cache);
+ &nested_column_iter11, &nested_storage_type, target_column, &storageReadOptions,
+ false, false, &column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
st = nested_column_iter11->init(nested_column_iter_opts);
EXPECT_TRUE(st.ok()) << st.msg();
@@ -2141,9 +2209,9 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
<< target_column._column_path->has_nested_part();
ColumnIteratorUPtr nested_column_iter1;
- st = variant_column_reader->_new_iterator_with_flat_leaves(&nested_column_iter1, target_column,
- &storageReadOptions, false, false,
- &column_reader_cache);
+ st = variant_column_reader->_new_iterator_with_flat_leaves(
+ &nested_column_iter1, &nested_storage_type, target_column, &storageReadOptions, false,
+ false, &column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
// check iter for read_by_rowids, next_batch
// dst is array<nullable(string)>
@@ -2161,8 +2229,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
// make read by nested_iter1 directly
ColumnIteratorUPtr nested_column_iter11;
st = variant_column_reader->_new_iterator_with_flat_leaves(
- &nested_column_iter11, target_column, &storageReadOptions, false, false,
- &column_reader_cache);
+ &nested_column_iter11, &nested_storage_type, target_column, &storageReadOptions,
+ false, false, &column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
st = nested_column_iter11->init(nested_column_iter_opts);
EXPECT_TRUE(st.ok()) << st.msg();
@@ -2191,13 +2259,9 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2348,11 +2412,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter_nullable) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2434,6 +2495,10 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -2506,10 +2571,8 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
@@ -2573,4 +2636,121 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
}
}
-} // namespace doris
\ No newline at end of file
+// Concurrently trigger external meta loading and subcolumn meta access to guard against
+// data races between `load_external_meta_once` writer and readers like
+// `get_subcolumn_meta_by_path` / `get_metadata_size`. This roughly simulates the
+// production crash stack where one thread was loading external meta while another
+// thread was reading from `_subcolumns_meta_info`.
+TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get_subcolumn_meta) {
+ // 1. create tablet_schema
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "V1");
+ _tablet_schema = std::make_shared<TabletSchema>();
+ _tablet_schema->init_from_pb(schema_pb);
+
+ // 2. create tablet with external segment meta explicitly enabled so that
+ // VariantColumnReader builds a VariantExternalMetaReader.
+ TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = true;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_meta->_tablet_id = 20000;
+ _tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
+
+ EXPECT_TRUE(_tablet->init().ok());
+ EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+
+ // 3. create file_writer
+ io::FileWriterPtr file_writer;
+ auto file_path = local_segment_path(_tablet->tablet_path(), "0", 0);
+ auto st = io::global_local_filesystem()->create_file(file_path, &file_writer);
+ EXPECT_TRUE(st.ok()) << st.msg();
+
+ // 4. create column_writer
+ SegmentFooterPB footer;
+ ColumnWriterOptions opts;
+ opts.meta = footer.add_columns();
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.file_writer = file_writer.get();
+ opts.footer = &footer;
+ RowsetWriterContext rowset_ctx;
+ rowset_ctx.write_type = DataWriteType::TYPE_DIRECT;
+ opts.rowset_ctx = &rowset_ctx;
+ opts.rowset_ctx->tablet_schema = _tablet_schema;
+ TabletColumn column = _tablet_schema->column(0);
+ _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+
+ std::unique_ptr<ColumnWriter> writer;
+ EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
+ EXPECT_TRUE(writer->init().ok());
+ EXPECT_TRUE(assert_cast<VariantColumnWriter*>(writer.get()) != nullptr);
+
+ // 5. write a small amount of data to build some subcolumns
+ auto olap_data_convertor = std::make_unique<vectorized::OlapBlockDataConvertor>();
+ auto block = _tablet_schema->create_block();
+ auto column_object = (*std::move(block.get_by_position(0).column)).mutate();
+ std::unordered_map<int, std::string> inserted_jsonstr;
+ auto path_with_size =
+ VariantUtil::fill_object_column_with_test_data(column_object, 200, &inserted_jsonstr);
+ olap_data_convertor->add_column_data_convertor(column);
+ olap_data_convertor->set_source_content(&block, 0, 200);
+ auto [result, accessor] = olap_data_convertor->convert_column_data(0);
+ EXPECT_TRUE(result.ok());
+ EXPECT_TRUE(accessor != nullptr);
+ EXPECT_TRUE(writer->append(accessor->get_nullmap(), accessor->get_data(), 200).ok());
+ st = writer->finish();
+ EXPECT_TRUE(st.ok()) << st.msg();
+ st = writer->write_data();
+ EXPECT_TRUE(st.ok()) << st.msg();
+ st = writer->write_ordinal_index();
+ EXPECT_TRUE(st.ok()) << st.msg();
+ st = writer->write_zone_map();
+ EXPECT_TRUE(st.ok()) << st.msg();
+ EXPECT_TRUE(file_writer->close().ok());
+ footer.set_num_rows(200);
+
+ // 6. open a VariantColumnReader on this segment
+ io::FileReaderSPtr file_reader;
+ st = io::global_local_filesystem()->open_file(file_path, &file_reader);
+ EXPECT_TRUE(st.ok()) << st.msg();
+ std::shared_ptr<ColumnReader> column_reader;
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
+ EXPECT_TRUE(st.ok()) << st.msg();
+
+ auto* variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
+ EXPECT_TRUE(variant_column_reader != nullptr);
+
+ // 7. run load_external_meta_once and subcolumn meta access concurrently.
+ const int rounds = 200;
+ std::atomic<bool> failed {false};
+ Status writer_status = Status::OK();
+
+ std::thread writer_thread([&] {
+ for (int i = 0; i < rounds && !failed.load(); ++i) {
+ Status s = variant_column_reader->load_external_meta_once();
+ if (!s.ok()) {
+ writer_status = s;
+ failed.store(true);
+ break;
+ }
+ }
+ });
+
+ std::thread reader_thread([&] {
+ for (int i = 0; i < rounds && !failed.load(); ++i) {
+ // Access subcolumn meta and metadata size repeatedly.
+ auto* node = variant_column_reader->get_subcolumn_meta_by_path(PathInData("key0"));
+ (void)node;
+ auto meta_size = variant_column_reader->get_metadata_size();
+ (void)meta_size;
+ }
+ });
+
+ writer_thread.join();
+ reader_thread.join();
+
+ EXPECT_TRUE(writer_status.ok());
+}
+
+} // namespace doris
diff --git a/be/test/vec/common/schema_util_rowset_test.cpp b/be/test/vec/common/schema_util_rowset_test.cpp
index f453aeff8d9..925bb093813 100644
--- a/be/test/vec/common/schema_util_rowset_test.cpp
+++ b/be/test/vec/common/schema_util_rowset_test.cpp
@@ -177,7 +177,8 @@ static void fill_block_with_test_data(vectorized::Block* block, int size) {
static int64_t inc_id = 1000;
static RowsetWriterContext rowset_writer_context(const std::unique_ptr<DataDir>& data_dir,
const TabletSchemaSPtr& schema,
- const std::string& tablet_path) {
+ const std::string& tablet_path,
+ const TabletSharedPtr& tablet) {
RowsetWriterContext context;
RowsetId rowset_id;
rowset_id.init(inc_id);
@@ -187,6 +188,8 @@ static RowsetWriterContext rowset_writer_context(const std::unique_ptr<DataDir>&
context.rowset_state = VISIBLE;
context.tablet_schema = schema;
context.tablet_path = tablet_path;
+ context.tablet_id = tablet->tablet_id();
+ context.tablet = tablet;
context.version = Version(inc_id, inc_id);
context.max_rows_per_segment = 200;
inc_id++;
@@ -221,6 +224,12 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ // set seed
+ std::srand(42);
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
@@ -237,7 +246,8 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -264,6 +274,12 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ // set seed
+ std::srand(42);
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows1");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
@@ -280,7 +296,8 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -294,6 +311,73 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
EXPECT_FALSE(st.ok());
}
+// Mixed-format compatibility: one tablet contains both old (inline meta) and new
+// (external column meta + variant ext meta) segments, and path stats / scanning
+// logic should work across all rowsets.
+TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) {
+ // 1. create tablet schema (same layout as check_path_stats_agg_key)
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(AGG_KEYS);
+ construct_column(schema_pb.add_column(), 0, "INT", "key", true);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "v1");
+ construct_column(schema_pb.add_column(), 2, "STRING", "v2");
+ construct_column(schema_pb.add_column(), 3, "VARIANT", "v3");
+ construct_column(schema_pb.add_column(), 4, "INT", "v4");
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->init_from_pb(schema_pb);
+
+ // 2. create tablet and data dir
+ TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ // First write a few rowsets with external_segment_meta_used_default = false (old format)
+ tablet_schema->set_external_segment_meta_used_default(false);
+ std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows_mixed");
+ EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
+ std::unique_ptr<DataDir> data_dir = std::make_unique<DataDir>(*_engine_ref, absolute_dir);
+ static_cast<void>(data_dir->update_capacity());
+ EXPECT_TRUE(data_dir->init(true).ok());
+
+ TabletSharedPtr tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, data_dir.get());
+ EXPECT_TRUE(tablet->init().ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ // 3. create multiple rowsets: first batch uses old footer format, second batch uses new format
+ std::vector<RowsetSharedPtr> rowsets;
+
+ auto create_and_add_rowset = [&](int /*idx*/) {
+ const auto& res = RowsetFactory::create_rowset_writer(
+ *_engine_ref,
+ rowset_writer_context(data_dir, tablet_schema, tablet->tablet_path(), tablet),
+ false);
+ EXPECT_TRUE(res.has_value()) << res.error();
+ const auto& rowset_writer = res.value();
+ auto rowset = create_rowset(rowset_writer, tablet_schema);
+ EXPECT_TRUE(tablet->add_rowset(rowset).ok());
+ rowsets.push_back(rowset);
+ };
+
+ // 3.1 write a few old-format rowsets (inline meta only)
+ for (int i = 0; i < 3; ++i) {
+ create_and_add_rowset(i);
+ }
+
+ // 3.2 flip tablet default to enable external column meta for subsequent segments
+ tablet_schema->set_external_segment_meta_used_default(true);
+
+ // 3.3 write a few new-format rowsets (external meta enabled)
+ for (int i = 0; i < 3; ++i) {
+ create_and_add_rowset(i + 3);
+ }
+
+ EXPECT_EQ(rowsets.size(), 6);
+
+ // 4. check that VariantCompactionUtil::check_path_stats works across mixed segments
+ // This will internally create Segment / ColumnReader instances and should be
+ // insensitive to whether a particular segment uses inline or external meta.
+ EXPECT_TRUE(
+ schema_util::VariantCompactionUtil::check_path_stats(rowsets, rowsets[0], tablet).ok());
+}
+
TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_schema) {
all_path_stats.clear();
// 1.create tablet schema
@@ -308,6 +392,10 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 12345;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -318,7 +406,8 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -373,6 +462,9 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
rowset_writer_context.rowset_state = VISIBLE;
rowset_writer_context.tablet_schema = tablet_schema;
rowset_writer_context.tablet_path = _tablet->tablet_path();
+ rowset_writer_context.data_dir = _data_dir.get();
+ rowset_writer_context.tablet_id = _tablet->tablet_id();
+ rowset_writer_context.tablet = _tablet;
rowset_writer_context.version = version;
rowset_writer_context.segments_overlap = overlap;
rowset_writer_context.max_rows_per_segment = max_rows_per_segment;
@@ -426,7 +518,10 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
EXPECT_TRUE(st.ok()) << st.msg();
// key3 is in the sparse column, return variant type
- auto data_type = segment->get_data_type_of(subcolumn_in_sparse, true);
+ StorageReadOptions type_opts;
+ type_opts.tablet_schema = out_rowset->tablet_schema();
+ type_opts.io_ctx.reader_type = ReaderType::READER_QUERY;
+ auto data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts);
EXPECT_TRUE(data_type != nullptr);
EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
@@ -435,7 +530,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
subcolumn_in_sparse.set_path_info(PathInData("v1.keya"));
// keya is not in the segment, return string type;
- data_type = segment->get_data_type_of(subcolumn_in_sparse, true);
+ data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts);
EXPECT_TRUE(data_type != nullptr);
EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_STRING);
@@ -463,7 +558,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
value_col.set_is_nullable(true);
sparse_typed_col.add_sub_column(value_col);
}
- data_type = segment->get_data_type_of(sparse_typed_col, true);
+ data_type = segment->get_data_type_of(sparse_typed_col, type_opts);
EXPECT_TRUE(data_type != nullptr);
EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_MAP);
@@ -473,7 +568,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
subcolumn_in_sparse._column_path->has_nested = true;
// keyb has nested part, return int type;
- data_type = segment->get_data_type_of(subcolumn_in_sparse, true);
+ data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts);
EXPECT_TRUE(data_type != nullptr);
EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_INT);
@@ -481,7 +576,7 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
subcolumn_in_sparse.set_name("v1.key1");
subcolumn_in_sparse.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
subcolumn_in_sparse.set_path_info(PathInData("v1.key1"));
- data_type = segment->get_data_type_of(subcolumn_in_sparse, true);
+ data_type = segment->get_data_type_of(subcolumn_in_sparse, type_opts);
EXPECT_TRUE(data_type != nullptr);
EXPECT_TRUE(data_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_STRING);
@@ -503,6 +598,10 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref,
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
std::unique_ptr<DataDir> _data_dir = std::make_unique<DataDir>(*_engine_ref, _absolute_dir);
@@ -519,7 +618,8 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref,
for (int i = 0; i < 1; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -632,6 +732,10 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
@@ -641,7 +745,8 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -698,6 +803,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
rowset_writer_context.rowset_state = VISIBLE;
rowset_writer_context.tablet_schema = tablet_schema;
rowset_writer_context.tablet_path = _absolute_dir + "/../";
+ rowset_writer_context.data_dir = _data_dir.get();
+ rowset_writer_context.tablet_id = _tablet->tablet_id();
+ rowset_writer_context.tablet = _tablet;
rowset_writer_context.version = version;
rowset_writer_context.segments_overlap = overlap;
rowset_writer_context.max_rows_per_segment = max_rows_per_segment;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 8a77a342546..3ab025079a3 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -81,7 +81,7 @@ public class Config extends ConfigBase {
options = {"NORMAL", "ASYNC", "BRIEF"})
public static String sys_log_mode = "ASYNC";
- @ConfField(description = {"FE 在 sys_log_roll_interval （日志滚动间隔）内允许保留的最大日志文件数。"
+ @ConfField(description = {"FE 在 sys_log_roll_interval（日志滚动间隔）内允许保留的最大日志文件数。"
+ "默认值为 10，意味着在每个日志滚动周期内，系统最多会保留 10 个日志文件。",
"This parameter defines the maximum number of FE log files that can be retained within the "
+ "sys_log_roll_interval (log roll interval). The default value is 10, which means the system"
@@ -140,7 +140,7 @@ public class Config extends ConfigBase {
public static boolean enable_file_logger = true;
@ConfField(mutable = false, masterOnly = false,
- description = {"是否检查table锁泄漏", "Whether to check table lock leaky"})
+ description = {"是否检查 table 锁泄漏", "Whether to check table lock leaky"})
public static boolean check_table_lock_leaky = false;
@ConfField(mutable = true, masterOnly = false,
@@ -302,18 +302,18 @@ public class Config extends ConfigBase {
+ "If there are many ReplicaWriteException in FE WARN log, you can try to increase this value"})
public static int bdbje_replica_ack_timeout_second = 10;
- @ConfField(description = {"在HA模式下，BDBJE 中保留的预留空间字节数的期望上限。非 HA 模式下无效",
+ @ConfField(description = {"在 HA 模式下，BDBJE 中保留的预留空间字节数的期望上限。非 HA 模式下无效",
"The desired upper limit on the number of bytes of reserved space to retain "
+ "in a replicated JE Environment. "
+ "This parameter is ignored in a non-replicated JE Environment."})
public static long bdbje_reserved_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G
- @ConfField(description = {"BDBJE 所需的空闲磁盘空间大小。如果空闲磁盘空间小于这个值，则BDBJE将无法写入。",
+ @ConfField(description = {"BDBJE 所需的空闲磁盘空间大小。如果空闲磁盘空间小于这个值，则 BDBJE 将无法写入。",
"Amount of free disk space required by BDBJE. "
+ "If the free disk space is less than this value, BDBJE will not be able to write."})
public static long bdbje_free_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G
- @ConfField(description = {"BDBJE Cache 内存大小， 最小值为 96KB。", "Amount of memory used by by BDBJE as cache. "})
+ @ConfField(description = {"BDBJE Cache 内存大小，最小值为 96KB。", "Amount of memory used by by BDBJE as cache. "})
public static long bdbje_cache_size_bytes = 10 * 1024 * 1024; // 10 MB
@ConfField(description = {"BDBJE Message 大小限制。", "Max message size of BDBJE. "})
@@ -327,11 +327,11 @@ public class Config extends ConfigBase {
"Queue size to store heartbeat task in heartbeat_mgr"})
public static int heartbeat_mgr_blocking_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"TabletStatMgr线程数",
+ @ConfField(masterOnly = true, description = {"TabletStatMgr 线程数",
"Num of thread to update tablet stat"})
public static int tablet_stat_mgr_threads_num = -1;
- @ConfField(masterOnly = true, description = {"Agent任务线程池的线程数",
+ @ConfField(masterOnly = true, description = {"Agent 任务线程池的线程数",
"Num of thread to handle agent task in agent task thread-pool"})
public static int max_agent_task_threads_num = 4096;
@@ -403,11 +403,11 @@ public class Config extends ConfigBase {
varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_https = false;
- @ConfField(description = {"Jetty 的 acceptor 线程数。Jetty的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。"
- + "acceptor 负责接受新的连接，然后交给 selector 处理HTTP报文协议的解包，最后由 worker 处理请求。"
- + "前两个线程池采用非阻塞模型，并且一个线程可以处理很多socket的读写，所以线程池的数量少。"
- + "对于大多数项目，只需要 1-2 个 acceptor 线程，2 到 4 个就足够了。Worker 的数量取决于应用的QPS和IO事件的比例。"
- + "越高QPS，或者IO占比越高，等待的线程越多，需要的线程总数越多。",
+ @ConfField(description = {"Jetty 的 acceptor 线程数。Jetty 的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。"
+ + "acceptor 负责接受新的连接，然后交给 selector 处理 HTTP 报文协议的解包，最后由 worker 处理请求。"
+ + "前两个线程池采用非阻塞模型，并且一个线程可以处理很多 socket 的读写，所以线程池的数量少。"
+ + "对于大多数项目，只需要 1-2 个 acceptor 线程，2 到 4 个就足够了。Worker 的数量取决于应用的 QPS 和 IO 事件的比例。"
+ + "越高 QPS，或者 IO 占比越高，等待的线程越多，需要的线程总数越多。",
"The number of acceptor threads for Jetty. Jetty's thread architecture model is very simple, "
+ "divided into three thread pools: acceptor, selector and worker. "
+ "The acceptor is responsible for accepting new connections, "
@@ -493,7 +493,7 @@ public class Config extends ConfigBase {
@ConfField(description = {"MySQL 服务的最大任务线程数", "The max number of task threads in MySQL service"})
public static int max_mysql_service_task_threads_num = 4096;
- @ConfField(description = {"BackendServiceProxy数量, 用于池化GRPC channel",
+ @ConfField(description = {"BackendServiceProxy 数量，用于池化 GRPC channel",
"BackendServiceProxy pool size for pooling GRPC channels."})
public static int backend_proxy_num = 48;
@@ -529,7 +529,7 @@ public class Config extends ConfigBase {
public static int publish_version_timeout_second = 30; // 30 seconds
@ConfField(mutable = true, masterOnly = true, description = {"导入 Publish 阶段的等待时间，单位是秒。超过此时间，"
- + "则只需每个tablet包含一个成功副本，则导入成功。值为 -1 时，表示无限等待。",
+ + "则只需每个 tablet 包含一个成功副本，则导入成功。值为 -1 时，表示无限等待。",
"Waiting time for one transaction changing to \"at least one replica success\", in seconds."
+ "If time exceeds this, and for each tablet it has at least one replica publish successful, "
+ "then the load task will be successful." })
@@ -584,7 +584,7 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true, description = {
"If the number of publishing transactions of a table exceeds this value, new transactions will "
+ "be rejected. Set to -1 to disable this limit.",
- "当一个表的发布事务数量超过该值时，新的事务将被拒绝。设置为-1表示不限制。"})
+ "当一个表的发布事务数量超过该值时，新的事务将被拒绝。设置为 -1 表示不限制。"})
public static long max_publishing_txn_num_per_table = 500;
@ConfField(description = {"thrift server 的最大 worker 线程数", "The max worker threads of thrift server"})
@@ -622,12 +622,17 @@ public class Config extends ConfigBase {
"Default timeout for insert load job, in seconds."})
public static int insert_load_default_timeout_second = 14400; // 4 hour
- @ConfField(mutable = true, masterOnly = true, description = {"对mow表随机设置cluster keys，用于测试",
+ @ConfField(mutable = true, masterOnly = true, description = {"对 mow 表随机设置 cluster keys，用于测试",
"random set cluster keys for mow table for test"})
public static boolean random_add_cluster_keys_for_mow = false;
@ConfField(mutable = true, masterOnly = true, description = {
- "等内部攒批真正写入完成才返回；insert into和stream load默认开启攒批",
+ "在 fuzzy 测试中随机选择部分表使用 V3 storage_format（ext_meta），用于增强覆盖",
+ "Randomly use V3 storage_format (ext_meta) for some tables in fuzzy tests to increase coverage"})
+ public static boolean random_use_v3_storage_format = true;
+
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "等内部攒批真正写入完成才返回；insert into 和 stream load 默认开启攒批",
"Wait for the internal batch to be written before returning; "
+ "insert into and stream load use group commit by default."})
public static boolean wait_internal_group_commit_finish = false;
@@ -636,12 +641,12 @@ public class Config extends ConfigBase {
"Default commit interval in ms for group commit"})
public static int group_commit_interval_ms_default_value = 10000;
- @ConfField(mutable = false, masterOnly = true, description = {"攒批的默认提交数据量，单位是字节，默认128M",
+ @ConfField(mutable = false, masterOnly = true, description = {"攒批的默认提交数据量，单位是字节，默认 128M",
"Default commit data bytes for group commit"})
public static int group_commit_data_bytes_default_value = 134217728;
@ConfField(mutable = true, masterOnly = true, description = {
- "内部攒批的超时时间为table的group_commit_interval_ms的倍数",
+ "内部攒批的超时时间为 table 的 group_commit_interval_ms 的倍数",
"The internal group commit timeout is the multiple of table's group_commit_interval_ms"})
public static int group_commit_timeout_multipler = 10;
@@ -694,7 +699,7 @@ public class Config extends ConfigBase {
public static boolean disable_show_stream_load = false;
@ConfField(mutable = true, description = {
- "是否开启stream load profile",
+ "是否开启 stream load profile",
"Whether to enable stream load profile"
})
public static boolean enable_stream_load_profile = false;
@@ -748,7 +753,7 @@ public class Config extends ConfigBase {
public static double used_capacity_percent_max_diff = 0.30;
@ConfField(mutable = true, masterOnly = true, description = {
- "设置固定的 BE 负载分数中磁盘使用率系数。BE 负载分数会综合磁盘使用率和副本数而得。有效值范围为[0, 1]，"
+ "设置固定的 BE 负载分数中磁盘使用率系数。BE 负载分数会综合磁盘使用率和副本数而得。有效值范围为 [0, 1]，"
+ "当超出此范围时，则使用其他方法自动计算此系数。",
"Sets a fixed disk usage factor in the BE load fraction. The BE load score is a combination of disk usage "
+ "and replica count. The valid value range is [0, 1]. When it is out of this range, other "
@@ -767,13 +772,13 @@ public class Config extends ConfigBase {
+ "and disable storage cool down function."})
public static boolean disable_storage_medium_check = false;
- @ConfField(description = {"创建表或分区时，可以指定存储介质(HDD 或 SSD)。如果未指定，"
+ @ConfField(description = {"创建表或分区时，可以指定存储介质 (HDD 或 SSD)。如果未指定，"
+ "则使用此配置指定的默认介质。",
"When create a table(or partition), you can specify its storage medium(HDD or SSD)."})
public static String default_storage_medium = "HDD";
@ConfField(mutable = true, masterOnly = true, description = {
- "删除数据库(表/分区)后，可以使用 RECOVER 语句恢复。此配置指定了数据的最大保留时间。"
+ "删除数据库 (表/分区) 后，可以使用 RECOVER 语句恢复。此配置指定了数据的最大保留时间。"
+ "超过此时间，数据将被永久删除。",
"After dropping database(table/partition), you can recover it by using RECOVER stmt.",
"And this specifies the maximal data retention time. After time, the data will be deleted permanently."})
@@ -836,7 +841,7 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true) public static boolean disable_colocate_balance = false;
- @ConfField(mutable = true, masterOnly = true, description = {"是否启用group间的均衡",
+ @ConfField(mutable = true, masterOnly = true, description = {"是否启用 group 间的均衡",
"is allow colocate balance between all groups"})
public static boolean disable_colocate_balance_between_groups = false;
@@ -1198,7 +1203,7 @@ public class Config extends ConfigBase {
// if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc
@ConfField(mutable = true, masterOnly = true, description = {
- "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
+ "重新发送 agent task 时，单次 RPC 分配给每个 be 的任务最大个数，默认值为 10000 个。",
"The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, "
+ "the default value is 10000."
})
@@ -1439,7 +1444,7 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig",
description = {
- "当前默认设置为 300，用来控制控制NereidsSqlCacheManager中sql cache过期时间，超过一段时间不访问cache会被回收",
+ "当前默认设置为 300，用来控制控制 NereidsSqlCacheManager 中 sql cache 过期时间，超过一段时间不访问 cache 会被回收",
"The current default setting is 300, which is used to control the expiration time of SQL cache"
+ "in NereidsSqlCacheManager. If the cache is not accessed for a period of time, "
+ "it will be reclaimed"
@@ -1455,7 +1460,7 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.nereids.stats.MemoryHboPlanStatisticsProvider$UpdateConfig",
description = {
- "当前默认设置为 86400，用来控制控制MemoryHboPlanStatisticsProvider中stats. cache过期时间，超过不访问会被回收",
+ "当前默认设置为 86400，用来控制控制 MemoryHboPlanStatisticsProvider 中 stats. cache 过期时间，超过不访问会被回收",
"The default setting is 86400, which is used to control the expiration time of plan stats. cache"
+ "in MemoryHboPlanStatisticsProvider. If the cache is not accessed for a period of time, "
+ "it will be reclaimed."
@@ -1471,7 +1476,7 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.nereids.stats.HboPlanInfoProvider$UpdateConfig",
description = {
- "当前默认设置为1000，用来控制控制HboPlanInfoProvider中plan info cache过期时间，超过一段时间不访问cache会被回收",
+ "当前默认设置为 1000，用来控制控制 HboPlanInfoProvider 中 plan info cache 过期时间，超过一段时间不访问 cache 会被回收",
"The default setting is 100, which is used to control the expiration time of hbo plan info cache"
+ "in HboPlanInfoProvider. If the cache is not accessed for a period of time, "
+ "it will be reclaimed."
@@ -1487,8 +1492,8 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",
description = {
- "当前默认设置为 300，用来控制控制NereidsSortedPartitionsCacheManager中分区元数据缓存过期时间，"
- + "超过一段时间不访问cache会被回收",
+ "当前默认设置为 300，用来控制控制 NereidsSortedPartitionsCacheManager 中分区元数据缓存过期时间，"
+ + "超过一段时间不访问 cache 会被回收",
"The current default setting is 300, which is used to control the expiration time of "
+ "the partition metadata cache in NereidsSortedPartitionsCheManager. "
+ "If the cache is not accessed for a period of time, it will be reclaimed"
@@ -1499,14 +1504,14 @@ public class Config extends ConfigBase {
/**
* Set the maximum number of rows that can be cached
*/
- @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大行数。",
+ @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache 可以缓存的最大行数。",
"Maximum number of rows that can be cached in SQL/Partition Cache, is 3000 by default."})
public static int cache_result_max_row_count = 3000;
/**
* Set the maximum data size that can be cached
*/
- @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大数据大小。",
+ @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache 可以缓存的最大数据大小。",
"Maximum data size of rows that can be cached in SQL/Partition Cache, is 3000 by default."})
public static int cache_result_max_data_size = 31457280; // 30M
@@ -1581,7 +1586,7 @@ public class Config extends ConfigBase {
@ConfField
public static boolean enable_bdbje_debug_mode = false;
- @ConfField(mutable = false, masterOnly = true, description = {"是否开启debug point模式，测试使用",
+ @ConfField(mutable = false, masterOnly = true, description = {"是否开启 debug point 模式，测试使用",
"is enable debug points, use in test."})
public static boolean enable_debug_points = false;
@@ -1615,7 +1620,7 @@ public class Config extends ConfigBase {
* sets the time without read activity before sending a keepalive ping
* the smaller the value, the sooner the channel is unavailable, but it will increase network io
*/
- @ConfField(description = { "设置grpc连接发送 keepalive ping 之前没有数据传输的时间。",
+ @ConfField(description = { "设置 grpc 连接发送 keepalive ping 之前没有数据传输的时间。",
"The time without grpc read activity before sending a keepalive ping" })
public static int grpc_keep_alive_second = 10;
@@ -1734,7 +1739,7 @@ public class Config extends ConfigBase {
public static boolean enable_cloud_restore_job = false;
@ConfField(mutable = true, masterOnly = true, description = {
- "存算分离恢复过程中，一次 create tablets rpc 创建的 tablet 数量上限，默认值为256个",
+ "存算分离恢复过程中，一次 create tablets rpc 创建的 tablet 数量上限，默认值为 256 个",
"During the cloud restore job, the maximum number of tablets created by one "
+ "create tablets RPC, 256 by default."})
public static int cloud_restore_create_tablet_batch_size = 256;
@@ -1780,7 +1785,7 @@ public class Config extends ConfigBase {
public static int column_comment_length_limit = -1;
@ConfField(mutable = true, description = {
- "内部表的默认压缩类型。支持的值有: LZ4, LZ4F, LZ4HC, ZLIB, ZSTD, SNAPPY, NONE。",
+ "内部表的默认压缩类型。支持的值有：LZ4, LZ4F, LZ4HC, ZLIB, ZSTD, SNAPPY, NONE。",
"Default compression type for internal tables. Supported values: LZ4, LZ4F, LZ4HC, ZLIB, ZSTD,"
+ " SNAPPY, NONE."})
public static String default_compression_type = "LZ4F";
@@ -1940,8 +1945,8 @@ public class Config extends ConfigBase {
*/
@ConfField(masterOnly = true, description = {"任务堆积时用于存放定时任务的队列大小", "The number of timer jobs that can be queued."})
public static int job_dispatch_timer_job_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"一个 Job 的 task 最大的持久化数量，超过这个限制将会丢弃旧的 task 记录, 如果值 < 1, 将不会持久化。",
- "Maximum number of persistence allowed per task in a job,exceeding which old tasks will be discarded，"
+ @ConfField(masterOnly = true, description = {"一个 Job 的 task 最大的持久化数量，超过这个限制将会丢弃旧的 task 记录，如果值 < 1, 将不会持久化。",
+ "Maximum number of persistence allowed per task in a job,exceeding which old tasks will be discarded,"
+ "If the value is less than 1, it will not be persisted." })
public static int max_persistence_task_count = 100;
@@ -1963,36 +1968,36 @@ public class Config extends ConfigBase {
+ "be automatically selected." })
public static int dictionary_task_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"finished 状态的 job 最长保存时间，超过这个时间将会被删除, 单位：小时",
+ @ConfField(masterOnly = true, description = {"finished 状态的 job 最长保存时间，超过这个时间将会被删除，单位：小时",
"The longest time to save the job in finished status, it will be deleted after this time. Unit: hour"})
public static int finished_job_cleanup_threshold_time_hour = 24;
- @ConfField(masterOnly = true, description = {"用于执行 Insert 任务的线程数,值应该大于0，否则默认为10",
+ @ConfField(masterOnly = true, description = {"用于执行 Insert 任务的线程数，值应该大于 0，否则默认为 10",
"The number of threads used to consume Insert tasks, "
+ "the value should be greater than 0, if it is <=0, default is 10."})
public static int job_insert_task_consumer_thread_num = 10;
- @ConfField(masterOnly = true, description = {"用于执行 MTMV 任务的线程数,值应该大于0，否则默认为10",
+ @ConfField(masterOnly = true, description = {"用于执行 MTMV 任务的线程数，值应该大于 0，否则默认为 10",
"The number of threads used to consume MTMV tasks, "
+ "the value should be greater than 0, if it is <=0, default is 10."})
public static int job_mtmv_task_consumer_thread_num = 10;
- @ConfField(masterOnly = true, description = { "用于执行字典导入和删除任务的线程数,值应该大于0，否则默认为3",
+ @ConfField(masterOnly = true, description = { "用于执行字典导入和删除任务的线程数，值应该大于 0，否则默认为 3",
"The number of threads used to perform the dictionary import and delete tasks, which should be"
+ " greater than 0, otherwise it defaults to 3." })
public static int job_dictionary_task_consumer_thread_num = 3;
- @ConfField(masterOnly = true, description = {"用于执行 Streaming 任务的线程数,值应该大于0，否则默认为100",
+ @ConfField(masterOnly = true, description = {"用于执行 Streaming 任务的线程数，值应该大于 0，否则默认为 100",
"The number of threads used to execute Streaming Tasks, "
+ "the value should be greater than 0, if it is <=0, default is 100."})
public static int job_streaming_task_exec_thread_num = 100;
- @ConfField(masterOnly = true, description = {"最大的 Streaming 作业数量,值应该大于0，否则默认为1024",
+ @ConfField(masterOnly = true, description = {"最大的 Streaming 作业数量，值应该大于 0，否则默认为 1024",
"The maximum number of Streaming jobs, "
+ "the value should be greater than 0, if it is <=0, default is 1024."})
public static int max_streaming_job_num = 1024;
- @ConfField(masterOnly = true, description = {"一个 Streaming Job 在内存中最多保留的 task的数量，超过将丢弃旧的记录",
+ @ConfField(masterOnly = true, description = {"一个 Streaming Job 在内存中最多保留的 task 的数量，超过将丢弃旧的记录",
"The maximum number of tasks a Streaming Job can keep in memory. If the number exceeds the limit, "
+ "old records will be discarded."})
public static int max_streaming_task_show_count = 100;
@@ -2039,12 +2044,12 @@ public class Config extends ConfigBase {
public static long query_queue_update_interval_ms = 5000;
@ConfField(mutable = true, description = {
- "当BE内存用量大于该值时，查询会进入排队逻辑，默认值为-1，代表该值不生效。取值范围0~1的小数",
+ "当 BE 内存用量大于该值时，查询会进入排队逻辑，默认值为 -1，代表该值不生效。取值范围 0~1 的小数",
"When be memory usage bigger than this value, query could queue, "
+ "default value is -1, means this value not work. Decimal value range from 0 to 1"})
public static double query_queue_by_be_used_memory = -1;
- @ConfField(mutable = true, description = {"基于内存反压场景FE定时拉取BE内存用量的时间间隔",
+ @ConfField(mutable = true, description = {"基于内存反压场景 FE 定时拉取 BE 内存用量的时间间隔",
"In the scenario of memory backpressure, "
+ "the time interval for obtaining BE memory usage at regular intervals"})
public static long get_be_resource_usage_interval_ms = 10000;
@@ -2116,9 +2121,9 @@ public class Config extends ConfigBase {
* And the max number of compute node is controlled by min_backend_num_for_external_table.
* If set to false, query on external table will assign to any node.
*/
- @ConfField(mutable = true, description = {"如果设置为true，外部表的查询将优先分配给计算节点。",
- "并且计算节点的最大数量由min_backend_num_for_external_table控制。",
- "如果设置为false，外部表的查询将分配给任何节点。"
+ @ConfField(mutable = true, description = {"如果设置为 true，外部表的查询将优先分配给计算节点。",
+ "并且计算节点的最大数量由 min_backend_num_for_external_table 控制。",
+ "如果设置为 false，外部表的查询将分配给任何节点。"
+ "如果集群内没有计算节点，则该参数不生效。",
"If set to true, query on external table will prefer to assign to compute node. "
+ "And the max number of compute node is controlled by min_backend_num_for_external_table. "
@@ -2126,9 +2131,9 @@ public class Config extends ConfigBase {
+ "If there is no compute node in cluster, this config takes no effect."})
public static boolean prefer_compute_node_for_external_table = false;
- @ConfField(mutable = true, description = {"只有当prefer_compute_node_for_external_table为true时生效，"
+ @ConfField(mutable = true, description = {"只有当 prefer_compute_node_for_external_table 为 true 时生效，"
+ "如果计算节点数小于这个值，外部表的查询会尝试获取一些混合节点来分配，以使节点总数达到这个值。"
- + "如果计算节点数大于这个值，外部表的查询将只分配给计算节点。-1表示只是用当前数量的计算节点",
+ + "如果计算节点数大于这个值，外部表的查询将只分配给计算节点。-1 表示只是用当前数量的计算节点",
"Only take effect when prefer_compute_node_for_external_table is true. "
+ "If the compute node number is less than this value, "
+ "query on external table will try to get some mix de to assign, "
@@ -2239,11 +2244,11 @@ public class Config extends ConfigBase {
"Max cache number of database and table instance in external catalog."})
public static long max_meta_object_cache_num = 1000;
- @ConfField(description = {"Hive分区表缓存的最大数量",
+ @ConfField(description = {"Hive 分区表缓存的最大数量",
"Max cache number of hive partition table"})
public static long max_hive_partition_table_cache_num = 1000;
- @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量，-1代表没有限制。",
+ @ConfField(mutable = false, masterOnly = false, description = {"获取 Hive 分区值时候的最大返回数量，-1 代表没有限制。",
"Max number of hive partition values to return while list partitions, -1 means no limitation."})
public static short max_hive_list_partition_num = -1;
@@ -2356,7 +2361,7 @@ public class Config extends ConfigBase {
* otherwise it will throw an AnalysisException.
*/
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL, description = {
- "当前默认设置为 false，开启后支持使用新优化器的load语句导入数据，失败后会降级旧的load语句。",
+ "当前默认设置为 false，开启后支持使用新优化器的 load 语句导入数据，失败后会降级旧的 load 语句。",
"Now default set to true, After this function is enabled, the load statement of "
+ "the new optimizer can be used to import data. If this function fails, "
+ "the old load statement will be degraded."})
@@ -2369,7 +2374,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig",
description = {
- "当前默认设置为 100，用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
+ "当前默认设置为 100，用来控制控制 NereidsSqlCacheManager 管理的 sql cache 数量。",
"Now default set to 100, this config is used to control the number of "
+ "sql cache managed by NereidsSqlCacheManager"
}
@@ -2380,7 +2385,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",
description = {
- "当前默认设置为 100，用来控制控制NereidsSortedPartitionsCacheManager中有序分区元数据的缓存个数，"
+ "当前默认设置为 100，用来控制控制 NereidsSortedPartitionsCacheManager 中有序分区元数据的缓存个数，"
+ "用于加速分区裁剪",
"The current default setting is 100, which is used to control the number of ordered "
+ "partition metadata caches in NereidsSortedPartitionsCacheManager, "
@@ -2396,7 +2401,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.nereids.stats.MemoryHboPlanStatisticsProvider$UpdateConfig",
description = {
- "当前默认设置为 100000，用来控制控制MemoryHboPlanStatisticsProvider管理的plan stats. cache数量。",
+ "当前默认设置为 100000，用来控制控制 MemoryHboPlanStatisticsProvider 管理的 plan stats. cache 数量。",
"Now default set to 100000, this config is used to control the number of "
+ "hbo plan stats. cache"
}
@@ -2409,7 +2414,7 @@ public class Config extends ConfigBase {
@ConfField(
mutable = true,
description = {
- "当前默认设置为 10，用来控制控制MemoryHboPlanStatisticsProvider管理的plan stats. cache recent runs数量。",
+ "当前默认设置为 10，用来控制控制 MemoryHboPlanStatisticsProvider 管理的 plan stats. cache recent runs 数量。",
"Now default set to 10, this config is used to control the number of "
+ "hbo plan stats. cache recent runs' entry number."
}
@@ -2423,7 +2428,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.nereids.stats.HboPlanInfoProvider$UpdateConfig",
description = {
- "当前默认设置为 1000，用来控制控制HboPlanInfoProvider管理的plan info cache数量。",
+ "当前默认设置为 1000，用来控制控制 HboPlanInfoProvider 管理的 plan info cache 数量。",
"Now default set to 1000, this config is used to control the number of "
+ "hbo plan info cache"
}
@@ -2556,7 +2561,7 @@ public class Config extends ConfigBase {
public static boolean enable_round_robin_create_tablet = true;
@ConfField(mutable = true, masterOnly = true, description = {
- "创建分区时，总是从第一个 BE 开始创建。注意：这种方式可能造成BE不均衡",
+ "创建分区时，总是从第一个 BE 开始创建。注意：这种方式可能造成 BE 不均衡",
"When creating tablet of a partition, always start from the first BE. "
+ "Note: This method may cause BE imbalance"})
public static boolean create_tablet_round_robin_from_start = false;
@@ -2619,19 +2624,19 @@ public class Config extends ConfigBase {
public static boolean enable_query_hit_stats = false;
@ConfField(mutable = true, description = {
- "设置为 true，如果查询无法选择到健康副本时，会打印出该tablet所有副本的详细信息，" + "以及不可查询的具体原因。",
+ "设置为 true，如果查询无法选择到健康副本时，会打印出该 tablet 所有副本的详细信息，" + "以及不可查询的具体原因。",
"When set to true, if a query is unable to select a healthy replica, "
+ "the detailed information of all the replicas of the tablet,"
+ " including the specific reason why they are unqueryable, will be printed out."})
public static boolean show_details_for_unaccessible_tablet = true;
@ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
- "是否启用binlog特性",
+ "是否启用 binlog 特性",
"Whether to enable binlog feature"})
public static boolean enable_feature_binlog = false;
@ConfField(mutable = false, description = {
- "是否默认为 Database/Table 启用binlog特性",
+ "是否默认为 Database/Table 启用 binlog 特性",
"Whether to enable binlog feature for Database/Table by default"})
public static boolean force_enable_feature_binlog = false;
@@ -2646,17 +2651,17 @@ public class Config extends ConfigBase {
public static boolean disallow_create_catalog_with_resource = true;
@ConfField(mutable = true, masterOnly = false, description = {
- "Hive行数估算分区采样数",
+ "Hive 行数估算分区采样数",
"Sample size for hive row count estimation."})
public static int hive_stats_partition_sample_size = 30;
@ConfField(mutable = true, masterOnly = true, description = {
- "启用Hive分桶表",
+ "启用 Hive 分桶表",
"Enable external hive bucket table"})
public static boolean enable_create_hive_bucket_table = false;
@ConfField(mutable = true, masterOnly = true, description = {
- "Hive创建外部表默认指定的文件格式",
+ "Hive 创建外部表默认指定的文件格式",
"Default hive file format for creating table."})
public static String hive_default_file_format = "orc";
@@ -2705,13 +2710,13 @@ public class Config extends ConfigBase {
public static boolean force_sample_analyze = false; // avoid full analyze for performance reason
@ConfField(mutable = true, description = {
- "Export任务允许的最大分区数量",
+ "Export 任务允许的最大分区数量",
"The maximum number of partitions allowed by Export job"})
public static int maximum_number_of_export_partitions = 2000;
@Deprecated
@ConfField(mutable = true, description = {
- "Export任务允许的最大并行数",
+ "Export 任务允许的最大并行数",
"The maximum parallelism allowed by Export job"})
public static int maximum_parallelism_of_export_job = 50;
@@ -2724,20 +2729,20 @@ public class Config extends ConfigBase {
public static boolean forbid_running_alter_job = false;
@ConfField(description = {
- "暂时性配置项，开启后会自动将所有的olap表修改为可light schema change",
+ "暂时性配置项，开启后会自动将所有的 olap 表修改为可 light schema change",
"temporary config filed, will make all olap tables enable light schema change"
})
public static boolean enable_convert_light_weight_schema_change = false;
@ConfField(mutable = true, masterOnly = false, description = {
- "查询information_schema.metadata_name_ids表时,获取一个数据库中所有表用的时间",
+ "查询 information_schema.metadata_name_ids 表时，获取一个数据库中所有表用的时间",
"When querying the information_schema.metadata_name_ids table,"
+ " the time used to obtain all tables in one database"
})
public static long query_metadata_name_ids_timeout = 3;
@ConfField(mutable = true, masterOnly = true, description = {
- "是否禁止LocalDeployManager删除节点",
+ "是否禁止 LocalDeployManager 删除节点",
"Whether to disable LocalDeployManager drop node"})
public static boolean disable_local_deploy_manager_drop_node = true;
@@ -2777,13 +2782,13 @@ public class Config extends ConfigBase {
public static long analyze_record_limit = 20000;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中最小的buckets数目",
+ "Auto Buckets 中最小的 buckets 数目",
"min buckets of auto bucket"
})
public static int autobucket_min_buckets = 1;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中最大的buckets数目",
+ "Auto Buckets 中最大的 buckets 数目",
"max buckets of auto bucket"
})
public static int autobucket_max_buckets = 128;
@@ -2793,8 +2798,8 @@ public class Config extends ConfigBase {
public static int arrow_flight_max_connections = 4096;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中按照partition size去估算bucket数，存算一体partition size 5G估算一个bucket，"
- + "但存算分离下partition size 10G估算一个bucket。 若配置小于0，会在在代码中会自适应存算一体模式默认5G，在存算分离默认10G",
+ "Auto Buckets 中按照 partition size 去估算 bucket 数，存算一体 partition size 5G 估算一个 bucket，"
+ + "但存算分离下 partition size 10G 估算一个 bucket。若配置小于 0，会在在代码中会自适应存算一体模式默认 5G，在存算分离默认 10G",
"In Auto Buckets, the number of buckets is estimated based on the partition size. "
+ "For storage and computing integration, a partition size of 5G is estimated as one bucket."
+ " but for cloud, a partition size of 10G is estimated as one bucket. "
@@ -2803,14 +2808,14 @@ public class Config extends ConfigBase {
})
public static int autobucket_partition_size_per_bucket_gb = -1;
- @ConfField(mutable = true, masterOnly = true, description = {"Auto bucket中计算出的新的分区bucket num超过前一个分区的"
- + "bucket num的百分比，被认为是异常case报警",
+ @ConfField(mutable = true, masterOnly = true, description = {"Auto bucket 中计算出的新的分区 bucket num 超过前一个分区的"
+ + "bucket num 的百分比，被认为是异常 case 报警",
"The new partition bucket number calculated in the auto bucket exceeds the percentage "
+ "of the previous partition's bucket number, which is considered an abnormal case alert."})
public static double autobucket_out_of_bounds_percent_threshold = 0.5;
- @ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow Flight Server中所有用户token的缓存上限，"
- + "超过后LRU淘汰, arrow flight sql是无状态的协议，连接通常不会主动断开，"
+ @ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow Flight Server 中所有用户 token 的缓存上限，"
+ + "超过后 LRU 淘汰，arrow flight sql 是无状态的协议，连接通常不会主动断开，"
+ "bearer token 从 cache 淘汰的同时会 unregister Connection.",
"(Deprecated, replaced by arrow_flight_max_connection) The cache limit of all user tokens in "
+ "Arrow Flight Server. which will be eliminated by LRU rules after exceeding the limit, "
@@ -2818,7 +2823,7 @@ public class Config extends ConfigBase {
+ "bearer token is evict from the cache will unregister ConnectContext."})
public static int arrow_flight_token_cache_size = 4096;
- @ConfField(description = {"Arrow Flight Server中用户token的存活时间，自上次写入后过期时间，单位秒，默认值为86400，即1天",
+ @ConfField(description = {"Arrow Flight Server 中用户 token 的存活时间，自上次写入后过期时间，单位秒，默认值为 86400，即 1 天",
"The alive time of the user token in Arrow Flight Server, expire after write, unit second,"
+ "the default value is 86400, which is 1 days"})
public static int arrow_flight_token_alive_time_second = 86400;
@@ -2832,14 +2837,14 @@ public class Config extends ConfigBase {
public static String mysqldb_replace_name = "mysql";
@ConfField(description = {
- "设置允许跨域访问的特定域名,默认允许任何域名跨域访问",
+ "设置允许跨域访问的特定域名，默认允许任何域名跨域访问",
"Set the specific domain name that allows cross-domain access. "
+ "By default, any domain name is allowed cross-domain access"
})
public static String access_control_allowed_origin_domain = "*";
@ConfField(description = {
- "开启java_udf, 默认为true。如果该配置为false，则禁止创建和使用java_udf。在一些场景下关闭该配置可防止命令注入攻击。",
+ "开启 java_udf, 默认为 true。如果该配置为 false，则禁止创建和使用 java_udf。在一些场景下关闭该配置可防止命令注入攻击。",
"Used to enable java_udf, default is true. if this configuration is false, creation and use of java_udf is "
+ "disabled. in some scenarios it may be necessary to disable this configuration to prevent "
+ "command injection attacks."
@@ -2847,7 +2852,7 @@ public class Config extends ConfigBase {
public static boolean enable_java_udf = true;
@ConfField(mutable = true, masterOnly = true, description = {
- "开启后，可以在导入时，利用创建的全局java_udf函数处理数据, 默认为false。",
+ "开启后，可以在导入时，利用创建的全局 java_udf 函数处理数据，默认为 false。",
"When enabled, data can be processed using the globally created java_udf function during import."
+ " The default setting is false."
})
@@ -2867,8 +2872,8 @@ public class Config extends ConfigBase {
public static boolean ignore_unknown_metadata_module = false;
@ConfField(mutable = true, description = {
- "从主节点同步image文件的超时时间，用户可根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，"
- + "单位为秒，默认值300",
+ "从主节点同步 image 文件的超时时间，用户可根据${meta_dir}/image 文件夹下面的 image 文件大小和节点间的网络环境调整，"
+ + "单位为秒，默认值 300",
"The timeout for FE Follower/Observer synchronizing an image file from the FE Master, can be adjusted by "
+ "the user on the size of image file in the ${meta_dir}/image and the network environment between "
+ "nodes. The default values is 300."
@@ -2876,9 +2881,9 @@ public class Config extends ConfigBase {
public static int sync_image_timeout_second = 300;
@ConfField(mutable = true, description = {
- "FE启动时加载image文件某个模块的二进制内容到字节数组，并将字节数组反序列化为utf8编码字符串时单批次（单位：byte, 至少500MB）"
- + "的大小。等于-1的值表示一次性读取完整的字节数组后反序列化反序列化为utf8编码字符串；"
- + "不等于-1的值（至少16MB）表示分批每次读取多大的字节数组后反序列化为utf8编码字符串，最后合并成完成的字符串。默认值为-1",
+ "FE 启动时加载 image 文件某个模块的二进制内容到字节数组，并将字节数组反序列化为 utf8 编码字符串时单批次（单位：byte, 至少 500MB）"
+ + "的大小。等于 -1 的值表示一次性读取完整的字节数组后反序列化反序列化为 utf8 编码字符串；"
+ + "不等于 -1 的值（至少 16MB）表示分批每次读取多大的字节数组后反序列化为 utf8 编码字符串，最后合并成完成的字符串。默认值为 -1",
"The size of a single batch (in bytes) when loading the binary content of a module of the "
+ "image file into a byte array and deserializing the byte array into a utf8 encoded string when FE starts."
+ " A value equal to -1 means reading the entire byte array at once and "
@@ -2926,12 +2931,12 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int workload_group_max_num = 15;
- @ConfField(description = {"查询be wal_queue 的超时阈值(ms)",
+ @ConfField(description = {"查询 be wal_queue 的超时阈值 (ms)",
"the timeout threshold of checking wal_queue on be(ms)"})
public static int check_wal_queue_timeout_threshold = 180000; // 3 min
@ConfField(mutable = true, masterOnly = true, description = {
- "对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。",
+ "对于自动分区表，防止用户意外创建大量分区，每个 OLAP 表允许的分区数量为`max_auto_partition_num`。默认 2000。",
"For auto-partitioned tables to prevent users from accidentally creating a large number of partitions, "
+ "the number of partitions allowed per OLAP table is `max_auto_partition_num`. Default 2000."
})
@@ -2952,9 +2957,9 @@ public class Config extends ConfigBase {
public static double diagnose_balance_max_tablet_num_ratio = 1.1;
@ConfField(masterOnly = true, description = {
- "设置 root 用户初始化2阶段 SHA-1 加密密码，默认为''，即不设置 root 密码。"
+ "设置 root 用户初始化 2 阶段 SHA-1 加密密码，默认为''，即不设置 root 密码。"
+ "后续 root 用户的 `set password` 操作会将 root 初始化密码覆盖。"
- + "示例：如要配置密码的明文是 `root@123`，可在Doris执行SQL `select password('root@123')` "
+ + "示例：如要配置密码的明文是 `root@123`，可在 Doris 执行 SQL `select password('root@123')` "
+ "获取加密密码 `*A00C34073A26B40AB4307650BFB9309D6BFA6999`",
"Set root user initial 2-staged SHA-1 encrypted password, default as '', means no root password. "
+ "Subsequent `set password` operations for root user will overwrite the initial root password. "
@@ -2963,25 +2968,25 @@ public class Config extends ConfigBase {
+ "password `*A00C34073A26B40AB4307650BFB9309D6BFA6999`"})
public static String initial_root_password = "";
- @ConfField(description = {"nereids trace文件的存放路径。",
+ @ConfField(description = {"nereids trace 文件的存放路径。",
"The path of the nereids trace file."})
public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace";
@ConfField(mutable = true, masterOnly = true, description = {
- "备份过程中，一个 upload 任务上传的快照数量上限，默认值为10个",
+ "备份过程中，一个 upload 任务上传的快照数量上限，默认值为 10 个",
"The max number of snapshots assigned to a upload task during the backup process, the default value is 10."
})
public static int backup_upload_snapshot_batch_size = 10;
@ConfField(mutable = true, masterOnly = true, description = {
- "恢复过程中，一个 download 任务下载的快照数量上限，默认值为10个",
+ "恢复过程中，一个 download 任务下载的快照数量上限，默认值为 10 个",
"The max number of snapshots assigned to a download task during the restore process, "
+ "the default value is 10."
})
public static int restore_download_snapshot_batch_size = 10;
@ConfField(mutable = true, masterOnly = true, description = {
- "备份恢复过程中，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
+ "备份恢复过程中，单次 RPC 分配给每个 be 的任务最大个数，默认值为 10000 个。",
"The max number of batched tasks per RPC assigned to each be during the backup/restore process, "
+ "the default value is 10000."
})
@@ -2992,11 +2997,11 @@ public class Config extends ConfigBase {
"The number of concurrent restore tasks per be"})
public static int restore_task_concurrency_per_be = 5000;
- @ConfField(mutable = true, description = {"执行 agent task 时，BE心跳超过多长时间，认为BE不可用",
+ @ConfField(mutable = true, description = {"执行 agent task 时，BE 心跳超过多长时间，认为 BE 不可用",
"The time after which BE is considered unavailable if the heartbeat is not received"})
public static int agent_task_be_unavailable_heartbeat_timeout_second = 300;
- @ConfField(description = {"是否开启通过http接口获取log文件的功能",
+ @ConfField(description = {"是否开启通过 http 接口获取 log 文件的功能",
"Whether to enable the function of getting log files through http interface"})
public static boolean enable_get_log_file_api = false;
@@ -3006,20 +3011,20 @@ public class Config extends ConfigBase {
public static boolean enable_collect_internal_query_profile = false;
@ConfField(mutable = false, masterOnly = false, description = {
- "http请求处理/api/query中sql任务的最大线程池。",
+ "http 请求处理/api/query 中 sql 任务的最大线程池。",
"The max number work threads of http sql submitter."
})
public static int http_sql_submitter_max_worker_threads = 2;
@ConfField(mutable = false, masterOnly = false, description = {
- "http请求处理/api/upload任务的最大线程池。",
+ "http 请求处理/api/upload 任务的最大线程池。",
"The max number work threads of http upload submitter."
})
public static int http_load_submitter_max_worker_threads = 2;
@ConfField(mutable = true, masterOnly = true, description = {
- "load label个数阈值，超过该个数后，对于已经完成导入作业或者任务，"
- + "其label会被删除，被删除的 label 可以被重用。 值为 -1 时，表示此阈值不生效。",
+ "load label 个数阈值，超过该个数后，对于已经完成导入作业或者任务，"
+ + "其 label 会被删除，被删除的 label 可以被重用。值为 -1 时，表示此阈值不生效。",
"The threshold of load labels' number. After this number is exceeded, "
+ "the labels of the completed import jobs or tasks will be deleted, "
+ "and the deleted labels can be reused. "
@@ -3041,7 +3046,7 @@ public class Config extends ConfigBase {
@ConfField
public static boolean ignore_bdbje_log_checksum_read = false;
- @ConfField(description = {"指定 mysql登录身份认证类型",
+ @ConfField(description = {"指定 mysql 登录身份认证类型",
"Specifies the authentication type"},
options = {"default", "ldap"})
public static String authentication_type = "default";
@@ -3060,8 +3065,8 @@ public class Config extends ConfigBase {
public static String inverted_index_storage_format = "V3";
@ConfField(mutable = true, masterOnly = true, description = {
- "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，"
- + "但delete后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低delete语句的性能来保证正确性。",
+ "是否在 unique 表 mow 上开启 delete 语句写 delete predicate。若开启，会提升 delete 语句的性能，"
+ + "但 delete 后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低 delete 语句的性能来保证正确性。",
"Enable the 'delete predicate' for DELETE statements. If enabled, it will enhance the performance of "
+ "DELETE statements, but partial column updates after a DELETE may result in erroneous data. "
+ "If disabled, it will reduce the performance of DELETE statements to ensure accuracy."
@@ -3111,7 +3116,7 @@ public class Config extends ConfigBase {
public static int checkpoint_manual_gc_threshold = 0;
@ConfField(mutable = true, description = {
- "是否在每个请求开始之前打印一遍请求内容, 主要是query语句",
+ "是否在每个请求开始之前打印一遍请求内容，主要是 query 语句",
"Should the request content be logged before each request starts, specifically the query statements"})
public static boolean enable_print_request_before_execution = false;
@@ -3136,17 +3141,17 @@ public class Config extends ConfigBase {
public static int profile_waiting_time_for_spill_seconds = 10;
@ConfField(mutable = true, description = {
- "是否通过检测协调者BE心跳来 abort 事务",
+ "是否通过检测协调者 BE 心跳来 abort 事务",
"SHould abort txn by checking coorinator be heartbeat"})
public static boolean enable_abort_txn_by_checking_coordinator_be = true;
@ConfField(mutable = true, description = {
- "是否在 schema change 过程中, 检测冲突事物并 abort 它",
+ "是否在 schema change 过程中，检测冲突事物并 abort 它",
"SHould abort txn by checking conflick txn in schema change"})
public static boolean enable_abort_txn_by_checking_conflict_txn = true;
@ConfField(mutable = true, description = {
- "内表自动收集时间间隔，当某一列上次收集时间距离当前时间大于该值，则会触发一次新的收集，0表示不会触发。",
+ "内表自动收集时间间隔，当某一列上次收集时间距离当前时间大于该值，则会触发一次新的收集，0 表示不会触发。",
"Columns that have not been collected within the specified interval will trigger automatic analyze. "
+ "0 means not trigger."
})
@@ -3174,7 +3179,7 @@ public class Config extends ConfigBase {
//==========================================================================
// begin of cloud config
//==========================================================================
- @ConfField(description = {"是否启用FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认为按照时间策略删除",
+ @ConfField(description = {"是否启用 FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的 log。默认为按照时间策略删除",
"Whether to enable the FE log file deletion policy based on size, "
+ "where logs exceeding the specified size are deleted. "
+ "It is disabled by default and follows a time-based deletion policy."},
@@ -3342,13 +3347,13 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int cloud_min_balance_tablet_num_per_run = 2;
- @ConfField(description = {"指定存算分离模式下所有Compute group的扩缩容预热方式。"
- + "without_warmup: 直接修改tablet分片映射，首次读从S3拉取，均衡最快但性能波动最大；"
- + "async_warmup: 异步预热，尽力而为拉取cache，均衡较快但可能cache miss；"
- + "sync_warmup: 同步预热，确保cache迁移完成，均衡较慢但无cache miss；"
- + "peer_read_async_warmup: 直接修改tablet分片映射，首次读从Peer BE拉取，均衡最快可能会影响同计算组中其他BE性能。"
- + "注意：此为全局FE配置，也可通过SQL（ALTER COMPUTE GROUP cg PROPERTIES）"
- + "设置compute group维度的balance类型，compute group维度配置优先级更高",
+ @ConfField(description = {"指定存算分离模式下所有 Compute group 的扩缩容预热方式。"
+ + "without_warmup: 直接修改 tablet 分片映射，首次读从 S3 拉取，均衡最快但性能波动最大；"
+ + "async_warmup: 异步预热，尽力而为拉取 cache，均衡较快但可能 cache miss；"
+ + "sync_warmup: 同步预热，确保 cache 迁移完成，均衡较慢但无 cache miss；"
+ + "peer_read_async_warmup: 直接修改 tablet 分片映射，首次读从 Peer BE 拉取，均衡最快可能会影响同计算组中其他 BE 性能。"
+ + "注意：此为全局 FE 配置，也可通过 SQL（ALTER COMPUTE GROUP cg PROPERTIES）"
+ + "设置 compute group 维度的 balance 类型，compute group 维度配置优先级更高",
"Specify the scaling and warming methods for all Compute groups in a cloud mode. "
+ "without_warmup: Directly modify shard mapping, first read from S3,"
+ "fastest re-balance but largest fluctuation; "
@@ -3369,7 +3374,7 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static int mow_calculate_delete_bitmap_retry_times = 10;
- @ConfField(mutable = true, description = {"指定S3 Load endpoint白名单, 举例: s3_load_endpoint_white_list=a,b,c",
+ @ConfField(mutable = true, description = {"指定 S3 Load endpoint 白名单，举例：s3_load_endpoint_white_list=a,b,c",
"the white list for the s3 load endpoint, if it is empty, no white list will be set,"
+ "for example: s3_load_endpoint_white_list=a,b,c"})
public static String[] s3_load_endpoint_white_list = {};
@@ -3383,7 +3388,7 @@ public class Config extends ConfigBase {
})
public static boolean force_azure_blob_global_endpoint = false;
- @ConfField(mutable = true, description = {"指定Jdbc driver url白名单, 举例: jdbc_driver_url_white_list=a,b,c",
+ @ConfField(mutable = true, description = {"指定 Jdbc driver url 白名单，举例：jdbc_driver_url_white_list=a,b,c",
"the white list for jdbc driver url, if it is empty, no white list will be set"
+ "for example: jdbc_driver_url_white_list=a,b,c"
})
@@ -3435,33 +3440,33 @@ public class Config extends ConfigBase {
public static int audit_event_log_queue_size = 250000;
@ConfField(mutable = true, description = {
- "streamload导入使用的转发策略, 可选值为public-private/public/private/direct/random-be/空",
+ "streamload 导入使用的转发策略，可选值为 public-private/public/private/direct/random-be/空",
"streamload route policy, available options are "
+ "public-private/public/private/direct/random-be and empty string" })
public static String streamload_redirect_policy = "";
@ConfField(mutable = true, description = {
- "存算分离模式下是否启用group commit的streamload BE转发功能。"
- + "解决LB随机转发导致group commit攒批失效的问题，通过BE二次转发确保同表请求到达同一BE节点。",
+ "存算分离模式下是否启用 group commit 的 streamload BE 转发功能。"
+ + "解决 LB 随机转发导致 group commit 攒批失效的问题，通过 BE 二次转发确保同表请求到达同一 BE 节点。",
"Whether to enable group commit streamload BE forward feature in cloud mode. "
+ "Solves the issue where LB random forwarding breaks group commit batching "
+ "by implementing BE-level forwarding to ensure same-table requests reach the same BE node." })
public static boolean enable_group_commit_streamload_be_forward = false;
- @ConfField(description = {"存算分离模式下建表是否检查残留recycler key, 默认true",
+ @ConfField(description = {"存算分离模式下建表是否检查残留 recycler key, 默认 true",
"create table in cloud mode, check recycler key remained, default true"})
public static boolean check_create_table_recycle_key_remained = true;
- @ConfField(mutable = true, description = {"存算分离模式下fe向ms请求锁的过期时间，默认60s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 fe 向 ms 请求锁的过期时间，默认 60s"})
public static int delete_bitmap_lock_expiration_seconds = 60;
- @ConfField(mutable = true, description = {"存算分离模式下calculate delete bitmap task 超时时间，默认60s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 calculate delete bitmap task 超时时间，默认 60s"})
public static int calculate_delete_bitmap_task_timeout_seconds = 60;
- @ConfField(mutable = true, description = {"存算分离模式下事务导入calculate delete bitmap task 超时时间，默认300s"})
+ @ConfField(mutable = true, description = {"存算分离模式下事务导入 calculate delete bitmap task 超时时间，默认 300s"})
public static int calculate_delete_bitmap_task_timeout_seconds_for_transaction_load = 300;
- @ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 commit 阶段等锁超时时间，默认 5s"})
public static int try_commit_lock_timeout_seconds = 5;
@ConfField(mutable = true, description = {"是否在事务提交时对所有表启用提交锁。设置为 true 时，所有表都会使用提交锁。"
@@ -3472,28 +3477,28 @@ public class Config extends ConfigBase {
+ "Default value is true." })
public static boolean enable_commit_lock_for_all_tables = true;
- @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"})
+ @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认 false"})
public static boolean enable_cloud_txn_lazy_commit = false;
@ConfField(mutable = true, masterOnly = true,
- description = {"存算分离模式下，当tablet分布的be异常，是否立即映射tablet到新的be上，默认false"})
+ description = {"存算分离模式下，当 tablet 分布的 be 异常，是否立即映射 tablet 到新的 be 上，默认 false"})
public static boolean enable_immediate_be_assign = false;
@ConfField(mutable = true, masterOnly = false,
- description = { "存算分离模式下，一个BE挂掉多长时间后，它的tablet彻底转移到其他BE上" })
+ description = { "存算分离模式下，一个 BE 挂掉多长时间后，它的 tablet 彻底转移到其他 BE 上" })
public static int rehash_tablet_after_be_dead_seconds = 3600;
- @ConfField(mutable = true, description = {"存算分离模式下是否启用自动启停功能，默认true",
+ @ConfField(mutable = true, description = {"存算分离模式下是否启用自动启停功能，默认 true",
"Whether to enable the automatic start-stop feature in cloud model, default is true."})
public static boolean enable_auto_start_for_cloud_cluster = true;
- @ConfField(mutable = true, description = {"存算分离模式下自动启停等待cluster唤醒退避重试次数，默认300次大约5分钟",
+ @ConfField(mutable = true, description = {"存算分离模式下自动启停等待 cluster 唤醒退避重试次数，默认 300 次大约 5 分钟",
"The automatic start-stop wait time for cluster wake-up backoff retry count in the cloud "
+ "model is set to 300 times, which is approximately 5 minutes by default."})
public static int auto_start_wait_to_resume_times = 300;
- @ConfField(description = {"Get tablet stat task的最大并发数。",
+ @ConfField(description = {"Get tablet stat task 的最大并发数。",
"Maximal concurrent num of get tablet stat job."})
public static int max_get_tablet_stat_task_threads_num = 4;
@@ -3505,25 +3510,25 @@ public class Config extends ConfigBase {
"Max retry times when a schema change job fails, default is 3."})
public static int schema_change_max_retry_time = 3;
- @ConfField(mutable = true, description = {"是否允许使用ShowCacheHotSpotStmt语句",
+ @ConfField(mutable = true, description = {"是否允许使用 ShowCacheHotSpotStmt 语句",
"Whether to enable the use of ShowCacheHotSpotStmt, default is false."})
public static boolean enable_show_file_cache_hotspot_stmt = false;
- @ConfField(mutable = true, description = {"存算分离模式下FE连接meta service的请求超时, 默认30000ms",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 连接 meta service 的请求超时，默认 30000ms",
"Request timeout for FE connecting to meta service in cloud mode, default is 30000ms."})
public static int meta_service_brpc_timeout_ms = 30000;
- @ConfField(mutable = true, description = {"存算分离模式下FE连接meta service的连接超时，默认500ms",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 连接 meta service 的连接超时，默认 500ms",
"Connection timeout for FE connecting to meta service in cloud mode., default is 500ms."})
public static int meta_service_brpc_connect_timeout_ms = 500;
- @ConfField(mutable = true, description = {"存算分离模式下FE请求meta service超时的重试次数，默认1次",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 请求 meta service 超时的重试次数，默认 1 次",
"In cloud mode, the retry number when the FE requests the meta service times out is 1 by default"})
public static int meta_service_rpc_timeout_retry_times = 1;
@ConfField(mutable = true, description = {"存算分离模式下自动启停功能，对于该配置中的数据库名不进行唤醒操作，"
+ "用于内部作业的数据库，例如统计信息用到的数据库，"
- + "举例: auto_start_ignore_db_names=__internal_schema, information_schema",
+ + "举例：auto_start_ignore_db_names=__internal_schema, information_schema",
"In the cloud mode, the automatic start and stop ignores the DB name of the internal job,"
+ "used for databases involved in internal jobs, such as those used for statistics, "
+ "For example: auto_start_ignore_db_names=__internal_schema, information_schema"
@@ -3571,7 +3576,7 @@ public class Config extends ConfigBase {
public static long resource_not_ready_sleep_seconds = 5;
@ConfField(mutable = true, description = {
- "设置为 true，如果查询无法选择到健康副本时，会打印出该tablet所有副本的详细信息，"})
+ "设置为 true，如果查询无法选择到健康副本时，会打印出该 tablet 所有副本的详细信息，"})
public static boolean sql_block_rule_ignore_admin = false;
@ConfField(description = {"认证插件目录",
@@ -3594,12 +3599,12 @@ public class Config extends ConfigBase {
+ "default is conf/authentication.conf"})
public static String authentication_config_file_path = "/conf/authentication.conf";
- @ConfField(description = {"用于测试，强制将所有的查询forward到master以验证forward query的行为",
+ @ConfField(description = {"用于测试，强制将所有的查询 forward 到 master 以验证 forward query 的行为",
"For testing purposes, all queries are forcibly forwarded to the master to verify"
+ "the behavior of forwarding queries."})
public static boolean force_forward_all_queries = false;
- @ConfField(description = {"用于禁用某些SQL，配置项为AST的class simple name列表(例如CreateRepositoryStmt,"
+ @ConfField(description = {"用于禁用某些 SQL，配置项为 AST 的 class simple name 列表 (例如 CreateRepositoryStmt,"
+ "CreatePolicyCommand)，用逗号间隔开",
"For disabling certain SQL queries, the configuration item is a list of simple class names of AST"
+ "(for example CreateRepositoryStmt, CreatePolicyCommand), separated by commas."})
@@ -3652,14 +3657,14 @@ public class Config extends ConfigBase {
public static String doris_tde_algorithm = "PLAINTEXT";
@ConfField(mutable = true, description = {
- "数据加密自动rotate master key的时间间隔，单位为毫秒，默认间隔是一个月",
+ "数据加密自动 rotate master key 的时间间隔，单位为毫秒，默认间隔是一个月",
"The time interval for automatic rotation of the master key in data encryption, in milliseconds."
+ "The default interval is one month."
})
public static long doris_tde_rotate_master_key_interval_ms = 30 * 24 * 3600 * 1000L;
@ConfField(mutable = true, description = {
- "数据加密检查是否要rotate的时间间隔，单位为毫秒，默认间隔是五分钟",
+ "数据加密检查是否要 rotate 的时间间隔，单位为毫秒，默认间隔是五分钟",
"Data encryption checks whether to rotate the time interval in milliseconds, "
+ "and the default interval is five minutes."
})
diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
index c7e361cbf53..ace11547572 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
@@ -548,7 +548,7 @@ public class DorisFE {
releaseFileLockAndCloseFileChannel();
throw new RuntimeException("Try to lock process failed", e);
}
- throw new RuntimeException("FE process has been started，please do not start multiple FE processes at the "
+ throw new RuntimeException("FE process has been started, please do not start multiple FE processes at the "
+ "same time");
}
@@ -580,7 +580,10 @@ public class DorisFE {
if (!Config.use_fuzzy_conf) {
return;
}
- if (Config.fuzzy_test_type.equalsIgnoreCase("daily") || Config.fuzzy_test_type.equalsIgnoreCase("rqg")) {
+
+ // Keep global fuzzy knobs that are not session-based.
+ if (Config.fuzzy_test_type.equalsIgnoreCase("daily")
+ || Config.fuzzy_test_type.equalsIgnoreCase("rqg")) {
Config.random_add_cluster_keys_for_mow = (LocalDate.now().getDayOfMonth() % 2 == 0);
LOG.info("fuzzy set random_add_cluster_keys_for_mow={}", Config.random_add_cluster_keys_for_mow);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index ee176c28dc0..31f60edfc54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -217,8 +217,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
partitionId, rollupTablet, tabletType, rollupSchemaHash,
rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
tbl.getBfFpp(), null, rollupSchema,
- tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(),
- tbl.isInMemory(), true,
+ tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(),
+ tbl.getStoragePolicy(), tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 6e113eb5650..fcf4d7db209 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -239,7 +239,7 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
tbl.getPartitionInfo().getTabletType(partitionId),
shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns,
bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(),
- tbl.getCompressionType(),
+ tbl.getCompressionType(), tbl.getStorageFormat(),
tbl.getStoragePolicy(), tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 6a49ce90f00..145e0d9f473 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1378,6 +1378,11 @@ public class SchemaChangeHandler extends AlterHandler {
TStorageFormat storageFormat = PropertyAnalyzer.analyzeStorageFormat(propertyMap);
+ // if not explicitly specified in ALTER statement, use the current storage format of the table
+ if (!propertyMap.containsKey("storage_format")) {
+ storageFormat = olapTable.getStorageFormat();
+ }
+
// property store_row_column && row_store_columns
// eg. "store_row_column" = "true"
// eg. "row_store_columns" = "k1, k2"
@@ -1620,14 +1625,7 @@ public class SchemaChangeHandler extends AlterHandler {
schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
schemaChangeJob.setStoreRowColumnInfo(hasRowStoreChanged, storeRowColumn, rsColumns);
-
- // If StorageFormat is set to TStorageFormat.V2
- // which will create tablet with preferred_rowset_type set to BETA
- // for both base table and rollup index
- if (hasIndexChange) {
- // only V2 support index, so if there is index changed, storage format must be V2
- storageFormat = TStorageFormat.V2;
- }
+ // set storage format of schema change job
schemaChangeJob.setStorageFormat(storageFormat);
// the following operations are done outside the 'for indices' loop
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index 80bd91d9d54..f9b1840f574 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -386,7 +386,8 @@ public class CloudRestoreJob extends RestoreJob {
indexMeta.getSchemaHash(), indexMeta.getKeysType(),
indexMeta.getShortKeyColumnCount(), localTbl.getCopiedBfColumns(),
localTbl.getBfFpp(), indexes, indexMeta.getSchema(), localTbl.getDataSortInfo(),
- localTbl.getCompressionType(), localTbl.getStoragePolicy(),
+ localTbl.getCompressionType(), localTbl.getStorageFormat(),
+ localTbl.getStoragePolicy(),
localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(),
localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(),
localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index b41570a25fc..7ac160f4b18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -30,7 +30,6 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
forceProperties = ImmutableList.of(
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_INMEMORY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM),
- RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 61e4263015f..e10fead5adf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -65,6 +65,7 @@ import org.apache.doris.rpc.RpcException;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
+import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Preconditions;
@@ -171,7 +172,8 @@ public class CloudInternalCatalog extends InternalCatalog {
OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId,
partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount,
bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(),
- tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false,
+ tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(),
tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(),
tbl.getTimeSeriesCompactionFileCountThreshold(),
@@ -208,7 +210,7 @@ public class CloudInternalCatalog extends InternalCatalog {
long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType,
short shortKeyColumnCount, Set<String> bfColumns, double bfFpp, List<Index> indexes,
List<Column> schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType,
- String storagePolicy, boolean isInMemory, boolean isShadow,
+ TStorageFormat storageFormat, String storagePolicy, boolean isInMemory, boolean isShadow,
String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite,
boolean storeRowColumn, int schemaVersion, String compactionPolicy,
Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold,
@@ -319,6 +321,15 @@ public class CloudInternalCatalog extends InternalCatalog {
break;
}
+ // Enable external column meta layout when storage_format is V3 (Cloud mode).
+ switch (storageFormat) {
+ case V3:
+ schemaBuilder.setIsExternalSegmentColumnMetaUsed(true);
+ break;
+ default:
+ break;
+ }
+
schemaBuilder.setSortColNum(dataSortInfo.getColNum());
for (int i = 0; i < schemaColumns.size(); i++) {
Column column = schemaColumns.get(i);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index f6fc9cbb3ea..e99a6e477a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1195,11 +1195,14 @@ public class PropertyAnalyzer {
return TStorageFormat.V2;
}
- if (storageFormat.equalsIgnoreCase("v1")) {
+ if (storageFormat.equalsIgnoreCase("V1")) {
throw new AnalysisException("Storage format V1 has been deprecated since version 0.14, "
+ "please use V2 instead");
- } else if (storageFormat.equalsIgnoreCase("v2")) {
+ } else if (storageFormat.equalsIgnoreCase("V2")) {
return TStorageFormat.V2;
+ } else if (storageFormat.equalsIgnoreCase("V3")) {
+ // V3 (V2 + external column meta feature)
+ return TStorageFormat.V3;
} else if (storageFormat.equalsIgnoreCase("default")) {
return TStorageFormat.V2;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index d417c908d78..53bd7f773f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -77,6 +77,7 @@ import org.apache.doris.nereids.types.VariantType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.base.Preconditions;
@@ -488,6 +489,19 @@ public class CreateTableInfo {
if (engineName.equalsIgnoreCase(ENGINE_OLAP)) {
boolean enableDuplicateWithoutKeysByDefault = false;
properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(ctlName, dbName, properties);
+
+ // In fuzzy tests, randomly set storage_format=V3 (ext_meta) for some tables.
+ SessionVariable sv = ctx.getSessionVariable();
+ boolean randomUseV3 =
+ sv != null && sv.useV3StorageFormat;
+ if (randomUseV3
+ && properties != null
+ && !properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V3");
+ LOG.info("Randomly set storage_format=V3 for table {}.{} in fuzzy mode (session={})",
+ dbName, tableName,
+ ctx != null ? ctx.getSessionId() : "<null>");
+ }
try {
if (properties != null) {
enableDuplicateWithoutKeysByDefault =
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index dc30cc1d5ea..45ed90e5866 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -993,7 +993,7 @@ public class SessionVariable implements Serializable, Writable {
public int minScannersConcurrency = 1;
@VariableMgr.VarAttr(name = MIN_FILE_SCANNERS_CONCURRENCY, needForward = true, description = {
- "外表Scanner 的最小并发度，默认为 1", "The min concurrency of Remote Scanner, default 1"
+ "外表 Scanner 的最小并发度，默认为 1", "The min concurrency of Remote Scanner, default 1"
})
public int minFileScannersConcurrency = 1;
@@ -1313,7 +1313,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = SKIP_PRUNE_PREDICATE, fuzzy = true,
description = {
- "是否跳过“在分区裁剪后删除恒真谓词”的优化。默认为OFF（即执行此优化）。",
+ "是否跳过“在分区裁剪后删除恒真谓词”的优化。默认为 OFF（即执行此优化）。",
"Skips the removal of always-true predicates after partition pruning. "
+ "Defaults to OFF (optimization is active)."
}
@@ -3101,7 +3101,7 @@ public class SessionVariable implements Serializable, Writable {
public int defaultVariantMaxSparseColumnStatisticsSize = 10000;
@VariableMgr.VarAttr(name = ENABLE_EXTENDED_REGEX, needForward = true, affectQueryResultInExecution = true,
- description = {"是否启用扩展的正则表达式, 支持如 look-around 类的零宽断言",
+ description = {"是否启用扩展的正则表达式，支持如 look-around 类的零宽断言",
"Enable extended regular expressions, support look-around zero-width assertions"})
public boolean enableExtendedRegex = false;
@@ -3112,6 +3112,15 @@ public class SessionVariable implements Serializable, Writable {
)
public int defaultVariantSparseHashShardCount = 0;
+ @VariableMgr.VarAttr(
+ name = "use_v3_storage_format",
+ fuzzy = true,
+ description = {
+ "In fuzzy tests, randomly use V3 storage_format (ext_meta) for some tables.",
+ "Only takes effect when user does not explicitly specify storage_format."}
+ )
+ public boolean useV3StorageFormat = false;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
@SuppressWarnings("checkstyle:Indentation")
@@ -3135,6 +3144,7 @@ public class SessionVariable implements Serializable, Writable {
this.exchangeMultiBlocksByteSize = minBytes + (int) (random.nextDouble() * (maxBytes - minBytes));
this.defaultVariantMaxSubcolumnsCount = random.nextInt(10);
this.defaultVariantSparseHashShardCount = random.nextInt(5) + 1;
+ this.useV3StorageFormat = random.nextBoolean();
int randomInt = random.nextInt(4);
if (randomInt % 2 == 0) {
this.rewriteOrToInPredicateThreshold = 100000;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 868a0677355..5dfe410dbd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -362,7 +362,6 @@ public class CreateReplicaTask extends AgentTask {
}
}
tSchema.setIndexes(tIndexes);
- storageFormat = TStorageFormat.V2;
}
if (bfColumns != null) {
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 73f626dbbc5..0cc1b180491 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -448,6 +448,9 @@ message TabletSchemaPB {
optional int64 storage_page_size = 30 [default=65536];
optional int64 storage_dict_page_size = 31 [default=262144];
+ // Whether new segments of this tablet use externalized ColumnMetaPB layout (CMO)
+ optional bool is_external_segment_column_meta_used = 32 [default = false];
+
optional SplitSchemaPB __split_schema = 1000; // A special field, DO NOT change it.
}
@@ -484,6 +487,9 @@ message TabletSchemaCloudPB {
optional int64 storage_page_size = 31 [default=65536];
optional int64 storage_dict_page_size = 32 [default=262144];
+ // Whether new segments of this tablet use externalized ColumnMetaPB layout (CMO)
+ optional bool is_external_segment_column_meta_used = 33 [default = false];
+
optional bool is_dynamic_schema = 100 [default=false];
// FIXME(gavin): deprecate and remove in the future
diff --git a/gensrc/proto/segment_v2.proto b/gensrc/proto/segment_v2.proto
index c88d7e50c11..abd1f451033 100644
--- a/gensrc/proto/segment_v2.proto
+++ b/gensrc/proto/segment_v2.proto
@@ -215,6 +215,16 @@ message ColumnMetaPB {
optional uint64 none_null_size = 23 [default = 0];
}
+// External column meta entry describing one top-level column's externalized
+// ColumnMetaPB in the Column Meta Region.
+message ColumnMetaEntryPB {
+ // unique column id (top-level only, including variant roots)
+ optional int32 unique_id = 1;
+ // serialized length in bytes of this column's ColumnMetaPB in the
+ // external Column Meta Region.
+ optional uint32 length = 2;
+}
+
message PrimaryKeyIndexMetaPB {
// primary key index
optional IndexedColumnMetaPB primary_key_index = 1;
@@ -225,8 +235,17 @@ message PrimaryKeyIndexMetaPB {
optional bytes max_key = 4;
}
+// SegmentFooterPB versioning (must stay in sync with be/src/olap/rowset/segment_v2/segment.h).
+enum SegmentFooterVersionPB {
+ // 1 = V2 baseline
+ SEGMENT_FOOTER_VERSION_V2_BASELINE = 1;
+ // 2 = V3 (externalized ColumnMetaPB region + CMO present)
+ SEGMENT_FOOTER_VERSION_V3_EXT_COL_META = 2;
+}
+
message SegmentFooterPB {
- optional uint32 version = 1 [default = 1]; // file version
+ // file version; see SegmentFooterVersionPB for the semantic meaning of values.
+ optional uint32 version = 1 [default = 1];
repeated ColumnMetaPB columns = 2; // tablet schema
optional uint32 num_rows = 3; // number of values
optional uint64 index_footprint = 4; // total index footprint of all columns
@@ -241,6 +260,14 @@ message SegmentFooterPB {
// Primary key index meta
optional PrimaryKeyIndexMetaPB primary_key_index_meta = 10;
+
+ // Externalized ColumnMetaPB layout (optional; presence indicates v2 meta layout available)
+ // Start offset of the contiguous per-column ColumnMetaPB region (absolute file offset).
+ optional uint64 col_meta_region_start = 11;
+ // Per-column external meta entries, in col_id order. Each entry contains the top-level
+ // unique id and the serialized length (in bytes) of its ColumnMetaPB in the external
+ // Column Meta Region.
+ repeated ColumnMetaEntryPB column_meta_entries = 12;
}
message BTreeMetaPB {
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 6870263c51d..d23f310780d 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -59,7 +59,9 @@ struct TTabletSchema {
enum TStorageFormat {
DEFAULT = 0,
V1 = 1,
- V2 = 2
+ V2 = 2,
+ // V3 stands externalized column meta (CMO)
+ V3 = 3
}
enum TEncryptionAlgorithm {
diff --git a/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out b/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out
new file mode 100644
index 00000000000..a04a08cbf67
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+1 1
+2 10
+
+-- !sql2 --
+1 2
+2 20
+
+-- !sql3 --
+2 40
+
+-- !sql4 --
+3 500
+
+-- !sql5 --
+1 100
+
+-- !sql6 --
+1 value
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out
new file mode 100644
index 00000000000..293c7c5371d
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !concurrent_1 --
+20
+
+-- !concurrent_2 --
+20
+
+-- !write_read_1 --
+16
+
+-- !write_read_2 --
+10
+
+-- !write_read_3 --
+6
+
+-- !stress_1 --
+15
+
+-- !stress_2 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !cache_1 --
+5
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out
new file mode 100644
index 00000000000..b95f6f8f269
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out
@@ -0,0 +1,174 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !empty_1 --
+1 \N
+2 \N
+3 \N
+
+-- !empty_2 --
+
+-- !empty_3 --
+1
+2
+3
+
+-- !many_1 --
+0 0
+1 0
+2 0
+3 0
+4 0
+5 0
+6 0
+7 0
+8 0
+9 0
+
+-- !many_2 --
+5 50
+6 50
+7 50
+8 50
+9 50
+
+-- !many_3 --
+9 99
+
+-- !many_4 --
+10
+
+-- !mixed_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !mixed_2 --
+1 2
+
+-- !mixed_3 --
+2 3
+
+-- !mixed_4 --
+3 4
+
+-- !mixed_5 --
+4 5
+
+-- !mixed_after_compact_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !mixed_after_compact_2 --
+4
+
+-- !nested_1 --
+1 value
+2 value2
+3 \N
+4 \N
+
+-- !nested_2 --
+2 123
+
+-- !nested_3 --
+1
+2
+
+-- !nested_4 --
+1
+2
+
+-- !array_1 --
+1 [1, 2, 3]
+2 [10, 20, 30, 40]
+3 \N
+4 \N
+
+-- !array_2 --
+1
+2
+
+-- !evolution_1 --
+1 1
+2 2
+3 3
+4 \N
+
+-- !evolution_2 --
+1 \N
+2 2
+3 3
+4 \N
+
+-- !evolution_3 --
+1 \N
+2 \N
+3 3
+4 \N
+
+-- !evolution_4 --
+1 \N
+2 \N
+3 \N
+4 10
+
+-- !evolution_5 --
+1 \N
+2 \N
+3 \N
+4 20
+
+-- !special_1 --
+1 1
+
+-- !special_2 --
+2 2
+
+-- !special_3 --
+3 3
+
+-- !null_1 --
+1 \N
+2 1
+3 \N
+
+-- !null_2 --
+3
+
+-- !null_3 --
+1
+3
+
+-- !null_4 --
+2
+
+-- !large_1 --
+1 10000
+2 5
+
+-- !large_2 --
+1
+
+-- !toggle_1 --
+1 1
+2 2
+3 3
+
+-- !toggle_2 --
+2 2
+3 3
+
+-- !toggle_3 --
+3 3
+
+-- !toggle_after_compact_1 --
+1 1
+2 2
+3 3
+
+-- !toggle_after_compact_2 --
+2 2
+3 3
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out
new file mode 100644
index 00000000000..0926c7eb167
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out
@@ -0,0 +1,120 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !inverted_1 --
+1 Alice Smith
+3 Charlie Smith
+
+-- !inverted_2 --
+1 Alice Smith
+
+-- !inverted_3 --
+2 30
+3 35
+
+-- !inverted_after_compact_1 --
+1 Alice Smith
+3 Charlie Smith
+
+-- !inverted_after_compact_2 --
+1
+
+-- !parent_idx_1 --
+1 Hello World
+3 World News
+
+-- !parent_idx_2 --
+1 This is a test
+2 Another test here
+
+-- !parent_idx_3 --
+1
+
+-- !multi_var_1 --
+1 1 10
+2 100 1000
+3 \N \N
+
+-- !multi_var_2 --
+1 2
+
+-- !multi_var_3 --
+1 20
+
+-- !multi_var_4 --
+3 4 40
+
+-- !multi_var_after_compact_1 --
+1 1 10
+2 100 1000
+3 \N \N
+
+-- !multi_var_after_compact_2 --
+3
+
+-- !agg_1 --
+A 300
+B 400
+
+-- !agg_2 --
+A 1.5
+B 2
+
+-- !agg_3 --
+2
+
+-- !join_1 --
+1 Alice Engineering
+2 Bob Sales
+3 Charlie Engineering
+
+-- !join_2 --
+2
+
+-- !union_1 --
+1 table1 100
+2 table1 200
+3 table2 150
+4 table2 250
+
+-- !union_2 --
+table1 2
+table2 2
+
+-- !subquery_1 --
+6 60 A
+7 70 A
+8 80 A
+9 90 A
+10 100 A
+
+-- !subquery_2 --
+A 5
+B 1
+
+-- !having_1 --
+A 600
+
+-- !order_limit_1 --
+1 19 item_1
+2 18 item_2
+3 17 item_3
+4 16 item_4
+5 15 item_5
+
+-- !order_limit_2 --
+9 item_9
+8 item_8
+7 item_7
+
+-- !distinct_1 --
+A
+B
+C
+
+-- !distinct_2 --
+100
+150
+200
+
+-- !distinct_3 --
+3
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out
new file mode 100644
index 00000000000..d5f580cabc9
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out
@@ -0,0 +1,197 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sparse_1 --
+1 1
+2 10
+3 100
+
+-- !sparse_2 --
+1 2
+2 20
+
+-- !sparse_3 --
+1 3
+2 30
+
+-- !sparse_4 --
+2 40
+
+-- !sparse_5 --
+3 500
+
+-- !sparse_6 --
+3 600
+
+-- !sparse_7 --
+3 700
+
+-- !sparse_after_compact_1 --
+1 1
+2 10
+3 100
+
+-- !sparse_after_compact_2 --
+2 40
+
+-- !sparse_after_compact_3 --
+3 700
+
+-- !bucket_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+
+-- !bucket_2 --
+0 0
+1 10
+2 20
+3 30
+4 40
+5 50
+6 60
+7 70
+8 80
+9 90
+
+-- !bucket_3 --
+0 0
+
+-- !bucket_4 --
+5 503
+
+-- !bucket_5 --
+10
+
+-- !bucket_after_compact_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+
+-- !bucket_after_compact_2 --
+0 0
+
+-- !mixed_1 --
+1 1
+2 10
+3 100
+
+-- !mixed_2 --
+1 5
+2 50
+
+-- !mixed_3 --
+2 60
+
+-- !mixed_4 --
+3 200
+
+-- !mixed_5 --
+1 \N
+2 \N
+3 \N
+
+-- !threshold_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !threshold_2 --
+1 2
+2 20
+3 200
+
+-- !threshold_3 --
+2 30
+3 300
+
+-- !threshold_4 --
+3 400
+
+-- !threshold_5 --
+4 5000
+
+-- !threshold_after_compact_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !threshold_after_compact_2 --
+3 400
+
+-- !threshold_after_compact_3 --
+4 5000
+
+-- !toggle_sparse_1 --
+1 1
+2 10
+3 100
+
+-- !toggle_sparse_2 --
+1 2
+2 20
+
+-- !toggle_sparse_3 --
+1 3
+2 30
+
+-- !toggle_sparse_4 --
+1 4
+
+-- !toggle_sparse_5 --
+2 50
+
+-- !toggle_sparse_6 --
+3 600
+
+-- !toggle_sparse_after_compact_1 --
+1 1
+2 10
+3 100
+
+-- !toggle_sparse_after_compact_2 --
+1 4
+
+-- !toggle_sparse_after_compact_3 --
+3
+
+-- !high_cardinality_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !high_cardinality_2 --
+0 0
+
+-- !high_cardinality_3 --
+10 1005
+
+-- !high_cardinality_4 --
+20
+
+-- !high_cardinality_after_compact_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !high_cardinality_after_compact_2 --
+20
+
diff --git a/regression-test/data/variant_p0/query_subcolumns.out b/regression-test/data/variant_p0/query_subcolumns.out
new file mode 100644
index 00000000000..1abc0a027c9
--- /dev/null
+++ b/regression-test/data/variant_p0/query_subcolumns.out
@@ -0,0 +1,211 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
diff --git a/regression-test/suites/query_p0/system/test_table_properties.groovy b/regression-test/suites/query_p0/system/test_table_properties.groovy
index a656108f626..e1bc4859d9b 100644
--- a/regression-test/suites/query_p0/system/test_table_properties.groovy
+++ b/regression-test/suites/query_p0/system/test_table_properties.groovy
@@ -37,7 +37,8 @@ suite("test_table_properties") {
DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
PROPERTIES (
"enable_mow_light_delete" = "false",
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
);
"""
sql """
@@ -52,7 +53,8 @@ suite("test_table_properties") {
)
DISTRIBUTED BY HASH(`type`) BUCKETS 1
PROPERTIES (
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
);
"""
sql """
@@ -80,7 +82,8 @@ suite("test_table_properties") {
DISTRIBUTED BY HASH(`user_id`) BUCKETS 16
PROPERTIES
(
- "replication_num" = "1"
+ "replication_num" = "1",
+ "storage_format" = "V2"
);
"""
diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy
index c2829c8ea37..8f5297bfa29 100644
--- a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy
+++ b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy
@@ -76,7 +76,8 @@ suite("test_show_create_table_and_views_nereids", "show") {
)
DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
PROPERTIES (
- "replication_num" = "1"
+ "replication_num" = "1",
+ "storage_format" = "V2"
)
"""
diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
index 6938948330b..1374160f32a 100644
--- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
+++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
@@ -156,6 +156,20 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){
qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where v["repo"]["name"] match 'xpressengine' order by 1;"""
qt_sql """select count() from github_events where v["repo"]["name"] match 'apache' order by 1;"""
+ sql """ALTER TABLE github_events SET("bloom_filter_columns" = "k,v")"""
+
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='github_events' ORDER BY createtime DESC LIMIT 1"""
+ time 600
+ }
+
+ sql """ALTER TABLE github_events2 SET("bloom_filter_columns" = "k,v")"""
+
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='github_events' ORDER BY createtime DESC LIMIT 1"""
+ time 600
+ }
+
// specify schema
// sql "alter table github_events2 modify column v variant<`payload.comment.id`:int,`payload.commits.url`:text,`payload.forkee.has_pages`:tinyint>"
// load_json_data.call("github_events2", """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""")
diff --git a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
new file mode 100644
index 00000000000..ece39caed92
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_storage_format_v2_1") {
+ def tableName = "test_storage_format_v2_1_table"
+
+ // Test 1: Create table with storage_format = V2.1
+ sql "DROP TABLE IF EXISTS test_storage_format_v2_1_table"
+ sql """
+ CREATE TABLE test_storage_format_v2_1_table (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_format" = "V3"
+ );
+ """
+
+ // Insert some data
+ sql """insert into ${tableName} values (1, '{"a": 1, "b": 2, "c": 3}')"""
+ sql """insert into ${tableName} values (2, '{"a": 10, "b": 20, "d": 40}')"""
+ sql """insert into ${tableName} values (3, '{"e": 500, "f": 600}')"""
+
+ // Query to verify data is correct
+ qt_sql1 "select k, v['a'] from ${tableName} where cast(v['a'] as int) is not null order by k"
+ qt_sql2 "select k, v['b'] from ${tableName} where cast(v['b'] as int) is not null order by k"
+ qt_sql3 "select k, v['d'] from ${tableName} where cast(v['d'] as int) is not null order by k"
+ qt_sql4 "select k, v['e'] from ${tableName} where cast(v['e'] as int) is not null order by k"
+
+ // Verify table properties
+ def result = sql "SHOW CREATE TABLE ${tableName}"
+ logger.info("Show create table result: ${result}")
+ assertTrue(result[0][1].contains("\"storage_format\" = \"V3\""),
+ "Table should be created with storage_format V3")
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS ${tableName}"
+
+ // Test 2: Create table without specifying storage_format (should default to V2)
+ def tableName2 = "test_storage_format_default"
+ sql "DROP TABLE IF EXISTS ${tableName2}"
+ sql """
+ CREATE TABLE ${tableName2} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """insert into ${tableName2} values (1, '{"x": 100}')"""
+ qt_sql5 "select k, v['x'] from ${tableName2} order by k"
+
+ sql "DROP TABLE IF EXISTS ${tableName2}"
+
+ // Test 3: Verify that storage_format is case-insensitive
+ def tableName3 = "test_storage_format_case_insensitive"
+ sql "DROP TABLE IF EXISTS ${tableName3}"
+ sql """
+ CREATE TABLE ${tableName3} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_format" = "V3"
+ );
+ """
+
+ sql """insert into ${tableName3} values (1, '{"test": "value"}')"""
+ qt_sql6 "select k, v['test'] from ${tableName3} order by k"
+
+ sql "DROP TABLE IF EXISTS ${tableName3}"
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy
new file mode 100644
index 00000000000..8ba7c427d35
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy
@@ -0,0 +1,307 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_concurrent", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Concurrent reads of external meta
+ sql "DROP TABLE IF EXISTS test_concurrent_read"
+ sql """
+ CREATE TABLE test_concurrent_read (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 4
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data with many subcolumns to stress test external meta loading
+ for (int i = 0; i < 20; i++) {
+ def fields = []
+ for (int j = 0; j < 50; j++) {
+ fields.add("\"field_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_concurrent_read values (${i}, '${json}')"""
+ }
+
+ // Run concurrent queries that will trigger external meta loading
+ def threads = []
+ def results = [].asSynchronized()
+
+ for (int t = 0; t < 5; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ // Each thread queries different fields
+ def fieldIdx = threadId * 10
+ def result = sql """
+ select count(*) from test_concurrent_read
+ where cast(v['field_${fieldIdx}'] as int) is not null
+ """
+ results.add([threadId: threadId, success: true, result: result])
+ } catch (Exception e) {
+ results.add([threadId: threadId, success: false, error: e.message])
+ logger.error("Thread ${threadId} failed: ${e.message}")
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ // Verify all threads succeeded
+ def failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ logger.error("Concurrent read test failed: ${failures}")
+ throw new Exception("Concurrent read test had failures: ${failures}")
+ }
+
+ qt_concurrent_1 "select count(*) from test_concurrent_read"
+ qt_concurrent_2 "select count(distinct k) from test_concurrent_read"
+
+ // Test 2: Multiple tables with external meta
+ for (int tableIdx = 0; tableIdx < 5; tableIdx++) {
+ def tableName = "test_multi_table_${tableIdx}"
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into ${tableName} values (1, '{"table_${tableIdx}": ${tableIdx}}')"""
+ sql """insert into ${tableName} values (2, '{"table_${tableIdx}": ${tableIdx * 10}}')"""
+ }
+
+ // Query all tables concurrently
+ threads = []
+ results = [].asSynchronized()
+
+ for (int t = 0; t < 5; t++) {
+ def tableIdx = t
+ def tableName = "test_multi_table_${tableIdx}"
+ threads.add(Thread.start {
+ try {
+ def result = sql """
+ select k, v['table_${tableIdx}'] from ${tableName} order by k
+ """
+ results.add([tableIdx: tableIdx, success: true, rowCount: result.size()])
+ } catch (Exception e) {
+ results.add([tableIdx: tableIdx, success: false, error: e.message])
+ logger.error("Table ${tableIdx} query failed: ${e.message}")
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Multi-table concurrent test had failures: ${failures}")
+ }
+
+ // Test 3: Concurrent writes and reads
+ sql "DROP TABLE IF EXISTS test_concurrent_write_read"
+ sql """
+ CREATE TABLE test_concurrent_write_read (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 4
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Initial data
+ for (int i = 0; i < 10; i++) {
+ sql """insert into test_concurrent_write_read values (${i}, '{"initial": ${i}}')"""
+ }
+
+ threads = []
+ results = [].asSynchronized()
+
+ // Start reader threads
+ for (int t = 0; t < 3; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 5; i++) {
+ def result = sql """select count(*) from test_concurrent_write_read"""
+ Thread.sleep(100) // Small delay between reads
+ }
+ results.add([type: 'reader', threadId: threadId, success: true])
+ } catch (Exception e) {
+ results.add([type: 'reader', threadId: threadId, success: false, error: e.message])
+ }
+ })
+ }
+
+ // Start writer threads
+ for (int t = 0; t < 2; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 3; i++) {
+ def k = 100 + threadId * 10 + i
+ sql """insert into test_concurrent_write_read values (${k}, '{"writer": ${threadId}}')"""
+ Thread.sleep(150)
+ }
+ results.add([type: 'writer', threadId: threadId, success: true])
+ } catch (Exception e) {
+ results.add([type: 'writer', threadId: threadId, success: false, error: e.message])
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Concurrent write-read test had failures: ${failures}")
+ }
+
+ qt_write_read_1 "select count(*) from test_concurrent_write_read"
+ qt_write_read_2 "select count(*) from test_concurrent_write_read where cast(v['initial'] as int) is not null"
+ qt_write_read_3 "select count(*) from test_concurrent_write_read where cast(v['writer'] as int) is not null"
+
+ // Test 4: Stress test with frequent compaction triggers
+ sql "DROP TABLE IF EXISTS test_compaction_stress"
+ sql """
+ CREATE TABLE test_compaction_stress (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 2
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert multiple small segments
+ for (int i = 0; i < 15; i++) {
+ sql """insert into test_compaction_stress values (${i}, '{"segment": ${i}, "value": ${i * 100}}')"""
+ }
+
+ // Trigger compaction while querying
+ threads = []
+ results = [].asSynchronized()
+
+ // Query thread
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 10; i++) {
+ def result = sql """select count(*) from test_compaction_stress"""
+ Thread.sleep(200)
+ }
+ results.add([type: 'query', success: true])
+ } catch (Exception e) {
+ results.add([type: 'query', success: false, error: e.message])
+ }
+ })
+
+ // Give query thread a head start
+ Thread.sleep(100)
+
+ // Compaction thread
+ threads.add(Thread.start {
+ try {
+ trigger_and_wait_compaction("test_compaction_stress", "full")
+ results.add([type: 'compaction', success: true])
+ } catch (Exception e) {
+ results.add([type: 'compaction', success: false, error: e.message])
+ }
+ })
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Compaction stress test had failures: ${failures}")
+ }
+
+ qt_stress_1 "select count(*) from test_compaction_stress"
+ qt_stress_2 "select k, v['segment'] from test_compaction_stress order by k limit 5"
+
+ // Test 5: Cache behavior - repeated queries should hit cache
+ sql "DROP TABLE IF EXISTS test_cache_behavior"
+ sql """
+ CREATE TABLE test_cache_behavior (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data with many subcolumns
+ def fields = []
+ for (int i = 0; i < 100; i++) {
+ fields.add("\"cache_field_${i}\": ${i}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+
+ for (int i = 0; i < 5; i++) {
+ sql """insert into test_cache_behavior values (${i}, '${json}')"""
+ }
+
+ // First query - should load external meta
+ def startTime = System.currentTimeMillis()
+ sql """select k, v['cache_field_0'], v['cache_field_50'], v['cache_field_99']
+ from test_cache_behavior order by k"""
+ def firstQueryTime = System.currentTimeMillis() - startTime
+
+ // Subsequent queries - should use cached meta (should be faster)
+ startTime = System.currentTimeMillis()
+ for (int i = 0; i < 5; i++) {
+ sql """select k, v['cache_field_0'], v['cache_field_50'], v['cache_field_99']
+ from test_cache_behavior order by k"""
+ }
+ def cachedQueriesTime = System.currentTimeMillis() - startTime
+ def avgCachedTime = cachedQueriesTime / 5
+
+ logger.info("First query time: ${firstQueryTime}ms, Avg cached query time: ${avgCachedTime}ms")
+
+ qt_cache_1 "select count(*) from test_cache_behavior"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_concurrent_read"
+ for (int i = 0; i < 5; i++) {
+ sql "DROP TABLE IF EXISTS test_multi_table_${i}"
+ }
+ sql "DROP TABLE IF EXISTS test_concurrent_write_read"
+ sql "DROP TABLE IF EXISTS test_compaction_stress"
+ sql "DROP TABLE IF EXISTS test_cache_behavior"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy
new file mode 100644
index 00000000000..029200519e9
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy
@@ -0,0 +1,296 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_edge_cases", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Empty subcolumns (only root variant column)
+ sql "DROP TABLE IF EXISTS test_empty_subcolumns"
+ sql """
+ CREATE TABLE test_empty_subcolumns (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_empty_subcolumns values (1, '{}')"""
+ sql """insert into test_empty_subcolumns values (2, 'null')"""
+ sql """insert into test_empty_subcolumns values (3, '[]')"""
+
+ qt_empty_1 "select k, cast(v as string) from test_empty_subcolumns order by k"
+ qt_empty_2 "select k from test_empty_subcolumns where v is null order by k"
+ qt_empty_3 "select k from test_empty_subcolumns where v is not null order by k"
+
+ // Test 2: Many subcolumns (test performance and correctness)
+ sql "DROP TABLE IF EXISTS test_many_subcolumns"
+ sql """
+ CREATE TABLE test_many_subcolumns (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Generate JSON with 100 subcolumns
+ def generateLargeJson = { int numFields ->
+ def fields = []
+ for (int i = 0; i < numFields; i++) {
+ fields.add("\"field_${i}\": ${i}")
+ }
+ return "{" + fields.join(", ") + "}"
+ }
+
+ // external meta enabled by table property
+ // Insert rows with different numbers of subcolumns
+ for (int row = 0; row < 10; row++) {
+ int numFields = (row + 1) * 10 // 10, 20, 30, ..., 100 fields
+ def json = generateLargeJson(numFields)
+ sql """insert into test_many_subcolumns values (${row}, '${json}')"""
+ }
+
+ qt_many_1 "select k, v['field_0'] from test_many_subcolumns order by k"
+ qt_many_2 "select k, v['field_50'] from test_many_subcolumns where cast(v['field_50'] as int) is not null order by k"
+ qt_many_3 "select k, v['field_99'] from test_many_subcolumns where cast(v['field_99'] as int) is not null order by k"
+ qt_many_4 "select count(*) from test_many_subcolumns"
+
+ // Test 3: Mixed old and new format segments
+ sql "DROP TABLE IF EXISTS test_mixed_format"
+ sql """
+ CREATE TABLE test_mixed_format (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Write with old format (V2)
+ sql """insert into test_mixed_format values (1, '{"a": 1, "b": 2}')"""
+ sql """insert into test_mixed_format values (2, '{"a": 10, "c": 3}')"""
+
+ // Switch to new format (V2.1) and write more rows
+ sql """insert into test_mixed_format values (3, '{"a": 100, "d": 4}')"""
+ sql """insert into test_mixed_format values (4, '{"a": 1000, "e": 5}')"""
+
+ // Query should work across all segments
+ qt_mixed_1 "select k, v['a'] from test_mixed_format order by k"
+ qt_mixed_2 "select k, v['b'] from test_mixed_format where cast(v['b'] as int) is not null order by k"
+ qt_mixed_3 "select k, v['c'] from test_mixed_format where cast(v['c'] as int) is not null order by k"
+ qt_mixed_4 "select k, v['d'] from test_mixed_format where cast(v['d'] as int) is not null order by k"
+ qt_mixed_5 "select k, v['e'] from test_mixed_format where cast(v['e'] as int) is not null order by k"
+
+ // Trigger compaction and verify
+ trigger_and_wait_compaction("test_mixed_format", "full")
+
+ qt_mixed_after_compact_1 "select k, v['a'] from test_mixed_format order by k"
+ qt_mixed_after_compact_2 "select count(distinct k) from test_mixed_format"
+
+ // Test 4: Nested structures with external meta
+ sql "DROP TABLE IF EXISTS test_nested_external"
+ sql """
+ CREATE TABLE test_nested_external (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_nested_external values (1, '{"nested": {"level1": {"level2": "value"}}}')"""
+ sql """insert into test_nested_external values (2, '{"nested": {"level1": {"level2": "value2", "level2_b": 123}}}')"""
+ sql """insert into test_nested_external values (3, '{"nested": {"level1": null}}')"""
+ sql """insert into test_nested_external values (4, '{"nested": null}')"""
+
+ qt_nested_1 "select k, v['nested']['level1']['level2'] from test_nested_external order by k"
+ qt_nested_2 "select k, v['nested']['level1']['level2_b'] from test_nested_external where cast(v['nested']['level1']['level2_b'] as int) is not null order by k"
+ qt_nested_3 "select k from test_nested_external where v['nested'] is not null order by k"
+ qt_nested_4 "select k from test_nested_external where v['nested']['level1'] is not null order by k"
+
+ // Test 5: Array types with external meta
+ sql "DROP TABLE IF EXISTS test_array_external"
+ sql """
+ CREATE TABLE test_array_external (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_array_external values (1, '{"arr": [1, 2, 3]}')"""
+ sql """insert into test_array_external values (2, '{"arr": [10, 20, 30, 40]}')"""
+ sql """insert into test_array_external values (3, '{"arr": []}')"""
+ sql """insert into test_array_external values (4, '{"arr": null}')"""
+
+ qt_array_1 "select k, v['arr'] from test_array_external order by k"
+ qt_array_2 "select k from test_array_external where v['arr'] is not null order by k"
+
+ // Test 6: Schema evolution - adding subcolumns over time
+ sql "DROP TABLE IF EXISTS test_schema_evolution"
+ sql """
+ CREATE TABLE test_schema_evolution (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_schema_evolution values (1, '{"a": 1}')"""
+
+ // Segment 2: add field 'b'
+ sql """insert into test_schema_evolution values (2, '{"a": 2, "b": 2}')"""
+
+ // Segment 3: add field 'c'
+ sql """insert into test_schema_evolution values (3, '{"a": 3, "b": 3, "c": 3}')"""
+
+ // Segment 4: completely different schema
+ sql """insert into test_schema_evolution values (4, '{"x": 10, "y": 20}')"""
+
+ qt_evolution_1 "select k, v['a'] from test_schema_evolution order by k"
+ qt_evolution_2 "select k, v['b'] from test_schema_evolution order by k"
+ qt_evolution_3 "select k, v['c'] from test_schema_evolution order by k"
+ qt_evolution_4 "select k, v['x'] from test_schema_evolution order by k"
+ qt_evolution_5 "select k, v['y'] from test_schema_evolution order by k"
+
+ // Test 7: Special characters in field names
+ sql "DROP TABLE IF EXISTS test_special_chars"
+ sql """
+ CREATE TABLE test_special_chars (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_special_chars values (1, '{"field-with-dash": 1}')"""
+ sql """insert into test_special_chars values (2, '{"field.with.dot": 2}')"""
+ sql """insert into test_special_chars values (3, '{"field_with_underscore": 3}')"""
+
+ qt_special_1 "select k, v['field-with-dash'] from test_special_chars where cast(v['field-with-dash'] as int) is not null order by k"
+ qt_special_2 "select k, v['field.with.dot'] from test_special_chars where cast(v['field.with.dot'] as int) is not null order by k"
+ qt_special_3 "select k, v['field_with_underscore'] from test_special_chars where cast(v['field_with_underscore'] as int) is not null order by k"
+
+ // Test 8: NULL handling
+ sql "DROP TABLE IF EXISTS test_null_handling"
+ sql """
+ CREATE TABLE test_null_handling (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_null_handling values (1, '{"a": null}')"""
+ sql """insert into test_null_handling values (2, '{"a": 1}')"""
+ sql """insert into test_null_handling values (3, null)"""
+
+ qt_null_1 "select k, v['a'] from test_null_handling order by k"
+ qt_null_2 "select k from test_null_handling where v is null order by k"
+ qt_null_3 "select k from test_null_handling where v['a'] is null order by k"
+ qt_null_4 "select k from test_null_handling where v['a'] is not null order by k"
+
+ // Test 9: Large string values
+ sql "DROP TABLE IF EXISTS test_large_strings"
+ sql """
+ CREATE TABLE test_large_strings (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ def largeString = "x" * 10000 // 10KB string
+ sql """insert into test_large_strings values (1, '{"large_field": "${largeString}"}')"""
+ sql """insert into test_large_strings values (2, '{"large_field": "small"}')"""
+
+ qt_large_1 "select k, length(cast(v['large_field'] as string)) from test_large_strings order by k"
+ qt_large_2 "select k from test_large_strings where length(cast(v['large_field'] as string)) > 100 order by k"
+
+ // Test 10: Config toggle during operation
+ sql "DROP TABLE IF EXISTS test_config_toggle"
+ sql """
+ CREATE TABLE test_config_toggle (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Start with new format (V2.1)
+ sql """insert into test_config_toggle values (1, '{"a": 1}')"""
+
+ // Switch to legacy (V2)
+ sql """insert into test_config_toggle values (2, '{"a": 2, "b": 2}')"""
+
+ // Enable again (V2.1)
+ sql """insert into test_config_toggle values (3, '{"a": 3, "b": 3, "c": 3}')"""
+
+ // All queries should work regardless of config state
+ qt_toggle_1 "select k, v['a'] from test_config_toggle order by k"
+ qt_toggle_2 "select k, v['b'] from test_config_toggle where cast(v['b'] as int) is not null order by k"
+ qt_toggle_3 "select k, v['c'] from test_config_toggle where cast(v['c'] as int) is not null order by k"
+
+ // Compact and verify
+ trigger_and_wait_compaction("test_config_toggle", "full")
+
+ qt_toggle_after_compact_1 "select k, v['a'] from test_config_toggle order by k"
+ qt_toggle_after_compact_2 "select k, v['b'] from test_config_toggle where cast(v['b'] as int) is not null order by k"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_empty_subcolumns"
+ sql "DROP TABLE IF EXISTS test_many_subcolumns"
+ sql "DROP TABLE IF EXISTS test_mixed_format"
+ sql "DROP TABLE IF EXISTS test_nested_external"
+ sql "DROP TABLE IF EXISTS test_array_external"
+ sql "DROP TABLE IF EXISTS test_schema_evolution"
+ sql "DROP TABLE IF EXISTS test_special_chars"
+ sql "DROP TABLE IF EXISTS test_null_handling"
+ sql "DROP TABLE IF EXISTS test_large_strings"
+ sql "DROP TABLE IF EXISTS test_config_toggle"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy
new file mode 100644
index 00000000000..fa68799b041
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy
@@ -0,0 +1,355 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_integration", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ sql "set enable_match_without_inverted_index = false"
+ sql "set enable_common_expr_pushdown_for_inverted_index = true"
+ sql "set enable_common_expr_pushdown = true"
+
+ // Test 1: External meta with inverted index on extracted columns
+ sql "DROP TABLE IF EXISTS test_inverted_index_extracted"
+ sql """
+ CREATE TABLE test_inverted_index_extracted (
+ k bigint,
+ v variant<'name':string, 'age':int>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english")
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_inverted_index_extracted values (1, '{"name": "Alice Smith", "age": 25}')"""
+ sql """insert into test_inverted_index_extracted values (2, '{"name": "Bob Johnson", "age": 30}')"""
+ sql """insert into test_inverted_index_extracted values (3, '{"name": "Charlie Smith", "age": 35}')"""
+
+ // Test inverted index on name field (should work with external meta)
+ qt_inverted_1 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Smith' order by k"
+ qt_inverted_2 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Alice' order by k"
+ qt_inverted_3 "select k, v['age'] from test_inverted_index_extracted where cast(v['age'] as int) > 25 order by k"
+
+ // After compaction, inverted index should still work
+ trigger_and_wait_compaction("test_inverted_index_extracted", "full")
+
+ qt_inverted_after_compact_1 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Smith' order by k"
+ qt_inverted_after_compact_2 "select count(*) from test_inverted_index_extracted where v['name'] match 'Johnson'"
+
+ // Test 2: External meta with parent column inverted index
+ sql "DROP TABLE IF EXISTS test_parent_index"
+ sql """
+ CREATE TABLE test_parent_index (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "5")>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english")
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_parent_index values (1, '{"title": "Hello World", "content": "This is a test"}')"""
+ sql """insert into test_parent_index values (2, '{"title": "Test Document", "content": "Another test here"}')"""
+ sql """insert into test_parent_index values (3, '{"title": "World News", "content": "Breaking news today"}')"""
+
+ // Query with inherited inverted index
+ qt_parent_idx_1 "select k, v['title'] from test_parent_index where v['title'] match 'World' order by k"
+ qt_parent_idx_2 "select k, v['content'] from test_parent_index where v['content'] match 'test' order by k"
+ qt_parent_idx_3 "select count(*) from test_parent_index where v['title'] match 'Document'"
+
+ // Test 3: Multiple variant columns with external meta
+ sql "DROP TABLE IF EXISTS test_multiple_variants"
+ sql """
+ CREATE TABLE test_multiple_variants (
+ k bigint,
+ v1 variant,
+ v2 variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_multiple_variants values (1, '{"a": 1, "b": 2}', '{"x": 10, "y": 20}')"""
+ sql """insert into test_multiple_variants values (2, '{"a": 100, "c": 300}', '{"x": 1000, "z": 3000}')"""
+ sql """insert into test_multiple_variants values (3, '{"d": 4}', '{"w": 40}')"""
+
+ // Both variant columns should use external meta independently
+ qt_multi_var_1 "select k, v1['a'], v2['x'] from test_multiple_variants order by k"
+ qt_multi_var_2 "select k, v1['b'] from test_multiple_variants where cast(v1['b'] as int) is not null order by k"
+ qt_multi_var_3 "select k, v2['y'] from test_multiple_variants where cast(v2['y'] as int) is not null order by k"
+ qt_multi_var_4 "select k, v1['d'], v2['w'] from test_multiple_variants where cast(v1['d'] as int) is not null order by k"
+
+ trigger_and_wait_compaction("test_multiple_variants", "full")
+
+ qt_multi_var_after_compact_1 "select k, v1['a'], v2['x'] from test_multiple_variants order by k"
+ qt_multi_var_after_compact_2 "select count(*) from test_multiple_variants"
+
+ // Test 4: External meta with aggregation functions
+ sql "DROP TABLE IF EXISTS test_aggregation"
+ sql """
+ CREATE TABLE test_aggregation (
+ k bigint,
+ category varchar(30),
+ v variant
+ )
+ DUPLICATE KEY(`k`, `category`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_aggregation values (1, 'A', '{"value": 100, "count": 1}')"""
+ sql """insert into test_aggregation values (2, 'A', '{"value": 200, "count": 2}')"""
+ sql """insert into test_aggregation values (3, 'B', '{"value": 150, "count": 1}')"""
+ sql """insert into test_aggregation values (4, 'B', '{"value": 250, "count": 3}')"""
+
+ qt_agg_1 "select category, sum(cast(v['value'] as int)) from test_aggregation group by category order by category"
+ qt_agg_2 "select category, avg(cast(v['count'] as double)) from test_aggregation group by category order by category"
+ qt_agg_3 "select count(distinct category) from test_aggregation where cast(v['value'] as int) > 100"
+
+ // Test 5: External meta with JOIN operations
+ sql "DROP TABLE IF EXISTS test_join_left"
+ sql "DROP TABLE IF EXISTS test_join_right"
+
+ sql """
+ CREATE TABLE test_join_left (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """
+ CREATE TABLE test_join_right (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_join_left values (1, '{"name": "Alice", "dept_id": 10}')"""
+ sql """insert into test_join_left values (2, '{"name": "Bob", "dept_id": 20}')"""
+ sql """insert into test_join_left values (3, '{"name": "Charlie", "dept_id": 10}')"""
+
+ sql """insert into test_join_right values (10, '{"dept_name": "Engineering", "location": "Building A"}')"""
+ sql """insert into test_join_right values (20, '{"dept_name": "Sales", "location": "Building B"}')"""
+
+ qt_join_1 """
+ select l.k, l.v['name'], r.v['dept_name']
+ from test_join_left l
+ join test_join_right r on cast(l.v['dept_id'] as int) = r.k
+ order by l.k
+ """
+
+ qt_join_2 """
+ select count(*)
+ from test_join_left l
+ join test_join_right r on cast(l.v['dept_id'] as int) = r.k
+ where r.v['location'] = 'Building A'
+ """
+
+ // Test 6: External meta with UNION operations
+ sql "DROP TABLE IF EXISTS test_union_1"
+ sql "DROP TABLE IF EXISTS test_union_2"
+
+ sql """
+ CREATE TABLE test_union_1 (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """
+ CREATE TABLE test_union_2 (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_union_1 values (1, '{"source": "table1", "value": 100}')"""
+ sql """insert into test_union_1 values (2, '{"source": "table1", "value": 200}')"""
+
+ sql """insert into test_union_2 values (3, '{"source": "table2", "value": 150}')"""
+ sql """insert into test_union_2 values (4, '{"source": "table2", "value": 250}')"""
+
+ qt_union_1 """
+ select k, v['source'], v['value'] from test_union_1
+ union all
+ select k, v['source'], v['value'] from test_union_2
+ order by k
+ """
+
+ qt_union_2 """
+ select cast(v['source'] as string) as src, count(*)
+ from (
+ select v from test_union_1
+ union all
+ select v from test_union_2
+ ) t
+ group by src
+ order by src
+ """
+
+ // Test 7: External meta with subquery and WHERE clause
+ sql "DROP TABLE IF EXISTS test_subquery"
+ sql """
+ CREATE TABLE test_subquery (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ for (int i = 1; i <= 10; i++) {
+ sql """insert into test_subquery values (${i}, '{"score": ${i * 10}, "grade": "${i > 5 ? 'A' : 'B'}"}')"""
+ }
+
+ qt_subquery_1 """
+ select k, v['score'], v['grade']
+ from test_subquery
+ where cast(v['score'] as int) > (select avg(cast(v['score'] as int)) from test_subquery)
+ order by k
+ """
+
+ qt_subquery_2 """
+ select cast(v['grade'] as string) as grade, count(*) as cnt
+ from test_subquery
+ where k in (select k from test_subquery where cast(v['score'] as int) >= 50)
+ group by grade
+ order by grade
+ """
+
+ // Test 8: External meta with HAVING clause
+ sql "DROP TABLE IF EXISTS test_having"
+ sql """
+ CREATE TABLE test_having (
+ k bigint,
+ category varchar(30),
+ v variant
+ )
+ DUPLICATE KEY(`k`, `category`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_having values (1, 'A', '{"amount": 100}')"""
+ sql """insert into test_having values (2, 'A', '{"amount": 200}')"""
+ sql """insert into test_having values (3, 'A', '{"amount": 300}')"""
+ sql """insert into test_having values (4, 'B', '{"amount": 50}')"""
+ sql """insert into test_having values (5, 'B', '{"amount": 60}')"""
+
+ qt_having_1 """
+ select category, sum(cast(v['amount'] as int)) as total
+ from test_having
+ group by category
+ having total > 150
+ order by category
+ """
+
+ // Test 9: External meta with ORDER BY and LIMIT on variant fields
+ sql "DROP TABLE IF EXISTS test_order_limit"
+ sql """
+ CREATE TABLE test_order_limit (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ for (int i = 1; i <= 20; i++) {
+ sql """insert into test_order_limit values (${i}, '{"priority": ${20 - i}, "name": "item_${i}"}')"""
+ }
+
+ qt_order_limit_1 """
+ select k, v['priority'], v['name']
+ from test_order_limit
+ order by cast(v['priority'] as int) desc
+ limit 5
+ """
+
+ qt_order_limit_2 """
+ select k, v['name']
+ from test_order_limit
+ where cast(v['priority'] as int) > 10
+ order by cast(v['priority'] as int)
+ limit 3
+ """
+
+ // Test 10: External meta with DISTINCT on variant fields
+ sql "DROP TABLE IF EXISTS test_distinct"
+ sql """
+ CREATE TABLE test_distinct (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_distinct values (1, '{"category": "A", "value": 100}')"""
+ sql """insert into test_distinct values (2, '{"category": "B", "value": 200}')"""
+ sql """insert into test_distinct values (3, '{"category": "A", "value": 150}')"""
+ sql """insert into test_distinct values (4, '{"category": "C", "value": 100}')"""
+
+ qt_distinct_1 "select distinct cast(v['category'] as string) from test_distinct order by cast(v['category'] as string)"
+ qt_distinct_2 "select distinct cast(v['value'] as int) from test_distinct order by cast(v['value'] as int)"
+ qt_distinct_3 "select count(distinct cast(v['category'] as string)) from test_distinct"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_inverted_index_extracted"
+ sql "DROP TABLE IF EXISTS test_parent_index"
+ sql "DROP TABLE IF EXISTS test_multiple_variants"
+ sql "DROP TABLE IF EXISTS test_aggregation"
+ sql "DROP TABLE IF EXISTS test_join_left"
+ sql "DROP TABLE IF EXISTS test_join_right"
+ sql "DROP TABLE IF EXISTS test_union_1"
+ sql "DROP TABLE IF EXISTS test_union_2"
+ sql "DROP TABLE IF EXISTS test_subquery"
+ sql "DROP TABLE IF EXISTS test_having"
+ sql "DROP TABLE IF EXISTS test_order_limit"
+ sql "DROP TABLE IF EXISTS test_distinct"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy
new file mode 100644
index 00000000000..676366758b1
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_with_sparse", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Sparse columns should be embedded in root's children_columns
+ sql "DROP TABLE IF EXISTS test_sparse_embedded"
+ sql """
+ CREATE TABLE test_sparse_embedded (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "3")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data that will exceed subcolumn limit, creating sparse columns
+ sql """insert into test_sparse_embedded values (1, '{"a": 1, "b": 2, "c": 3}')"""
+ sql """insert into test_sparse_embedded values (2, '{"a": 10, "b": 20, "c": 30, "d": 40}')"""
+ sql """insert into test_sparse_embedded values (3, '{"a": 100, "e": 500, "f": 600, "g": 700}')"""
+
+ // Query extracted columns (should be in external meta)
+ qt_sparse_1 "select k, v['a'] from test_sparse_embedded order by k"
+ qt_sparse_2 "select k, v['b'] from test_sparse_embedded where cast(v['b'] as int) is not null order by k"
+ qt_sparse_3 "select k, v['c'] from test_sparse_embedded where cast(v['c'] as int) is not null order by k"
+
+ // Query sparse columns (should be embedded in root's children_columns)
+ qt_sparse_4 "select k, v['d'] from test_sparse_embedded where cast(v['d'] as int) is not null order by k"
+ qt_sparse_5 "select k, v['e'] from test_sparse_embedded where cast(v['e'] as int) is not null order by k"
+ qt_sparse_6 "select k, v['f'] from test_sparse_embedded where cast(v['f'] as int) is not null order by k"
+ qt_sparse_7 "select k, v['g'] from test_sparse_embedded where cast(v['g'] as int) is not null order by k"
+
+ // Compaction should handle both extracted and sparse columns correctly
+ trigger_and_wait_compaction("test_sparse_embedded", "full")
+
+ qt_sparse_after_compact_1 "select k, v['a'] from test_sparse_embedded order by k"
+ qt_sparse_after_compact_2 "select k, v['d'] from test_sparse_embedded where cast(v['d'] as int) is not null order by k"
+ qt_sparse_after_compact_3 "select k, v['g'] from test_sparse_embedded where cast(v['g'] as int) is not null order by k"
+
+ // Test 2: Bucketed sparse columns with external meta
+ sql "DROP TABLE IF EXISTS test_bucket_sparse"
+ sql """
+ CREATE TABLE test_bucket_sparse (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "2")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Create multiple rows that will generate bucketed sparse columns
+ for (int i = 0; i < 10; i++) {
+ def fields = ["\"extracted_1\": ${i}", "\"extracted_2\": ${i * 10}"]
+ // Add varying sparse fields
+ for (int j = 0; j < 5; j++) {
+ fields.add("\"sparse_${i}_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_bucket_sparse values (${i}, '${json}')"""
+ }
+
+ // Query extracted columns
+ qt_bucket_1 "select k, v['extracted_1'] from test_bucket_sparse order by k"
+ qt_bucket_2 "select k, v['extracted_2'] from test_bucket_sparse order by k"
+
+ // Query sparse columns from different rows
+ qt_bucket_3 "select k, v['sparse_0_0'] from test_bucket_sparse where cast(v['sparse_0_0'] as int) is not null order by k"
+ qt_bucket_4 "select k, v['sparse_5_3'] from test_bucket_sparse where cast(v['sparse_5_3'] as int) is not null order by k"
+ qt_bucket_5 "select count(*) from test_bucket_sparse"
+
+ // Compact and verify buckets are handled correctly
+ trigger_and_wait_compaction("test_bucket_sparse", "full")
+
+ qt_bucket_after_compact_1 "select k, v['extracted_1'] from test_bucket_sparse order by k"
+ qt_bucket_after_compact_2 "select k, v['sparse_0_0'] from test_bucket_sparse where cast(v['sparse_0_0'] as int) is not null order by k"
+
+ // Test 3: Mixed extracted, sparse, and non-existent columns
+ sql "DROP TABLE IF EXISTS test_mixed_column_types"
+ sql """
+ CREATE TABLE test_mixed_column_types (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "5")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Row 1: Only extracted columns
+ sql """insert into test_mixed_column_types values (1, '{"e1": 1, "e2": 2, "e3": 3, "e4": 4, "e5": 5}')"""
+
+ // Row 2: Extracted + sparse
+ sql """insert into test_mixed_column_types values (2, '{"e1": 10, "e2": 20, "e3": 30, "e4": 40, "e5": 50, "s1": 60, "s2": 70}')"""
+
+ // Row 3: Different set of fields
+ sql """insert into test_mixed_column_types values (3, '{"e1": 100, "new1": 200, "new2": 300}')"""
+
+ // Query all types
+ qt_mixed_1 "select k, v['e1'] from test_mixed_column_types order by k" // Exists in all rows (extracted)
+ qt_mixed_2 "select k, v['e5'] from test_mixed_column_types where cast(v['e5'] as int) is not null order by k" // Exists in row 1,2 (extracted)
+ qt_mixed_3 "select k, v['s1'] from test_mixed_column_types where cast(v['s1'] as int) is not null order by k" // Exists in row 2 (sparse)
+ qt_mixed_4 "select k, v['new1'] from test_mixed_column_types where cast(v['new1'] as int) is not null order by k" // Exists in row 3 (sparse or extracted)
+ qt_mixed_5 "select k, v['non_existent'] from test_mixed_column_types order by k" // Doesn't exist anywhere
+
+ // Test 4: Transition from extracted to sparse
+ sql "DROP TABLE IF EXISTS test_extraction_threshold"
+ sql """
+ CREATE TABLE test_extraction_threshold (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "3")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Segment 1: Within extraction limit
+ sql """insert into test_extraction_threshold values (1, '{"field_a": 1, "field_b": 2}')"""
+ sql """insert into test_extraction_threshold values (2, '{"field_a": 10, "field_b": 20, "field_c": 30}')"""
+
+ // Segment 2: Exceeds limit, triggers sparse columns
+ sql """insert into test_extraction_threshold values (3, '{"field_a": 100, "field_b": 200, "field_c": 300, "field_d": 400}')"""
+ sql """insert into test_extraction_threshold values (4, '{"field_a": 1000, "field_e": 5000}')"""
+
+ // All fields should be queryable
+ qt_threshold_1 "select k, v['field_a'] from test_extraction_threshold order by k"
+ qt_threshold_2 "select k, v['field_b'] from test_extraction_threshold where cast(v['field_b'] as int) is not null order by k"
+ qt_threshold_3 "select k, v['field_c'] from test_extraction_threshold where cast(v['field_c'] as int) is not null order by k"
+ qt_threshold_4 "select k, v['field_d'] from test_extraction_threshold where cast(v['field_d'] as int) is not null order by k"
+ qt_threshold_5 "select k, v['field_e'] from test_extraction_threshold where cast(v['field_e'] as int) is not null order by k"
+
+ // After compaction, schema should be merged correctly
+ trigger_and_wait_compaction("test_extraction_threshold", "full")
+
+ qt_threshold_after_compact_1 "select k, v['field_a'] from test_extraction_threshold order by k"
+ qt_threshold_after_compact_2 "select k, v['field_d'] from test_extraction_threshold where cast(v['field_d'] as int) is not null order by k"
+ qt_threshold_after_compact_3 "select k, v['field_e'] from test_extraction_threshold where cast(v['field_e'] as int) is not null order by k"
+
+ // Test 5: Sparse columns with config toggle
+ sql "DROP TABLE IF EXISTS test_sparse_config_toggle"
+ sql """
+ CREATE TABLE test_sparse_config_toggle (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "2")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Write with external meta enabled (V2.1)
+ sql """insert into test_sparse_config_toggle values (1, '{"a": 1, "b": 2, "c": 3, "d": 4}')"""
+
+ // Switch to legacy V2
+ sql """insert into test_sparse_config_toggle values (2, '{"a": 10, "b": 20, "c": 30, "e": 50}')"""
+
+ // Enable external meta again (V2.1)
+ sql """insert into test_sparse_config_toggle values (3, '{"a": 100, "f": 600, "g": 700}')"""
+
+ // All queries should work regardless of which format was used
+ qt_toggle_sparse_1 "select k, v['a'] from test_sparse_config_toggle order by k"
+ qt_toggle_sparse_2 "select k, v['b'] from test_sparse_config_toggle where cast(v['b'] as int) is not null order by k"
+ qt_toggle_sparse_3 "select k, v['c'] from test_sparse_config_toggle where cast(v['c'] as int) is not null order by k"
+ qt_toggle_sparse_4 "select k, v['d'] from test_sparse_config_toggle where cast(v['d'] as int) is not null order by k"
+ qt_toggle_sparse_5 "select k, v['e'] from test_sparse_config_toggle where cast(v['e'] as int) is not null order by k"
+ qt_toggle_sparse_6 "select k, v['f'] from test_sparse_config_toggle where cast(v['f'] as int) is not null order by k"
+
+ // Compact
+ trigger_and_wait_compaction("test_sparse_config_toggle", "full")
+
+ qt_toggle_sparse_after_compact_1 "select k, v['a'] from test_sparse_config_toggle order by k"
+ qt_toggle_sparse_after_compact_2 "select k, v['d'] from test_sparse_config_toggle where cast(v['d'] as int) is not null order by k"
+ qt_toggle_sparse_after_compact_3 "select count(*) from test_sparse_config_toggle"
+
+ // Test 6: Very high cardinality sparse columns
+ sql "DROP TABLE IF EXISTS test_high_cardinality_sparse"
+ sql """
+ CREATE TABLE test_high_cardinality_sparse (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "1")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Each row has one extracted column and many unique sparse columns
+ for (int i = 0; i < 20; i++) {
+ def fields = ["\"common\": ${i}"]
+ // Each row has unique sparse fields
+ for (int j = 0; j < 10; j++) {
+ fields.add("\"unique_${i}_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_high_cardinality_sparse values (${i}, '${json}')"""
+ }
+
+ // Query common extracted field
+ qt_high_cardinality_1 "select k, v['common'] from test_high_cardinality_sparse order by k limit 5"
+
+ // Query sparse fields (each exists in only one row)
+ qt_high_cardinality_2 "select k, v['unique_0_0'] from test_high_cardinality_sparse where cast(v['unique_0_0'] as int) is not null order by k"
+ qt_high_cardinality_3 "select k, v['unique_10_5'] from test_high_cardinality_sparse where cast(v['unique_10_5'] as int) is not null order by k"
+ qt_high_cardinality_4 "select count(*) from test_high_cardinality_sparse"
+
+ // Compaction with high cardinality sparse columns
+ trigger_and_wait_compaction("test_high_cardinality_sparse", "full")
+
+ qt_high_cardinality_after_compact_1 "select k, v['common'] from test_high_cardinality_sparse order by k limit 5"
+ qt_high_cardinality_after_compact_2 "select count(*) from test_high_cardinality_sparse"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_sparse_embedded"
+ sql "DROP TABLE IF EXISTS test_bucket_sparse"
+ sql "DROP TABLE IF EXISTS test_mixed_column_types"
+ sql "DROP TABLE IF EXISTS test_extraction_threshold"
+ sql "DROP TABLE IF EXISTS test_sparse_config_toggle"
+ sql "DROP TABLE IF EXISTS test_high_cardinality_sparse"
+
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy b/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
index d47c486047e..5b129ac1363 100644
--- a/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
+++ b/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
@@ -130,8 +130,8 @@ suite("test_compaction_variant_predefine_with_sparse_limit", "nonConcurrent") {
order_qt_select "select * from ${tableName} order by k, cast(v as string) limit 5;"
}
} finally {
- sql "DROP TABLE IF EXISTS simple_variant_DUPLICATE"
- sql "DROP TABLE IF EXISTS simple_variant_UNIQUE"
- sql "DROP TABLE IF EXISTS simple_variant_AGGREGATE"
+ // sql "DROP TABLE IF EXISTS simple_variant_DUPLICATE"
+ // sql "DROP TABLE IF EXISTS simple_variant_UNIQUE"
+ // sql "DROP TABLE IF EXISTS simple_variant_AGGREGATE"
}
}
diff --git a/regression-test/suites/variant_p0/query_subcolumns.groovy b/regression-test/suites/variant_p0/query_subcolumns.groovy
new file mode 100644
index 00000000000..5590bcaab6a
--- /dev/null
+++ b/regression-test/suites/variant_p0/query_subcolumns.groovy
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("regression_test_query_subcolumns", "nonConcurrent"){
+
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+ sql """DROP TABLE IF EXISTS query_subcolumns"""
+
+ // query must use inverted index for match operator
+ sql """ set enable_common_expr_pushdown = true; """
+ sql "set enable_match_without_inverted_index = false"
+ for (int i = 0; i < 5; i++) {
+ int max_subcolumns_count = Math.floor(Math.random() * 10)
+ // int max_subcolumns_count =1
+ def var = """variant<properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") """
+ if (max_subcolumns_count % 2 == 0) {
+ var = """variant <'d' : int, 'b' : int, 'e' : string, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")>,
+ INDEX idx_v_e (v) USING INVERTED PROPERTIES("field_pattern" = "e", "parser" = "english", "support_phrase" = "true"),
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") """
+ }
+ sql "DROP TABLE IF EXISTS query_subcolumns"
+ sql """
+ CREATE TABLE IF NOT EXISTS query_subcolumns (
+ k bigint,
+ v ${var}
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+ sql """insert into query_subcolumns values (1, '{"a" : 1, "b" : "2"}')"""
+ // legacy V2 write
+ sql """insert into query_subcolumns values (2, '{"a" : 1, "b" : "2", "c" : 3}')"""
+ // switch to V2.1
+ sql """insert into query_subcolumns values (3, '{"a" : 1, "b" : "2", "c" : 3, "d" : 4}')"""
+ // keep V2.1
+ sql """insert into query_subcolumns values (4, '{"a" : 1}')"""
+ // switch back to V2
+ sql """insert into query_subcolumns values (5, '{"e" : "hello, world"}')"""
+ // and again V2.1
+ sql """insert into query_subcolumns values (6, '{"f" : "make it work"}')"""
+
+ qt_sql "select v['a'] from query_subcolumns where cast(v['a'] as int) is not null order by k"
+ qt_sql "select v['b'] from query_subcolumns where cast(v['b'] as int) is not null order by k"
+ qt_sql "select v['c'] from query_subcolumns where cast(v['c'] as int) is not null order by k"
+ qt_sql "select v['d'] from query_subcolumns where cast(v['d'] as int) is not null order by k"
+ qt_sql "select v['e'] from query_subcolumns where v['e'] like 'hello%' order by k"
+ if (max_subcolumns_count >= 6) {
+ sql "select v['f'] from query_subcolumns where v['f'] match 'make' order by k"
+ }
+
+ // if (max_subcolumns_count % 2) {
+ // // sql """alter table query_subcolumns set ("storage_format" = "V3")"""
+ // } else {
+ // // sql """alter table query_subcolumns set ("storage_format" = "V2")"""
+ // }
+
+ // triger compaction
+ trigger_and_wait_compaction("query_subcolumns", "full")
+
+ qt_sql "select v['a'] from query_subcolumns where cast(v['a'] as int) is not null order by k"
+ qt_sql "select v['b'] from query_subcolumns where cast(v['b'] as int) is not null order by k"
+ qt_sql "select v['c'] from query_subcolumns where cast(v['c'] as int) is not null order by k"
+ qt_sql "select v['d'] from query_subcolumns where cast(v['d'] as int) is not null order by k"
+ qt_sql "select v['e'] from query_subcolumns where v['e'] like 'hello%' order by k"
+ if (max_subcolumns_count >= 6) {
+ sql "select v['f'] from query_subcolumns where v['f'] match 'make' order by k"
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org