You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2025/11/26 15:31:26 UTC
(doris) 04/04: [feature][segment] Support external meta V3 with common format
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch cs_opt_version-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f6b14986719362c5534e31ae81ecbbe09b289234
Author: eldenmoon <li...@selectdb.com>
AuthorDate: Wed Oct 29 18:02:19 2025 +0800
[feature][segment] Support external meta V3 with common format
---
be/src/cloud/pb_convert.cpp | 4 +
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 3 +-
.../rowset/segment_v2/column_meta_accessor.cpp | 253 +++++++++
.../olap/rowset/segment_v2/column_meta_accessor.h | 93 ++++
be/src/olap/rowset/segment_v2/column_reader.cpp | 26 +-
be/src/olap/rowset/segment_v2/column_reader.h | 18 +-
.../olap/rowset/segment_v2/column_reader_cache.cpp | 180 ++++---
.../olap/rowset/segment_v2/column_reader_cache.h | 37 +-
.../rowset/segment_v2/external_col_meta_util.cpp | 199 +++++++
.../rowset/segment_v2/external_col_meta_util.h | 91 ++++
be/src/olap/rowset/segment_v2/segment.cpp | 120 ++---
be/src/olap/rowset/segment_v2/segment.h | 11 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 16 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 12 +-
be/src/olap/rowset/segment_v2/stream_reader.h | 3 +-
.../segment_v2/variant/variant_column_reader.cpp | 405 +++++++++++---
.../segment_v2/variant/variant_column_reader.h | 62 ++-
.../variant/variant_column_writer_impl.h | 169 ++++++
.../segment_v2/variant/variant_ext_meta_writer.cpp | 199 +++++++
.../segment_v2/variant/variant_ext_meta_writer.h | 99 ++++
.../variant/variant_external_meta_reader.cpp | 246 +++++++++
.../variant/variant_external_meta_reader.h | 95 ++++
.../segment_v2/variant_column_writer_impl.cpp | 8 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 13 +
be/src/olap/tablet_manager.cpp | 3 +-
be/src/olap/tablet_meta.cpp | 30 +-
be/src/olap/tablet_meta.h | 7 +-
be/src/olap/tablet_schema.cpp | 8 +
be/src/olap/tablet_schema.h | 12 +
be/src/service/internal_service.cpp | 4 +
be/src/vec/common/schema_util.cpp | 28 +-
be/src/vec/data_types/data_type_factory.cpp | 12 +
be/test/olap/date_bloom_filter_test.cpp | 19 +-
be/test/olap/delete_bitmap_calculator_test.cpp | 13 +
.../segment_v2/column_meta_accessor_test.cpp | 593 +++++++++++++++++++++
.../rowset/segment_v2/column_reader_cache_test.cpp | 40 +-
.../segment_v2/external_col_meta_util_test.cpp | 517 ++++++++++++++++++
be/test/olap/rowset/segment_v2/mock/mock_segment.h | 6 -
.../variant_column_writer_reader_test.cpp | 129 +++--
be/test/vec/common/schema_util_rowset_test.cpp | 126 ++++-
.../main/java/org/apache/doris/common/Config.java | 241 +++++----
.../src/main/java/org/apache/doris/DorisFE.java | 2 +-
.../apache/doris/alter/SchemaChangeHandler.java | 14 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 7 +-
.../trees/plans/commands/info/CreateTableInfo.java | 19 +-
.../java/org/apache/doris/qe/SessionVariable.java | 10 +
.../org/apache/doris/task/CreateReplicaTask.java | 1 -
gensrc/proto/olap_file.proto | 6 +
gensrc/proto/segment_v2.proto | 18 +
gensrc/thrift/AgentService.thrift | 8 +-
.../ext_meta/test_storage_format_v2_1.out | 21 +
.../test_variant_external_meta_concurrent.out | 29 +
.../test_variant_external_meta_edge_cases.out | 174 ++++++
.../test_variant_external_meta_integration.out | 120 +++++
.../test_variant_external_meta_with_sparse.out | 197 +++++++
.../data/variant_p0/query_subcolumns.out | 211 ++++++++
.../load.groovy | 14 +
regression-test/suites/variant_p0/agg.groovy | 33 +-
.../ext_meta/test_storage_format_v2_1.groovy | 98 ++++
.../test_variant_external_meta_concurrent.groovy | 307 +++++++++++
.../test_variant_external_meta_edge_cases.groovy | 296 ++++++++++
.../test_variant_external_meta_integration.groovy | 355 ++++++++++++
.../test_variant_external_meta_with_sparse.groovy | 251 +++++++++
...est_variant_compaction_with_sparse_limit.groovy | 6 +-
.../suites/variant_p0/query_subcolumns.groovy | 94 ++++
65 files changed, 5900 insertions(+), 541 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 352b9ae9356..7c9ac5f35c9 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -342,6 +342,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_meta_used_default(in.is_external_segment_meta_used_default());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
@@ -372,6 +373,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_meta_used_default(in.is_external_segment_meta_used_default());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
@@ -415,6 +417,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_meta_used_default(in.is_external_segment_meta_used_default());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
@@ -446,6 +449,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
out->set_skip_write_index_on_load(in.skip_write_index_on_load());
out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
out->set_is_dynamic_schema(in.is_dynamic_schema());
+ out->set_is_external_segment_meta_used_default(in.is_external_segment_meta_used_default());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
index 34eb14951ae..0e5c1fdba45 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
@@ -210,7 +210,8 @@ Status BinaryPrefixPageDecoder::seek_at_or_after_value(const void* value, bool*
_cur_pos++;
auto st = _read_next_value();
if (st.is<ErrorCode::END_OF_FILE>()) {
- return Status::Error<ErrorCode::ENTRY_NOT_FOUND>("all value small than the value");
+ return Status::Error<ErrorCode::ENTRY_NOT_FOUND, false>(
+ "all value small than the value");
}
if (!st.ok()) {
return st;
diff --git a/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp b/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp
new file mode 100644
index 00000000000..3009633b9b6
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/column_meta_accessor.cpp
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
+
+#include <cstdint>
+#include <string>
+
+#include "io/io_common.h"
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+#include "olap/rowset/segment_v2/segment.h" // kSegmentFooterVersionV3_ExtColMeta
+
+namespace doris::segment_v2 {
+
+// Abstract base implementation shared by both V2 and V3 layouts.
+class ColumnMetaAccessor::Impl {
+public:
+ virtual ~Impl() = default;
+ // Build uid -> col_id mapping based on the provided footer.
+ virtual Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const = 0;
+ // Lookup ColumnMetaPB by logical ordinal id using the provided footer.
+ virtual Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const = 0;
+ // Traverse all ColumnMetaPBs using the provided footer.
+ virtual Status traverse_metas(
+ const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const = 0;
+};
+
+// V2: inline layout using only footer.columns().
+class ColumnMetaAccessorV2 : public ColumnMetaAccessor::Impl {
+public:
+ explicit ColumnMetaAccessorV2(io::FileReaderSPtr file_reader)
+ : _file_reader(std::move(file_reader)) {}
+
+ Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const override {
+ uid_to_colid->clear();
+ uint32_t ordinal = 0;
+ for (const auto& column_meta : footer.columns()) {
+ // 跳过 variant 子列（unique_id == -1）
+ if (column_meta.unique_id() == -1) {
+ ordinal++;
+ continue;
+ }
+ uid_to_colid->try_emplace(column_meta.unique_id(), ordinal++);
+ }
+ return Status::OK();
+ }
+
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const override {
+ if (footer.columns_size() == 0 ||
+ column_ordinal_id >= static_cast<uint32_t>(footer.columns_size())) {
+ return Status::Corruption(
+ "no inline column meta available for column_id={} (columns_size={})",
+ column_ordinal_id, footer.columns_size());
+ }
+ *out = footer.columns(static_cast<int>(column_ordinal_id));
+ return Status::OK();
+ }
+
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const override {
+ if (footer.columns_size() == 0) {
+ return Status::Corruption("no column meta found in footer (inline V2)");
+ }
+ for (const auto& column : footer.columns()) {
+ visitor(column);
+ }
+ return Status::OK();
+ }
+
+private:
+ io::FileReaderSPtr _file_reader;
+};
+
+// V3: use external Column Meta Region + column_meta_entries layout.
+class ColumnMetaAccessorV3 : public ColumnMetaAccessor::Impl {
+public:
+ ColumnMetaAccessorV3(io::FileReaderSPtr file_reader,
+ const ExternalColMetaUtil::ExternalMetaPointers& ptrs)
+ : _file_reader(std::move(file_reader)), _ptrs(ptrs) {}
+
+ Status build_uid_to_colid_map(
+ const SegmentFooterPB& footer,
+ std::unordered_map<int32_t, size_t>* uid_to_colid) const override {
+ uid_to_colid->clear();
+
+ // Prefer external uid->col_id map when available.
+ Status st = ExternalColMetaUtil::parse_uid_to_colid_map(footer, _ptrs, uid_to_colid);
+
+ // Compatibility: if external map is broken, fall back to inline columns.
+ if (!st.ok() && footer.columns_size() > 0) {
+ uint32_t ordinal = 0;
+ for (const auto& column_meta : footer.columns()) {
+ if (column_meta.unique_id() == -1) {
+ ordinal++;
+ continue;
+ }
+ uid_to_colid->try_emplace(column_meta.unique_id(), ordinal++);
+ }
+ return Status::OK();
+ }
+
+ if (!st.ok()) {
+ return Status::Corruption(
+ "segment external meta mapping missing or corrupted and no inline columns; "
+ "uid->col_id cannot be resolved. path={}, error={}",
+ _file_reader ? _file_reader->path().native() : std::string("<unknown>"),
+ st.to_string());
+ }
+
+ return Status::OK();
+ }
+
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const override {
+ // Prefer external Column Meta Region first.
+ if (column_ordinal_id < _ptrs.num_columns) {
+ return ExternalColMetaUtil::read_col_meta(_file_reader, footer, _ptrs,
+ column_ordinal_id, out);
+ }
+
+ // Fallback: inline footer.columns().
+ if (footer.columns_size() > 0 &&
+ column_ordinal_id < static_cast<uint32_t>(footer.columns_size())) {
+ *out = footer.columns(static_cast<int>(column_ordinal_id));
+ return Status::OK();
+ }
+
+ return Status::Corruption(
+ "no column meta available for column_id={} (inline/external missing)",
+ column_ordinal_id);
+ }
+
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const override {
+ const uint64_t region_size = (_ptrs.region_end > _ptrs.region_start)
+ ? (_ptrs.region_end - _ptrs.region_start)
+ : 0;
+ if (region_size == 0) {
+ return Status::Corruption("invalid external meta region size");
+ }
+
+ // Read entire meta region once to reduce random I/O.
+ std::string region_buf;
+ region_buf.resize(static_cast<size_t>(region_size));
+ size_t br = 0;
+ io::IOContext io_ctx {.is_index_data = true};
+ RETURN_IF_ERROR(_file_reader->read_at(_ptrs.region_start, Slice(region_buf), &br, &io_ctx));
+ if (br != region_size) {
+ return Status::Corruption("short read on meta region");
+ }
+
+ if (footer.column_meta_entries_size() != _ptrs.num_columns) {
+ return Status::Corruption("column_meta_entries size mismatch");
+ }
+
+ uint64_t offset = _ptrs.region_start;
+ for (uint32_t i = 0; i < _ptrs.num_columns; ++i) {
+ const auto& entry = footer.column_meta_entries(static_cast<int>(i));
+ const auto sz = static_cast<uint64_t>(entry.length());
+ const uint64_t pos = offset;
+ if (!ExternalColMetaUtil::is_valid_meta_slice(pos, sz, _ptrs)) {
+ return Status::Corruption("external meta entry out of region bounds");
+ }
+ const uint64_t rel = pos - _ptrs.region_start;
+ ColumnMetaPB meta;
+ if (!meta.ParseFromArray(region_buf.data() + rel, static_cast<int>(sz))) {
+ return Status::Corruption("failed parse ColumnMetaPB from region");
+ }
+ visitor(meta);
+ offset += sz;
+ }
+ return Status::OK();
+ }
+
+private:
+ io::FileReaderSPtr _file_reader;
+ ExternalColMetaUtil::ExternalMetaPointers _ptrs;
+};
+
+ColumnMetaAccessor::ColumnMetaAccessor() = default;
+ColumnMetaAccessor::~ColumnMetaAccessor() = default;
+ColumnMetaAccessor::ColumnMetaAccessor(ColumnMetaAccessor&&) noexcept = default;
+ColumnMetaAccessor& ColumnMetaAccessor::operator=(ColumnMetaAccessor&&) noexcept = default;
+
+Status ColumnMetaAccessor::init(const SegmentFooterPB& footer, io::FileReaderSPtr file_reader) {
+ // First check footer version to see if external Column Meta Region might exist,
+ // then try to parse external layout; if parsing fails, fall back to V2.
+ if (footer.version() >= kSegmentFooterVersionV3_ExtColMeta) {
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs));
+ _impl = std::make_unique<ColumnMetaAccessorV3>(std::move(file_reader), ptrs);
+ } else {
+ _impl = std::make_unique<ColumnMetaAccessorV2>(std::move(file_reader));
+ }
+ // Build inner uid -> column ordinal id mapping once during init.
+ _column_uid_to_column_ordinal.clear();
+ RETURN_IF_ERROR(_impl->build_uid_to_colid_map(footer, &_column_uid_to_column_ordinal));
+ return Status::OK();
+}
+
+Status ColumnMetaAccessor::get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const {
+ return _impl->get_column_meta_by_column_ordinal_id(footer, column_ordinal_id, out);
+}
+
+Status ColumnMetaAccessor::get_column_meta_by_uid(const SegmentFooterPB& footer, int32_t column_uid,
+ ColumnMetaPB* out) const {
+ auto it = _column_uid_to_column_ordinal.find(column_uid);
+ if (it == _column_uid_to_column_ordinal.end()) {
+ *out = ColumnMetaPB();
+ return Status::Error<ErrorCode::NOT_FOUND, false>(
+ "column not found in segment meta, col_uid={}", column_uid);
+ }
+ return _impl->get_column_meta_by_column_ordinal_id(footer, static_cast<uint32_t>(it->second),
+ out);
+}
+
+bool ColumnMetaAccessor::has_column_uid(int32_t column_uid) const {
+ return _column_uid_to_column_ordinal.contains(column_uid);
+}
+
+Status ColumnMetaAccessor::traverse_metas(
+ const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const {
+ return _impl->traverse_metas(footer, visitor);
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/column_meta_accessor.h b/be/src/olap/rowset/segment_v2/column_meta_accessor.h
new file mode 100644
index 00000000000..a99f8f314dc
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/column_meta_accessor.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <unordered_map>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+
+namespace doris::segment_v2 {
+
+// Centralized accessor for column metadata layout of a Segment.
+// It selects V2 (inline footer.columns) or V3 (external Column Meta Region)
+// implementation internally and exposes a unified API to callers, hiding
+// the physical layout (inline vs external CMO).
+class ColumnMetaAccessor {
+public:
+ class Impl;
+ ColumnMetaAccessor();
+ ~ColumnMetaAccessor();
+ ColumnMetaAccessor(ColumnMetaAccessor&&) noexcept;
+ ColumnMetaAccessor& operator=(ColumnMetaAccessor&&) noexcept;
+ ColumnMetaAccessor(const ColumnMetaAccessor&) = delete;
+ ColumnMetaAccessor& operator=(const ColumnMetaAccessor&) = delete;
+ // Initialize accessor and choose V2 / V3 implementation based on footer.
+ //
+ // The caller is responsible for ensuring that `footer` outlives this accessor.
+ Status init(const SegmentFooterPB& footer, io::FileReaderSPtr file_reader);
+
+ // Get ColumnMetaPB for the given column ordinal id using the provided footer.
+ //
+ // Semantics:
+ // - In legacy V2 segments, column_ordinal_id is the index into footer.columns().
+ // - In V3 segments with external CMO, column_ordinal_id indexes the external
+ // Column Meta Region in column order; when external meta is missing or
+ // we fall back to inline columns, the same logical ordinal is reused.
+ //
+ // Error convention:
+ // - If the ordinal is out of range or the corresponding meta is missing/corrupted,
+ // returns ErrorCode::CORRUPTION.
+ Status get_column_meta_by_column_ordinal_id(const SegmentFooterPB& footer,
+ uint32_t column_ordinal_id,
+ ColumnMetaPB* out) const;
+
+ // Get ColumnMetaPB for the given column unique id using the provided footer.
+ // column_uid is TabletColumn.unique_id / ColumnMetaPB.unique_id.
+ //
+ // Mapping scope:
+ // - Only top-level physical columns (including Variant root columns) are indexed.
+ // - Variant extracted subcolumns (unique_id == -1) and path subcolumns are excluded.
+ //
+ // Error convention:
+ // - If the uid does not exist in this segment (no physical column for this uid),
+ // returns ErrorCode::NOT_FOUND.
+ Status get_column_meta_by_uid(const SegmentFooterPB& footer, int32_t column_uid,
+ ColumnMetaPB* out) const;
+
+ // Fast existence check for a given column unique id.
+ //
+ // Only checks the in-memory uid -> ordinal map; does not trigger any I/O and
+ // does not parse ColumnMetaPB. Semantics are consistent with
+ // get_column_meta_by_uid(...) in terms of uid presence (NOT_FOUND vs OK).
+ bool has_column_uid(int32_t column_uid) const;
+
+ // Traverse all ColumnMetaPBs using the provided footer.
+ Status traverse_metas(const SegmentFooterPB& footer,
+ const std::function<void(const ColumnMetaPB&)>& visitor) const;
+
+private:
+ std::unique_ptr<Impl> _impl;
+ // map column unique id ---> column ordinal id (footer ordinal in legacy mode, or external CMO column id)
+ std::unordered_map<int32_t, size_t> _column_uid_to_column_ordinal;
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 4fe1a0cae57..698b6730d98 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -35,9 +35,7 @@
#include "io/io_common.h"
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
-#include "olap/comparison_predicate.h"
#include "olap/decimal12.h"
-#include "olap/inverted_index_parser.h"
#include "olap/iterators.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/binary_dict_page.h" // for BinaryDictPageDecoder
@@ -45,6 +43,7 @@
#include "olap/rowset/segment_v2/bitmap_index_reader.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
#include "olap/rowset/segment_v2/bloom_filter_index_reader.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo
#include "olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h"
#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
@@ -224,28 +223,6 @@ bool ColumnReader::is_compaction_reader_type(ReaderType type) {
type == ReaderType::READER_FULL_COMPACTION;
}
-Status ColumnReader::create_variant(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader) {
- std::unique_ptr<VariantColumnReader> reader_local(new VariantColumnReader());
- RETURN_IF_ERROR(reader_local->init(opts, footer, column_id, num_rows, file_reader));
- *reader = std::move(reader_local);
- return Status::OK();
-}
-
-Status ColumnReader::create(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader) {
- // create normal column reader or variant subcolumn reader with extracted columns info in footer
- if ((FieldType)footer.columns(column_id).type() != FieldType::OLAP_FIELD_TYPE_VARIANT) {
- return ColumnReader::create(opts, footer.columns(column_id), num_rows, file_reader, reader);
- }
- // create variant column reader with extracted columns info in footer with hierarchical data info
- return create_variant(opts, footer, column_id, num_rows, file_reader, reader);
-}
-
Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader) {
@@ -299,6 +276,7 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
_meta_children_column_type = (FieldType)meta.children_columns(0).type();
}
+ _data_type = vectorized::DataTypeFactory::instance().create_data_type(meta);
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 76dd06ccbbc..45a85c6210f 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -69,7 +69,6 @@ struct Slice;
struct StringRef;
namespace segment_v2 {
-
class EncodingInfo;
class ColumnIterator;
class BloomFilterIndexReader;
@@ -81,7 +80,6 @@ class InvertedIndexFileReader;
class PageDecoder;
class RowRanges;
class ZoneMapIndexReader;
-struct VariantStatistics;
struct ColumnReaderOptions {
// whether verify checksum when read page
@@ -133,10 +131,7 @@ public:
static Status create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
- static Status create(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader);
+
static Status create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
@@ -149,10 +144,7 @@ public:
static Status create_agg_state(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader);
- static Status create_variant(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- const io::FileReaderSPtr& file_reader,
- std::shared_ptr<ColumnReader>* reader);
+
enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING, ALL_DICT_ENCODING };
static bool is_compaction_reader_type(ReaderType type);
@@ -234,11 +226,15 @@ public:
void disable_index_meta_cache() { _use_index_page_cache = false; }
+ vectorized::DataTypePtr get_vec_data_type() { return _data_type; }
+
virtual FieldType get_meta_type() { return _meta_type; }
int64_t get_metadata_size() const override;
private:
+ friend class VariantColumnReader;
+
ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows,
io::FileReaderSPtr file_reader);
Status init(const ColumnMetaPB* meta);
@@ -299,6 +295,8 @@ private:
DictEncodingType _dict_encoding_type;
+ vectorized::DataTypePtr _data_type;
+
TypeInfoPtr _type_info =
TypeInfoPtr(nullptr, nullptr); // initialized in init(), may changed by subclasses.
const EncodingInfo* _encoding_info =
diff --git a/be/src/olap/rowset/segment_v2/column_reader_cache.cpp b/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
index 30b6e9528f5..366683aed8b 100644
--- a/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader_cache.cpp
@@ -17,8 +17,10 @@
#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
+#
bvar::Adder<int64_t> g_segment_column_cache_count("segment_column_cache_count");
bvar::Adder<int64_t> g_segment_column_cache_hit_count("segment_column_cache_hit_count");
@@ -27,7 +29,18 @@ bvar::Adder<int64_t> g_segment_column_cache_evict_count("segment_column_cache_ev
namespace doris::segment_v2 {
-ColumnReaderCache::ColumnReaderCache(Segment* segment) : _segment(segment) {}
+#include "common/compile_check_begin.h"
+
+ColumnReaderCache::ColumnReaderCache(
+ ColumnMetaAccessor* accessor, TabletSchemaSPtr tablet_schema,
+ io::FileReaderSPtr file_reader, uint64_t num_rows,
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)>
+ get_footer_cb)
+ : _accessor(accessor),
+ _tablet_schema(std::move(tablet_schema)),
+ _file_reader(std::move(file_reader)),
+ _num_rows(num_rows),
+ _get_footer_cb(std::move(get_footer_cb)) {}
ColumnReaderCache::~ColumnReaderCache() {
g_segment_column_cache_count << -_cache_map.size();
@@ -47,20 +60,8 @@ std::shared_ptr<ColumnReader> ColumnReaderCache::_lookup(const ColumnReaderCache
return it->second->reader;
}
-Status ColumnReaderCache::_insert(const ColumnReaderCacheKey& key, const ColumnReaderOptions& opts,
- const SegmentFooterPB& footer, int32_t column_id,
- const io::FileReaderSPtr& file_reader, size_t num_rows,
- std::shared_ptr<ColumnReader>* column_reader) {
- std::lock_guard<std::mutex> lock(_cache_mutex);
- // If already exists, and move it to the front
- if (_cache_map.find(key) != _cache_map.end()) {
- g_segment_column_cache_hit_count << 1;
- auto it = _cache_map[key];
- DCHECK_EQ(it->key.first, key.first);
- _lru_list.splice(_lru_list.begin(), _lru_list, it);
- *column_reader = it->reader;
- return Status::OK();
- }
+void ColumnReaderCache::_insert_locked_nocheck(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& reader) {
// If capacity exceeded, remove least recently used (tail)
if (_cache_map.size() >= config::max_segment_partial_column_cache_size) {
g_segment_column_cache_count << -1;
@@ -70,21 +71,15 @@ Status ColumnReaderCache::_insert(const ColumnReaderCacheKey& key, const ColumnR
_cache_map.erase(last_it->key);
_lru_list.pop_back();
}
- std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(ColumnReader::create(opts, footer, column_id, num_rows, file_reader, &reader));
- // Insert new node at the front
- std::shared_ptr<ColumnReader> reader_ptr = std::move(reader);
- g_segment_column_cache_count << 1;
- DCHECK(key.first >= 0) << " col_uid: " << key.first
- << " relative_path: " << key.second.get_path();
- _lru_list.push_front(CacheNode {key, reader_ptr, std::chrono::steady_clock::now()});
+ _lru_list.push_front(CacheNode {
+ .key = key, .reader = reader, .last_access = std::chrono::steady_clock::now()});
_cache_map[key] = _lru_list.begin();
- *column_reader = reader_ptr;
- VLOG_DEBUG << "insert cache: " << key.first << " " << key.second.get_path()
- << ", type: " << (int)reader_ptr->get_meta_type()
- << ", cache_size: " << _cache_map.size() << ", list_size: " << _lru_list.size()
- << ", cache_map: " << _cache_map.size() << ", lru_list: " << _lru_list.size();
- return Status::OK();
+}
+
+void ColumnReaderCache::_insert_direct(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& column_reader) {
+ std::lock_guard<std::mutex> lock(_cache_mutex);
+ _insert_locked_nocheck(key, column_reader);
}
std::map<int32_t, std::shared_ptr<ColumnReader>> ColumnReaderCache::get_available_readers(
@@ -107,78 +102,97 @@ Status ColumnReaderCache::get_column_reader(int32_t col_uid,
*column_reader = cached;
return Status::OK();
}
- auto it = _segment->_column_uid_to_footer_ordinal.find(col_uid);
- if (it == _segment->_column_uid_to_footer_ordinal.end()) {
- // no such column in this segment, return nullptr
- *column_reader = nullptr;
- return Status::OK();
- }
+ // Load footer once under cache mutex (not thread-safe otherwise)
std::shared_ptr<SegmentFooterPB> footer_pb_shared;
{
std::lock_guard<std::mutex> lock(_cache_mutex);
- // keep the lock until the footer is loaded, since _get_segment_footer is not thread safe
- RETURN_IF_ERROR(_segment->_get_segment_footer(footer_pb_shared, stats));
+ RETURN_IF_ERROR(_get_footer_cb(footer_pb_shared, stats));
+ }
+
+ // Lookup column meta by uid via ColumnMetaAccessor. If not initialized or not found, return NOT_FOUND.
+ ColumnMetaPB meta;
+ Status st_meta = _accessor->get_column_meta_by_uid(*footer_pb_shared, col_uid, &meta);
+ if (st_meta.is<ErrorCode::NOT_FOUND>()) {
+ *column_reader = nullptr;
+ return st_meta;
+ }
+ RETURN_IF_ERROR(st_meta);
+
+ ColumnReaderOptions opts {.kept_in_memory = _tablet_schema->is_in_memory(),
+ .be_exec_version = _be_exec_version,
+ .tablet_schema = _tablet_schema};
+
+ std::shared_ptr<ColumnReader> reader;
+ if ((FieldType)meta.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) {
+ // Variant root columns require VariantColumnReader, which encapsulates
+ // subcolumn layout, sparse columns and external meta.
+ std::unique_ptr<VariantColumnReader> variant_reader(new VariantColumnReader());
+ RETURN_IF_ERROR(variant_reader->init(opts, _accessor, footer_pb_shared, col_uid, _num_rows,
+ _file_reader));
+ reader.reset(variant_reader.release());
+ VLOG_DEBUG << "insert cache (variant): uid=" << col_uid << " col_id=" << meta.column_id();
+ } else {
+ // For non-variant columns, we can create reader directly from ColumnMetaPB.
+ VLOG_DEBUG << "insert cache: uid=" << col_uid << " col_id=" << meta.column_id();
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta, _num_rows, _file_reader, &reader));
}
- // lazy create column reader from footer
- const auto& col_footer_pb = footer_pb_shared->columns(it->second);
- ColumnReaderOptions opts {
- .kept_in_memory = _segment->tablet_schema()->is_in_memory(),
- .be_exec_version = _be_exec_version,
- .tablet_schema = _segment->tablet_schema(),
- };
- VLOG_DEBUG << "insert cache: " << col_uid << " "
- << ""
- << ", type: " << (int)col_footer_pb.type() << ", footer_ordinal: " << it->second;
- RETURN_IF_ERROR(_insert({col_uid, {}}, opts, *footer_pb_shared, it->second,
- _segment->_file_reader, _segment->num_rows(), column_reader));
+
+ _insert_direct({col_uid, {}}, reader);
+ *column_reader = std::move(reader);
return Status::OK();
}
-Status ColumnReaderCache::get_path_column_reader(uint32_t col_uid,
+Status ColumnReaderCache::get_path_column_reader(int32_t col_uid,
vectorized::PathInData relative_path,
std::shared_ptr<ColumnReader>* column_reader,
OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint) {
- // Attempt to find in cache
+ // Attempt to find in cache first
if (auto cached = _lookup({col_uid, relative_path})) {
*column_reader = cached;
return Status::OK();
}
- const SubcolumnColumnMetaInfo::Node* node = node_hint;
- std::shared_ptr<ColumnReader> variant_column_reader;
- if (node == nullptr) {
- RETURN_IF_ERROR(get_column_reader(col_uid, &variant_column_reader, stats));
- node = variant_column_reader
- ? static_cast<VariantColumnReader*>(variant_column_reader.get())
- ->get_subcolumn_meta_by_path(relative_path)
- : nullptr;
+
+ if (!_accessor->has_column_uid(col_uid)) {
+ *column_reader = nullptr;
+ return Status::Error<ErrorCode::NOT_FOUND, false>("column not found in segment, col_uid={}",
+ col_uid);
}
- if (node != nullptr) {
- // lazy create column reader from footer
- DCHECK_GE(node->data.footer_ordinal, 0);
- std::shared_ptr<SegmentFooterPB> footer_pb_shared;
- {
- std::lock_guard<std::mutex> lock(_cache_mutex);
- // keep the lock until the footer is loaded, since _get_segment_footer is not thread safe
- RETURN_IF_ERROR(_segment->_get_segment_footer(footer_pb_shared, stats));
- }
- const auto& col_footer_pb = footer_pb_shared->columns(node->data.footer_ordinal);
- ColumnReaderOptions opts {
- .kept_in_memory = _segment->tablet_schema()->is_in_memory(),
- .be_exec_version = _be_exec_version,
- .tablet_schema = _segment->tablet_schema(),
- };
- VLOG_DEBUG << "insert cache: " << col_uid << " "
- << ""
- << ", type: " << (int)col_footer_pb.type()
- << ", footer_ordinal: " << node->data.footer_ordinal;
- RETURN_IF_ERROR(_insert({col_uid, node->path}, opts, *footer_pb_shared,
- node->data.footer_ordinal, _segment->_file_reader,
- _segment->num_rows(), column_reader));
+
+ // Load footer once under cache mutex (not thread-safe otherwise)
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ {
+ std::lock_guard<std::mutex> lock(_cache_mutex);
+ RETURN_IF_ERROR(_get_footer_cb(footer_pb_shared, stats));
+ }
+
+ // Ensure variant root reader is available in cache.
+ ColumnReaderOptions opts {.kept_in_memory = _tablet_schema->is_in_memory(),
+ .be_exec_version = _be_exec_version,
+ .tablet_schema = _tablet_schema};
+ std::shared_ptr<ColumnReader> variant_column_reader;
+ RETURN_IF_ERROR(get_column_reader(col_uid, &variant_column_reader, stats));
+
+ if (relative_path.empty()) {
+ *column_reader = std::move(variant_column_reader);
return Status::OK();
}
- // no such column in this segment, return nullptr
- *column_reader = nullptr;
+
+ // Delegate path-level reader creation to VariantColumnReader, which hides
+ // inline vs external meta details.
+ std::shared_ptr<ColumnReader> path_reader;
+ auto* vreader = static_cast<VariantColumnReader*>(variant_column_reader.get());
+ Status st = vreader->create_path_reader(relative_path, opts, _accessor, *footer_pb_shared,
+ _file_reader, _num_rows, &path_reader);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ *column_reader = nullptr;
+ return st;
+ }
+ RETURN_IF_ERROR(st);
+
+ // Cache and return
+ _insert_direct({col_uid, relative_path}, path_reader);
+ *column_reader = std::move(path_reader);
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader_cache.h b/be/src/olap/rowset/segment_v2/column_reader_cache.h
index ce4060a5f26..8efd829c3c1 100644
--- a/be/src/olap/rowset/segment_v2/column_reader_cache.h
+++ b/be/src/olap/rowset/segment_v2/column_reader_cache.h
@@ -23,8 +23,7 @@
namespace doris::segment_v2 {
class ColumnReaderCache;
-class Segment;
-using SegmentSPtr = std::shared_ptr<Segment>;
+class ColumnMetaAccessor;
// Key: pair of column uid and variant path (empty for normal column)
using ColumnReaderCacheKey = std::pair<int32_t, vectorized::PathInData>;
@@ -40,7 +39,14 @@ struct CacheNode {
// Limit the column reader count to config::max_segment_partial_column_cache_size
class ColumnReaderCache {
public:
- explicit ColumnReaderCache(Segment* segment);
+ // Main constructor used in production: cache is bound to a specific segment's
+ // ColumnMetaAccessor, TabletSchema, file reader and row count, plus a footer
+ // getter callback (Segment::_get_segment_footer).
+ ColumnReaderCache(
+ ColumnMetaAccessor* accessor, TabletSchemaSPtr tablet_schema,
+ io::FileReaderSPtr file_reader, uint64_t num_rows,
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)>
+ get_footer_cb);
virtual ~ColumnReaderCache();
// Get all available readers
// if include_subcolumns is true, return all available readers, including subcolumn readers
@@ -52,7 +58,7 @@ public:
OlapReaderStatistics* stats);
// Get column reader by column unique id and path(leaf node of variant's subcolumn)
- virtual Status get_path_column_reader(uint32_t col_uid, vectorized::PathInData relative_path,
+ virtual Status get_path_column_reader(int32_t col_uid, vectorized::PathInData relative_path,
std::shared_ptr<ColumnReader>* column_reader,
OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint = nullptr);
@@ -60,11 +66,13 @@ public:
private:
// Lookup function remains similar
std::shared_ptr<ColumnReader> _lookup(const ColumnReaderCacheKey& key);
- // Insert and create column reader if not exists
- Status _insert(const ColumnReaderCacheKey& key, const ColumnReaderOptions& opts,
- const SegmentFooterPB& footer, int32_t column_id,
- const io::FileReaderSPtr& file_reader, size_t num_rows,
- std::shared_ptr<ColumnReader>* column_reader);
+ // Insert helper: assumes _cache_mutex already locked, will evict if needed
+ void _insert_locked_nocheck(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& reader);
+
+ // Insert an already-created reader directly into cache
+ void _insert_direct(const ColumnReaderCacheKey& key,
+ const std::shared_ptr<ColumnReader>& column_reader);
// keep _lru_list and _cache_map thread safe
std::mutex _cache_mutex;
// Doubly-linked list to maintain LRU order
@@ -72,8 +80,15 @@ private:
// Map from key to list iterator for O(1) access
std::unordered_map<ColumnReaderCacheKey, std::list<CacheNode>::iterator> _cache_map;
int _be_exec_version = BeExecVersionManager::get_newest_version();
- // ATTN: Do not use share_ptr, reference will be recycled
- Segment* _segment;
+
+ // Non-owning pointer to the segment's ColumnMetaAccessor.
+ ColumnMetaAccessor* _accessor = nullptr;
+ // Segment-level context needed to construct ColumnReader.
+ TabletSchemaSPtr _tablet_schema;
+ io::FileReaderSPtr _file_reader;
+ uint64_t _num_rows = 0;
+ // Callback to get footer, usually bound to Segment::_get_segment_footer.
+ std::function<Status(std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*)> _get_footer_cb;
};
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp b/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp
new file mode 100644
index 00000000000..315713a44b9
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/external_col_meta_util.cpp
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "io/io_common.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+
+namespace doris::segment_v2 {
+
+Status ExternalColMetaUtil::parse_external_meta_pointers(
+ const SegmentFooterPB& footer, ExternalColMetaUtil::ExternalMetaPointers* out) {
+ out->region_start = footer.col_meta_region_start();
+ out->num_columns = static_cast<uint32_t>(footer.column_meta_entries_size());
+
+ // Validate pointers and basic consistency to ensure external meta is actually present.
+ if (out->region_start == 0) {
+ return Status::Corruption("external meta region_start is 0");
+ }
+ if (out->num_columns == 0) {
+ return Status::Corruption("external meta num_columns is 0");
+ }
+
+ // Compute region_end = region_start + sum(length_i) with overflow guards.
+ uint64_t region_size = 0;
+ for (int i = 0; i < footer.column_meta_entries_size(); ++i) {
+ const auto& entry = footer.column_meta_entries(i);
+ const auto len = static_cast<uint64_t>(entry.length());
+ if (len == 0) {
+ return Status::Corruption("column_meta_entries[{}] has zero length", i);
+ }
+ if (region_size + len < region_size) {
+ return Status::Corruption("region_size overflow at column_meta_entries[{}]", i);
+ }
+ region_size += len;
+ }
+
+ out->region_end = out->region_start + region_size;
+ if (out->region_end <= out->region_start) {
+ return Status::Corruption("invalid region_end: start={}, end={}", out->region_start,
+ out->region_end);
+ }
+ return Status::OK();
+}
+
+Status ExternalColMetaUtil::parse_uid_to_colid_map(
+ const SegmentFooterPB& footer, const ExternalColMetaUtil::ExternalMetaPointers& ptrs,
+ std::unordered_map<int32_t, size_t>* out) {
+ out->clear();
+ out->reserve(ptrs.num_columns);
+
+ for (int i = 0; i < footer.column_meta_entries_size(); ++i) {
+ const auto& entry = footer.column_meta_entries(i);
+ // Skip subcolumns (unique_id == -1) for top-level map
+ if (entry.unique_id() != -1) {
+ (*out)[entry.unique_id()] = static_cast<size_t>(i);
+ }
+ }
+ return Status::OK();
+}
+
+bool ExternalColMetaUtil::is_valid_meta_slice(uint64_t pos, uint64_t size,
+ const ExternalColMetaUtil::ExternalMetaPointers& p) {
+ if (size == 0) {
+ return false;
+ }
+ if (pos < p.region_start) {
+ return false;
+ }
+ if (pos + size < pos) {
+ return false;
+ } // overflow guard
+ if (pos + size > p.region_end) {
+ return false;
+ }
+ return true;
+}
+
+Status ExternalColMetaUtil::read_col_meta(const io::FileReaderSPtr& file_reader,
+ const SegmentFooterPB& footer,
+ const ExternalColMetaUtil::ExternalMetaPointers& p,
+ uint32_t col_id, ColumnMetaPB* out_meta) {
+ if (col_id >= p.num_columns) {
+ return Status::Corruption("col_id {} out of range {}", col_id, p.num_columns);
+ }
+ if (footer.column_meta_entries_size() != p.num_columns) {
+ return Status::Corruption("column_meta_entries size mismatch: entries={}, num_columns={}",
+ footer.column_meta_entries_size(), p.num_columns);
+ }
+ // compute meta offset via prefix-sum of entry lengths.
+ // TODO: prepare position and size in advance, and then read the meta.
+ uint64_t pos = p.region_start;
+ for (uint32_t i = 0; i < col_id; ++i) {
+ pos += static_cast<uint64_t>(footer.column_meta_entries(i).length());
+ }
+ const auto size = static_cast<uint64_t>(footer.column_meta_entries(col_id).length());
+ if (!ExternalColMetaUtil::is_valid_meta_slice(pos, size, p)) {
+ return Status::Corruption(
+ "invalid ColumnMetaPB bounds: col_id={}, pos={}, size={}, region_start={}, "
+ "region_end={}",
+ col_id, pos, size, p.region_start, p.region_end);
+ }
+ if (size > static_cast<uint64_t>(std::numeric_limits<size_t>::max())) {
+ return Status::Corruption("ColumnMetaPB size too large to allocate: {}", size);
+ }
+ std::string buf;
+ buf.resize(static_cast<size_t>(size));
+ size_t meta_read = 0;
+ io::IOContext io_ctx {.is_index_data = true};
+ RETURN_IF_ERROR(file_reader->read_at(pos, Slice(buf.data(), buf.size()), &meta_read, &io_ctx));
+ if (meta_read != size) {
+ return Status::Corruption("short read ColumnMetaPB: expect={}, actual={}", size, meta_read);
+ }
+ if (!out_meta->ParseFromArray(buf.data(), static_cast<int>(buf.size()))) {
+ return Status::Corruption("failed to parse ColumnMetaPB, col_id={}", col_id);
+ }
+ return Status::OK();
+}
+
+Status ExternalColMetaUtil::write_external_column_meta(
+ io::FileWriter* file_writer, SegmentFooterPB* footer, CompressionTypePB compression_type,
+ const std::function<Status(const std::vector<Slice>&)>& write_cb) {
+ if (footer->columns_size() == 0) {
+ return Status::OK();
+ }
+
+ std::vector<ColumnMetaPB> all_metas;
+ // 0) Externalize variant subcolumns first, then proceed with per-column meta region.
+ {
+ auto variant_ext_meta_agg =
+ std::make_unique<VariantExtMetaWriter>(file_writer, compression_type);
+ RETURN_IF_ERROR(variant_ext_meta_agg->externalize_from_footer(footer, &all_metas));
+ }
+
+ // Use the reorganized metas (including subcolumns)
+ const uint64_t meta_region_start = file_writer->bytes_appended();
+
+ footer->clear_column_meta_entries();
+
+ // 1) write contiguous meta blobs in col_id order and record lengths in footer.
+ for (const auto& meta : all_metas) {
+ std::string meta_bytes;
+ if (!meta.SerializeToString(&meta_bytes)) {
+ return Status::InternalError("failed to serialize column meta, unique_id={}",
+ meta.unique_id());
+ }
+ Slice s(meta_bytes);
+ RETURN_IF_ERROR(write_cb({s}));
+ auto* entry = footer->add_column_meta_entries();
+ entry->set_unique_id(static_cast<int32_t>(meta.unique_id()));
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ // 2) write pointers via proto fields
+ footer->set_col_meta_region_start(meta_region_start);
+
+ // 3) clear inline columns to enable true on-demand meta loading
+ // Note: footer->columns has already been pruned to only Top Level Columns in externalize_from_footer
+ // But for full externalization, we might want to clear them all or keep only necessary info?
+ // The original logic was footer->clear_columns().
+ // If we clear columns, the Reader needs to know how to reconstruct the schema.
+ // Currently, SegmentFooterPB.columns is used as the schema source if present.
+ // If we clear it, Reader must rely on External Meta.
+ // However, the Reader typically reads footer first. If columns is empty, it assumes V3 and reads external.
+ // So yes, we should clear it.
+ // But wait, in externalize_from_footer we carefully put Top Level columns back into footer->columns.
+ // Why? Because in previous logic, we might want to keep roots in footer?
+ // The previous logic: "replace Footer.columns with only the kept top-level columns".
+ // BUT then `write_external_column_meta` calls `footer->clear_columns()` at the end!
+ // So `footer->columns` will be empty anyway.
+ // The only reason to reconstruct `footer->columns` in `externalize_from_footer` is if `write_external_column_meta` logic depended on it.
+ // In my updated `write_external_column_meta`, I iterate over `all_metas` which is returned by `externalize_from_footer`.
+ // So I don't strictly need `footer->columns` to be correct in between.
+ // However, strictly following protocol: `externalize_from_footer` modifies footer to reflect "logical" columns (Top Level).
+ // And then `write_external_column_meta` finalizes it by clearing them and setting pointers.
+
+ footer->clear_columns();
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/external_col_meta_util.h b/be/src/olap/rowset/segment_v2/external_col_meta_util.h
new file mode 100644
index 00000000000..6a0e245c2f9
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/external_col_meta_util.h
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+#include <vector>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_writer.h"
+#include "util/slice.h"
+
+namespace doris::segment_v2 {
+// -----------------------------------------------------------------------------
+// External Column Meta layout and footer pointers
+// -----------------------------------------------------------------------------
+//
+// Segment tail:
+//
+// ┌──────────────────────────────────────────────────────────────────────┐
+// │ ... data pages ... │
+// │ │
+// │ Column Meta Region (variable-size, per top-level column, col_id asc) │
+// │ [Meta[col0]] [Meta[col1]] ... [Meta[colN-1]] │
+// │ │
+// │ Footer (SegmentFooterPB) │
+// │ fields: │
+// │ - col_meta_region_start : uint64 │
+// │ - column_meta_entries : repeated ColumnMetaEntryPB │
+// │ - unique_id : int32 │
+// │ - length : uint32 │
+// │ - num_columns : uint32 │
+// └──────────────────────────────────────────────────────────────────────┘
+//
+
+// Wrapper class to expose static methods for external column meta utilities.
+class ExternalColMetaUtil {
+public:
+ struct ExternalMetaPointers {
+ uint64_t region_start = 0;
+ // region_end is one-past-the-end file offset of the Column Meta Region.
+ uint64_t region_end = 0;
+ uint32_t num_columns = 0;
+ };
+
+ // Parse three external meta pointers from Footer.
+ // Returns Status::OK() on success, or Status::Corruption() with detailed error message.
+ static Status parse_external_meta_pointers(const SegmentFooterPB& footer,
+ ExternalMetaPointers* out);
+
+ // Parse uid->col_id mapping from footer.column_meta_entries.
+ // Returns Status::OK() on success, or Status::Corruption() with detailed error message.
+ static Status parse_uid_to_colid_map(const SegmentFooterPB& footer,
+ const ExternalMetaPointers& ptrs,
+ std::unordered_map<int32_t, size_t>* out);
+
+ static bool is_valid_meta_slice(uint64_t pos, uint64_t size, const ExternalMetaPointers& p);
+
+ // Read one ColumnMetaPB from external Column Meta Region; use INDEX cache lane because
+ // it's small metadata. Column offsets are derived from footer.column_meta_lengths via
+ // prefix sums.
+ static Status read_col_meta(const io::FileReaderSPtr& file_reader,
+ const SegmentFooterPB& footer, const ExternalMetaPointers& p,
+ uint32_t col_id, ColumnMetaPB* out_meta);
+
+ // Write external ColumnMetaPB contiguous region + footer pointers + column_meta_entries.
+ // write_cb: a callback that appends slices to the underlying file (e.g., SegmentWriter::_write_raw_data).
+ static Status write_external_column_meta(
+ io::FileWriter* file_writer, SegmentFooterPB* footer,
+ CompressionTypePB compression_type,
+ const std::function<Status(const std::vector<Slice>&)>& write_cb);
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index d3d368b8fa4..d13aa16cb34 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -42,6 +42,7 @@
#include "olap/olap_common.h"
#include "olap/primary_key_index.h"
#include "olap/rowset/rowset_reader_context.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
#include "olap/rowset/segment_v2/empty_segment_iterator.h"
@@ -240,7 +241,11 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
}
const TabletColumn& col = read_options.tablet_schema->column(column_id);
std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(get_column_reader(col, &reader, read_options.stats));
+ auto st = get_column_reader(col, &reader, read_options.stats);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ continue;
+ }
+ RETURN_IF_ERROR(st);
if (!reader || !reader->has_zone_map()) {
continue;
}
@@ -266,9 +271,13 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
and_predicate.add_column_predicate(
SingleColumnBlockPredicate::create_unique(runtime_predicate.get()));
std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(get_column_reader(
+ auto st = get_column_reader(
read_options.tablet_schema->column(runtime_predicate->column_id()), &reader,
- read_options.stats));
+ read_options.stats);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ continue;
+ }
+ RETURN_IF_ERROR(st);
if (reader &&
can_apply_predicate_safely(runtime_predicate->column_id(), runtime_predicate.get(),
*schema, read_options.io_ctx.reader_type) &&
@@ -580,68 +589,24 @@ vectorized::DataTypePtr Segment::get_data_type_of(const TabletColumn& column,
// Path exists, proceed with variant logic.
vectorized::PathInData relative_path = path->copy_pop_front();
- int32_t unique_id = column.unique_id() > 0 ? column.unique_id() : column.parent_unique_id();
+ int32_t unique_id = column.unique_id() >= 0 ? column.unique_id() : column.parent_unique_id();
- // Find the reader for the base variant column.
- if (!_column_uid_to_footer_ordinal.contains(unique_id)) {
+ // If this uid does not exist in segment meta, fallback to schema type.
+ if (!_column_meta_accessor->has_column_uid(unique_id)) {
return vectorized::DataTypeFactory::instance().create_data_type(column);
}
- std::shared_ptr<ColumnReader> reader;
- // get the parent variant column reader
+ std::shared_ptr<ColumnReader> v_reader;
+
+ // Get the parent variant column reader
OlapReaderStatistics stats;
// If status is not ok, it will throw exception(data corruption)
- THROW_IF_ERROR(get_column_reader(unique_id, &reader, &stats));
- const auto* variant_reader = static_cast<const VariantColumnReader*>(reader.get());
-
- // Find the specific node within the variant structure using the relative path.
- const auto* node = variant_reader->get_subcolumn_meta_by_path(relative_path);
-
- if (relative_path.get_path() == SPARSE_COLUMN_PATH) {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
-
- // Case 1: Node not found for the given path within the variant reader.
- // If relative_path is empty, it means the original path pointed to the root
- // of the variant column itself. We should return the Variant type.
- // If node is nullptr, it means the path is not exist in the variant sub columns.
- if (node == nullptr || relative_path.empty()) {
- // when the path is in the sparse column or exceeded the limit, return the variant type.
- if (variant_reader->exist_in_sparse_column(relative_path) ||
- variant_reader->is_exceeded_sparse_column_limit()) {
- if (column.is_nested_subcolumn()) {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
- return column.is_nullable()
- ? vectorized::make_nullable(std::make_shared<vectorized::DataTypeObject>(
- column.variant_max_subcolumns_count()))
- : std::make_shared<vectorized::DataTypeObject>(
- column.variant_max_subcolumns_count());
- }
- // now, path is not in this segment, return the default type from column.
- else {
- return vectorized::DataTypeFactory::instance().create_data_type(column);
- }
- }
-
- bool exist_in_sparse = variant_reader->exist_in_sparse_column(relative_path);
- bool is_physical_leaf = node->children.empty();
-
- if (is_physical_leaf && column.is_nested_subcolumn()) {
- return node->data.file_column_type;
- }
-
- // Condition to return the specific underlying type of the node:
- // 1. We are reading flat leaves (ignoring hierarchy).
- // 2. OR It's a leaf in the physical column structure AND it doesn't *also* exist
- // in the sparse column (meaning it's purely a materialized leaf).
- if (read_flat_leaves || (is_physical_leaf && !exist_in_sparse &&
- !variant_reader->is_exceeded_sparse_column_limit())) {
- return node->data.file_column_type;
- }
-
- // not the compaction read, return the default type from column.
- return vectorized::DataTypeFactory::instance().create_data_type(column);
+ THROW_IF_ERROR(get_column_reader(unique_id, &v_reader, &stats));
+ DCHECK(v_reader != nullptr);
+ const auto* variant_reader = static_cast<const VariantColumnReader*>(v_reader.get());
+ // Delegate type inference for variant paths to VariantColumnReader.
+ return variant_reader->infer_data_type_for_path(column, relative_path, read_flat_leaves,
+ _column_reader_cache.get(), unique_id);
}
Status Segment::_create_column_meta_once(OlapReaderStatistics* stats) {
@@ -654,17 +619,14 @@ Status Segment::_create_column_meta_once(OlapReaderStatistics* stats) {
}
Status Segment::_create_column_meta(const SegmentFooterPB& footer) {
- // unique_id -> idx in footer.columns()
- uint32_t ordinal = 0;
- for (const auto& column_meta : footer.columns()) {
- // no need to create column reader for variant's subcolumn
- if (column_meta.unique_id() == -1) {
- ordinal++;
- continue;
- }
- _column_uid_to_footer_ordinal.try_emplace(column_meta.unique_id(), ordinal++);
- }
- _column_reader_cache = std::make_unique<ColumnReaderCache>(this);
+ // Initialize column meta accessor which internally maintains uid -> column_ordinal mapping.
+ _column_meta_accessor = std::make_unique<ColumnMetaAccessor>();
+ RETURN_IF_ERROR(_column_meta_accessor->init(footer, _file_reader));
+ _column_reader_cache = std::make_unique<ColumnReaderCache>(
+ _column_meta_accessor.get(), _tablet_schema, _file_reader, _num_rows,
+ [this](std::shared_ptr<SegmentFooterPB>& footer_pb, OlapReaderStatistics* stats) {
+ return _get_segment_footer(footer_pb, stats);
+ });
return Status::OK();
}
@@ -706,11 +668,13 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
// For compability reason unique_id may less than 0 for variant extracted column
int32_t unique_id = tablet_column.unique_id() >= 0 ? tablet_column.unique_id()
: tablet_column.parent_unique_id();
- // init default iterator
- if (!_column_uid_to_footer_ordinal.contains(unique_id)) {
+
+ // If column meta for this uid is not found in this segment, use default iterator.
+ if (!_column_meta_accessor->has_column_uid(unique_id)) {
RETURN_IF_ERROR(new_default_iterator(tablet_column, iter));
return Status::OK();
}
+
// init iterator by unique id
std::shared_ptr<ColumnReader> reader;
RETURN_IF_ERROR(get_column_reader(unique_id, &reader, opt->stats));
@@ -773,7 +737,11 @@ Status Segment::new_bitmap_index_iterator(const TabletColumn& tablet_column,
std::unique_ptr<BitmapIndexIterator>* iter) {
RETURN_IF_ERROR(_create_column_meta_once(read_options.stats));
std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(get_column_reader(tablet_column, &reader, read_options.stats));
+ auto st = get_column_reader(tablet_column, &reader, read_options.stats);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
if (reader != nullptr && reader->has_bitmap_index()) {
BitmapIndexIterator* it;
RETURN_IF_ERROR(reader->new_bitmap_index_iterator(&it));
@@ -792,7 +760,11 @@ Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column,
}
RETURN_IF_ERROR(_create_column_meta_once(read_options.stats));
std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(get_column_reader(tablet_column, &reader, read_options.stats));
+ auto st = get_column_reader(tablet_column, &reader, read_options.stats);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
if (reader != nullptr && index_meta) {
if (_inverted_index_file_reader == nullptr) {
RETURN_IF_ERROR(
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index be74c28f68b..c2616dcd809 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -62,12 +62,19 @@ struct RowLocation;
namespace segment_v2 {
+// SegmentFooterPB versioning:
+// 1 = V2 baseline
+// 2 = V3 (externalized ColumnMetaPB region + CMO present)
+static constexpr uint32_t kSegmentFooterVersionV2 = 1;
+static constexpr uint32_t kSegmentFooterVersionV3_ExtColMeta = 2;
+
class BitmapIndexIterator;
class Segment;
class InvertedIndexIterator;
class InvertedIndexFileReader;
struct VariantStatistics;
class ColumnReaderCache;
+class ColumnMetaAccessor;
using SegmentSharedPtr = std::shared_ptr<Segment>;
// A Segment is used to represent a segment in memory format. When segment is
@@ -265,8 +272,8 @@ private:
// Limited cache for column readers
std::unique_ptr<ColumnReaderCache> _column_reader_cache;
- // map column unique id ---> it's footer ordinal
- std::unordered_map<int32_t, size_t> _column_uid_to_footer_ordinal;
+ // Centralized accessor for column metadata layout and uid->column_ordinal mapping.
+ std::unique_ptr<ColumnMetaAccessor> _column_meta_accessor;
// Init from ColumnMetaPB in SegmentFooterPB
// map column unique id ---> it's inner data type
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index de9a6ba35a3..1c720b5948d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -306,8 +306,6 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
_record_rowids = true;
}
- RETURN_IF_ERROR(init_iterators());
-
if (opts.output_columns != nullptr) {
_output_columns = *(opts.output_columns);
}
@@ -345,6 +343,8 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
}
}
+ RETURN_IF_ERROR(init_iterators());
+
RETURN_IF_ERROR(_construct_compound_expr_context());
_enable_common_expr_pushdown = !_common_expr_ctxs_push_down.empty();
_initialize_predicate_results();
@@ -1089,6 +1089,8 @@ Status SegmentIterator::_init_inverted_index_iterators() {
// We use this column to locate the metadata for the inverted index, which requires a unique_id and path.
const auto& column = _opts.tablet_schema->column(cid);
std::vector<const TabletIndex*> inverted_indexs;
+ // Keep shared_ptr alive to prevent use-after-free when accessing raw pointers
+ TabletIndexes inverted_indexs_holder;
// If the column is an extracted column, we need to find the sub-column in the parent column reader.
std::shared_ptr<ColumnReader> column_reader;
if (column.is_extracted_column()) {
@@ -1097,8 +1099,14 @@ Status SegmentIterator::_init_inverted_index_iterators() {
column_reader == nullptr) {
continue;
}
- inverted_indexs = assert_cast<VariantColumnReader*>(column_reader.get())
- ->find_subcolumn_tablet_indexes(column.suffix_path());
+ inverted_indexs_holder =
+ assert_cast<VariantColumnReader*>(column_reader.get())
+ ->find_subcolumn_tablet_indexes(column,
+ _storage_name_and_type[cid].second);
+ // Extract raw pointers from shared_ptr for iteration
+ for (const auto& index_ptr : inverted_indexs_holder) {
+ inverted_indexs.push_back(index_ptr.get());
+ }
}
// If the column is not an extracted column, we can directly get the inverted index metadata from the tablet schema.
else {
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 3046871864a..b3c0c987207 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -46,10 +46,12 @@
#include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "olap/rowset/segment_creator.h"
#include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_writer.h"
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/page_pointer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
#include "olap/rowset/segment_v2/variant_stats_calculator.h"
#include "olap/segment_loader.h"
#include "olap/short_key_index.h"
@@ -73,7 +75,6 @@
#include "vec/jsonb/serialize.h"
#include "vec/olap/olap_data_convertor.h"
#include "vec/runtime/vdatetime_value.h"
-
namespace doris {
namespace segment_v2 {
@@ -1207,6 +1208,15 @@ Status SegmentWriter::_write_primary_key_index() {
Status SegmentWriter::_write_footer() {
_footer.set_num_rows(_row_count);
+ // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
+ if (_tablet_schema->is_external_segment_meta_used_default()) {
+ _footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+ VLOG_DEBUG << "use external column meta";
+ // External ColumnMetaPB writing (optional)
+ RETURN_IF_ERROR(ExternalColMetaUtil::write_external_column_meta(
+ _file_writer, &_footer, _opts.compression_type,
+ [this](const std::vector<Slice>& slices) { return _write_raw_data(slices); }));
+ }
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4)
std::string footer_buf;
diff --git a/be/src/olap/rowset/segment_v2/stream_reader.h b/be/src/olap/rowset/segment_v2/stream_reader.h
index 134792b1ffd..b009df3b20e 100644
--- a/be/src/olap/rowset/segment_v2/stream_reader.h
+++ b/be/src/olap/rowset/segment_v2/stream_reader.h
@@ -18,8 +18,7 @@
#pragma once
#include <memory>
-
-// #include "olap/rowset/segment_v2/column_reader.h"
+#include "olap/rowset/segment_v2/column_reader.h"
#include "vec/columns/column.h"
#include "vec/columns/subcolumn_tree.h"
#include "vec/data_types/data_type.h"
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
index f4a1431bcfe..d38533fe0ed 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
@@ -26,14 +26,17 @@
#include "common/config.h"
#include "common/status.h"
#include "io/fs/file_reader.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/hierarchical_data_iterator.h"
#include "olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h"
#include "olap/rowset/segment_v2/variant/sparse_column_merge_iterator.h"
#include "olap/tablet_schema.h"
+#include "util/slice.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_object.h"
@@ -114,6 +117,10 @@ Status VariantColumnReader::_create_hierarchical_reader(ColumnIteratorUPtr* read
const SubcolumnColumnMetaInfo::Node* root,
ColumnReaderCache* column_reader_cache,
OlapReaderStatistics* stats) {
+ // make sure external meta is loaded otherwise can't find any meta data for extracted columns
+ // TODO(lhy): this will load all external meta if not loaded, and memory will be consumed.
+ RETURN_IF_ERROR(load_external_meta_once());
+
// Node contains column with children columns or has correspoding sparse columns
// Create reader with hirachical data.
std::unique_ptr<SubstreamIterator> sparse_iter;
@@ -124,6 +131,9 @@ Status VariantColumnReader::_create_hierarchical_reader(ColumnIteratorUPtr* read
sparse_iter = std::make_unique<SubstreamIterator>(
vectorized::ColumnObject::create_sparse_column_fn(), std::move(iter), nullptr);
}
+ if (node == nullptr) {
+ node = _subcolumns_meta_info->find_exact(path);
+ }
// If read the full path of variant read in MERGE_ROOT, otherwise READ_DIRECT
HierarchicalDataIterator::ReadType read_type =
(path == root->path) ? HierarchicalDataIterator::ReadType::MERGE_ROOT
@@ -229,6 +239,9 @@ Status VariantColumnReader::_new_iterator_with_flat_leaves(ColumnIteratorUPtr* i
bool exceeded_sparse_column_limit,
bool existed_in_sparse_column,
ColumnReaderCache* column_reader_cache) {
+ // make sure external meta is loaded otherwise can't find any meta data for extracted columns
+ RETURN_IF_ERROR(load_external_meta_once());
+
DCHECK(opts != nullptr);
auto relative_path = target_col.path_info_ptr()->copy_pop_front();
// compaction need to read flat leaves nodes data to prevent from amplification
@@ -289,6 +302,43 @@ Status VariantColumnReader::_new_iterator_with_flat_leaves(ColumnIteratorUPtr* i
return Status::OK();
}
+bool VariantColumnReader::has_prefix_path(const vectorized::PathInData& relative_path) const {
+ if (relative_path.empty()) {
+ return true;
+ }
+ const std::string path = relative_path.get_path();
+ const std::string dot_prefix = relative_path.get_path() + ".";
+
+ // 1) exact node exists and has children.
+ if (const auto* node = _subcolumns_meta_info->find_exact(relative_path)) {
+ if (!node->children.empty()) {
+ return true;
+ }
+ }
+
+ // 2) Check sparse column stats: use lower_bound to test the `p.` prefix range
+ // example sparse columns path: a.b.c, a.b.e, access prefix: a.b.
+ // then we must read the sparse columns
+ if (_statistics && !_statistics->sparse_column_non_null_size.empty()) {
+ auto it = _statistics->sparse_column_non_null_size.lower_bound(dot_prefix);
+ if (it != _statistics->sparse_column_non_null_size.end() &&
+ it->first.starts_with(dot_prefix)) {
+ return true;
+ }
+ }
+
+ // 3) Check external meta store (if available).
+ if (_ext_meta_reader && _ext_meta_reader->available()) {
+ bool has = false;
+ // Pass strict prefix `p.` to avoid false positives like `a.b` matching `a.bc`.
+ if (_ext_meta_reader->has_prefix(dot_prefix, &has).ok() && has) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
const TabletColumn* target_col,
const StorageReadOptions* opt) {
@@ -319,8 +369,7 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
// Check if path exist in sparse column
bool existed_in_sparse_column =
!_statistics->sparse_column_non_null_size.empty() &&
- _statistics->sparse_column_non_null_size.find(relative_path.get_path()) !=
- _statistics->sparse_column_non_null_size.end();
+ _statistics->sparse_column_non_null_size.contains(relative_path.get_path());
// Otherwise the prefix is not exist and the sparse column size is reached limit
// which means the path maybe exist in sparse_column
@@ -346,15 +395,8 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
}
// Check if path is prefix, example sparse columns path: a.b.c, a.b.e, access prefix: a.b.
- // then we must read the sparse columns
- const std::string& prefix = relative_path.get_path() + ".";
- bool prefix_existed_in_sparse_column =
- !_statistics->sparse_column_non_null_size.empty() &&
- (_statistics->sparse_column_non_null_size.lower_bound(prefix) !=
- _statistics->sparse_column_non_null_size.end()) &&
- _statistics->sparse_column_non_null_size.lower_bound(prefix)->first.starts_with(prefix);
- // if prefix exists in sparse column, read sparse column with hierarchical reader
- if (prefix_existed_in_sparse_column || exceeded_sparse_column_limit) {
+ // Or access root path
+ if (has_prefix_path(relative_path)) {
// Example {"b" : {"c":456,"e":7.111}}
// b.c is sparse column, b.e is subcolumn, so b is both the prefix of sparse column and subcolumn
return _create_hierarchical_reader(iterator, col_uid, relative_path, node, root,
@@ -376,42 +418,125 @@ Status VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
return Status::OK();
}
+ // read leaf node
if (node != nullptr) {
- // relative_path means the root node, should always use HierarchicalDataIterator
- if (node->is_leaf_node() && !relative_path.empty()) {
- // Node contains column without any child sub columns and no corresponding sparse columns
- // Direct read extracted columns
- const auto* node = _subcolumns_meta_info->find_leaf(relative_path);
- std::shared_ptr<ColumnReader> column_reader;
- RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
- col_uid, node->path, &column_reader, opt->stats, node));
- RETURN_IF_ERROR(column_reader->new_iterator(iterator, nullptr));
- } else {
- RETURN_IF_ERROR(_create_hierarchical_reader(iterator, col_uid, relative_path, node,
- root, column_reader_cache, opt->stats));
- }
+ DCHECK(node->is_leaf_node());
+ // Node contains column without any child sub columns and no corresponding sparse columns
+ // Direct read extracted columns
+ const auto* leaf_node = _subcolumns_meta_info->find_leaf(relative_path);
+ std::shared_ptr<ColumnReader> leaf_column_reader;
+ RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
+ col_uid, leaf_node->path, &leaf_column_reader, opt->stats, leaf_node));
+ RETURN_IF_ERROR(leaf_column_reader->new_iterator(iterator, nullptr));
} else {
+ if (_ext_meta_reader && _ext_meta_reader->available()) {
+ // Get path reader from external meta
+ std::shared_ptr<ColumnReader> leaf_column_reader;
+ Status st = column_reader_cache->get_path_column_reader(
+ col_uid, relative_path, &leaf_column_reader, opt->stats, nullptr);
+ DCHECK(!has_prefix_path(relative_path));
+ if (st.ok()) {
+ // Try external meta fallback: build a leaf reader on demand from externalized meta
+ RETURN_IF_ERROR(leaf_column_reader->new_iterator(iterator, nullptr));
+ return Status::OK();
+ }
+ if (!st.is<ErrorCode::NOT_FOUND>()) {
+ return st;
+ }
+ // not found, need continue
+ }
+ if (exceeded_sparse_column_limit) {
+ // maybe exist prefix path in sparse column
+ return _create_hierarchical_reader(iterator, col_uid, relative_path, node, root,
+ column_reader_cache, opt->stats);
+ }
// Sparse column not exists and not reached stats limit, then the target path is not exist, get a default iterator
RETURN_IF_ERROR(Segment::new_default_iterator(*target_col, iterator));
}
return Status::OK();
}
-Status VariantColumnReader::init(const ColumnReaderOptions& opts, const SegmentFooterPB& footer,
- uint32_t column_id, uint64_t num_rows,
- io::FileReaderSPtr file_reader) {
+Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const std::shared_ptr<SegmentFooterPB>& footer, int32_t column_uid,
+ uint64_t num_rows, io::FileReaderSPtr file_reader) {
// init sub columns
_subcolumns_meta_info = std::make_unique<SubcolumnColumnMetaInfo>();
_statistics = std::make_unique<VariantStatistics>();
- const ColumnMetaPB& self_column_pb = footer.columns(column_id);
+
+ // Prefer external root ColumnMetaPB via ColumnMetaAccessor when available.
+ ColumnMetaPB self_column_pb;
+ {
+ // Locate root column meta by unique id; this hides inline vs external CMO layout.
+ RETURN_IF_ERROR(accessor->get_column_meta_by_uid(*footer, column_uid, &self_column_pb));
+ // root column
+ // root subcolumn is ColumnVariant::MostCommonType which is jsonb
+ vectorized::DataTypePtr root_type =
+ self_column_pb.is_nullable()
+ ? make_nullable(
+ std::make_unique<vectorized::ColumnObject::MostCommonType>())
+ : std::make_unique<vectorized::ColumnObject::MostCommonType>();
+ int32_t root_footer_ordinal = -1;
+ if (self_column_pb.has_column_id()) {
+ // Narrow explicitly to avoid implicit narrowing warnings.
+ root_footer_ordinal = static_cast<int32_t>(self_column_pb.column_id());
+ }
+ _subcolumns_meta_info->create_root(SubcolumnMeta {
+ .file_column_type = root_type,
+ // Use column_id from meta as footer ordinal when inline footer is available.
+ .footer_ordinal = root_footer_ordinal});
+ RETURN_IF_ERROR(ColumnReader::create(opts, self_column_pb, num_rows, file_reader,
+ &_root_column_reader));
+ }
+
+ _data_type = vectorized::DataTypeFactory::instance().create_data_type(self_column_pb);
+ _root_unique_id = self_column_pb.unique_id();
const auto& parent_index = opts.tablet_schema->inverted_indexs(self_column_pb.unique_id());
// record variant_sparse_column_statistics_size from parent column
_variant_sparse_column_statistics_size =
opts.tablet_schema->column_by_uid(self_column_pb.unique_id())
.variant_max_sparse_column_statistics_size();
+ _tablet_schema = opts.tablet_schema;
+ // helper to handle sparse meta (single or bucket) from a ColumnMetaPB
+ auto handle_sparse_meta = [&](const ColumnMetaPB& col, bool* handled) -> Status {
+ *handled = false;
+ if (!col.has_column_path_info()) {
+ return Status::OK();
+ }
+ vectorized::PathInData path;
+ path.from_protobuf(col.column_path_info());
+ auto relative = path.copy_pop_front();
+ if (relative.empty()) {
+ return Status::OK();
+ }
+ std::string rel_str = relative.get_path();
+ if (rel_str == SPARSE_COLUMN_PATH) {
+ DCHECK(col.has_variant_statistics()) << col.DebugString();
+ const auto& variant_stats = col.variant_statistics();
+ for (const auto& [subpath, size] : variant_stats.sparse_column_non_null_size()) {
+ _statistics->sparse_column_non_null_size.emplace(subpath, size);
+ }
+ std::shared_ptr<ColumnReader> single_reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, col, footer->num_rows(), file_reader,
+ &single_reader));
+ *handled = true;
+ _sparse_column_reader = single_reader;
+ return Status::OK();
+ }
+ return Status::OK();
+ };
+
+ // First try initialize sparse from root's embedded children_columns (new segments)
+ if (self_column_pb.children_columns_size() > 0) {
+ for (int i = 0; i < self_column_pb.children_columns_size(); ++i) {
+ const ColumnMetaPB& child_pb = self_column_pb.children_columns(i);
+ bool handled = false;
+ RETURN_IF_ERROR(handle_sparse_meta(child_pb, &handled));
+ }
+ }
- for (int32_t ordinal = 0; ordinal < footer.columns_size(); ++ordinal) {
- const ColumnMetaPB& column_pb = footer.columns(ordinal);
+ // init from inline columns meta
+ for (int32_t ordinal = 0; ordinal < footer->columns_size(); ++ordinal) {
+ const ColumnMetaPB& column_pb = footer->columns(ordinal);
// Find all columns belonging to the current variant column
// 1. not the variant column
if (!column_pb.has_column_path_info()) {
@@ -433,84 +558,124 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, const SegmentF
vectorized::PathInData path;
path.from_protobuf(column_pb.column_path_info());
- // init sparse column
- if (path.copy_pop_front().get_path() == SPARSE_COLUMN_PATH) {
- DCHECK(column_pb.has_variant_statistics()) << column_pb.DebugString();
- const auto& variant_stats = column_pb.variant_statistics();
- for (const auto& [path, size] : variant_stats.sparse_column_non_null_size()) {
- _statistics->sparse_column_non_null_size.emplace(path, size);
+ // init sparse column readers
+ auto relative_sparse = path.copy_pop_front();
+ auto rel_str = relative_sparse.get_path();
+ {
+ bool handled = false;
+ RETURN_IF_ERROR(handle_sparse_meta(column_pb, &handled));
+ if (handled) {
+ continue;
}
- RETURN_IF_ERROR(ColumnReader::create(opts, column_pb, footer.num_rows(), file_reader,
- &_sparse_column_reader));
- continue;
}
// init subcolumns
auto relative_path = path.copy_pop_front();
- auto get_data_type_fn = [&]() {
- // root subcolumn is ColumnObject::MostCommonType which is jsonb
- if (relative_path.empty()) {
- return self_column_pb.is_nullable()
- ? make_nullable(std::make_unique<
- vectorized::ColumnObject::MostCommonType>())
- : std::make_unique<vectorized::ColumnObject::MostCommonType>();
- }
- return vectorized::DataTypeFactory::instance().create_data_type(column_pb);
- };
- // init subcolumns
- if (_subcolumns_meta_info->get_root() == nullptr) {
- _subcolumns_meta_info->create_root(SubcolumnMeta {});
- }
if (relative_path.empty()) {
- // root column
- _subcolumns_meta_info->get_mutable_root()->modify_to_scalar(
- SubcolumnMeta {get_data_type_fn(), ordinal});
- RETURN_IF_ERROR(ColumnReader::create(opts, column_pb, num_rows, file_reader,
- &_root_column_reader));
- } else {
- // check the root is already a leaf node
- if (column_pb.has_none_null_size()) {
- _statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
- column_pb.none_null_size());
- }
- _subcolumns_meta_info->add(relative_path, SubcolumnMeta {get_data_type_fn(), ordinal});
- TabletSchema::SubColumnInfo sub_column_info;
- // if subcolumn has index, add index to _variant_subcolumns_indexes
- if (vectorized::schema_util::generate_sub_column_info(
- *opts.tablet_schema, self_column_pb.unique_id(), relative_path.get_path(),
- &sub_column_info) &&
- !sub_column_info.indexes.empty()) {
- _variant_subcolumns_indexes[path.get_path()] = std::move(sub_column_info.indexes);
- }
- // if parent column has index, add index to _variant_subcolumns_indexes
- else if (!parent_index.empty()) {
- vectorized::schema_util::inherit_index(
- parent_index, _variant_subcolumns_indexes[path.get_path()], column_pb);
- }
+ continue;
+ }
+ // check the root is already a leaf node
+ if (column_pb.has_none_null_size()) {
+ _statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
+ column_pb.none_null_size());
}
+ _subcolumns_meta_info->add(
+ relative_path,
+ SubcolumnMeta {
+ .file_column_type =
+ vectorized::DataTypeFactory::instance().create_data_type(column_pb),
+ .footer_ordinal = ordinal});
}
// init sparse column set in stats
if (self_column_pb.has_variant_statistics()) {
- _statistics = std::make_unique<VariantStatistics>();
const auto& variant_stats = self_column_pb.variant_statistics();
for (const auto& [path, size] : variant_stats.sparse_column_non_null_size()) {
_statistics->sparse_column_non_null_size.emplace(path, size);
}
}
+ _segment_file_reader = file_reader;
+ _num_rows = num_rows;
+ // try build external meta readers (optional)
+ _ext_meta_reader = std::make_unique<VariantExternalMetaReader>();
+ RETURN_IF_ERROR(_ext_meta_reader->init_from_footer(footer, file_reader, _root_unique_id));
return Status::OK();
}
+
+Status VariantColumnReader::create_reader_from_external_meta(const std::string& path,
+ const ColumnReaderOptions& opts,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows,
+ std::shared_ptr<ColumnReader>* out) {
+ if (!_ext_meta_reader || !_ext_meta_reader->available()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("no external variant meta");
+ }
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(_ext_meta_reader->lookup_meta_by_path(path, &meta));
+ return ColumnReader::create(opts, meta, num_rows, file_reader, out);
+}
-std::vector<const TabletIndex*> VariantColumnReader::find_subcolumn_tablet_indexes(
- const std::string& path) {
- auto it = _variant_subcolumns_indexes.find(path);
- std::vector<const TabletIndex*> indexes;
- if (it != _variant_subcolumns_indexes.end()) {
- for (const auto& index : it->second) {
- indexes.push_back(index.get());
- }
+Status VariantColumnReader::create_path_reader(const vectorized::PathInData& relative_path,
+ const ColumnReaderOptions& opts,
+ ColumnMetaAccessor* accessor,
+ const SegmentFooterPB& footer,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows,
+ std::shared_ptr<ColumnReader>* out) {
+ // 1) Try inline subcolumn meta if available (footer_ordinal >= 0)
+ const auto* node = get_subcolumn_meta_by_path(relative_path);
+ if (node != nullptr && node->data.footer_ordinal >= 0) {
+ // leaf node, get the column meta by footer ordinal
+ const int32_t column_ordinal = node->data.footer_ordinal;
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(
+ accessor->get_column_meta_by_column_ordinal_id(footer, column_ordinal, &meta));
+ return ColumnReader::create(opts, meta, num_rows, file_reader, out);
+ }
+
+ // 2) Try external meta layout (if available)
+ Status st = create_reader_from_external_meta(relative_path.get_path(), opts, file_reader,
+ num_rows, out);
+ if (st.is<ErrorCode::NOT_FOUND>()) {
+ *out = nullptr;
+ return st;
+ }
+ return st;
+}
+
+Status VariantColumnReader::load_external_meta_once() {
+ if (!_ext_meta_reader || !_ext_meta_reader->available()) {
+ return Status::OK();
+ }
+ return _ext_meta_reader->load_all_once(_subcolumns_meta_info.get(), _statistics.get());
+}
+
+TabletIndexes VariantColumnReader::find_subcolumn_tablet_indexes(
+ const TabletColumn& column, const vectorized::DataTypePtr& data_type) {
+ TabletSchema::SubColumnInfo sub_column_info;
+ const auto& parent_index = _tablet_schema->inverted_indexs(column.parent_unique_id());
+ auto relative_path = column.path_info_ptr()->copy_pop_front();
+ // if subcolumn has index, add index to _variant_subcolumns_indexes
+ if (vectorized::schema_util::generate_sub_column_info(
+ *_tablet_schema, column.parent_unique_id(), relative_path.get_path(),
+ &sub_column_info) &&
+ !sub_column_info.indexes.empty()) {
+ }
+ // if parent column has index, add index to _variant_subcolumns_indexes
+ else if (!parent_index.empty() &&
+ data_type->get_storage_field_type() != FieldType::OLAP_FIELD_TYPE_VARIANT &&
+ data_type->get_storage_field_type() != FieldType::OLAP_FIELD_TYPE_MAP /*SPARSE COLUMN*/) {
+ // type in column maynot be real type, so use data_type to get the real type
+ TabletColumn target_column = vectorized::schema_util::get_column_by_type(
+ data_type, column.name(),
+ {.unique_id = -1,
+ .parent_unique_id = column.parent_unique_id(),
+ .path_info = *column.path_info_ptr()});
+ vectorized::schema_util::inherit_index(parent_index, sub_column_info.indexes,
+ target_column);
}
- return indexes;
+ // Return shared_ptr directly to maintain object lifetime
+ return sub_column_info.indexes;
}
void VariantColumnReader::get_subcolumns_types(
@@ -540,6 +705,70 @@ void VariantColumnReader::get_nested_paths(
}
}
+vectorized::DataTypePtr VariantColumnReader::infer_data_type_for_path(
+ const TabletColumn& column, const vectorized::PathInData& relative_path,
+ bool read_flat_leaves, ColumnReaderCache* cache, int32_t col_uid) const {
+ // english only in comments
+ // Locate the subcolumn node by path.
+ const auto* node = get_subcolumn_meta_by_path(relative_path);
+
+ // If path explicitly refers to sparse column internal path, fall back to schema type.
+ if (relative_path.get_path().find(SPARSE_COLUMN_PATH) != std::string::npos) {
+ return vectorized::DataTypeFactory::instance().create_data_type(column);
+ }
+
+ // Use variant type when the path is a prefix of any existing subcolumn path.
+ if (has_prefix_path(relative_path)) {
+ return vectorized::DataTypeFactory::instance().create_data_type(column);
+ }
+
+ // Try to reuse an existing leaf ColumnReader to infer its vectorized data type.
+ if (cache != nullptr) {
+ std::shared_ptr<ColumnReader> leaf_reader;
+ if (cache->get_path_column_reader(col_uid, relative_path, &leaf_reader, nullptr).ok() &&
+ leaf_reader != nullptr) {
+ return leaf_reader->get_vec_data_type();
+ }
+ }
+
+ // Node not found for the given path within the variant subcolumns.
+ if (node == nullptr) {
+ // Nested subcolumn is not present in sparse column: keep schema type.
+ if (column.is_nested_subcolumn()) {
+ return vectorized::DataTypeFactory::instance().create_data_type(column);
+ }
+
+ // When the path is in the sparse column or exceeded the limit, return variant type.
+ if (exist_in_sparse_column(relative_path) || is_exceeded_sparse_column_limit()) {
+ return column.is_nullable() ? vectorized::make_nullable(
+ std::make_shared<vectorized::DataTypeObject>(
+ column.variant_max_subcolumns_count()))
+ : std::make_shared<vectorized::DataTypeObject>(
+ column.variant_max_subcolumns_count());
+ }
+ // Path is not present in this segment: use default schema type.
+ return vectorized::DataTypeFactory::instance().create_data_type(column);
+ }
+
+ // For nested subcolumns, always use the physical file_column_type recorded in meta.
+ if (column.is_nested_subcolumn()) {
+ return node->data.file_column_type;
+ }
+
+ const bool exist_in_sparse = exist_in_sparse_column(relative_path);
+
+ // Condition to return the specific underlying type of the node:
+ // 1. We are reading flat leaves (ignoring hierarchy).
+ // 2. OR the path does not exist in sparse column and sparse column limit is not exceeded
+ // (meaning it is a pure materialized leaf).
+ if (read_flat_leaves || (!exist_in_sparse && !is_exceeded_sparse_column_limit())) {
+ return node->data.file_column_type;
+ }
+
+ // For non-compaction reads where we still treat this as logical VARIANT.
+ return vectorized::DataTypeFactory::instance().create_data_type(column);
+}
+
Status VariantRootColumnIterator::_process_root_column(
vectorized::MutableColumnPtr& dst, vectorized::MutableColumnPtr& root_column,
const vectorized::DataTypePtr& most_common_type) {
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
index f22809eed52..a217a7eee48 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
@@ -27,8 +27,10 @@
#include <vector>
#include "olap/rowset/segment_v2/column_reader.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/rowset/segment_v2/variant_statistics.h"
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
#include "olap/tablet_schema.h"
#include "vec/columns/column_object.h"
#include "vec/columns/subcolumn_tree.h"
@@ -38,6 +40,7 @@ namespace doris {
class TabletIndex;
class StorageReadOptions;
+class TabletSchema;
namespace segment_v2 {
@@ -45,12 +48,14 @@ class ColumnIterator;
class InvertedIndexIterator;
class InvertedIndexFileReader;
class ColumnReaderCache;
+class ColumnMetaAccessor;
class VariantColumnReader : public ColumnReader {
public:
VariantColumnReader() = default;
- Status init(const ColumnReaderOptions& opts, const SegmentFooterPB& footer, uint32_t column_id,
+ Status init(const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const std::shared_ptr<SegmentFooterPB>& footer, int32_t column_uid,
uint64_t num_rows, io::FileReaderSPtr file_reader);
Status new_iterator(ColumnIteratorUPtr* iterator, const TabletColumn* col,
@@ -70,7 +75,9 @@ public:
int64_t get_metadata_size() const override;
- std::vector<const TabletIndex*> find_subcolumn_tablet_indexes(const std::string&);
+ // Return shared_ptr to ensure the lifetime of TabletIndex objects
+ TabletIndexes find_subcolumn_tablet_indexes(const TabletColumn& target_column,
+ const vectorized::DataTypePtr& data_type);
bool exist_in_sparse_column(const vectorized::PathInData& path) const;
@@ -80,15 +87,54 @@ public:
return _subcolumns_meta_info.get();
}
+ // Get the types of all subcolumns in the variant column.
void get_subcolumns_types(
std::unordered_map<vectorized::PathInData, vectorized::DataTypes,
vectorized::PathInData::Hash>* subcolumns_types) const;
+ // Get the typed paths in the variant column.
void get_typed_paths(std::unordered_set<std::string>* typed_paths) const;
+ // Get the nested paths in the variant column.
void get_nested_paths(std::unordered_set<vectorized::PathInData, vectorized::PathInData::Hash>*
nested_paths) const;
+ // Infer the storage data type for a variant subcolumn identified by `relative_path`.
+ // This encapsulates the same decision logic as Segment::get_data_type_of for variant paths,
+ // including handling of sparse columns, prefix paths, and flat-leaf compaction mode.
+ vectorized::DataTypePtr infer_data_type_for_path(const TabletColumn& column,
+ const vectorized::PathInData& relative_path,
+ bool read_flat_leaves,
+ ColumnReaderCache* cache,
+ int32_t col_uid) const;
+
+ // Create a ColumnReader for a sub-column identified by `relative_path`.
+ // This method will first try inline footer.columns via footer_ordinal and then
+ // fall back to external meta if available. Callers do not need to care about
+ // the underlying layout (inline vs external).
+ Status create_path_reader(const vectorized::PathInData& relative_path,
+ const ColumnReaderOptions& opts, ColumnMetaAccessor* accessor,
+ const SegmentFooterPB& footer, const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows, std::shared_ptr<ColumnReader>* out);
+
+ // Try create a ColumnReader from externalized meta (path -> ColumnMetaPB bytes) if present.
+ // Only used internally by create_path_reader. External callers should not rely
+ // on external meta details directly.
+ Status create_reader_from_external_meta(const std::string& path,
+ const ColumnReaderOptions& opts,
+ const io::FileReaderSPtr& file_reader,
+ uint64_t num_rows, std::shared_ptr<ColumnReader>* out);
+
+ // Ensure external meta is loaded only once across concurrent callers.
+ Status load_external_meta_once();
+
+ // Determine whether `path` is a strict prefix of any existing subcolumn path.
+ // Consider three sources:
+ // 1) Extracted subcolumns in `_subcolumns_meta_info`
+ // 2) Sparse column statistics in `_statistics->sparse_column_non_null_size`
+ // 3) Externalized metas via `_ext_meta_reader`
+ bool has_prefix_path(const vectorized::PathInData& relative_path) const;
+
private:
// init for compaction read
Status _new_default_iter_with_same_nested(ColumnIteratorUPtr* iterator, const TabletColumn& col,
@@ -114,11 +160,19 @@ private:
std::shared_ptr<ColumnReader> _sparse_column_reader;
std::shared_ptr<ColumnReader> _root_column_reader;
std::unique_ptr<VariantStatistics> _statistics;
- // key: subcolumn path, value: subcolumn indexes
- std::unordered_map<std::string, TabletIndexes> _variant_subcolumns_indexes;
+ std::shared_ptr<TabletSchema> _tablet_schema;
// variant_sparse_column_statistics_size
size_t _variant_sparse_column_statistics_size =
BeConsts::DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATS_SIZE;
+
+ // Externalized meta reader (optional)
+ std::unique_ptr<VariantExternalMetaReader> _ext_meta_reader;
+
+ io::FileReaderSPtr _segment_file_reader;
+ uint64_t _num_rows {0};
+ uint32_t _root_unique_id {0};
+
+ // call-once guard moved into VariantExternalMetaReader
};
class VariantRootColumnIterator : public ColumnIterator {
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
new file mode 100644
index 00000000000..d04c6be1d2c
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/segment_v2.pb.h>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/column_writer.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/rowset/segment_v2/variant_statistics.h"
+#include "olap/tablet_schema.h"
+#include "vec/columns/column.h"
+#include "vec/common/schema_util.h"
+
+namespace doris {
+
+namespace vectorized {
+class ColumnVariant;
+class OlapBlockDataConvertor;
+} // namespace vectorized
+namespace segment_v2 {
+
+#include "common/compile_check_begin.h"
+
+class ColumnWriter;
+class ScalarColumnWriter;
+
+// Unifies writing of Variant sparse data in two modes:
+// 1) Single sparse column: one Map(String,String) column `__DORIS_VARIANT_SPARSE__`
+// 2) Bucketized sparse columns: N Map columns `__DORIS_VARIANT_SPARSE__.b{i}`
+//
+// Responsibilities:
+// - Initialize column writers and metas (consuming column_id identically to previous logic)
+// - Convert and append Variant's serialized sparse data to storage writers
+// - Emit per-column (or per-bucket) sparse path statistics and set meta num_rows
+//
+// Invariants:
+// - `_first_column_id` stores the column_id assigned to the first sparse meta created by init*
+// and is used to bind converter column ids deterministically.
+class UnifiedSparseColumnWriter {
+public:
+ // Initialize single sparse column writer and consume one column_id.
+ Status init_single(const TabletColumn& sparse_column, int& column_id,
+ const ColumnWriterOptions& base_opts, SegmentFooterPB* footer);
+
+ // Initialize N bucket writers and consume N column_ids.
+ Status init_buckets(int bucket_num, const TabletColumn& parent_column, int& column_id,
+ const ColumnWriterOptions& base_opts, SegmentFooterPB* footer);
+
+ bool has_single() const { return static_cast<bool>(_single_writer); }
+ bool has_buckets() const { return !_bucket_writers.empty(); }
+ bool is_bucket_mode() const { return has_buckets(); }
+ int bucket_num() const { return static_cast<int>(_bucket_writers.size()); }
+
+ ColumnWriter* single_writer() const { return _single_writer.get(); }
+ ColumnWriterOptions* single_opts() { return &_single_opts; }
+ const ColumnWriterOptions* single_opts() const { return &_single_opts; }
+
+ std::vector<std::unique_ptr<ColumnWriter>>& bucket_writers() { return _bucket_writers; }
+ const std::vector<std::unique_ptr<ColumnWriter>>& bucket_writers() const {
+ return _bucket_writers;
+ }
+ std::vector<ColumnWriterOptions>& bucket_opts() { return _bucket_opts; }
+ const std::vector<ColumnWriterOptions>& bucket_opts() const { return _bucket_opts; }
+
+ // Delegate lifecycle operations to the underlying writer(s).
+ uint64_t estimate_buffer_size() const;
+ Status finish();
+ Status write_data();
+ Status write_ordinal_index();
+
+ // Append data from ColumnVariant into single/bucket writers and fill statistics/meta.
+ // - For single mode: updates out_statistics and writes stats to single meta, sets num_rows
+ // - For bucket mode: writes per-bucket stats to each bucket meta, sets num_rows
+ // Convert and append Variant's sparse data:
+ // - Single mode: convert src.get_sparse_column() and append to the single writer, populate
+ // out_stats (merged) and write stats to meta
+ // - Bucket mode: materialize N ColumnMap temporaries, distribute entries by
+ // schema_util::variant_sparse_shard_of(path), convert and append per bucket, and write
+ // per-bucket stats to metas
+ Status append_from_variant(const vectorized::ColumnVariant& src, size_t num_rows,
+ vectorized::OlapBlockDataConvertor* converter,
+ const TabletColumn& tablet_column, VariantStatistics* out_stats);
+
+private:
+ // Single sparse writer and its options/meta
+ std::unique_ptr<ColumnWriter> _single_writer;
+ ColumnWriterOptions _single_opts;
+ // Bucketized sparse writers and their options/metas (size == bucket_num)
+ std::vector<std::unique_ptr<ColumnWriter>> _bucket_writers;
+ std::vector<ColumnWriterOptions> _bucket_opts;
+ // remember assigned column ids for conversion
+ int _first_column_id = -1;
+};
+
+class VariantColumnWriterImpl {
+public:
+ VariantColumnWriterImpl(const ColumnWriterOptions& opts, const TabletColumn* column);
+ Status finalize();
+ Status init();
+ bool is_finalized() const;
+
+ Status append_data(const uint8_t** ptr, size_t num_rows);
+
+ Status finish();
+ Status write_data();
+ Status write_ordinal_index();
+ Status write_zone_map();
+ Status write_bitmap_index();
+ Status write_inverted_index();
+ Status write_bloom_filter_index();
+ uint64_t estimate_buffer_size();
+ Status append_nullable(const uint8_t* null_map, const uint8_t** ptr, size_t num_rows);
+
+private:
+ Status _process_root_column(vectorized::ColumnVariant* ptr,
+ vectorized::OlapBlockDataConvertor* converter, size_t num_rows,
+ int& column_id);
+ Status _process_sparse_column(vectorized::ColumnVariant* ptr,
+ vectorized::OlapBlockDataConvertor* converter, size_t num_rows,
+ int& column_id);
+ Status _process_subcolumns(vectorized::ColumnVariant* ptr,
+ vectorized::OlapBlockDataConvertor* converter, size_t num_rows,
+ int& column_id);
+ // prepare a column for finalize
+ doris::vectorized::MutableColumnPtr _column;
+ doris::vectorized::ColumnUInt8 _null_column;
+ ColumnWriterOptions _opts;
+ const TabletColumn* _tablet_column = nullptr;
+ bool _is_finalized = false;
+ // for root column
+ std::unique_ptr<ColumnWriter> _root_writer;
+ // unified sparse writers (single or bucket mode)
+ UnifiedSparseColumnWriter _sparse_writer;
+ std::vector<std::unique_ptr<ColumnWriter>> _subcolumn_writers;
+ std::vector<ColumnWriterOptions> _subcolumn_opts;
+
+ // staticstics which will be persisted in the footer
+ VariantStatistics _statistics;
+
+ // hold the references of subcolumns indexes
+ std::vector<TabletIndexes> _subcolumns_indexes;
+
+ // hold the references of subcolumns info
+ std::unordered_map<std::string, TabletSchema::SubColumnInfo> _subcolumns_info;
+};
+
+void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column,
+ CompressionTypePB compression_type);
+
+#include "common/compile_check_end.h"
+
+} // namespace segment_v2
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp
new file mode 100644
index 00000000000..b31a9ceff20
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.cpp
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/types.h"
+#include "vec/common/schema_util.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+
+Status VariantExtMetaWriter::_ensure_inited(Writers* w) {
+ if (w->inited) {
+ return Status::OK();
+ }
+
+ // key writer: VARCHAR, value index ON, ordinal index ON
+ IndexedColumnWriterOptions dict_opts;
+ dict_opts.write_value_index = true;
+ dict_opts.write_ordinal_index = true;
+ dict_opts.encoding = PREFIX_ENCODING;
+ dict_opts.compression = _comp;
+ const TypeInfo* dict_type = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
+ w->key_writer = std::make_unique<IndexedColumnWriter>(dict_opts, dict_type, _fw);
+ RETURN_IF_ERROR(w->key_writer->init());
+
+ w->inited = true;
+ return Status::OK();
+}
+
+Status VariantExtMetaWriter::add(int32_t root_uid, const Slice& key) {
+ auto& w = _writers_by_uid[root_uid];
+ RETURN_IF_ERROR(_ensure_inited(&w));
+
+ // Only store the relative path string as key. The concrete column_id in
+ // Column Meta Region will be derived on the reader side using:
+ // col_id = root_column_id + 1 + key_ordinal
+ // based on the agreed layout of ColumnMetaEntryPB in SegmentFooterPB.
+ RETURN_IF_ERROR(w.key_writer->add(&key));
+ ++w.count;
+ return Status::OK();
+}
+
+Status VariantExtMetaWriter::flush_to_footer(SegmentFooterPB* footer) {
+ for (auto& [uid, w] : _writers_by_uid) {
+ if (!w.inited || w.count == 0) {
+ continue;
+ }
+ doris::segment_v2::IndexedColumnMetaPB key_meta;
+ RETURN_IF_ERROR(w.key_writer->finish(&key_meta));
+
+ // keys
+ std::string k = std::string("variant_meta_keys.") + std::to_string(uid);
+ std::string v;
+ key_meta.AppendToString(&v);
+ auto* p1 = footer->add_file_meta_datas();
+ p1->set_key(k);
+ p1->set_value(v);
+ }
+ _writers_by_uid.clear();
+ return Status::OK();
+}
+
+// Helper struct to manage column classification
+struct ColumnClassification {
+ std::vector<ColumnMetaPB> top_level_columns;
+ std::unordered_map<int32_t, std::vector<std::pair<std::string, ColumnMetaPB>>> pending_subcols;
+ std::unordered_map<int32_t, std::vector<ColumnMetaPB>> pending_sparse;
+ size_t externalized_count = 0;
+};
+
+// Separate function to classify columns
+static void classify_columns(const SegmentFooterPB& footer, ColumnClassification& classification) {
+ classification.top_level_columns.reserve(footer.columns_size());
+
+ for (int i = 0; i < footer.columns_size(); ++i) {
+ const ColumnMetaPB& col = footer.columns(i);
+ bool is_subcol = false;
+ if (col.has_column_path_info()) {
+ vectorized::PathInData full_path;
+ full_path.from_protobuf(col.column_path_info());
+ vectorized::PathInData rel = full_path.copy_pop_front();
+ if (!rel.empty()) {
+ is_subcol = true;
+ std::string rel_path = rel.get_path();
+ int32_t root_uid = col.column_path_info().parrent_column_unique_id();
+
+ if (rel_path.find("__DORIS_VARIANT_SPARSE__") != std::string::npos) {
+ classification.pending_sparse[root_uid].emplace_back(col);
+ } else {
+ classification.pending_subcols[root_uid].emplace_back(rel_path, col);
+ classification.externalized_count++;
+ }
+ }
+ }
+
+ if (!is_subcol) {
+ classification.top_level_columns.emplace_back(col);
+ }
+ }
+}
+
+Status VariantExtMetaWriter::externalize_from_footer(SegmentFooterPB* footer,
+ std::vector<ColumnMetaPB>* out_metas) {
+ // Variant meta pre-processing and reorganization:
+ // 1. Identify Variant Root columns and their subcolumns.
+ // 2. Collect sparse subcolumns to embed in Root.
+ // 3. Collect non-sparse subcolumns to externalize to Meta Region.
+ // 4. Rebuild the column list (out_metas) ensuring subcolumns follow their Root.
+ // 5. Record mapping (Path -> Index in out_metas) for externalized subcolumns.
+
+ ColumnClassification classification;
+ classify_columns(*footer, classification);
+
+ // Rebuild out_metas
+ out_metas->clear();
+ out_metas->reserve(footer->columns_size()); // Rough estimate
+
+ std::vector<ColumnMetaPB> kept_footer_columns;
+ kept_footer_columns.reserve(footer->columns_size());
+
+ for (auto& col : classification.top_level_columns) {
+ // Add Top Level Column
+ out_metas->emplace_back(col);
+ // Keep it in footer columns as well
+ kept_footer_columns.emplace_back(col);
+ ColumnMetaPB* current_root_meta = &out_metas->back();
+ ColumnMetaPB* footer_root_meta = &kept_footer_columns.back();
+
+ // Check if it is a Variant Root
+ if (col.type() == int(FieldType::OLAP_FIELD_TYPE_VARIANT)) {
+ int32_t root_uid = col.unique_id();
+
+ // 1. Embed sparse subcolumns
+ if (auto it = classification.pending_sparse.find(root_uid);
+ it != classification.pending_sparse.end()) {
+ for (const auto& sparse_meta : it->second) {
+ current_root_meta->add_children_columns()->CopyFrom(sparse_meta);
+ footer_root_meta->add_children_columns()->CopyFrom(sparse_meta);
+ }
+ }
+
+ // 2. Append non-sparse subcolumns to out_metas and record path-only keys
+ if (auto it = classification.pending_subcols.find(root_uid);
+ it != classification.pending_subcols.end()) {
+ // Sort by path for consistent index order
+ std::sort(it->second.begin(), it->second.end(),
+ [](const auto& a, const auto& b) { return a.first < b.first; });
+
+ for (auto& [path, meta] : it->second) {
+ // Append to out_metas
+ out_metas->emplace_back(meta);
+ // Mark as subcolumn placeholder for footer logic (unique_id = -1)
+ out_metas->back().set_unique_id(-1);
+
+ // Record path key for this subcolumn. Column id will be derived
+ // by reader as (root_col_id + 1 + ordinal).
+ RETURN_IF_ERROR(add(root_uid, Slice(path)));
+ }
+ }
+ }
+ }
+
+ // Write the index to footer
+ RETURN_IF_ERROR(flush_to_footer(footer));
+
+ // Update footer.columns to only contain Top Level Columns (with embedded sparse cols)
+ footer->clear_columns();
+ for (const auto& c : kept_footer_columns) {
+ footer->add_columns()->CopyFrom(c);
+ }
+
+ VLOG_DEBUG << "VariantExtMetaWriter::externalize_from_footer, externalized subcolumns: "
+ << classification.externalized_count
+ << ", kept columns: " << kept_footer_columns.size()
+ << ", total meta entries: " << out_metas->size();
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h
new file mode 100644
index 00000000000..e70429983dd
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_ext_meta_writer.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "common/status.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/types.h"
+#include "util/slice.h"
+
+namespace doris::segment_v2 {
+
+// -----------------------------------------------------------------------------
+// Variant External Meta (path -> ColumnMetaPB bytes) storage format
+// -----------------------------------------------------------------------------
+// Purpose: externalize ColumnMetaPB of variant subcolumns as a mapping from
+// relative path to column meta entries in the external Column Meta Region,
+// instead of packing all into Footer.columns. This enables on-demand subcolumn
+// loading and fast path lookup.
+//
+// Physical layout (per variant root uid): one IndexedColumn of keys is written
+// - variant_meta_keys.<root_uid> : IndexedColumn(VARCHAR)
+// • value index: ON (fast existence/lookup by path)
+// • ordinal index: ON (enable full scan / load_all)
+// • encoding: PREFIX_ENCODING
+// • content: Key = Path (relative path string only)
+//
+// Footer carrier:
+// footer.file_meta_datas contains one entry per root uid:
+// - key = "variant_meta_keys.<root_uid>" , value = IndexedColumnMetaPB bytes
+//
+// Interop with external column meta (column_meta_entries):
+// - Variant root ColumnMetaPB is externalized via column_meta_entries (addressed by col_id)
+// - Variant subcolumn ColumnMetaPB:
+// • non-sparse: stored in Column Meta Region, addressed by a unique col_id (in column_meta_entries).
+// The mapping from Path -> col_id is stored in this class (VariantExtMetaWriter).
+// • sparse (including buckets): embedded into root's ColumnMetaPB.children_columns and
+// loaded together with root meta via column_meta_entries
+//
+class VariantExtMetaWriter {
+public:
+ VariantExtMetaWriter(io::FileWriter* fw, CompressionTypePB comp) : _fw(fw), _comp(comp) {}
+
+ // Add one path entry for the specified variant root uid.
+ // key: subcolumn path (VARCHAR)
+ // The concrete ColumnMetaPB location (column_id) is derived on read side
+ // via (root_column_id + 1 + ordinal) based on the agreed Column Meta Region
+ // layout.
+ Status add(int32_t root_uid, const Slice& key);
+
+ // Finish writers and append their metas into footer.file_meta_datas.
+ Status flush_to_footer(SegmentFooterPB* footer);
+
+ // Scan footer.columns, find variant extracted subcolumns, externalize them into
+ // ext meta index and remove them from footer.columns.
+ // This method reorganizes the columns:
+ // 1. Separates Top Level Columns (kept in footer) and Variant Subcolumns.
+ // 2. Records Subcolumns' indices (relative to the combined list) into the Variant Index.
+ // 3. Populates out_metas with the combined list of all columns (Top Level + Subcolumns),
+ // ordered such that Subcolumns immediately follow their Root Variant.
+ Status externalize_from_footer(SegmentFooterPB* footer, std::vector<ColumnMetaPB>* out_metas);
+
+ bool empty() const { return _writers_by_uid.empty(); }
+
+private:
+ struct Writers {
+ std::unique_ptr<IndexedColumnWriter> key_writer; // value index
+ size_t count = 0;
+ bool inited = false;
+ };
+
+ Status _ensure_inited(Writers* w);
+
+ io::FileWriter* _fw;
+ CompressionTypePB _comp;
+ std::unordered_map<int32_t, Writers> _writers_by_uid;
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp
new file mode 100644
index 00000000000..9673f0093d3
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.cpp
@@ -0,0 +1,246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
+
+#include <gen_cpp/segment_v2.pb.h>
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
+#include "vec/columns/column_string.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+
+#include "common/compile_check_begin.h"
+
+Status VariantExternalMetaReader::_find_key_meta(const SegmentFooterPB& footer, int32_t root_uid,
+ const MetadataPairPB** keys_meta_pair) const {
+ *keys_meta_pair = nullptr;
+ // prefer suffixed pairs
+ std::string suffix = "." + std::to_string(root_uid);
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == std::string("variant_meta_keys") + suffix) {
+ *keys_meta_pair = &m;
+ return Status::OK();
+ }
+ }
+ // fallback: legacy single-variant footer
+ if (!*keys_meta_pair) {
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == "variant_meta_keys") {
+ *keys_meta_pair = &m;
+ return Status::OK();
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::init_from_footer(std::shared_ptr<const SegmentFooterPB> footer,
+ const io::FileReaderSPtr& file_reader,
+ int32_t root_uid) {
+ _footer = std::move(footer);
+ _file_reader = file_reader;
+
+ const MetadataPairPB* keys_meta_pair = nullptr;
+ RETURN_IF_ERROR(_find_key_meta(*_footer, root_uid, &keys_meta_pair));
+ // If keys meta is absent, external meta is considered unavailable.
+ if (!keys_meta_pair) {
+ return Status::OK();
+ }
+
+ // Parse external Column Meta Region pointers and uid->col_id map.
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_external_meta_pointers(*_footer, &_meta_ptrs));
+ std::unordered_map<int32_t, size_t> uid2colid;
+ RETURN_IF_ERROR(ExternalColMetaUtil::parse_uid_to_colid_map(*_footer, _meta_ptrs, &uid2colid));
+ auto it = uid2colid.find(root_uid);
+ if (it == uid2colid.end()) {
+ return Status::Corruption("root uid {} not found in external uid->col_id map", root_uid);
+ }
+ _root_col_id = static_cast<uint32_t>(it->second);
+
+ // Open the path-key indexed column.
+ doris::segment_v2::IndexedColumnMetaPB key_meta;
+ if (!key_meta.ParseFromArray(keys_meta_pair->value().data(),
+ static_cast<int>(keys_meta_pair->value().size()))) {
+ return Status::Corruption("bad variant_meta_keys meta");
+ }
+
+ _key_reader = std::make_unique<segment_v2::IndexedColumnReader>(file_reader, key_meta);
+ RETURN_IF_ERROR(_key_reader->load(true, false));
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::lookup_meta_by_path(const std::string& rel_path,
+ ColumnMetaPB* out_meta) const {
+ if (!available()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("no external variant meta");
+ }
+ // 1. Seek for Path
+ auto key_it = segment_v2::IndexedColumnIterator(_key_reader.get());
+ bool exact = false;
+ Status st = key_it.seek_at_or_after(&rel_path, &exact);
+ if (st.is<ErrorCode::ENTRY_NOT_FOUND>()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>("variant meta key not found");
+ }
+ if (!st.ok()) {
+ return st;
+ }
+
+ // Record ordinal immediately after seek; get_current_ordinal() is only valid
+ // while iterator is in "seeked" state (before calling next_batch()).
+ auto ordinal = static_cast<uint32_t>(key_it.get_current_ordinal());
+
+ // 2. Read key and verify exact match (keys store pure relative path).
+ size_t n = 1;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n != 1) {
+ return Status::Corruption("variant meta key read failed");
+ }
+ auto* s = assert_cast<vectorized::ColumnString*>(dst.get());
+ auto ref = s->get_data_at(0);
+
+ if (ref.size != rel_path.size() || memcmp(ref.data, rel_path.data(), ref.size) != 0) {
+ // seek_at_or_after found something greater, but path doesn't match
+ return Status::Error<ErrorCode::NOT_FOUND, false>("variant meta key not found");
+ }
+
+ // 3. Derive column id from root_col_id and key ordinal, then read ColumnMetaPB.
+ auto col_id = _root_col_id + 1 + ordinal;
+ return ExternalColMetaUtil::read_col_meta(_file_reader, *_footer, _meta_ptrs, col_id, out_meta);
+}
+
+Status VariantExternalMetaReader::load_all(SubcolumnColumnMetaInfo* out_meta_tree,
+ VariantStatistics* out_stats) {
+ if (!available()) {
+ return Status::OK();
+ }
+
+ auto key_it = segment_v2::IndexedColumnIterator(_key_reader.get());
+ RETURN_IF_ERROR(key_it.seek_to_ordinal(0));
+ auto total = static_cast<size_t>(_key_reader->num_values());
+ size_t built = 0;
+
+ while (built < total) {
+ size_t n = total - built;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n == 0) {
+ break;
+ }
+ for (size_t i = 0; i < n; ++i) {
+ // Derive column id for this key by its global ordinal.
+ auto ordinal = static_cast<uint32_t>(built + i);
+ auto col_id = _root_col_id + 1 + ordinal;
+
+ ColumnMetaPB meta;
+ RETURN_IF_ERROR(ExternalColMetaUtil::read_col_meta(_file_reader, *_footer, _meta_ptrs,
+ col_id, &meta));
+
+ if (!meta.has_column_path_info()) {
+ continue;
+ }
+ vectorized::PathInData full_path;
+ full_path.from_protobuf(meta.column_path_info());
+ auto relative_path = full_path.copy_pop_front();
+ if (relative_path.empty()) {
+ continue; // skip root
+ }
+ if (out_meta_tree->find_leaf(relative_path)) {
+ continue; // already exists
+ }
+ if (meta.has_none_null_size() && out_stats != nullptr) {
+ out_stats->subcolumns_non_null_size.emplace(relative_path.get_path(),
+ meta.none_null_size());
+ }
+ auto file_type = vectorized::DataTypeFactory::instance().create_data_type(meta);
+ out_meta_tree->add(relative_path,
+ SubcolumnMeta {.file_column_type = file_type, .footer_ordinal = -1});
+ }
+ built += n;
+ }
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::has_prefix(const std::string& prefix, bool* out) const {
+ // english only in comments
+ DCHECK(out != nullptr);
+ DCHECK(available());
+ *out = false;
+ // Empty prefix means everything matches; guard for safety
+ if (prefix.empty()) {
+ *out = true;
+ return Status::OK();
+ }
+
+ segment_v2::IndexedColumnIterator key_it(_key_reader.get());
+ bool exact = false;
+ Status st = key_it.seek_at_or_after(&prefix, &exact);
+ if (st.is<ErrorCode::ENTRY_NOT_FOUND>()) {
+ *out = false;
+ return Status::OK();
+ }
+ if (!st.ok()) {
+ return st;
+ }
+
+ size_t n = 1;
+ auto col = vectorized::ColumnString::create();
+ vectorized::MutableColumnPtr dst = std::move(col);
+ RETURN_IF_ERROR(key_it.next_batch(&n, dst));
+ if (n == 0) {
+ *out = false;
+ return Status::OK();
+ }
+ auto* s = assert_cast<vectorized::ColumnString*>(dst.get());
+ auto ref = s->get_data_at(0);
+ std::string_view key_sv(ref.data, ref.size);
+
+ // starts_with check. Key is Path + ID.
+ // If prefix is just path part, it matches.
+ // If prefix contains ID part (unlikely use case), it also matches.
+ *out = key_sv.size() >= prefix.size() && key_sv.starts_with(prefix);
+ return Status::OK();
+}
+
+Status VariantExternalMetaReader::load_all_once(SubcolumnColumnMetaInfo* out_meta_tree,
+ VariantStatistics* out_stats) {
+ if (!available()) {
+ return Status::OK();
+ }
+ return _load_once_call.call([&]() -> Status {
+ if (_loaded) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(load_all(out_meta_tree, out_stats));
+ _loaded = true;
+ return Status::OK();
+ });
+}
+
+#include "common/compile_check_end.h"
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h
new file mode 100644
index 00000000000..e07ff105e24
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/variant_external_meta_reader.h
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/segment_v2.pb.h>
+
+#include <memory>
+#include <string>
+
+#include "common/status.h"
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+#include "olap/rowset/segment_v2/indexed_column_reader.h"
+#include "olap/rowset/segment_v2/stream_reader.h"
+#include "olap/rowset/segment_v2/variant_statistics.h"
+#include "util/once.h"
+
+namespace doris::segment_v2 {
+
+#include "common/compile_check_begin.h"
+
+// Encapsulates reading of externalized Variant subcolumn metas.
+// It discovers key/value indexed-columns from SegmentFooterPB, supports:
+// - availability check
+// - lookup ColumnMetaPB by relative path
+// - one-time bulk load of all metas into SubcolumnColumnMetaInfo and VariantStatistics
+class VariantExternalMetaReader {
+public:
+ VariantExternalMetaReader() = default;
+
+ // Initialize by locating and opening ext meta key/value indexed columns.
+ // root_uid is used to find suffixed keys like: variant_meta_keys.<root_uid>
+ //
+ // NOTE: footer is kept as shared_ptr to ensure its lifetime is at least as
+ // long as this reader; callers should pass in the same footer shared_ptr
+ // they own (e.g. from Segment::_get_segment_footer).
+ Status init_from_footer(std::shared_ptr<const SegmentFooterPB> footer,
+ const io::FileReaderSPtr& file_reader, int32_t root_uid);
+
+ bool available() const { return _key_reader != nullptr; }
+
+ // Lookup a single ColumnMetaPB by relative path. Returns NOT_FOUND if missing/unavailable.
+ Status lookup_meta_by_path(const std::string& rel_path, ColumnMetaPB* out_meta) const;
+
+ // Check whether there exists any key in external meta that starts with `prefix`.
+ // This performs a lower_bound (seek_at_or_after) on the sorted key column
+ // and verifies the first key is prefixed by `prefix`.
+ Status has_prefix(const std::string& prefix, bool* out) const;
+
+ // Ensure external metas are loaded exactly once and merged into provided structures.
+ Status load_all_once(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics* out_stats);
+
+ // Load and merge all external metas without call-once guard.
+ Status load_all(SubcolumnColumnMetaInfo* out_meta_tree, VariantStatistics* out_stats);
+
+private:
+ // helpers
+ Status _find_key_meta(const SegmentFooterPB& footer, int32_t root_uid,
+ const MetadataPairPB** keys_meta_pair) const;
+
+ std::unique_ptr<IndexedColumnReader> _key_reader;
+
+ // Shared footer to access Column Meta Region info (column_meta_entries).
+ // Using shared_ptr prevents dangling pointer when the original owner releases it.
+ std::shared_ptr<const SegmentFooterPB> _footer;
+ ExternalColMetaUtil::ExternalMetaPointers _meta_ptrs;
+ io::FileReaderSPtr _file_reader;
+
+ // Global column_id of the variant root in external Column Meta Region.
+ // For its non-sparse subcolumns, their column_id is derived as:
+ // root_col_id + 1 + key_ordinal
+ uint32_t _root_col_id = 0;
+
+ // call-once guard for bulk loading
+ DorisCallOnce<Status> _load_once_call;
+ bool _loaded = false;
+};
+
+#include "common/compile_check_end.h"
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
index 6cb59d186da..c5fcefa0f5b 100644
--- a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
@@ -31,9 +31,12 @@
#include "olap/rowset/rowset_fwd.h"
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/column_writer.h"
+#include "olap/rowset/segment_v2/indexed_column_writer.h"
#include "olap/segment_loader.h"
#include "olap/tablet_schema.h"
+#include "olap/types.h"
#include "util/simd/bits.h"
+#include "util/slice.h"
#include "vec/columns/column.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_object.h"
@@ -250,6 +253,9 @@ Status VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnObject* pt
};
_subcolumns_indexes.resize(ptr->get_subcolumns().size());
// convert sub column data from engine format to storage layer format
+ // NOTE: We only keep up to variant_max_subcolumns_count as extracted columns; others are externalized.
+ // uint32_t extracted = 0;
+ // uint32_t extract_limit = _tablet_column->variant_max_subcolumns_count();
for (const auto& entry :
vectorized::schema_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
const auto& least_common_type = entry->data.get_least_common_type();
@@ -263,7 +269,7 @@ Status VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnObject* pt
}
CHECK(entry->data.is_finalized());
- // create subcolumn writer
+ // create subcolumn writer if under limit; otherwise externalize ColumnMetaPB via IndexedColumn
int current_column_id = column_id++;
TabletColumn tablet_column;
int64_t none_null_value_size = entry->data.get_non_null_value_size();
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index adb4986c2be..72a43873855 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -49,10 +49,12 @@
#include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "olap/rowset/segment_creator.h"
#include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/page_pointer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
#include "olap/segment_loader.h"
#include "olap/short_key_index.h"
#include "olap/tablet_schema.h"
@@ -1447,6 +1449,17 @@ Status VerticalSegmentWriter::_write_primary_key_index() {
Status VerticalSegmentWriter::_write_footer() {
_footer.set_num_rows(_row_count);
+ // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
+
+ if (_tablet_schema->is_external_segment_meta_used_default()) {
+ _footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+ VLOG_DEBUG << "use external column meta";
+ // External ColumnMetaPB writing (optional)
+ RETURN_IF_ERROR(ExternalColMetaUtil::write_external_column_meta(
+ _file_writer, &_footer, _opts.compression_type,
+ [this](const std::vector<Slice>& slices) { return _write_raw_data(slices); }));
+ }
+
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4)
VLOG_DEBUG << "footer " << _footer.DebugString();
std::string footer_buf;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 727b259a13b..f2672da5f6e 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1556,7 +1556,8 @@ Status TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& reque
(*tablet_meta)->set_preferred_rowset_type(_engine.default_rowset_type());
} else if (request.storage_format == TStorageFormat::V1) {
(*tablet_meta)->set_preferred_rowset_type(ALPHA_ROWSET);
- } else if (request.storage_format == TStorageFormat::V2) {
+ } else if (request.storage_format == TStorageFormat::V2 ||
+ request.storage_format == TStorageFormat::V3) {
(*tablet_meta)->set_preferred_rowset_type(BETA_ROWSET);
} else {
return Status::Error<CE_CMD_PARAMS_ERROR>("invalid TStorageFormat: {}",
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index da1aae33780..424ffa6bc51 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -108,12 +108,16 @@ TabletMetaSharedPtr TabletMeta::create(
break;
}
}
+ // Decide storage format for this tablet. DEFAULT / not-set fall back to V2 on BE side.
+ TStorageFormat::type storage_format =
+ request.__isset.storage_format ? request.storage_format : TStorageFormat::V2;
return std::make_shared<TabletMeta>(
request.table_id, request.partition_id, request.tablet_id, request.replica_id,
request.tablet_schema.schema_hash, shard_id, request.tablet_schema, next_unique_id,
col_ordinal_to_unique_id, tablet_uid,
request.__isset.tablet_type ? request.tablet_type : TTabletType::TABLET_TYPE_DISK,
- request.compression_type, request.storage_policy_id,
+ request.__isset.compression_type ? request.compression_type : TCompressionType::LZ4F,
+ request.__isset.storage_policy_id ? request.storage_policy_id : -1,
request.__isset.enable_unique_key_merge_on_write
? request.enable_unique_key_merge_on_write
: false,
@@ -123,7 +127,7 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
request.time_series_compaction_level_threshold, inverted_index_file_storage_format,
- request.tde_algorithm);
+ request.tde_algorithm, storage_format);
}
TabletMeta::~TabletMeta() {
@@ -151,10 +155,12 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
int64_t time_series_compaction_empty_rowsets_threshold,
int64_t time_series_compaction_level_threshold,
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format,
- TEncryptionAlgorithm::type tde_algorithm)
+ TEncryptionAlgorithm::type tde_algorithm,
+ TStorageFormat::type storage_format)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
- _delete_bitmap(new DeleteBitmap(tablet_id)) {
+ _delete_bitmap(new DeleteBitmap(tablet_id)),
+ _storage_format(storage_format) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(table_id);
tablet_meta_pb.set_partition_id(partition_id);
@@ -395,6 +401,22 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
tablet_meta_pb.set_encryption_algorithm(EncryptionAlgorithmPB::PLAINTEXT);
}
+ // Initialize default external ColumnMeta usage according to storage format.
+ // V2: legacy behavior, inline ColumnMetaPB only.
+ // V3: V2 + external ColumnMetaPB (CMO) enabled by default.
+ switch (_storage_format) {
+ case TStorageFormat::V2:
+ case TStorageFormat::DEFAULT:
+ case TStorageFormat::V1:
+ break;
+ case TStorageFormat::V3:
+ schema->set_is_external_segment_meta_used_default(true);
+ _schema->set_external_segment_meta_used_default(true);
+ break;
+ default:
+ break;
+ }
+
init_from_pb(tablet_meta_pb);
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 68f6d323bbc..4846d046f0e 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -118,7 +118,8 @@ public:
int64_t time_series_compaction_level_threshold = 1,
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
TInvertedIndexFileStorageFormat::V2,
- TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT);
+ TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT,
+ TStorageFormat::type storage_format = TStorageFormat::V2);
// If need add a filed in TableMeta, filed init copy in copy construct function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -369,6 +370,10 @@ private:
EncryptionAlgorithmPB _encryption_algorithm = PLAINTEXT;
+ // Persisted storage format for this tablet (e.g. V2, V3). Used to derive
+ // schema-level defaults such as external ColumnMeta usage.
+ TStorageFormat::type _storage_format = TStorageFormat::V2;
+
mutable std::shared_mutex _meta_lock;
};
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 255e15546af..561a7a6ef57 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1129,6 +1129,11 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
schema.row_store_column_unique_ids().end());
_variant_enable_flatten_nested = schema.variant_enable_flatten_nested();
_vl_field_mem_size += _row_store_column_unique_ids.capacity() * sizeof(int32_t);
+ if (schema.has_is_external_segment_meta_used_default()) {
+ _is_external_segment_meta_used_default = schema.is_external_segment_meta_used_default();
+ } else {
+ _is_external_segment_meta_used_default = false;
+ }
update_metadata_size();
}
@@ -1384,6 +1389,8 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign(
_row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end());
tablet_schema_pb->set_variant_enable_flatten_nested(_variant_enable_flatten_nested);
+ tablet_schema_pb->set_is_external_segment_meta_used_default(
+ _is_external_segment_meta_used_default);
}
size_t TabletSchema::row_size() const {
@@ -1713,6 +1720,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
if (a._storage_dict_page_size != b._storage_dict_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false;
if (a._variant_enable_flatten_nested != b._variant_enable_flatten_nested) return false;
+ if (a._is_external_segment_meta_used_default != b._is_external_segment_meta_used_default) return false;
return true;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index ad4897adc9b..3bae9ed4f21 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -655,6 +655,15 @@ public:
void clear_path_set_info() { _path_set_info_map.clear(); }
+ // Whether new segments use externalized ColumnMetaPB layout (CMO) by default
+ bool is_external_segment_meta_used_default() const {
+ return _is_external_segment_meta_used_default;
+ }
+
+ void set_external_segment_meta_used_default(bool v) {
+ _is_external_segment_meta_used_default = v;
+ }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -731,6 +740,9 @@ private:
// value: index
using PatternToIndex = std::unordered_map<std::string, std::vector<TabletIndexPtr>>;
std::unordered_map<int32_t, PatternToIndex> _index_by_unique_id_with_pattern;
+
+ // Default behavior for new segments: use external ColumnMeta region + CMO table if true
+ bool _is_external_segment_meta_used_default = false;
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index a1c5c98ce45..15ddf30ce53 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1132,6 +1132,10 @@ void PInternalService::fetch_remote_tablet_schema(google::protobuf::RpcControlle
bool ret = _heavy_work_pool.try_offer([request, response, done]() {
brpc::ClosureGuard closure_guard(done);
Status st = Status::OK();
+ std::shared_ptr<MemTrackerLimiter> mem_tracker = MemTrackerLimiter::create_shared(
+ MemTrackerLimiter::Type::OTHER,
+ fmt::format("InternalService::fetch_remote_tablet_schema"));
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker);
if (request->is_coordinator()) {
// Spawn rpc request to none coordinator nodes, and finally merge them all
PFetchRemoteSchemaRequest remote_request(*request);
diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp
index aa21db80d29..49a94630a37 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -759,8 +759,10 @@ Status aggregate_path_to_stats(
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* variant_column_reader =
- assert_cast<const segment_v2::VariantColumnReader*>(column_reader.get());
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting stats
+ RETURN_IF_ERROR(variant_column_reader->load_external_meta_once());
const auto* source_stats = variant_column_reader->get_stats();
CHECK(source_stats);
@@ -798,8 +800,10 @@ Status aggregate_variant_extended_info(
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* variant_column_reader =
- assert_cast<const segment_v2::VariantColumnReader*>(column_reader.get());
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting stats
+ RETURN_IF_ERROR(variant_column_reader->load_external_meta_once());
const auto* source_stats = variant_column_reader->get_stats();
CHECK(source_stats);
@@ -819,14 +823,17 @@ Status aggregate_variant_extended_info(
auto& paths_types =
(*uid_to_variant_extended_info)[column->unique_id()].path_to_data_types;
variant_column_reader->get_subcolumns_types(&paths_types);
+ VLOG_DEBUG << "path_to_data_types size: " << paths_types.size();
// 3. extract typed paths
auto& typed_paths = (*uid_to_variant_extended_info)[column->unique_id()].typed_paths;
variant_column_reader->get_typed_paths(&typed_paths);
+ VLOG_DEBUG << "typed_paths size: " << typed_paths.size();
// 4. extract nested paths
auto& nested_paths = (*uid_to_variant_extended_info)[column->unique_id()].nested_paths;
variant_column_reader->get_nested_paths(&nested_paths);
+ VLOG_DEBUG << "nested_paths size: " << nested_paths.size();
}
}
return Status::OK();
@@ -1490,9 +1497,16 @@ TabletSchemaSPtr calculate_variant_extended_schema(const std::vector<RowsetShare
}
CHECK(column_reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT);
- const auto* subcolumn_meta_info =
- assert_cast<VariantColumnReader*>(column_reader.get())
- ->get_subcolumns_meta_info();
+ auto* variant_column_reader =
+ assert_cast<segment_v2::VariantColumnReader*>(column_reader.get());
+ // load external meta before getting subcolumn meta info
+ st = variant_column_reader->load_external_meta_once();
+ if (!st.ok()) {
+ LOG(WARNING) << "Failed to load external meta for column: " << column->name()
+ << " error: " << st.to_string();
+ continue;
+ }
+ const auto* subcolumn_meta_info = variant_column_reader->get_subcolumns_meta_info();
for (const auto& entry : *subcolumn_meta_info) {
if (entry->path.empty()) {
continue;
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index bfcda534ecc..5129376de19 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -384,6 +384,18 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
case FieldType::OLAP_FIELD_TYPE_INT:
result = std::make_shared<vectorized::DataTypeInt32>();
break;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT:
+ result = std::make_shared<vectorized::DataTypeUInt8>();
+ break;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT:
+ result = std::make_shared<vectorized::DataTypeUInt16>();
+ break;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT:
+ result = std::make_shared<vectorized::DataTypeUInt32>();
+ break;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
+ result = std::make_shared<vectorized::DataTypeUInt64>();
+ break;
case FieldType::OLAP_FIELD_TYPE_FLOAT:
result = std::make_shared<vectorized::DataTypeFloat32>();
break;
diff --git a/be/test/olap/date_bloom_filter_test.cpp b/be/test/olap/date_bloom_filter_test.cpp
index ddb413bc2fd..ccd51d0bfd0 100644
--- a/be/test/olap/date_bloom_filter_test.cpp
+++ b/be/test/olap/date_bloom_filter_test.cpp
@@ -18,6 +18,7 @@
#include <gen_cpp/segment_v2.pb.h>
#include <gtest/gtest.h>
+#include "common/status.h"
#include "olap/comparison_predicate.h"
#include "olap/in_list_predicate.h"
#include "olap/rowset/beta_rowset.h"
@@ -144,15 +145,21 @@ TEST_F(DateBloomFilterTest, query_index_test) {
columns[0]->insert_many_fix_len_data(reinterpret_cast<const char*>(&olap_date_value), 1);
columns[1]->insert_many_fix_len_data(reinterpret_cast<const char*>(&olap_datetime_value), 1);
- EXPECT_TRUE(rowset_writer->add_block(&block).ok());
- EXPECT_TRUE(rowset_writer->flush().ok());
- EXPECT_TRUE(rowset_writer->build(rowset).ok());
- EXPECT_TRUE(_tablet->add_rowset(rowset).ok());
+ Status st;
+ st = rowset_writer->add_block(&block);
+ ASSERT_TRUE(st.ok()) << st; // add_block failed, no valid segment will be written
+ st = rowset_writer->flush();
+ ASSERT_TRUE(st.ok()) << st; // flush failed
+ st = rowset_writer->build(rowset);
+ ASSERT_TRUE(st.ok()) << st; // build rowset failed
+ st = _tablet->add_rowset(rowset);
+ ASSERT_TRUE(st.ok()) << st; // add_rowset failed
segment_v2::SegmentSharedPtr segment;
- EXPECT_TRUE(((BetaRowset*)rowset.get())->load_segment(0, &segment).ok());
+ st = ((BetaRowset*)rowset.get())->load_segment(0, nullptr, &segment);
+ ASSERT_TRUE(st.ok()) << st; // load_segment failed, segment may be null
std::shared_ptr<SegmentFooterPB> footer_pb_shared;
- auto st = segment->_get_segment_footer(footer_pb_shared, nullptr);
+ st = segment->_get_segment_footer(footer_pb_shared, nullptr);
EXPECT_TRUE(st.ok());
st = segment->_create_column_meta(*footer_pb_shared);
EXPECT_TRUE(st.ok());
diff --git a/be/test/olap/delete_bitmap_calculator_test.cpp b/be/test/olap/delete_bitmap_calculator_test.cpp
index 42e1d279f0f..efd0906915e 100644
--- a/be/test/olap/delete_bitmap_calculator_test.cpp
+++ b/be/test/olap/delete_bitmap_calculator_test.cpp
@@ -109,6 +109,19 @@ void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, siz
EXPECT_EQ(nrows, (*res)->num_rows());
}
+// Convenience overload that also returns the on-disk path of the built segment.
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir, std::string* out_path) {
+ std::string filename = fmt::format("{}_{}.dat", rowset_id.to_string(), segment_id);
+ std::string path = fmt::format("{}/{}", segment_dir, filename);
+ if (out_path != nullptr) {
+ *out_path = path;
+ }
+ build_segment(std::move(opts), std::move(build_schema), segment_id, std::move(query_schema),
+ nrows, std::move(generator), res, std::move(segment_dir));
+}
+
TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
KeysType keys_type = UNIQUE_KEYS) {
TabletSchemaSPtr res = std::make_shared<TabletSchema>();
diff --git a/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp b/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp
new file mode 100644
index 00000000000..b8a8db1a2ff
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/column_meta_accessor_test.cpp
@@ -0,0 +1,593 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "io/fs/local_file_system.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+
+using namespace doris;
+using namespace doris::segment_v2;
+
+namespace {
+
+constexpr std::string_view kTestDir = "./ut_dir/column_meta_accessor_test";
+
+// Helper to create a clean test directory
+std::string make_test_file_path(const std::string& file_name) {
+ auto fs = io::global_local_filesystem();
+ static_cast<void>(fs->delete_directory(kTestDir));
+ CHECK(fs->create_directory(kTestDir).ok());
+ return std::string(kTestDir) + "/" + file_name;
+}
+
+// Helper to write segment footer trailer (footer + metadata)
+Status append_footer_trailer(io::FileWriter* fw, SegmentFooterPB* footer) {
+ std::string footer_buf;
+ if (!footer->SerializeToString(&footer_buf)) {
+ return Status::InternalError("failed to serialize SegmentFooterPB");
+ }
+ faststring fixed_buf;
+ put_fixed32_le(&fixed_buf, static_cast<uint32_t>(footer_buf.size()));
+ uint32_t checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ put_fixed32_le(&fixed_buf, checksum);
+ fixed_buf.append("D0R1", 4);
+ std::vector<Slice> slices {Slice(footer_buf), Slice(fixed_buf)};
+ RETURN_IF_ERROR(fw->appendv(slices.data(), slices.size()));
+ return Status::OK();
+}
+
+// Helper to read footer from file
+Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* footer) {
+ const auto file_size = fr->size();
+ if (file_size < 12) {
+ return Status::Corruption("file too small");
+ }
+ uint8_t fixed_buf[12];
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12, Slice(fixed_buf, sizeof(fixed_buf)), &bytes_read));
+ if (bytes_read != sizeof(fixed_buf)) {
+ return Status::Corruption("short read footer trailer");
+ }
+ const uint32_t footer_length = decode_fixed32_le(fixed_buf);
+ if (file_size < 12 + footer_length) {
+ return Status::Corruption("bad footer length");
+ }
+ std::string footer_buf;
+ footer_buf.resize(footer_length);
+ bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12 - footer_length,
+ Slice(footer_buf.data(), footer_buf.size()), &bytes_read));
+ if (bytes_read != footer_length) {
+ return Status::Corruption("short read footer");
+ }
+ if (!footer->ParseFromArray(footer_buf.data(), static_cast<int>(footer_buf.size()))) {
+ return Status::Corruption("failed to parse footer");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+// Test V2 (inline) mode with valid column metadata
+TEST(ColumnMetaAccessorTest, V2_BasicInlineColumns) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+
+ // Add 3 columns with different types
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+ col0->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ auto* col1 = footer.add_columns();
+ col1->set_unique_id(20);
+ col1->set_column_id(1);
+ col1->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ auto* col2 = footer.add_columns();
+ col2->set_unique_id(30);
+ col2->set_column_id(2);
+ col2->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 10);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 1, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 20);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 2, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 30);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ // Test has_column_uid for existing and non-existing uids
+ EXPECT_TRUE(accessor.has_column_uid(10));
+ EXPECT_TRUE(accessor.has_column_uid(20));
+ EXPECT_TRUE(accessor.has_column_uid(30));
+ EXPECT_FALSE(accessor.has_column_uid(999));
+
+ // Test get_column_meta_by_uid
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 10, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 20, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 1);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 30, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 2);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_BIGINT));
+
+ // Test traverse_metas
+ std::vector<int32_t> traversed_uids;
+ ASSERT_TRUE(accessor.traverse_metas(footer,
+ [&](const ColumnMetaPB& m) {
+ traversed_uids.push_back(m.unique_id());
+ })
+ .ok());
+ EXPECT_EQ(traversed_uids.size(), 3);
+ EXPECT_EQ(traversed_uids[0], 10);
+ EXPECT_EQ(traversed_uids[1], 20);
+ EXPECT_EQ(traversed_uids[2], 30);
+}
+
+// Test V2 mode with variant subcolumns (unique_id == -1 should be skipped)
+TEST(ColumnMetaAccessorTest, V2_SkipVariantSubcolumns) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+
+ // Regular column
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+
+ // Variant subcolumn (should be skipped in uid->colid map)
+ auto* col1 = footer.add_columns();
+ col1->set_unique_id(-1);
+ col1->set_column_id(1);
+
+ // Regular column
+ auto* col2 = footer.add_columns();
+ col2->set_unique_id(30);
+ col2->set_column_id(2);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Only regular columns should be accessible by uid; variant subcolumns (uid == -1) are skipped.
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 10, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 30, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 2);
+ Status st = accessor.get_column_meta_by_uid(footer, -1, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::NOT_FOUND>());
+
+ // has_column_uid should also reflect the same semantics
+ EXPECT_TRUE(accessor.has_column_uid(10));
+ EXPECT_TRUE(accessor.has_column_uid(30));
+ EXPECT_FALSE(accessor.has_column_uid(-1));
+}
+
+// Test V2 mode error cases
+TEST(ColumnMetaAccessorTest, V2_ErrorCases) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+
+ // Add one column
+ auto* col0 = footer.add_columns();
+ col0->set_unique_id(10);
+ col0->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id with invalid column id
+ ColumnMetaPB meta;
+ Status st = accessor.get_column_meta_by_column_ordinal_id(footer, 999, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+
+ // Test empty footer
+ SegmentFooterPB empty_footer;
+ empty_footer.set_version(kSegmentFooterVersionV2);
+ ColumnMetaAccessor empty_accessor;
+ ASSERT_TRUE(empty_accessor.init(empty_footer, nullptr).ok());
+
+ st = empty_accessor.get_column_meta_by_column_ordinal_id(empty_footer, 0, &meta);
+ EXPECT_FALSE(st.ok());
+
+ st = empty_accessor.traverse_metas(empty_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+}
+
+// Test V3 (external) mode with valid external meta region
+TEST(ColumnMetaAccessorTest, V3_ExternalMetaRegion) {
+ // 1. Create a V3 footer with external meta layout
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+
+ // Prepare 2 columns
+ std::vector<ColumnMetaPB> columns;
+ ColumnMetaPB col0;
+ col0.set_unique_id(100);
+ col0.set_column_id(0);
+ col0.set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ col0.set_length(4);
+ columns.push_back(col0);
+
+ ColumnMetaPB col1;
+ col1.set_unique_id(200);
+ col1.set_column_id(1);
+ col1.set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+ col1.set_length(16);
+ columns.push_back(col1);
+
+ // 2. Write to file with external meta region
+ std::string file_path = make_test_file_path("v3_external_meta.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ // Write some dummy data before meta region
+ std::string dummy_data(128, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write each column meta and record entry
+ for (const auto& col : columns) {
+ std::string meta_bytes;
+ ASSERT_TRUE(col.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(col.unique_id());
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ // Write footer trailer
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ // 3. Read back and test accessor
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Test get_column_meta_by_column_ordinal_id
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 100);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 1, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 200);
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ // Test get_column_meta_by_uid
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(loaded_footer, 100, &meta).ok());
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(loaded_footer, 200, &meta).ok());
+ EXPECT_EQ(meta.type(), static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING));
+
+ // Test traverse_metas
+ std::vector<int32_t> traversed_uids;
+ ASSERT_TRUE(accessor.traverse_metas(loaded_footer,
+ [&](const ColumnMetaPB& m) {
+ traversed_uids.push_back(m.unique_id());
+ })
+ .ok());
+ EXPECT_EQ(traversed_uids.size(), 2);
+ EXPECT_EQ(traversed_uids[0], 100);
+ EXPECT_EQ(traversed_uids[1], 200);
+}
+
+// Test V3 mode error cases
+TEST(ColumnMetaAccessorTest, V3_ErrorCases_InvalidColumnId) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+
+ // Create minimal valid external meta
+ std::string file_path = make_test_file_path("v3_error_test.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ // Write some padding before meta region so that region_start is non-zero.
+ // ExternalColMetaUtil::parse_external_meta_pointers() treats region_start == 0
+ // as invalid layout.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ ColumnMetaPB col0;
+ col0.set_unique_id(10);
+ col0.set_column_id(0);
+ std::string meta_bytes;
+ ASSERT_TRUE(col0.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10);
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Test invalid column_id
+ ColumnMetaPB meta;
+ Status st = accessor.get_column_meta_by_column_ordinal_id(loaded_footer, 999, &meta);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with empty external meta region
+TEST(ColumnMetaAccessorTest, V3_EmptyExternalMetaRegion) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+ footer.set_col_meta_region_start(100);
+ // No column_meta_entries added (empty)
+
+ ColumnMetaAccessor accessor;
+ // For an empty external meta region (num_columns == 0), the current
+ // implementation treats this as a corrupted layout and init() fails.
+ Status st = accessor.init(footer, nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with mismatched entries count
+TEST(ColumnMetaAccessorTest, V3_MismatchedEntriesCount) {
+ std::string file_path = make_test_file_path("v3_mismatch.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+
+ // Write some padding before meta region so that region_start is non-zero.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write 2 column metas
+ for (int i = 0; i < 2; ++i) {
+ ColumnMetaPB col;
+ col.set_unique_id(10 + i);
+ col.set_column_id(i);
+ std::string meta_bytes;
+ ASSERT_TRUE(col.SerializeToString(&meta_bytes));
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10 + i);
+ entry->set_length(static_cast<uint32_t>(meta_bytes.size()));
+ }
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ // Initialize accessor with the original footer (entries count matches num_columns).
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // Manually corrupt the footer by adding extra entry after init so that
+ // footer.column_meta_entries_size() != ptrs.num_columns, which should be
+ // detected by traverse_metas.
+ loaded_footer.add_column_meta_entries()->set_unique_id(999);
+
+ // traverse_metas should detect mismatch
+ Status st = accessor.traverse_metas(loaded_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test V3 mode with corrupted protobuf data
+TEST(ColumnMetaAccessorTest, V3_CorruptedProtobufData) {
+ std::string file_path = make_test_file_path("v3_corrupted.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ ASSERT_TRUE(fs->create_file(file_path, &fw).ok());
+
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+
+ // Write some padding before meta region so that region_start is non-zero.
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ uint64_t meta_region_start = fw->bytes_appended();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ // Write garbage data instead of valid protobuf
+ std::string garbage(100, 'Z');
+ ASSERT_TRUE(fw->append(Slice(garbage)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(10);
+ entry->set_length(100);
+
+ ASSERT_TRUE(append_footer_trailer(fw.get(), &footer).ok());
+ ASSERT_TRUE(fw->close().ok());
+
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ ASSERT_TRUE(fs->open_file(file_path, &fr, &opts).ok());
+
+ SegmentFooterPB loaded_footer;
+ ASSERT_TRUE(read_footer_from_file(fr, &loaded_footer).ok());
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(loaded_footer, fr).ok());
+
+ // traverse_metas should fail to parse corrupted data
+ Status st = accessor.traverse_metas(loaded_footer, [](const ColumnMetaPB&) {});
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+}
+
+// Test version detection and automatic V2/V3 selection
+TEST(ColumnMetaAccessorTest, VersionDetection) {
+ // V1 footer should use V2 accessor (inline mode)
+ {
+ SegmentFooterPB footer;
+ footer.set_version(1);
+ auto* col = footer.add_columns();
+ col->set_unique_id(10);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 10);
+ }
+
+ // V2 footer should use V2 accessor
+ {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+ auto* col = footer.add_columns();
+ col->set_unique_id(20);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), 20);
+ }
+
+ // V3 footer without valid external meta is treated as corrupted layout and
+ // init() should fail (no automatic fallback to V2 inline mode).
+ {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV3_ExtColMeta);
+ auto* col = footer.add_columns();
+ col->set_unique_id(30);
+ col->set_column_id(0);
+
+ ColumnMetaAccessor accessor;
+ Status st = accessor.init(footer, nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_TRUE(st.is<ErrorCode::CORRUPTION>());
+ }
+}
+
+// Test large number of columns
+TEST(ColumnMetaAccessorTest, LargeNumberOfColumns) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+
+ const int num_columns = 1000;
+ for (int i = 0; i < num_columns; ++i) {
+ auto* col = footer.add_columns();
+ col->set_unique_id(i);
+ col->set_column_id(i);
+ col->set_type(static_cast<int>(FieldType::OLAP_FIELD_TYPE_INT));
+ }
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Spot check mapping via get_column_meta_by_uid
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 0, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 0);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 500, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 500);
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, 999, &meta).ok());
+ EXPECT_EQ(meta.column_id(), 999);
+
+ // Test traverse_metas counts correctly
+ int count = 0;
+ ASSERT_TRUE(accessor.traverse_metas(footer, [&](const ColumnMetaPB&) { count++; }).ok());
+ EXPECT_EQ(count, num_columns);
+}
+
+// Test concurrent access (thread safety not guaranteed by ColumnMetaAccessor itself,
+// but test that multiple sequential calls work correctly)
+TEST(ColumnMetaAccessorTest, MultipleSequentialAccesses) {
+ SegmentFooterPB footer;
+ footer.set_version(kSegmentFooterVersionV2);
+
+ for (int i = 0; i < 10; ++i) {
+ auto* col = footer.add_columns();
+ col->set_unique_id(i);
+ col->set_column_id(i);
+ }
+
+ ColumnMetaAccessor accessor;
+ ASSERT_TRUE(accessor.init(footer, nullptr).ok());
+
+ // Multiple calls should give consistent results
+ for (int round = 0; round < 100; ++round) {
+ for (int i = 0; i < 10; ++i) {
+ ColumnMetaPB meta;
+ ASSERT_TRUE(accessor.get_column_meta_by_column_ordinal_id(footer, i, &meta).ok());
+ EXPECT_EQ(meta.unique_id(), i);
+ // ensure uid lookup also works repeatedly
+ ASSERT_TRUE(accessor.get_column_meta_by_uid(footer, i, &meta).ok());
+ }
+ }
+}
diff --git a/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
index 814877ad9b1..6a2448d0468 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_cache_test.cpp
@@ -24,12 +24,11 @@
#include <thread>
#include <vector>
-#include "agent/be_exec_version_manager.h"
#include "common/config.h"
-#include "gen_cpp/olap_file.pb.h"
#include "gen_cpp/segment_v2.pb.h"
#include "io/fs/file_reader.h"
#include "mock/mock_segment.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
@@ -65,11 +64,22 @@ protected:
// Create mock segment
_mock_segment = std::make_unique<StrictMock<MockSegment>>();
- // Create cache
- _cache = std::make_unique<ColumnReaderCache>(_mock_segment.get());
-
- // Set up basic mock expectations
+ // Set up basic mock expectations before constructing the cache, because
+ // ColumnReaderCache's ctor reads _mock_segment->num_rows().
setup_basic_mocks();
+
+ // Create cache bound to mock segment context.
+ // ColumnMetaAccessor will be initialized later in setup_segment_footer()
+ // once the test-specific footer has been constructed.
+ io::FileReaderSPtr file_reader; // nullptr is fine for inline-only tests
+ auto footer_cb = [this](std::shared_ptr<SegmentFooterPB>& footer_pb_shared,
+ OlapReaderStatistics* stats) -> Status {
+ // Delegate to MockSegment::_get_segment_footer
+ return _mock_segment->_get_segment_footer(footer_pb_shared, stats);
+ };
+ _cache = std::make_unique<ColumnReaderCache>(&_accessor, _mock_segment->tablet_schema(),
+ file_reader, _mock_segment->num_rows(),
+ footer_cb);
}
void TearDown() override {
@@ -108,9 +118,13 @@ protected:
*footer->add_columns() = col;
}
_mock_segment->set_footer(footer);
+ // Initialize ColumnMetaAccessor with the current footer; external meta is not
+ // used in these tests so we can pass a null file reader.
+ CHECK(_accessor.init(*footer, nullptr).ok());
}
std::unique_ptr<StrictMock<MockSegment>> _mock_segment;
+ ColumnMetaAccessor _accessor;
std::unique_ptr<ColumnReaderCache> _cache;
OlapReaderStatistics _stats;
};
@@ -354,17 +368,17 @@ TEST_F(ColumnReaderCacheTest, DifferentColumnTypes) {
FieldType::OLAP_FIELD_TYPE_DOUBLE, FieldType::OLAP_FIELD_TYPE_BOOL};
for (size_t i = 0; i < types.size(); ++i) {
- setup_column_uid_mapping(i + 1, 0);
+ setup_column_uid_mapping(static_cast<int32_t>(i + 1), 0);
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(types[i]));
- col_meta.set_unique_id(i + 1);
+ col_meta.set_unique_id(static_cast<int32_t>(i + 1));
col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
std::shared_ptr<ColumnReader> reader;
- Status status = _cache->get_column_reader(i + 1, &reader, &_stats);
+ Status status = _cache->get_column_reader(static_cast<int32_t>(i + 1), &reader, &_stats);
EXPECT_TRUE(status.ok());
EXPECT_NE(reader, nullptr);
}
@@ -396,13 +410,13 @@ TEST_F(ColumnReaderCacheTest, NodeHintParameter) {
vectorized::PathInData path("hint_field");
std::shared_ptr<ColumnReader> reader;
Status status = _cache->get_path_column_reader(1, path, &reader, &_stats, &node_hint);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(reader, nullptr);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_FOUND>());
+ EXPECT_EQ(reader, nullptr);
// Test cache hit
status = _cache->get_path_column_reader(1, path, &reader, &_stats, &node_hint);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(reader, nullptr);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_FOUND>());
+ EXPECT_EQ(reader, nullptr);
}
// Test cache performance under load
diff --git a/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp b/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp
new file mode 100644
index 00000000000..1453a18f0f7
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/external_col_meta_util_test.cpp
@@ -0,0 +1,517 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/external_col_meta_util.h"
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "io/fs/local_file_system.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_writer.h"
+#include "olap/rowset/segment_v2/variant/variant_ext_meta_writer.h"
+#include "olap/rowset/segment_v2/variant/variant_external_meta_reader.h"
+#include "olap/tablet_schema_helper.h"
+#include "olap/types.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+#include "vec/json/path_in_data.h"
+
+using namespace doris;
+using namespace doris::segment_v2;
+using namespace doris::vectorized;
+
+namespace {
+
+constexpr std::string_view kTestDir = "./ut_dir/external_col_meta_util_test";
+
+// Helper to create a clean local test directory and return absolute path + filename.
+std::string make_test_file_path(const std::string& file_name) {
+ auto fs = io::global_local_filesystem();
+ // english only in comments
+ static_cast<void>(fs->delete_directory(kTestDir));
+ CHECK(fs->create_directory(kTestDir).ok());
+ return std::string(kTestDir) + "/" + file_name;
+}
+
+// Helper to write a simple "segment-like" footer trailer. Current tests do not call
+// Segment::_parse_footer, so the exact magic value is not important as long as the
+// layout is self‑consistent.
+Status append_footer_trailer(io::FileWriter* fw, SegmentFooterPB* footer) {
+ std::string footer_buf;
+ if (!footer->SerializeToString(&footer_buf)) {
+ return Status::InternalError("failed to serialize SegmentFooterPB for test");
+ }
+ faststring fixed_buf;
+ // footer size (4 bytes)
+ put_fixed32_le(&fixed_buf, static_cast<uint32_t>(footer_buf.size()));
+ // footer checksum (4 bytes)
+ uint32_t checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ put_fixed32_le(&fixed_buf, checksum);
+ // magic number (4 bytes)
+ fixed_buf.append("D0R1", 4);
+ std::vector<Slice> slices {Slice(footer_buf), Slice(fixed_buf)};
+ RETURN_IF_ERROR(fw->appendv(slices.data(), slices.size()));
+ return Status::OK();
+}
+
+// Helper to read SegmentFooterPB back from a segment-like file and validate checksum.
+Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* footer) {
+ const auto file_size = fr->size();
+ if (file_size < 12) {
+ return Status::Corruption("test segment file too small: {}", file_size);
+ }
+ uint8_t fixed_buf[12];
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12, Slice(fixed_buf, sizeof(fixed_buf)), &bytes_read));
+ if (bytes_read != sizeof(fixed_buf)) {
+ return Status::Corruption("short read footer trailer, read={}, expect={}", bytes_read,
+ sizeof(fixed_buf));
+ }
+ const uint32_t footer_length = decode_fixed32_le(fixed_buf);
+ if (file_size < 12 + footer_length) {
+ return Status::Corruption("bad footer length in test segment: file_size={}, footer_len={}",
+ file_size, footer_length);
+ }
+ std::string footer_buf;
+ footer_buf.resize(footer_length);
+ bytes_read = 0;
+ RETURN_IF_ERROR(fr->read_at(file_size - 12 - footer_length,
+ Slice(footer_buf.data(), footer_buf.size()), &bytes_read));
+ if (bytes_read != footer_length) {
+ return Status::Corruption("short read footer pb, read={}, expect={}", bytes_read,
+ footer_length);
+ }
+ const uint32_t expect_checksum = decode_fixed32_le(fixed_buf + 4);
+ const uint32_t actual_checksum = crc32c::Value(footer_buf.data(), footer_buf.size());
+ if (actual_checksum != expect_checksum) {
+ return Status::Corruption("footer checksum mismatch, actual={}, expect={}", actual_checksum,
+ expect_checksum);
+ }
+ if (!footer->ParseFromArray(footer_buf.data(), static_cast<int>(footer_buf.size()))) {
+ return Status::Corruption("failed to parse SegmentFooterPB from test segment");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+namespace doris {
+
+using Generator = std::function<void(size_t rid, int cid, RowCursorCell& cell)>;
+
+// Helper declarations are defined in tablet_schema_helper.{h,cpp} and
+// delete_bitmap_calculator_test.cpp.
+TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
+ KeysType keys_type = UNIQUE_KEYS);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir);
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir, std::string* out_path);
+
+} // namespace doris
+
+// Basic unit test for ExternalColMetaUtil pointer parsing and uid‑>col_id map.
+TEST(ExternalColMetaUtilTest, PointersAndUidMapValidation) {
+ SegmentFooterPB footer;
+ footer.set_col_meta_region_start(100U);
+ auto* e0 = footer.add_column_meta_entries();
+ e0->set_unique_id(10);
+ e0->set_length(50);
+ auto* e1 = footer.add_column_meta_entries();
+ e1->set_unique_id(20);
+ e1->set_length(60);
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ EXPECT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ EXPECT_EQ(ptrs.region_start, 100U);
+ EXPECT_EQ(ptrs.num_columns, 2U);
+ EXPECT_EQ(ptrs.region_end, 210U);
+
+ std::unordered_map<int32_t, size_t> uid2colid;
+ EXPECT_TRUE(ExternalColMetaUtil::parse_uid_to_colid_map(footer, ptrs, &uid2colid).ok());
+ EXPECT_EQ(uid2colid.size(), 2U);
+ EXPECT_EQ(uid2colid[10], 0U);
+ EXPECT_EQ(uid2colid[20], 1U);
+
+ // Negative cases for pointers
+ footer.clear_column_meta_entries();
+ EXPECT_FALSE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ footer.add_column_meta_entries()->set_unique_id(10);
+ footer.add_column_meta_entries()->set_unique_id(20);
+ footer.mutable_column_meta_entries(0)->set_length(50);
+ footer.mutable_column_meta_entries(1)->set_length(60);
+ // zero length is illegal
+ footer.mutable_column_meta_entries(1)->set_length(0);
+ EXPECT_FALSE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ footer.mutable_column_meta_entries(1)->set_length(60);
+
+ // Slice validation
+ ExternalColMetaUtil::ExternalMetaPointers p;
+ p.region_start = 100;
+ p.region_end = 200;
+ EXPECT_TRUE(ExternalColMetaUtil::is_valid_meta_slice(100, 10, p));
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(100, 0, p)); // size == 0
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(50, 10, p)); // pos < region_start
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(195, 10, p)); // pos + size > region_end
+ // overflow guard: deliberately craft overflow
+ EXPECT_FALSE(ExternalColMetaUtil::is_valid_meta_slice(std::numeric_limits<uint64_t>::max() - 1,
+ 4, p));
+}
+
+// End‑to‑end test of VariantExtMetaWriter + VariantExternalMetaReader on a standalone file.
+// This does NOT involve full tablet/rowset plumbing, but exercises:
+// - externalize_from_footer(): classification of root/non‑sparse/sparse columns
+// - flush_to_footer(): writing IndexedColumn metas into footer.file_meta_datas
+// - init_from_footer()/lookup_meta_by_path(): reading back subcolumn ColumnMetaPB by path
+// - legacy unsuffixed keys: "variant_meta_keys" / "variant_meta_values"
+TEST(ExternalColMetaUtilTest, VariantExtMetaWriterAndReaderInteropAndCompatibility) {
+ // 1. Prepare a simple footer with:
+ // - one non‑variant column (kept as is)
+ // - one variant root (kept, path "v1")
+ // - one non‑sparse subcolumn "v1.key0" (externalized)
+ // - one sparse subcolumn "__DORIS_VARIANT_SPARSE__.key_sparse" (embedded into root)
+ SegmentFooterPB footer;
+ // non‑variant top level column
+ {
+ auto* col = footer.add_columns();
+ col->set_unique_id(0);
+ // use int(FieldType) to match production code
+ col->set_type(int(FieldType::OLAP_FIELD_TYPE_INT)); // simple scalar
+ }
+
+ const int32_t root_uid = 1;
+ // variant root column with path "v1"
+ ColumnMetaPB* root = footer.add_columns();
+ root->set_unique_id(root_uid);
+ root->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+ {
+ PathInData full_root_path("v1");
+ full_root_path.to_protobuf(root->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ }
+
+ // non‑sparse subcolumn: v1.key0
+ ColumnMetaPB* non_sparse = footer.add_columns();
+ non_sparse->set_unique_id(2);
+ non_sparse->set_type(int(FieldType::OLAP_FIELD_TYPE_INT));
+ {
+ PathInData full_path("v1.key0");
+ full_path.to_protobuf(non_sparse->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ non_sparse->set_none_null_size(123);
+ }
+
+ // sparse subcolumn: v1.__DORIS_VARIANT_SPARSE__
+ ColumnMetaPB* sparse = footer.add_columns();
+ sparse->set_unique_id(3);
+ sparse->set_type(int(FieldType::OLAP_FIELD_TYPE_VARIANT));
+ {
+ // For sparse column, the relative path (after pop_front) should contain
+ // "__DORIS_VARIANT_SPARSE__" so that VariantExtMetaWriter treats it as sparse.
+ PathInData full_path("v1.__DORIS_VARIANT_SPARSE__");
+ full_path.to_protobuf(sparse->mutable_column_path_info(),
+ /*parent_col_unique_id=*/root_uid);
+ // carry sparse statistics in VariantStatisticsPB
+ auto* stats = sparse->mutable_variant_statistics();
+ (*stats->mutable_sparse_column_non_null_size())["key_sparse"] = 10;
+ }
+
+ // 2. Externalize non‑sparse subcolumns into a dedicated file via VariantExtMetaWriter.
+ std::string file_path = make_test_file_path("variant_ext_meta.bin");
+ io::FileWriterPtr fw;
+ auto fs = io::global_local_filesystem();
+ Status st = fs->create_file(file_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ std::vector<ColumnMetaPB> out_metas;
+ {
+ VariantExtMetaWriter ext_writer(fw.get(), CompressionTypePB::LZ4);
+ st = ext_writer.externalize_from_footer(&footer, &out_metas);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ }
+
+ // Capture region start AFTER Variant Index is written
+ uint64_t meta_region_start = fw->bytes_appended();
+
+ // Manually write the Meta Region (simulating ExternalColMetaUtil::write_external_column_meta)
+ footer.clear_column_meta_entries();
+ footer.set_col_meta_region_start(meta_region_start);
+
+ for (const auto& meta : out_metas) {
+ std::string meta_bytes;
+ meta.SerializeToString(&meta_bytes);
+ ASSERT_TRUE(fw->append(Slice(meta_bytes)).ok());
+
+ auto* entry = footer.add_column_meta_entries();
+ entry->set_unique_id(meta.unique_id());
+ entry->set_length(meta_bytes.size());
+ }
+
+ // After externalize_from_footer:
+ // - footer.columns() should contain:
+ // * non‑variant column uid=0
+ // * root variant column uid=1, whose children_columns includes the sparse meta
+ // - "v1.key0" should be removed from footer.columns() and only exist in external meta.
+ ASSERT_EQ(footer.columns_size(), 2);
+ const ColumnMetaPB& kept0 = footer.columns(0);
+ const ColumnMetaPB& kept1 = footer.columns(1);
+ EXPECT_EQ(kept0.unique_id(), 0);
+ EXPECT_EQ(kept1.unique_id(), root_uid);
+ // root should have embedded sparse children
+ ASSERT_EQ(kept1.children_columns_size(), 1);
+ const ColumnMetaPB& embedded_sparse = kept1.children_columns(0);
+ EXPECT_TRUE(embedded_sparse.has_column_path_info());
+ PathInData embedded_path;
+ embedded_path.from_protobuf(embedded_sparse.column_path_info());
+ // full path should keep root prefix
+ EXPECT_EQ(embedded_path.get_path(), "v1.__DORIS_VARIANT_SPARSE__");
+ ASSERT_TRUE(embedded_sparse.has_variant_statistics());
+ EXPECT_EQ(embedded_sparse.variant_statistics().sparse_column_non_null_size().at("key_sparse"),
+ 10);
+
+ // 3. Verify that ext meta keys metas were flushed into footer.file_meta_datas.
+ bool found_keys_meta = false;
+ std::string keys_meta_bytes;
+ for (const auto& m : footer.file_meta_datas()) {
+ if (m.key() == "variant_meta_keys." + std::to_string(root_uid)) {
+ found_keys_meta = true;
+ keys_meta_bytes = m.value();
+ }
+ }
+ ASSERT_TRUE(found_keys_meta);
+ // variant_meta_values should NOT be present
+
+ // 4. Finish the "segment‑like" file by appending footer + trailer, then close.
+ st = append_footer_trailer(fw.get(), &footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(fw->close().ok());
+
+ // 5. Open a reader on the same file and build VariantExternalMetaReader from the footer.
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ st = fs->open_file(file_path, &fr, &opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ VariantExternalMetaReader reader_new;
+ auto footer_sp = std::make_shared<SegmentFooterPB>();
+ footer_sp->CopyFrom(footer);
+ st = reader_new.init_from_footer(footer_sp, fr, root_uid);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(reader_new.available());
+
+ // 6. Lookup meta by relative path and verify content.
+ ColumnMetaPB loaded_meta;
+ st = reader_new.lookup_meta_by_path("key0", &loaded_meta);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(loaded_meta.has_column_path_info());
+ PathInData loaded_path;
+ loaded_path.from_protobuf(loaded_meta.column_path_info());
+ // full path should still contain root prefix "v1.key0"
+ EXPECT_EQ(loaded_path.get_path(), "v1.key0");
+ EXPECT_EQ(loaded_meta.none_null_size(), 123);
+ EXPECT_EQ(loaded_meta.column_path_info().parrent_column_unique_id(), root_uid);
+
+ // 7. Legacy compatibility: construct a footer with unsuffixed keys:
+ // "variant_meta_keys"
+ SegmentFooterPB legacy_footer;
+ legacy_footer.CopyFrom(footer);
+ legacy_footer.clear_file_meta_datas();
+ {
+ auto* p1 = legacy_footer.add_file_meta_datas();
+ p1->set_key("variant_meta_keys");
+ p1->set_value(keys_meta_bytes);
+ }
+
+ VariantExternalMetaReader reader_legacy;
+ auto legacy_footer_sp = std::make_shared<SegmentFooterPB>();
+ legacy_footer_sp->CopyFrom(legacy_footer);
+ st = reader_legacy.init_from_footer(legacy_footer_sp, fr, root_uid);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(reader_legacy.available());
+
+ ColumnMetaPB loaded_meta_legacy;
+ st = reader_legacy.lookup_meta_by_path("key0", &loaded_meta_legacy);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ PathInData loaded_path_legacy;
+ loaded_path_legacy.from_protobuf(loaded_meta_legacy.column_path_info());
+ EXPECT_EQ(loaded_path_legacy.get_path(), "v1.key0");
+ EXPECT_EQ(loaded_meta_legacy.none_null_size(), 123);
+ EXPECT_EQ(loaded_meta_legacy.column_path_info().parrent_column_unique_id(), root_uid);
+}
+
+// End-to-end test: use ExternalColMetaUtil to write external ColumnMeta region + CMO
+// into a V3 segment-like file, then read footer back and verify:
+// - footer.version() is V3
+// - external meta pointers / uid->col_id map are present and valid
+// - ColumnMetaPB can be read back through CMO by col_id
+TEST(ExternalColMetaUtilTest, WriteAndReadV3SegmentExternalMetaRoundTrip) {
+ // 1. Build a simple footer with two top-level columns so uid->col_id map is deterministic.
+ SegmentFooterPB footer;
+ {
+ auto* col0 = footer.add_columns();
+ col0->set_column_id(0);
+ col0->set_unique_id(10);
+ col0->set_type(int(FieldType::OLAP_FIELD_TYPE_INT));
+ col0->set_length(4);
+ col0->set_encoding(DEFAULT_ENCODING);
+ col0->set_compression(CompressionTypePB::LZ4);
+ col0->set_is_nullable(false);
+
+ auto* col1 = footer.add_columns();
+ col1->set_column_id(1);
+ col1->set_unique_id(20);
+ col1->set_type(int(FieldType::OLAP_FIELD_TYPE_BIGINT));
+ col1->set_length(8);
+ col1->set_encoding(DEFAULT_ENCODING);
+ col1->set_compression(CompressionTypePB::LZ4);
+ col1->set_is_nullable(true);
+ }
+
+ // Stamp V3 version like SegmentWriter::_write_footer does for external meta enabled tablets.
+ footer.set_version(segment_v2::kSegmentFooterVersionV3_ExtColMeta);
+
+ std::string file_path = make_test_file_path("v3_ext_col_meta_segment.bin");
+ auto fs = io::global_local_filesystem();
+ io::FileWriterPtr fw;
+ Status st = fs->create_file(file_path, &fw);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ // Simulate real segment layout: write some dummy data pages before external meta region
+ // so that `col_meta_region_start` is non-zero (as in production segments).
+ {
+ std::string dummy_data(16, 'X');
+ ASSERT_TRUE(fw->append(Slice(dummy_data)).ok());
+ }
+
+ // 2. Write external ColumnMeta region + CMO + uid->col_id mapping.
+ st = ExternalColMetaUtil::write_external_column_meta(
+ fw.get(), &footer, CompressionTypePB::LZ4,
+ [fw_raw = fw.get()](const std::vector<Slice>& slices) {
+ return fw_raw->appendv(slices.data(), slices.size());
+ });
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ // After write_external_column_meta, inline footer.columns() are cleared in V3 mode.
+ ASSERT_EQ(footer.columns_size(), 0);
+
+ // 3. Finish by appending footer + trailer and close writer.
+ st = append_footer_trailer(fw.get(), &footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ ASSERT_TRUE(fw->close().ok());
+
+ // 4. Re-open file, parse footer back from on-disk trailer, and verify footer.version() is V3.
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions opts;
+ st = fs->open_file(file_path, &fr, &opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ SegmentFooterPB loaded_footer;
+ st = read_footer_from_file(fr, &loaded_footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(loaded_footer.version(), segment_v2::kSegmentFooterVersionV3_ExtColMeta);
+
+ // 5. Parse external meta pointers and uid->col_id mapping from the loaded footer.
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(loaded_footer, &ptrs).ok());
+ std::unordered_map<int32_t, size_t> uid2colid;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_uid_to_colid_map(loaded_footer, ptrs, &uid2colid).ok());
+ ASSERT_EQ(uid2colid.size(), 2U);
+ EXPECT_EQ(uid2colid[10], 0U);
+ EXPECT_EQ(uid2colid[20], 1U);
+
+ // 6. Read ColumnMetaPB back from external meta region and verify it matches the
+ // original unique_ids.
+ ColumnMetaPB meta0;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, /*col_id=*/0, &meta0);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta0.unique_id(), 10);
+ EXPECT_EQ(meta0.column_id(), 0);
+
+ ColumnMetaPB meta1;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, /*col_id=*/1, &meta1);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta1.unique_id(), 20);
+ EXPECT_EQ(meta1.column_id(), 1);
+}
+
+// Build a real Segment file with data rows, then verify that Segment::num_rows()
+// matches the written row count and that footer metadata is consistent with schema.
+TEST(ExternalColMetaUtilTest, BuildSegmentAndVerifyDataAndFooterMeta) {
+ // 1. Build a simple UNIQUE_KEYS tablet schema: one key column, one value column.
+ std::vector<TabletColumnPtr> columns;
+ columns.emplace_back(create_int_key(/*id=*/0));
+ columns.emplace_back(create_int_value(/*id=*/1));
+ TabletSchemaSPtr tablet_schema = create_schema(columns, UNIQUE_KEYS);
+ // Enable external ColumnMetaPB so that SegmentWriter produces a V3 footer with
+ // externalized column meta region + column_meta_entries.
+ tablet_schema->set_external_segment_meta_used_default(true);
+
+ // 2. Use existing build_segment helper (SegmentWriter-based) to write a segment file.
+ SegmentWriterOptions opts;
+ opts.enable_unique_key_merge_on_write = true;
+ const size_t nrows = 16;
+
+ auto generator = [](size_t rid, int cid, RowCursorCell& cell) {
+ cell.set_not_null();
+ // deterministic int payload: value = rid * 10 + cid
+ *reinterpret_cast<int*>(cell.mutable_cell_ptr()) = static_cast<int>(rid * 10 + cid);
+ };
+
+ std::shared_ptr<Segment> segment;
+ std::string segment_path;
+ build_segment(opts, tablet_schema,
+ /*segment_id=*/0, tablet_schema, nrows, generator, &segment,
+ std::string(kTestDir), &segment_path);
+
+ ASSERT_NE(segment, nullptr);
+
+ // 3. Verify basic segment data property (row count).
+ EXPECT_EQ(segment->num_rows(), nrows);
+
+ // 4. Verify external ColumnMetaPB region is actually written on disk and consistent with schema.
+ auto fs = io::global_local_filesystem();
+ io::FileReaderSPtr fr;
+ io::FileReaderOptions reader_opts;
+ Status st = fs->open_file(segment_path, &fr, &reader_opts);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+
+ SegmentFooterPB loaded_footer;
+ st = read_footer_from_file(fr, &loaded_footer);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(loaded_footer.version(), segment_v2::kSegmentFooterVersionV3_ExtColMeta);
+
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(loaded_footer, &ptrs).ok());
+ ASSERT_EQ(ptrs.num_columns, tablet_schema->num_columns());
+
+ // Read back per-column meta from external region and check uid/col_id mapping.
+ for (uint32_t cid = 0; cid < ptrs.num_columns; ++cid) {
+ ColumnMetaPB meta;
+ st = ExternalColMetaUtil::read_col_meta(fr, loaded_footer, ptrs, cid, &meta);
+ ASSERT_TRUE(st.ok()) << st.to_json();
+ EXPECT_EQ(meta.unique_id(), tablet_schema->column(cid).unique_id());
+ EXPECT_EQ(meta.column_id(), cid);
+ }
+}
diff --git a/be/test/olap/rowset/segment_v2/mock/mock_segment.h b/be/test/olap/rowset/segment_v2/mock/mock_segment.h
index 7efe16e7586..9f89a084a94 100644
--- a/be/test/olap/rowset/segment_v2/mock/mock_segment.h
+++ b/be/test/olap/rowset/segment_v2/mock/mock_segment.h
@@ -52,16 +52,10 @@ public:
_tablet_schema->_cols.push_back(std::make_shared<TabletColumn>());
_tablet_schema->_cols.back()->set_unique_id(col_uid);
_tablet_schema->_field_id_to_index[col_uid] = footer_ordinal;
- _column_uid_to_footer_ordinal[col_uid] = footer_ordinal;
}
void set_footer(std::shared_ptr<SegmentFooterPB> footer) { _footer = footer; }
- // Access to internal data for testing
- const std::unordered_map<int32_t, size_t>& get_column_uid_mapping() const {
- return _column_uid_to_footer_ordinal;
- }
-
std::shared_ptr<SegmentFooterPB> get_footer() const { return _footer; }
std::shared_ptr<SegmentFooterPB> _footer;
diff --git a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
index 117fc0579b5..42c8a9a4bc3 100644
--- a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
@@ -16,6 +16,7 @@
// under the License.
#include "gtest/gtest.h"
+#include "olap/rowset/segment_v2/column_meta_accessor.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_reader_cache.h"
#include "olap/rowset/segment_v2/variant/hierarchical_data_iterator.h"
@@ -40,13 +41,16 @@ class MockColumnReaderCache : public segment_v2::ColumnReaderCache {
public:
MockColumnReaderCache(const SegmentFooterPB& footer, const io::FileReaderSPtr& file_reader,
const std::shared_ptr<TabletSchema>& tablet_schema)
- : ColumnReaderCache(nullptr),
+ : ColumnReaderCache(nullptr, nullptr, nullptr, 0,
+ [](std::shared_ptr<SegmentFooterPB>&, OlapReaderStatistics*) {
+ return Status::OK();
+ }),
_footer(footer),
_file_reader(file_reader),
_tablet_schema(tablet_schema) {}
Status get_path_column_reader(
- uint32_t col_uid, vectorized::PathInData relative_path,
+ int32_t col_uid, vectorized::PathInData relative_path,
std::shared_ptr<segment_v2::ColumnReader>* column_reader, OlapReaderStatistics* stats,
const SubcolumnColumnMetaInfo::Node* node_hint = nullptr) override {
DCHECK(node_hint != nullptr);
@@ -74,6 +78,30 @@ private:
const std::shared_ptr<TabletSchema>& _tablet_schema;
};
+// Helper to create a root VariantColumnReader using ColumnMetaAccessor, which
+// hides inline vs external column meta layout (V2 vs V3).
+static Status create_variant_root_reader(const SegmentFooterPB& footer,
+ const io::FileReaderSPtr& file_reader,
+ const TabletSchemaSPtr& tablet_schema,
+ std::shared_ptr<segment_v2::ColumnReader>* out) {
+ segment_v2::ColumnMetaAccessor accessor;
+ RETURN_IF_ERROR(accessor.init(footer, file_reader));
+
+ segment_v2::ColumnReaderOptions opts;
+ opts.kept_in_memory = false;
+ opts.be_exec_version = BeExecVersionManager::get_newest_version();
+ opts.tablet_schema = tablet_schema;
+
+ auto variant_reader = std::make_shared<segment_v2::VariantColumnReader>();
+ int32_t root_uid = tablet_schema->column(0).unique_id();
+ // Wrap footer into shared_ptr to match VariantColumnReader::init signature.
+ auto footer_sp = std::make_shared<SegmentFooterPB>(footer);
+ RETURN_IF_ERROR(variant_reader->init(opts, &accessor, footer_sp, root_uid, footer.num_rows(),
+ file_reader));
+ *out = std::move(variant_reader);
+ return Status::OK();
+}
+
class VariantColumnWriterReaderTest : public testing::Test {
public:
void SetUp() override {
@@ -183,6 +211,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -254,10 +286,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ // create root variant reader using ColumnMetaAccessor (supports inline/external meta)
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
@@ -318,7 +349,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
{
auto iter = assert_cast<HierarchicalDataIterator*>(it.get());
std::shared_ptr<ColumnReader> column_reader1;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader1);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader1);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_ANY_THROW(st = iter->seek_to_first());
std::cout << "hier:" << iter->get_current_ordinal() << std::endl;
@@ -735,6 +766,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -805,10 +840,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -927,6 +960,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1000,6 +1037,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1094,10 +1135,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -1144,6 +1183,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize)
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1229,6 +1272,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1316,6 +1363,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1403,6 +1454,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1490,6 +1545,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1576,6 +1635,10 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1658,10 +1721,8 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions reader_opts;
- reader_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> reader;
- st = ColumnReader::create(reader_opts, footer, 0, 1000, file_reader, &reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(reader.get());
EXPECT_TRUE(variant_column_reader != nullptr);
@@ -1706,6 +1767,10 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1800,10 +1865,8 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions reader_opts;
- reader_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> reader;
- st = ColumnReader::create(reader_opts, footer, 0, 1000, file_reader, &reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(reader.get());
EXPECT_TRUE(variant_column_reader != nullptr);
@@ -1850,6 +1913,10 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
@@ -1956,11 +2023,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_subcolumn) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2134,13 +2198,9 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2290,11 +2350,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter_nullable) {
io::FileReaderSPtr file_reader;
Status st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
-
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
@@ -2375,6 +2432,10 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -2447,10 +2508,8 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
io::FileReaderSPtr file_reader;
st = io::global_local_filesystem()->open_file(file_path, &file_reader);
EXPECT_TRUE(st.ok()) << st.msg();
- ColumnReaderOptions read_opts;
- read_opts.tablet_schema = _tablet_schema;
std::shared_ptr<ColumnReader> column_reader;
- st = ColumnReader::create(read_opts, footer, 0, 1000, file_reader, &column_reader);
+ st = create_variant_root_reader(footer, file_reader, _tablet_schema, &column_reader);
EXPECT_TRUE(st.ok()) << st.msg();
auto* variant_column_reader = assert_cast<VariantColumnReader*>(column_reader.get());
diff --git a/be/test/vec/common/schema_util_rowset_test.cpp b/be/test/vec/common/schema_util_rowset_test.cpp
index 7d2ac9f1912..9b681f706eb 100644
--- a/be/test/vec/common/schema_util_rowset_test.cpp
+++ b/be/test/vec/common/schema_util_rowset_test.cpp
@@ -176,7 +176,8 @@ static void fill_block_with_test_data(vectorized::Block* block, int size) {
static int64_t inc_id = 1000;
static RowsetWriterContext rowset_writer_context(const std::unique_ptr<DataDir>& data_dir,
const TabletSchemaSPtr& schema,
- const std::string& tablet_path) {
+ const std::string& tablet_path,
+ const TabletSharedPtr& tablet) {
RowsetWriterContext context;
RowsetId rowset_id;
rowset_id.init(inc_id);
@@ -186,6 +187,8 @@ static RowsetWriterContext rowset_writer_context(const std::unique_ptr<DataDir>&
context.rowset_state = VISIBLE;
context.tablet_schema = schema;
context.tablet_path = tablet_path;
+ context.tablet_id = tablet->tablet_id();
+ context.tablet = tablet;
context.version = Version(inc_id, inc_id);
context.max_rows_per_segment = 200;
inc_id++;
@@ -220,7 +223,13 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
- string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows");
+ // set seed
+ std::srand(42);
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
std::unique_ptr<DataDir> _data_dir = std::make_unique<DataDir>(*_engine_ref, absolute_dir);
@@ -236,7 +245,8 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -262,7 +272,13 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
- string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows1");
+ // set seed
+ std::srand(42);
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows1");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
std::unique_ptr<DataDir> _data_dir = std::make_unique<DataDir>(*_engine_ref, absolute_dir);
@@ -278,7 +294,8 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -292,7 +309,74 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
EXPECT_FALSE(st.ok());
}
-TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_compaction_schema) {
+// Mixed-format compatibility: one tablet contains both old (inline meta) and new
+// (external column meta + variant ext meta) segments, and path stats / scanning
+// logic should work across all rowsets.
+TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) {
+ // 1. create tablet schema (same layout as check_path_stats_agg_key)
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(AGG_KEYS);
+ construct_column(schema_pb.add_column(), 0, "INT", "key", true);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "v1");
+ construct_column(schema_pb.add_column(), 2, "STRING", "v2");
+ construct_column(schema_pb.add_column(), 3, "VARIANT", "v3");
+ construct_column(schema_pb.add_column(), 4, "INT", "v4");
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->init_from_pb(schema_pb);
+
+ // 2. create tablet and data dir
+ TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ // First write a few rowsets with external_segment_meta_used_default = false (old format)
+ tablet_schema->set_external_segment_meta_used_default(false);
+ std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows_mixed");
+ EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
+ std::unique_ptr<DataDir> data_dir = std::make_unique<DataDir>(*_engine_ref, absolute_dir);
+ static_cast<void>(data_dir->update_capacity());
+ EXPECT_TRUE(data_dir->init(true).ok());
+
+ TabletSharedPtr tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, data_dir.get());
+ EXPECT_TRUE(tablet->init().ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ // 3. create multiple rowsets: first batch uses old footer format, second batch uses new format
+ std::vector<RowsetSharedPtr> rowsets;
+
+ auto create_and_add_rowset = [&](int /*idx*/) {
+ const auto& res = RowsetFactory::create_rowset_writer(
+ *_engine_ref,
+ rowset_writer_context(data_dir, tablet_schema, tablet->tablet_path(), tablet),
+ false);
+ EXPECT_TRUE(res.has_value()) << res.error();
+ const auto& rowset_writer = res.value();
+ auto rowset = create_rowset(rowset_writer, tablet_schema);
+ EXPECT_TRUE(tablet->add_rowset(rowset).ok());
+ rowsets.push_back(rowset);
+ };
+
+ // 3.1 write a few old-format rowsets (inline meta only)
+ for (int i = 0; i < 3; ++i) {
+ create_and_add_rowset(i);
+ }
+
+ // 3.2 flip tablet default to enable external column meta for subsequent segments
+ tablet_schema->set_external_segment_meta_used_default(true);
+
+ // 3.3 write a few new-format rowsets (external meta enabled)
+ for (int i = 0; i < 3; ++i) {
+ create_and_add_rowset(i + 3);
+ }
+
+ EXPECT_EQ(rowsets.size(), 6);
+
+ // 4. check that VariantCompactionUtil::check_path_stats works across mixed segments
+ // This will internally create Segment / ColumnReader instances and should be
+ // insensitive to whether a particular segment uses inline or external meta.
+ EXPECT_TRUE(
+ schema_util::VariantCompactionUtil::check_path_stats(rowsets, rowsets[0], tablet).ok());
+}
+
+TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_schema) {
all_path_stats.clear();
// 1.create tablet schema
TabletSchemaPB schema_pb;
@@ -306,6 +390,11 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_compaction_schema) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_meta->_tablet_id = 12345;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
@@ -315,7 +404,8 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_compaction_schema) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -386,7 +476,10 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_compaction_schema) {
rowset_writer_context.rowset_type = BETA_ROWSET;
rowset_writer_context.rowset_state = VISIBLE;
rowset_writer_context.tablet_schema = tablet_schema;
- rowset_writer_context.tablet_path = _absolute_dir + "/../";
+ rowset_writer_context.tablet_path = _tablet->tablet_path();
+ rowset_writer_context.data_dir = _data_dir.get();
+ rowset_writer_context.tablet_id = _tablet->tablet_id();
+ rowset_writer_context.tablet = _tablet;
rowset_writer_context.version = version;
rowset_writer_context.segments_overlap = overlap;
rowset_writer_context.max_rows_per_segment = max_rows_per_segment;
@@ -428,6 +521,10 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref,
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
std::unique_ptr<DataDir> _data_dir = std::make_unique<DataDir>(*_engine_ref, _absolute_dir);
@@ -444,7 +541,8 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref,
for (int i = 0; i < 1; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -551,6 +649,10 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
+ bool external_segment_meta_used_default = rand() % 2 == 0;
+ std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
+ << std::endl;
+ tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
@@ -560,7 +662,8 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
for (int i = 0; i < 5; i++) {
const auto& res = RowsetFactory::create_rowset_writer(
*_engine_ref,
- rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path()), false);
+ rowset_writer_context(_data_dir, tablet_schema, _tablet->tablet_path(), _tablet),
+ false);
EXPECT_TRUE(res.has_value()) << res.error();
const auto& rowset_writer = res.value();
auto rowset = create_rowset(rowset_writer, tablet_schema);
@@ -632,6 +735,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
rowset_writer_context.rowset_state = VISIBLE;
rowset_writer_context.tablet_schema = tablet_schema;
rowset_writer_context.tablet_path = _absolute_dir + "/../";
+ rowset_writer_context.data_dir = _data_dir.get();
+ rowset_writer_context.tablet_id = _tablet->tablet_id();
+ rowset_writer_context.tablet = _tablet;
rowset_writer_context.version = version;
rowset_writer_context.segments_overlap = overlap;
rowset_writer_context.max_rows_per_segment = max_rows_per_segment;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 3f6577e3669..7585126aa19 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -81,7 +81,7 @@ public class Config extends ConfigBase {
options = {"NORMAL", "ASYNC", "BRIEF"})
public static String sys_log_mode = "ASYNC";
- @ConfField(description = {"FE 在 sys_log_roll_interval （日志滚动间隔）内允许保留的最大日志文件数。"
+ @ConfField(description = {"FE 在 sys_log_roll_interval（日志滚动间隔）内允许保留的最大日志文件数。"
+ "默认值为 10，意味着在每个日志滚动周期内，系统最多会保留 10 个日志文件。",
"This parameter defines the maximum number of FE log files that can be retained within the "
+ "sys_log_roll_interval (log roll interval). The default value is 10, which means the system"
@@ -140,7 +140,7 @@ public class Config extends ConfigBase {
public static boolean enable_file_logger = true;
@ConfField(mutable = false, masterOnly = false,
- description = {"是否检查table锁泄漏", "Whether to check table lock leaky"})
+ description = {"是否检查 table 锁泄漏", "Whether to check table lock leaky"})
public static boolean check_table_lock_leaky = false;
@ConfField(mutable = true, masterOnly = false,
@@ -302,13 +302,13 @@ public class Config extends ConfigBase {
+ "If there are many ReplicaWriteException in FE WARN log, you can try to increase this value"})
public static int bdbje_replica_ack_timeout_second = 10;
- @ConfField(description = {"在HA模式下，BDBJE 中保留的预留空间字节数的期望上限。非 HA 模式下无效",
+ @ConfField(description = {"在 HA 模式下，BDBJE 中保留的预留空间字节数的期望上限。非 HA 模式下无效",
"The desired upper limit on the number of bytes of reserved space to retain "
+ "in a replicated JE Environment. "
+ "This parameter is ignored in a non-replicated JE Environment."})
public static long bdbje_reserved_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G
- @ConfField(description = {"BDBJE 所需的空闲磁盘空间大小。如果空闲磁盘空间小于这个值，则BDBJE将无法写入。",
+ @ConfField(description = {"BDBJE 所需的空闲磁盘空间大小。如果空闲磁盘空间小于这个值，则 BDBJE 将无法写入。",
"Amount of free disk space required by BDBJE. "
+ "If the free disk space is less than this value, BDBJE will not be able to write."})
public static long bdbje_free_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G
@@ -321,11 +321,11 @@ public class Config extends ConfigBase {
"Queue size to store heartbeat task in heartbeat_mgr"})
public static int heartbeat_mgr_blocking_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"TabletStatMgr线程数",
+ @ConfField(masterOnly = true, description = {"TabletStatMgr 线程数",
"Num of thread to update tablet stat"})
public static int tablet_stat_mgr_threads_num = -1;
- @ConfField(masterOnly = true, description = {"Agent任务线程池的线程数",
+ @ConfField(masterOnly = true, description = {"Agent 任务线程池的线程数",
"Num of thread to handle agent task in agent task thread-pool"})
public static int max_agent_task_threads_num = 4096;
@@ -397,11 +397,11 @@ public class Config extends ConfigBase {
varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_https = false;
- @ConfField(description = {"Jetty 的 acceptor 线程数。Jetty的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。"
- + "acceptor 负责接受新的连接，然后交给 selector 处理HTTP报文协议的解包，最后由 worker 处理请求。"
- + "前两个线程池采用非阻塞模型，并且一个线程可以处理很多socket的读写，所以线程池的数量少。"
- + "对于大多数项目，只需要 1-2 个 acceptor 线程，2 到 4 个就足够了。Worker 的数量取决于应用的QPS和IO事件的比例。"
- + "越高QPS，或者IO占比越高，等待的线程越多，需要的线程总数越多。",
+ @ConfField(description = {"Jetty 的 acceptor 线程数。Jetty 的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。"
+ + "acceptor 负责接受新的连接，然后交给 selector 处理 HTTP 报文协议的解包，最后由 worker 处理请求。"
+ + "前两个线程池采用非阻塞模型，并且一个线程可以处理很多 socket 的读写，所以线程池的数量少。"
+ + "对于大多数项目，只需要 1-2 个 acceptor 线程，2 到 4 个就足够了。Worker 的数量取决于应用的 QPS 和 IO 事件的比例。"
+ + "越高 QPS，或者 IO 占比越高，等待的线程越多，需要的线程总数越多。",
"The number of acceptor threads for Jetty. Jetty's thread architecture model is very simple, "
+ "divided into three thread pools: acceptor, selector and worker. "
+ "The acceptor is responsible for accepting new connections, "
@@ -483,7 +483,7 @@ public class Config extends ConfigBase {
@ConfField(description = {"MySQL 服务的最大任务线程数", "The max number of task threads in MySQL service"})
public static int max_mysql_service_task_threads_num = 4096;
- @ConfField(description = {"BackendServiceProxy数量, 用于池化GRPC channel",
+ @ConfField(description = {"BackendServiceProxy 数量，用于池化 GRPC channel",
"BackendServiceProxy pool size for pooling GRPC channels."})
public static int backend_proxy_num = 48;
@@ -519,7 +519,7 @@ public class Config extends ConfigBase {
public static int publish_version_timeout_second = 30; // 30 seconds
@ConfField(mutable = true, masterOnly = true, description = {"导入 Publish 阶段的等待时间，单位是秒。超过此时间，"
- + "则只需每个tablet包含一个成功副本，则导入成功。值为 -1 时，表示无限等待。",
+ + "则只需每个 tablet 包含一个成功副本，则导入成功。值为 -1 时，表示无限等待。",
"Waiting time for one transaction changing to \"at least one replica success\", in seconds."
+ "If time exceeds this, and for each tablet it has at least one replica publish successful, "
+ "then the load task will be successful." })
@@ -598,7 +598,12 @@ public class Config extends ConfigBase {
public static int insert_load_default_timeout_second = 14400; // 4 hour
@ConfField(mutable = true, masterOnly = true, description = {
- "等内部攒批真正写入完成才返回；insert into和stream load默认开启攒批",
+ "在 fuzzy 测试中随机选择部分表使用 V3 storage_format（ext_meta），用于增强覆盖",
+ "Randomly use V3 storage_format (ext_meta) for some tables in fuzzy tests to increase coverage"})
+ public static boolean random_use_v3_storage_format = true;
+
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "等内部攒批真正写入完成才返回；insert into 和 stream load 默认开启攒批",
"Wait for the internal batch to be written before returning; "
+ "insert into and stream load use group commit by default."})
public static boolean wait_internal_group_commit_finish = false;
@@ -607,12 +612,12 @@ public class Config extends ConfigBase {
"Default commit interval in ms for group commit"})
public static int group_commit_interval_ms_default_value = 10000;
- @ConfField(mutable = false, masterOnly = true, description = {"攒批的默认提交数据量，单位是字节，默认128M",
+ @ConfField(mutable = false, masterOnly = true, description = {"攒批的默认提交数据量，单位是字节，默认 128M",
"Default commit data bytes for group commit"})
public static int group_commit_data_bytes_default_value = 134217728;
@ConfField(mutable = true, masterOnly = true, description = {
- "内部攒批的超时时间为table的group_commit_interval_ms的倍数",
+ "内部攒批的超时时间为 table 的 group_commit_interval_ms 的倍数",
"The internal group commit timeout is the multiple of table's group_commit_interval_ms"})
public static int group_commit_timeout_multipler = 10;
@@ -713,7 +718,7 @@ public class Config extends ConfigBase {
public static double used_capacity_percent_max_diff = 0.30;
@ConfField(mutable = true, masterOnly = true, description = {
- "设置固定的 BE 负载分数中磁盘使用率系数。BE 负载分数会综合磁盘使用率和副本数而得。有效值范围为[0, 1]，"
+ "设置固定的 BE 负载分数中磁盘使用率系数。BE 负载分数会综合磁盘使用率和副本数而得。有效值范围为 [0, 1]，"
+ "当超出此范围时，则使用其他方法自动计算此系数。",
"Sets a fixed disk usage factor in the BE load fraction. The BE load score is a combination of disk usage "
+ "and replica count. The valid value range is [0, 1]. When it is out of this range, other "
@@ -732,13 +737,13 @@ public class Config extends ConfigBase {
+ "and disable storage cool down function."})
public static boolean disable_storage_medium_check = false;
- @ConfField(description = {"创建表或分区时，可以指定存储介质(HDD 或 SSD)。如果未指定，"
+ @ConfField(description = {"创建表或分区时，可以指定存储介质 (HDD 或 SSD)。如果未指定，"
+ "则使用此配置指定的默认介质。",
"When create a table(or partition), you can specify its storage medium(HDD or SSD)."})
public static String default_storage_medium = "HDD";
@ConfField(mutable = true, masterOnly = true, description = {
- "删除数据库(表/分区)后，可以使用 RECOVER 语句恢复。此配置指定了数据的最大保留时间。"
+ "删除数据库 (表/分区) 后，可以使用 RECOVER 语句恢复。此配置指定了数据的最大保留时间。"
+ "超过此时间，数据将被永久删除。",
"After dropping database(table/partition), you can recover it by using RECOVER stmt.",
"And this specifies the maximal data retention time. After time, the data will be deleted permanently."})
@@ -801,7 +806,7 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true) public static boolean disable_colocate_balance = false;
- @ConfField(mutable = true, masterOnly = true, description = {"是否启用group间的均衡",
+ @ConfField(mutable = true, masterOnly = true, description = {"是否启用 group 间的均衡",
"is allow colocate balance between all groups"})
public static boolean disable_colocate_balance_between_groups = false;
@@ -1163,7 +1168,7 @@ public class Config extends ConfigBase {
// if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc
@ConfField(mutable = true, masterOnly = true, description = {
- "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
+ "重新发送 agent task 时，单次 RPC 分配给每个 be 的任务最大个数，默认值为 10000 个。",
"The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, "
+ "the default value is 10000."
})
@@ -1395,7 +1400,7 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig",
description = {
- "当前默认设置为 300，用来控制控制NereidsSqlCacheManager中sql cache过期时间，超过一段时间不访问cache会被回收",
+ "当前默认设置为 300，用来控制控制 NereidsSqlCacheManager 中 sql cache 过期时间，超过一段时间不访问 cache 会被回收",
"The current default setting is 300, which is used to control the expiration time of SQL cache"
+ "in NereidsSqlCacheManager. If the cache is not accessed for a period of time, "
+ "it will be reclaimed"
@@ -1412,8 +1417,8 @@ public class Config extends ConfigBase {
masterOnly = false,
callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",
description = {
- "当前默认设置为 300，用来控制控制NereidsSortedPartitionsCacheManager中分区元数据缓存过期时间，"
- + "超过一段时间不访问cache会被回收",
+ "当前默认设置为 300，用来控制控制 NereidsSortedPartitionsCacheManager 中分区元数据缓存过期时间，"
+ + "超过一段时间不访问 cache 会被回收",
"The current default setting is 300, which is used to control the expiration time of "
+ "the partition metadata cache in NereidsSortedPartitionsCheManager. "
+ "If the cache is not accessed for a period of time, it will be reclaimed"
@@ -1424,14 +1429,14 @@ public class Config extends ConfigBase {
/**
* Set the maximum number of rows that can be cached
*/
- @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大行数。",
+ @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache 可以缓存的最大行数。",
"Maximum number of rows that can be cached in SQL/Partition Cache, is 3000 by default."})
public static int cache_result_max_row_count = 3000;
/**
* Set the maximum data size that can be cached
*/
- @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大数据大小。",
+ @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache 可以缓存的最大数据大小。",
"Maximum data size of rows that can be cached in SQL/Partition Cache, is 3000 by default."})
public static int cache_result_max_data_size = 31457280; // 30M
@@ -1506,7 +1511,7 @@ public class Config extends ConfigBase {
@ConfField
public static boolean enable_bdbje_debug_mode = false;
- @ConfField(mutable = false, masterOnly = true, description = {"是否开启debug point模式，测试使用",
+ @ConfField(mutable = false, masterOnly = true, description = {"是否开启 debug point 模式，测试使用",
"is enable debug points, use in test."})
public static boolean enable_debug_points = false;
@@ -1540,7 +1545,7 @@ public class Config extends ConfigBase {
* sets the time without read activity before sending a keepalive ping
* the smaller the value, the sooner the channel is unavailable, but it will increase network io
*/
- @ConfField(description = { "设置grpc连接发送 keepalive ping 之前没有数据传输的时间。",
+ @ConfField(description = { "设置 grpc 连接发送 keepalive ping 之前没有数据传输的时间。",
"The time without grpc read activity before sending a keepalive ping" })
public static int grpc_keep_alive_second = 10;
@@ -1659,7 +1664,7 @@ public class Config extends ConfigBase {
public static boolean enable_cloud_restore_job = false;
@ConfField(mutable = true, masterOnly = true, description = {
- "存算分离恢复过程中，一次 create tablets rpc 创建的 tablet 数量上限，默认值为256个",
+ "存算分离恢复过程中，一次 create tablets rpc 创建的 tablet 数量上限，默认值为 256 个",
"During the cloud restore job, the maximum number of tablets created by one "
+ "create tablets RPC, 256 by default."})
public static int cloud_restore_create_tablet_batch_size = 256;
@@ -1852,8 +1857,8 @@ public class Config extends ConfigBase {
*/
@ConfField(masterOnly = true, description = {"任务堆积时用于存放定时任务的队列大小", "The number of timer jobs that can be queued."})
public static int job_dispatch_timer_job_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"一个 Job 的 task 最大的持久化数量，超过这个限制将会丢弃旧的 task 记录, 如果值 < 1, 将不会持久化。",
- "Maximum number of persistence allowed per task in a job,exceeding which old tasks will be discarded，"
+ @ConfField(masterOnly = true, description = {"一个 Job 的 task 最大的持久化数量，超过这个限制将会丢弃旧的 task 记录，如果值 < 1, 将不会持久化。",
+ "Maximum number of persistence allowed per task in a job,exceeding which old tasks will be discarded,"
+ "If the value is less than 1, it will not be persisted." })
public static int max_persistence_task_count = 100;
@ConfField(masterOnly = true, description = {"MV task 的等待队列大小，如果是负数，则会使用 1024，如果不是 2 的幂，则会自动选择一个最接近的"
@@ -1866,16 +1871,16 @@ public class Config extends ConfigBase {
+ " If the size is not a power of two, the nearest power of the size will "
+ "be automatically selected."})
public static int insert_task_queue_size = 1024;
- @ConfField(masterOnly = true, description = {"finished 状态的 job 最长保存时间，超过这个时间将会被删除, 单位：小时",
+ @ConfField(masterOnly = true, description = {"finished 状态的 job 最长保存时间，超过这个时间将会被删除，单位：小时",
"The longest time to save the job in finished status, it will be deleted after this time. Unit: hour"})
public static int finished_job_cleanup_threshold_time_hour = 24;
- @ConfField(masterOnly = true, description = {"用于执行 Insert 任务的线程数,值应该大于0，否则默认为10",
+ @ConfField(masterOnly = true, description = {"用于执行 Insert 任务的线程数，值应该大于 0，否则默认为 10",
"The number of threads used to consume Insert tasks, "
+ "the value should be greater than 0, if it is <=0, default is 10."})
public static int job_insert_task_consumer_thread_num = 10;
- @ConfField(masterOnly = true, description = {"用于执行 MTMV 任务的线程数,值应该大于0，否则默认为10",
+ @ConfField(masterOnly = true, description = {"用于执行 MTMV 任务的线程数，值应该大于 0，否则默认为 10",
"The number of threads used to consume mtmv tasks, "
+ "the value should be greater than 0, if it is <=0, default is 10."})
public static int job_mtmv_task_consumer_thread_num = 10;
@@ -1929,12 +1934,12 @@ public class Config extends ConfigBase {
public static long query_queue_update_interval_ms = 5000;
@ConfField(mutable = true, description = {
- "当BE内存用量大于该值时，查询会进入排队逻辑，默认值为-1，代表该值不生效。取值范围0~1的小数",
+ "当 BE 内存用量大于该值时，查询会进入排队逻辑，默认值为 -1，代表该值不生效。取值范围 0~1 的小数",
"When be memory usage bigger than this value, query could queue, "
+ "default value is -1, means this value not work. Decimal value range from 0 to 1"})
public static double query_queue_by_be_used_memory = -1;
- @ConfField(mutable = true, description = {"基于内存反压场景FE定时拉取BE内存用量的时间间隔",
+ @ConfField(mutable = true, description = {"基于内存反压场景 FE 定时拉取 BE 内存用量的时间间隔",
"In the scenario of memory backpressure, "
+ "the time interval for obtaining BE memory usage at regular intervals"})
public static long get_be_resource_usage_interval_ms = 10000;
@@ -2030,9 +2035,9 @@ public class Config extends ConfigBase {
* And the max number of compute node is controlled by min_backend_num_for_external_table.
* If set to false, query on external table will assign to any node.
*/
- @ConfField(mutable = true, description = {"如果设置为true，外部表的查询将优先分配给计算节点。",
- "并且计算节点的最大数量由min_backend_num_for_external_table控制。",
- "如果设置为false，外部表的查询将分配给任何节点。"
+ @ConfField(mutable = true, description = {"如果设置为 true，外部表的查询将优先分配给计算节点。",
+ "并且计算节点的最大数量由 min_backend_num_for_external_table 控制。",
+ "如果设置为 false，外部表的查询将分配给任何节点。"
+ "如果集群内没有计算节点，则该参数不生效。",
"If set to true, query on external table will prefer to assign to compute node. "
+ "And the max number of compute node is controlled by min_backend_num_for_external_table. "
@@ -2040,9 +2045,9 @@ public class Config extends ConfigBase {
+ "If there is no compute node in cluster, this config takes no effect."})
public static boolean prefer_compute_node_for_external_table = false;
- @ConfField(mutable = true, description = {"只有当prefer_compute_node_for_external_table为true时生效，"
+ @ConfField(mutable = true, description = {"只有当 prefer_compute_node_for_external_table 为 true 时生效，"
+ "如果计算节点数小于这个值，外部表的查询会尝试获取一些混合节点来分配，以使节点总数达到这个值。"
- + "如果计算节点数大于这个值，外部表的查询将只分配给计算节点。-1表示只是用当前数量的计算节点",
+ + "如果计算节点数大于这个值，外部表的查询将只分配给计算节点。-1 表示只是用当前数量的计算节点",
"Only take effect when prefer_compute_node_for_external_table is true. "
+ "If the compute node number is less than this value, "
+ "query on external table will try to get some mix de to assign, "
@@ -2118,7 +2123,7 @@ public class Config extends ConfigBase {
public static int blacklist_duration_second = 120;
@ConfField(mutable = true, masterOnly = false, description = {
- "禁止创建odbc, mysql, broker类型的外表", "Disallow the creation of odbc, mysql, broker type external tables"})
+ "禁止创建 odbc, mysql, broker 类型的外表", "Disallow the creation of odbc, mysql, broker type external tables"})
public static boolean enable_odbc_mysql_broker_table = false;
/**
@@ -2157,11 +2162,11 @@ public class Config extends ConfigBase {
"Max cache number of database and table instance in external catalog."})
public static long max_meta_object_cache_num = 1000;
- @ConfField(description = {"Hive分区表缓存的最大数量",
+ @ConfField(description = {"Hive 分区表缓存的最大数量",
"Max cache number of hive partition table"})
public static long max_hive_partition_table_cache_num = 1000;
- @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量，-1代表没有限制。",
+ @ConfField(mutable = false, masterOnly = false, description = {"获取 Hive 分区值时候的最大返回数量，-1 代表没有限制。",
"Max number of hive partition values to return while list partitions, -1 means no limitation."})
public static short max_hive_list_partition_num = -1;
@@ -2264,7 +2269,7 @@ public class Config extends ConfigBase {
* otherwise it will throw an AnalysisException.
*/
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL, description = {
- "当前默认设置为 false，开启后支持使用新优化器的load语句导入数据，失败后会降级旧的load语句。",
+ "当前默认设置为 false，开启后支持使用新优化器的 load 语句导入数据，失败后会降级旧的 load 语句。",
"Now default set to true, After this function is enabled, the load statement of "
+ "the new optimizer can be used to import data. If this function fails, "
+ "the old load statement will be degraded."})
@@ -2277,7 +2282,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig",
description = {
- "当前默认设置为 100，用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
+ "当前默认设置为 100，用来控制控制 NereidsSqlCacheManager 管理的 sql cache 数量。",
"Now default set to 100, this config is used to control the number of "
+ "sql cache managed by NereidsSqlCacheManager"
}
@@ -2288,7 +2293,7 @@ public class Config extends ConfigBase {
mutable = true,
callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",
description = {
- "当前默认设置为 100，用来控制控制NereidsSortedPartitionsCacheManager中有序分区元数据的缓存个数，"
+ "当前默认设置为 100，用来控制控制 NereidsSortedPartitionsCacheManager 中有序分区元数据的缓存个数，"
+ "用于加速分区裁剪",
"The current default setting is 100, which is used to control the number of ordered "
+ "partition metadata caches in NereidsSortedPartitionsCacheManager, "
@@ -2423,7 +2428,7 @@ public class Config extends ConfigBase {
public static boolean enable_round_robin_create_tablet = true;
@ConfField(mutable = true, masterOnly = true, description = {
- "创建分区时，总是从第一个 BE 开始创建。注意：这种方式可能造成BE不均衡",
+ "创建分区时，总是从第一个 BE 开始创建。注意：这种方式可能造成 BE 不均衡",
"When creating tablet of a partition, always start from the first BE. "
+ "Note: This method may cause BE imbalance"})
public static boolean create_tablet_round_robin_from_start = false;
@@ -2486,19 +2491,19 @@ public class Config extends ConfigBase {
public static boolean enable_query_hit_stats = false;
@ConfField(mutable = true, description = {
- "设置为 true，如果查询无法选择到健康副本时，会打印出该tablet所有副本的详细信息，" + "以及不可查询的具体原因。",
+ "设置为 true，如果查询无法选择到健康副本时，会打印出该 tablet 所有副本的详细信息，" + "以及不可查询的具体原因。",
"When set to true, if a query is unable to select a healthy replica, "
+ "the detailed information of all the replicas of the tablet,"
+ " including the specific reason why they are unqueryable, will be printed out."})
public static boolean show_details_for_unaccessible_tablet = true;
@ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
- "是否启用binlog特性",
+ "是否启用 binlog 特性",
"Whether to enable binlog feature"})
public static boolean enable_feature_binlog = false;
@ConfField(mutable = false, description = {
- "是否默认为 Database/Table 启用binlog特性",
+ "是否默认为 Database/Table 启用 binlog 特性",
"Whether to enable binlog feature for Database/Table by default"})
public static boolean force_enable_feature_binlog = false;
@@ -2513,17 +2518,17 @@ public class Config extends ConfigBase {
public static boolean disallow_create_catalog_with_resource = true;
@ConfField(mutable = true, masterOnly = false, description = {
- "Hive行数估算分区采样数",
+ "Hive 行数估算分区采样数",
"Sample size for hive row count estimation."})
public static int hive_stats_partition_sample_size = 30;
@ConfField(mutable = true, masterOnly = true, description = {
- "启用Hive分桶表",
+ "启用 Hive 分桶表",
"Enable external hive bucket table"})
public static boolean enable_create_hive_bucket_table = false;
@ConfField(mutable = true, masterOnly = true, description = {
- "Hive创建外部表默认指定的文件格式",
+ "Hive 创建外部表默认指定的文件格式",
"Default hive file format for creating table."})
public static String hive_default_file_format = "orc";
@@ -2571,12 +2576,12 @@ public class Config extends ConfigBase {
public static boolean force_sample_analyze = false; // avoid full analyze for performance reason
@ConfField(mutable = true, description = {
- "Export任务允许的最大分区数量",
+ "Export 任务允许的最大分区数量",
"The maximum number of partitions allowed by Export job"})
public static int maximum_number_of_export_partitions = 2000;
@ConfField(mutable = true, description = {
- "Export任务允许的最大并行数",
+ "Export 任务允许的最大并行数",
"The maximum parallelism allowed by Export job"})
public static int maximum_parallelism_of_export_job = 50;
@@ -2589,20 +2594,20 @@ public class Config extends ConfigBase {
public static boolean forbid_running_alter_job = false;
@ConfField(description = {
- "暂时性配置项，开启后会自动将所有的olap表修改为可light schema change",
+ "暂时性配置项，开启后会自动将所有的 olap 表修改为可 light schema change",
"temporary config filed, will make all olap tables enable light schema change"
})
public static boolean enable_convert_light_weight_schema_change = false;
@ConfField(mutable = true, masterOnly = false, description = {
- "查询information_schema.metadata_name_ids表时,获取一个数据库中所有表用的时间",
+ "查询 information_schema.metadata_name_ids 表时，获取一个数据库中所有表用的时间",
"When querying the information_schema.metadata_name_ids table,"
+ " the time used to obtain all tables in one database"
})
public static long query_metadata_name_ids_timeout = 3;
@ConfField(mutable = true, masterOnly = true, description = {
- "是否禁止LocalDeployManager删除节点",
+ "是否禁止 LocalDeployManager 删除节点",
"Whether to disable LocalDeployManager drop node"})
public static boolean disable_local_deploy_manager_drop_node = true;
@@ -2642,13 +2647,13 @@ public class Config extends ConfigBase {
public static long analyze_record_limit = 20000;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中最小的buckets数目",
+ "Auto Buckets 中最小的 buckets 数目",
"min buckets of auto bucket"
})
public static int autobucket_min_buckets = 1;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中最大的buckets数目",
+ "Auto Buckets 中最大的 buckets 数目",
"max buckets of auto bucket"
})
public static int autobucket_max_buckets = 128;
@@ -2658,8 +2663,8 @@ public class Config extends ConfigBase {
public static int arrow_flight_max_connections = 4096;
@ConfField(mutable = true, masterOnly = true, description = {
- "Auto Buckets中按照partition size去估算bucket数，存算一体partition size 5G估算一个bucket，"
- + "但存算分离下partition size 10G估算一个bucket。 若配置小于0，会在在代码中会自适应存算一体模式默认5G，在存算分离默认10G",
+ "Auto Buckets 中按照 partition size 去估算 bucket 数，存算一体 partition size 5G 估算一个 bucket，"
+ + "但存算分离下 partition size 10G 估算一个 bucket。若配置小于 0，会在在代码中会自适应存算一体模式默认 5G，在存算分离默认 10G",
"In Auto Buckets, the number of buckets is estimated based on the partition size. "
+ "For storage and computing integration, a partition size of 5G is estimated as one bucket."
+ " but for cloud, a partition size of 10G is estimated as one bucket. "
@@ -2668,14 +2673,14 @@ public class Config extends ConfigBase {
})
public static int autobucket_partition_size_per_bucket_gb = -1;
- @ConfField(mutable = true, masterOnly = true, description = {"Auto bucket中计算出的新的分区bucket num超过前一个分区的"
- + "bucket num的百分比，被认为是异常case报警",
+ @ConfField(mutable = true, masterOnly = true, description = {"Auto bucket 中计算出的新的分区 bucket num 超过前一个分区的"
+ + "bucket num 的百分比，被认为是异常 case 报警",
"The new partition bucket number calculated in the auto bucket exceeds the percentage "
+ "of the previous partition's bucket number, which is considered an abnormal case alert."})
public static double autobucket_out_of_bounds_percent_threshold = 0.5;
- @ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow Flight Server中所有用户token的缓存上限，"
- + "超过后LRU淘汰, arrow flight sql是无状态的协议，连接通常不会主动断开，"
+ @ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow Flight Server 中所有用户 token 的缓存上限，"
+ + "超过后 LRU 淘汰，arrow flight sql 是无状态的协议，连接通常不会主动断开，"
+ "bearer token 从 cache 淘汰的同时会 unregister Connection.",
"(Deprecated, replaced by arrow_flight_max_connection) The cache limit of all user tokens in "
+ "Arrow Flight Server. which will be eliminated by LRU rules after exceeding the limit, "
@@ -2683,7 +2688,7 @@ public class Config extends ConfigBase {
+ "bearer token is evict from the cache will unregister ConnectContext."})
public static int arrow_flight_token_cache_size = 4096;
- @ConfField(description = {"Arrow Flight Server中用户token的存活时间，自上次写入后过期时间，单位秒，默认值为86400，即1天",
+ @ConfField(description = {"Arrow Flight Server 中用户 token 的存活时间，自上次写入后过期时间，单位秒，默认值为 86400，即 1 天",
"The alive time of the user token in Arrow Flight Server, expire after write, unit second,"
+ "the default value is 86400, which is 1 days"})
public static int arrow_flight_token_alive_time_second = 86400;
@@ -2697,14 +2702,14 @@ public class Config extends ConfigBase {
public static String mysqldb_replace_name = "mysql";
@ConfField(description = {
- "设置允许跨域访问的特定域名,默认允许任何域名跨域访问",
+ "设置允许跨域访问的特定域名，默认允许任何域名跨域访问",
"Set the specific domain name that allows cross-domain access. "
+ "By default, any domain name is allowed cross-domain access"
})
public static String access_control_allowed_origin_domain = "*";
@ConfField(description = {
- "开启java_udf, 默认为true。如果该配置为false，则禁止创建和使用java_udf。在一些场景下关闭该配置可防止命令注入攻击。",
+ "开启 java_udf, 默认为 true。如果该配置为 false，则禁止创建和使用 java_udf。在一些场景下关闭该配置可防止命令注入攻击。",
"Used to enable java_udf, default is true. if this configuration is false, creation and use of java_udf is "
+ "disabled. in some scenarios it may be necessary to disable this configuration to prevent "
+ "command injection attacks."
@@ -2712,7 +2717,7 @@ public class Config extends ConfigBase {
public static boolean enable_java_udf = true;
@ConfField(mutable = true, masterOnly = true, description = {
- "开启后，可以在导入时，利用创建的全局java_udf函数处理数据, 默认为false。",
+ "开启后，可以在导入时，利用创建的全局 java_udf 函数处理数据，默认为 false。",
"When enabled, data can be processed using the globally created java_udf function during import."
+ " The default setting is false."
})
@@ -2732,8 +2737,8 @@ public class Config extends ConfigBase {
public static boolean ignore_unknown_metadata_module = false;
@ConfField(mutable = true, description = {
- "从主节点同步image文件的超时时间，用户可根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，"
- + "单位为秒，默认值300",
+ "从主节点同步 image 文件的超时时间，用户可根据${meta_dir}/image 文件夹下面的 image 文件大小和节点间的网络环境调整，"
+ + "单位为秒，默认值 300",
"The timeout for FE Follower/Observer synchronizing an image file from the FE Master, can be adjusted by "
+ "the user on the size of image file in the ${meta_dir}/image and the network environment between "
+ "nodes. The default values is 300."
@@ -2775,12 +2780,12 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int workload_group_max_num = 15;
- @ConfField(description = {"查询be wal_queue 的超时阈值(ms)",
+ @ConfField(description = {"查询 be wal_queue 的超时阈值 (ms)",
"the timeout threshold of checking wal_queue on be(ms)"})
public static int check_wal_queue_timeout_threshold = 180000; // 3 min
@ConfField(mutable = true, masterOnly = true, description = {
- "对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。",
+ "对于自动分区表，防止用户意外创建大量分区，每个 OLAP 表允许的分区数量为`max_auto_partition_num`。默认 2000。",
"For auto-partitioned tables to prevent users from accidentally creating a large number of partitions, "
+ "the number of partitions allowed per OLAP table is `max_auto_partition_num`. Default 2000."
})
@@ -2801,9 +2806,9 @@ public class Config extends ConfigBase {
public static double diagnose_balance_max_tablet_num_ratio = 1.1;
@ConfField(masterOnly = true, description = {
- "设置 root 用户初始化2阶段 SHA-1 加密密码，默认为''，即不设置 root 密码。"
+ "设置 root 用户初始化 2 阶段 SHA-1 加密密码，默认为''，即不设置 root 密码。"
+ "后续 root 用户的 `set password` 操作会将 root 初始化密码覆盖。"
- + "示例：如要配置密码的明文是 `root@123`，可在Doris执行SQL `select password('root@123')` "
+ + "示例：如要配置密码的明文是 `root@123`，可在 Doris 执行 SQL `select password('root@123')` "
+ "获取加密密码 `*A00C34073A26B40AB4307650BFB9309D6BFA6999`",
"Set root user initial 2-staged SHA-1 encrypted password, default as '', means no root password. "
+ "Subsequent `set password` operations for root user will overwrite the initial root password. "
@@ -2812,25 +2817,25 @@ public class Config extends ConfigBase {
+ "password `*A00C34073A26B40AB4307650BFB9309D6BFA6999`"})
public static String initial_root_password = "";
- @ConfField(description = {"nereids trace文件的存放路径。",
+ @ConfField(description = {"nereids trace 文件的存放路径。",
"The path of the nereids trace file."})
public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace";
@ConfField(mutable = true, masterOnly = true, description = {
- "备份过程中，一个 upload 任务上传的快照数量上限，默认值为10个",
+ "备份过程中，一个 upload 任务上传的快照数量上限，默认值为 10 个",
"The max number of snapshots assigned to a upload task during the backup process, the default value is 10."
})
public static int backup_upload_snapshot_batch_size = 10;
@ConfField(mutable = true, masterOnly = true, description = {
- "恢复过程中，一个 download 任务下载的快照数量上限，默认值为10个",
+ "恢复过程中，一个 download 任务下载的快照数量上限，默认值为 10 个",
"The max number of snapshots assigned to a download task during the restore process, "
+ "the default value is 10."
})
public static int restore_download_snapshot_batch_size = 10;
@ConfField(mutable = true, masterOnly = true, description = {
- "备份恢复过程中，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
+ "备份恢复过程中，单次 RPC 分配给每个 be 的任务最大个数，默认值为 10000 个。",
"The max number of batched tasks per RPC assigned to each be during the backup/restore process, "
+ "the default value is 10000."
})
@@ -2841,11 +2846,11 @@ public class Config extends ConfigBase {
"The number of concurrent restore tasks per be"})
public static int restore_task_concurrency_per_be = 5000;
- @ConfField(mutable = true, description = {"执行 agent task 时，BE心跳超过多长时间，认为BE不可用",
+ @ConfField(mutable = true, description = {"执行 agent task 时，BE 心跳超过多长时间，认为 BE 不可用",
"The time after which BE is considered unavailable if the heartbeat is not received"})
public static int agent_task_be_unavailable_heartbeat_timeout_second = 300;
- @ConfField(description = {"是否开启通过http接口获取log文件的功能",
+ @ConfField(description = {"是否开启通过 http 接口获取 log 文件的功能",
"Whether to enable the function of getting log files through http interface"})
public static boolean enable_get_log_file_api = false;
@@ -2855,20 +2860,20 @@ public class Config extends ConfigBase {
public static boolean enable_collect_internal_query_profile = false;
@ConfField(mutable = false, masterOnly = false, description = {
- "http请求处理/api/query中sql任务的最大线程池。",
+ "http 请求处理/api/query 中 sql 任务的最大线程池。",
"The max number work threads of http sql submitter."
})
public static int http_sql_submitter_max_worker_threads = 2;
@ConfField(mutable = false, masterOnly = false, description = {
- "http请求处理/api/upload任务的最大线程池。",
+ "http 请求处理/api/upload 任务的最大线程池。",
"The max number work threads of http upload submitter."
})
public static int http_load_submitter_max_worker_threads = 2;
@ConfField(mutable = true, masterOnly = true, description = {
- "load label个数阈值，超过该个数后，对于已经完成导入作业或者任务，"
- + "其label会被删除，被删除的 label 可以被重用。 值为 -1 时，表示此阈值不生效。",
+ "load label 个数阈值，超过该个数后，对于已经完成导入作业或者任务，"
+ + "其 label 会被删除，被删除的 label 可以被重用。值为 -1 时，表示此阈值不生效。",
"The threshold of load labels' number. After this number is exceeded, "
+ "the labels of the completed import jobs or tasks will be deleted, "
+ "and the deleted labels can be reused. "
@@ -2890,7 +2895,7 @@ public class Config extends ConfigBase {
@ConfField
public static boolean ignore_bdbje_log_checksum_read = false;
- @ConfField(description = {"指定 mysql登录身份认证类型",
+ @ConfField(description = {"指定 mysql 登录身份认证类型",
"Specifies the authentication type"},
options = {"default", "ldap"})
public static String authentication_type = "default";
@@ -2914,8 +2919,8 @@ public class Config extends ConfigBase {
public static String inverted_index_storage_format = "V2";
@ConfField(mutable = true, masterOnly = true, description = {
- "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，"
- + "但delete后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低delete语句的性能来保证正确性。",
+ "是否在 unique 表 mow 上开启 delete 语句写 delete predicate。若开启，会提升 delete 语句的性能，"
+ + "但 delete 后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低 delete 语句的性能来保证正确性。",
"Enable the 'delete predicate' for DELETE statements. If enabled, it will enhance the performance of "
+ "DELETE statements, but partial column updates after a DELETE may result in erroneous data. "
+ "If disabled, it will reduce the performance of DELETE statements to ensure accuracy."
@@ -2965,7 +2970,7 @@ public class Config extends ConfigBase {
public static int checkpoint_manual_gc_threshold = 0;
@ConfField(mutable = true, description = {
- "是否在每个请求开始之前打印一遍请求内容, 主要是query语句",
+ "是否在每个请求开始之前打印一遍请求内容，主要是 query 语句",
"Should the request content be logged before each request starts, specifically the query statements"})
public static boolean enable_print_request_before_execution = false;
@@ -2990,17 +2995,17 @@ public class Config extends ConfigBase {
public static int profile_waiting_time_for_spill_seconds = 10;
@ConfField(mutable = true, description = {
- "是否通过检测协调者BE心跳来 abort 事务",
+ "是否通过检测协调者 BE 心跳来 abort 事务",
"SHould abort txn by checking coorinator be heartbeat"})
public static boolean enable_abort_txn_by_checking_coordinator_be = true;
@ConfField(mutable = true, description = {
- "是否在 schema change 过程中, 检测冲突事物并 abort 它",
+ "是否在 schema change 过程中，检测冲突事物并 abort 它",
"SHould abort txn by checking conflick txn in schema change"})
public static boolean enable_abort_txn_by_checking_conflict_txn = true;
@ConfField(mutable = true, description = {
- "内表自动收集时间间隔，当某一列上次收集时间距离当前时间大于该值，则会触发一次新的收集，0表示不会触发。",
+ "内表自动收集时间间隔，当某一列上次收集时间距离当前时间大于该值，则会触发一次新的收集，0 表示不会触发。",
"Columns that have not been collected within the specified interval will trigger automatic analyze. "
+ "0 means not trigger."
})
@@ -3015,7 +3020,7 @@ public class Config extends ConfigBase {
//==========================================================================
// begin of cloud config
//==========================================================================
- @ConfField(description = {"是否启用FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认为按照时间策略删除",
+ @ConfField(description = {"是否启用 FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的 log。默认为按照时间策略删除",
"Whether to enable the FE log file deletion policy based on size, "
+ "where logs exceeding the specified size are deleted. "
+ "It is disabled by default and follows a time-based deletion policy."},
@@ -3195,7 +3200,7 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static int mow_calculate_delete_bitmap_retry_times = 10;
- @ConfField(mutable = true, description = {"指定S3 Load endpoint白名单, 举例: s3_load_endpoint_white_list=a,b,c",
+ @ConfField(mutable = true, description = {"指定 S3 Load endpoint 白名单，举例：s3_load_endpoint_white_list=a,b,c",
"the white list for the s3 load endpoint, if it is empty, no white list will be set,"
+ "for example: s3_load_endpoint_white_list=a,b,c"})
public static String[] s3_load_endpoint_white_list = {};
@@ -3209,7 +3214,7 @@ public class Config extends ConfigBase {
})
public static boolean force_azure_blob_global_endpoint = false;
- @ConfField(mutable = true, description = {"指定Jdbc driver url白名单, 举例: jdbc_driver_url_white_list=a,b,c",
+ @ConfField(mutable = true, description = {"指定 Jdbc driver url 白名单，举例：jdbc_driver_url_white_list=a,b,c",
"the white list for jdbc driver url, if it is empty, no white list will be set"
+ "for example: jdbc_driver_url_white_list=a,b,c"
})
@@ -3261,30 +3266,30 @@ public class Config extends ConfigBase {
public static int audit_event_log_queue_size = 250000;
@ConfField(mutable = true, description = {
- "streamload导入使用的转发策略, 可选值为public-private/public/private/direct/random-be/空",
+ "streamload 导入使用的转发策略，可选值为 public-private/public/private/direct/random-be/空",
"streamload route policy, available options are "
+ "public-private/public/private/direct/random-be and empty string" })
public static String streamload_redirect_policy = "";
@ConfField(mutable = true, description = {
- "存算分离模式下是否启用group commit的streamload BE转发功能。"
- + "解决LB随机转发导致group commit攒批失效的问题，通过BE二次转发确保同表请求到达同一BE节点。",
+ "存算分离模式下是否启用 group commit 的 streamload BE 转发功能。"
+ + "解决 LB 随机转发导致 group commit 攒批失效的问题，通过 BE 二次转发确保同表请求到达同一 BE 节点。",
"Whether to enable group commit streamload BE forward feature in cloud mode. "
+ "Solves the issue where LB random forwarding breaks group commit batching "
+ "by implementing BE-level forwarding to ensure same-table requests reach the same BE node." })
public static boolean enable_group_commit_streamload_be_forward = false;
- @ConfField(description = {"存算分离模式下建表是否检查残留recycler key, 默认true",
+ @ConfField(description = {"存算分离模式下建表是否检查残留 recycler key, 默认 true",
"create table in cloud mode, check recycler key remained, default true"})
public static boolean check_create_table_recycle_key_remained = true;
- @ConfField(mutable = true, description = {"存算分离模式下fe向ms请求锁的过期时间，默认60s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 fe 向 ms 请求锁的过期时间，默认 60s"})
public static int delete_bitmap_lock_expiration_seconds = 60;
- @ConfField(mutable = true, description = {"存算分离模式下calculate delete bitmap task 超时时间，默认60s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 calculate delete bitmap task 超时时间，默认 60s"})
public static int calculate_delete_bitmap_task_timeout_seconds = 60;
- @ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"})
+ @ConfField(mutable = true, description = {"存算分离模式下 commit 阶段等锁超时时间，默认 5s"})
public static int try_commit_lock_timeout_seconds = 5;
@ConfField(mutable = true, description = {"是否在事务提交时对所有表启用提交锁。设置为 true 时，所有表都会使用提交锁。"
@@ -3295,28 +3300,28 @@ public class Config extends ConfigBase {
+ "Default value is true." })
public static boolean enable_commit_lock_for_all_tables = true;
- @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"})
+ @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认 false"})
public static boolean enable_cloud_txn_lazy_commit = false;
@ConfField(mutable = true, masterOnly = true,
- description = {"存算分离模式下，当tablet分布的be异常，是否立即映射tablet到新的be上，默认false"})
+ description = {"存算分离模式下，当 tablet 分布的 be 异常，是否立即映射 tablet 到新的 be 上，默认 false"})
public static boolean enable_immediate_be_assign = false;
@ConfField(mutable = true, masterOnly = false,
- description = { "存算分离模式下，一个BE挂掉多长时间后，它的tablet彻底转移到其他BE上" })
+ description = { "存算分离模式下，一个 BE 挂掉多长时间后，它的 tablet 彻底转移到其他 BE 上" })
public static int rehash_tablet_after_be_dead_seconds = 3600;
- @ConfField(mutable = true, description = {"存算分离模式下是否启用自动启停功能，默认true",
+ @ConfField(mutable = true, description = {"存算分离模式下是否启用自动启停功能，默认 true",
"Whether to enable the automatic start-stop feature in cloud model, default is true."})
public static boolean enable_auto_start_for_cloud_cluster = true;
- @ConfField(mutable = true, description = {"存算分离模式下自动启停等待cluster唤醒退避重试次数，默认300次大约5分钟",
+ @ConfField(mutable = true, description = {"存算分离模式下自动启停等待 cluster 唤醒退避重试次数，默认 300 次大约 5 分钟",
"The automatic start-stop wait time for cluster wake-up backoff retry count in the cloud "
+ "model is set to 300 times, which is approximately 5 minutes by default."})
public static int auto_start_wait_to_resume_times = 300;
- @ConfField(description = {"Get tablet stat task的最大并发数。",
+ @ConfField(description = {"Get tablet stat task 的最大并发数。",
"Maximal concurrent num of get tablet stat job."})
public static int max_get_tablet_stat_task_threads_num = 4;
@@ -3328,25 +3333,25 @@ public class Config extends ConfigBase {
"Max retry times when a schema change job fails, default is 3."})
public static int schema_change_max_retry_time = 3;
- @ConfField(mutable = true, description = {"是否允许使用ShowCacheHotSpotStmt语句",
+ @ConfField(mutable = true, description = {"是否允许使用 ShowCacheHotSpotStmt 语句",
"Whether to enable the use of ShowCacheHotSpotStmt, default is false."})
public static boolean enable_show_file_cache_hotspot_stmt = false;
- @ConfField(mutable = true, description = {"存算分离模式下FE连接meta service的请求超时, 默认30000ms",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 连接 meta service 的请求超时，默认 30000ms",
"Request timeout for FE connecting to meta service in cloud mode, default is 30000ms."})
public static int meta_service_brpc_timeout_ms = 30000;
- @ConfField(mutable = true, description = {"存算分离模式下FE连接meta service的连接超时，默认500ms",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 连接 meta service 的连接超时，默认 500ms",
"Connection timeout for FE connecting to meta service in cloud mode., default is 500ms."})
public static int meta_service_brpc_connect_timeout_ms = 500;
- @ConfField(mutable = true, description = {"存算分离模式下FE请求meta service超时的重试次数，默认1次",
+ @ConfField(mutable = true, description = {"存算分离模式下 FE 请求 meta service 超时的重试次数，默认 1 次",
"In cloud mode, the retry number when the FE requests the meta service times out is 1 by default"})
public static int meta_service_rpc_timeout_retry_times = 1;
@ConfField(mutable = true, description = {"存算分离模式下自动启停功能，对于该配置中的数据库名不进行唤醒操作，"
+ "用于内部作业的数据库，例如统计信息用到的数据库，"
- + "举例: auto_start_ignore_db_names=__internal_schema, information_schema",
+ + "举例：auto_start_ignore_db_names=__internal_schema, information_schema",
"In the cloud mode, the automatic start and stop ignores the DB name of the internal job,"
+ "used for databases involved in internal jobs, such as those used for statistics, "
+ "For example: auto_start_ignore_db_names=__internal_schema, information_schema"
@@ -3377,7 +3382,7 @@ public class Config extends ConfigBase {
public static long resource_not_ready_sleep_seconds = 5;
@ConfField(mutable = true, description = {
- "设置为 true，如果查询无法选择到健康副本时，会打印出该tablet所有副本的详细信息，"})
+ "设置为 true，如果查询无法选择到健康副本时，会打印出该 tablet 所有副本的详细信息，"})
public static boolean sql_block_rule_ignore_admin = false;
//==========================================================================
// start of lock config
@@ -3397,12 +3402,12 @@ public class Config extends ConfigBase {
"Is metadata synchronization enabled in safe mode"})
public static boolean meta_helper_security_mode = false;
- @ConfField(description = {"用于测试，强制将所有的查询forward到master以验证forward query的行为",
+ @ConfField(description = {"用于测试，强制将所有的查询 forward 到 master 以验证 forward query 的行为",
"For testing purposes, all queries are forcibly forwarded to the master to verify"
+ "the behavior of forwarding queries."})
public static boolean force_forward_all_queries = false;
- @ConfField(description = {"用于禁用某些SQL，配置项为AST的class simple name列表(例如CreateRepositoryStmt,"
+ @ConfField(description = {"用于禁用某些 SQL，配置项为 AST 的 class simple name 列表 (例如 CreateRepositoryStmt,"
+ "CreatePolicyCommand)，用逗号间隔开",
"For disabling certain SQL queries, the configuration item is a list of simple class names of AST"
+ "(for example CreateRepositoryStmt, CreatePolicyCommand), separated by commas."})
diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
index d5b7cd7354e..47b9fb3137d 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
@@ -501,7 +501,7 @@ public class DorisFE {
releaseFileLockAndCloseFileChannel();
throw new RuntimeException("Try to lock process failed", e);
}
- throw new RuntimeException("FE process has been started，please do not start multiple FE processes at the "
+ throw new RuntimeException("FE process has been started, please do not start multiple FE processes at the "
+ "same time");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 364cacab538..71b478ab12a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1395,6 +1395,11 @@ public class SchemaChangeHandler extends AlterHandler {
TStorageFormat storageFormat = PropertyAnalyzer.analyzeStorageFormat(propertyMap);
+ // if not explicitly specified in ALTER statement, use the current storage format of the table
+ if (!propertyMap.containsKey("storage_format")) {
+ storageFormat = olapTable.getStorageFormat();
+ }
+
// property store_row_column && row_store_columns
// eg. "store_row_column" = "true"
// eg. "row_store_columns" = "k1, k2"
@@ -1662,14 +1667,7 @@ public class SchemaChangeHandler extends AlterHandler {
schemaChangeJob.setStoreRowColumnInfo(hasRowStoreChanged, storeRowColumn, rsColumns);
schemaChangeJob.setEnableUniqueKeySkipBitmapInfo(enableUniqueKeySkipBitmap,
hasEnableUniqueKeySkipBitmapChanged);
-
- // If StorageFormat is set to TStorageFormat.V2
- // which will create tablet with preferred_rowset_type set to BETA
- // for both base table and rollup index
- if (hasIndexChange) {
- // only V2 support index, so if there is index changed, storage format must be V2
- storageFormat = TStorageFormat.V2;
- }
+ // set storage format of schema change job
schemaChangeJob.setStorageFormat(storageFormat);
// the following operations are done outside the 'for indices' loop
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index e50a40304be..3d971971dea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1160,11 +1160,14 @@ public class PropertyAnalyzer {
return TStorageFormat.V2;
}
- if (storageFormat.equalsIgnoreCase("v1")) {
+ if (storageFormat.equalsIgnoreCase("V1")) {
throw new AnalysisException("Storage format V1 has been deprecated since version 0.14, "
+ "please use V2 instead");
- } else if (storageFormat.equalsIgnoreCase("v2")) {
+ } else if (storageFormat.equalsIgnoreCase("V2")) {
return TStorageFormat.V2;
+ } else if (storageFormat.equalsIgnoreCase("V3")) {
+ // V3 (V2 + external column meta feature)
+ return TStorageFormat.V3;
} else if (storageFormat.equalsIgnoreCase("default")) {
return TStorageFormat.V2;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index a9fa6d42df0..664d6305457 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -79,6 +79,7 @@ import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.base.Preconditions;
@@ -88,6 +89,8 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashMap;
@@ -104,7 +107,6 @@ import java.util.stream.Collectors;
* table info in creating table.
*/
public class CreateTableInfo {
-
public static final String ENGINE_OLAP = "olap";
public static final String ENGINE_JDBC = "jdbc";
public static final String ENGINE_ELASTICSEARCH = "elasticsearch";
@@ -113,6 +115,8 @@ public class CreateTableInfo {
public static final String ENGINE_BROKER = "broker";
public static final String ENGINE_HIVE = "hive";
public static final String ENGINE_ICEBERG = "iceberg";
+
+ private static final Logger LOG = LogManager.getLogger(CreateTableInfo.class);
private static final ImmutableSet<AggregateType> GENERATED_COLUMN_ALLOW_AGG_TYPE =
ImmutableSet.of(AggregateType.REPLACE, AggregateType.REPLACE_IF_NOT_NULL);
@@ -382,6 +386,19 @@ public class CreateTableInfo {
if (engineName.equalsIgnoreCase(ENGINE_OLAP)) {
boolean enableDuplicateWithoutKeysByDefault = false;
properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(ctlName, dbName, properties);
+
+ // In fuzzy tests, randomly set storage_format=V3 (ext_meta) for some tables.
+ SessionVariable sv = ctx.getSessionVariable();
+ boolean randomUseV3 =
+ sv != null && sv.randomUseV3StorageFormat && Config.use_fuzzy_session_variable;
+ if (randomUseV3
+ && properties != null
+ && !properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V3");
+ LOG.info("Randomly set storage_format=V3 for table {}.{} in fuzzy mode (session={})",
+ dbName, tableName,
+ ctx != null ? ctx.getSessionId() : "<null>");
+ }
try {
if (properties != null) {
enableDuplicateWithoutKeysByDefault =
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index bc0c440bfec..ec1c67d1ee8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -2635,6 +2635,15 @@ public class SessionVariable implements Serializable, Writable {
)
public int defaultVariantMaxSparseColumnStatisticsSize = 10000;
+ @VariableMgr.VarAttr(
+ name = "random_use_v3_storage_format",
+ fuzzy = true,
+ description = {
+ "In fuzzy tests, randomly use V3 storage_format (ext_meta) for some tables.",
+ "Only takes effect when user does not explicitly specify storage_format."}
+ )
+ public boolean randomUseV3StorageFormat = false;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
@SuppressWarnings("checkstyle:Indentation")
@@ -2656,6 +2665,7 @@ public class SessionVariable implements Serializable, Writable {
this.enableShareHashTableForBroadcastJoin = random.nextBoolean();
this.defaultVariantMaxSubcolumnsCount = random.nextInt(10);
this.defaultEnableTypedPathsToSparse = random.nextBoolean();
+ this.randomUseV3StorageFormat = random.nextBoolean();
int randomInt = random.nextInt(4);
if (randomInt % 2 == 0) {
this.rewriteOrToInPredicateThreshold = 100000;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index a5b4ef1a1c3..352db545a66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -362,7 +362,6 @@ public class CreateReplicaTask extends AgentTask {
}
}
tSchema.setIndexes(tIndexes);
- storageFormat = TStorageFormat.V2;
}
if (bfColumns != null) {
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 1e97d5ad476..6ace01a9257 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -427,6 +427,9 @@ message TabletSchemaPB {
optional int32 skip_bitmap_col_idx = 29 [default = -1];
optional int64 storage_page_size = 30 [default=65536];
optional int64 storage_dict_page_size = 31 [default=262144];
+
+ // Whether new segments of this tablet use externalized ColumnMetaPB layout (CMO)
+ optional bool is_external_segment_meta_used_default = 32 [default = false];
}
message TabletSchemaCloudPB {
@@ -463,6 +466,9 @@ message TabletSchemaCloudPB {
optional int64 storage_page_size = 31 [default=65536];
optional int64 storage_dict_page_size = 32 [default=262144];
+ // Whether new segments of this tablet use externalized ColumnMetaPB layout (CMO)
+ optional bool is_external_segment_meta_used_default = 33 [default = false];
+
optional bool is_dynamic_schema = 100 [default=false];
}
diff --git a/gensrc/proto/segment_v2.proto b/gensrc/proto/segment_v2.proto
index d2f2199e5ba..e902a5989e1 100644
--- a/gensrc/proto/segment_v2.proto
+++ b/gensrc/proto/segment_v2.proto
@@ -209,6 +209,16 @@ message ColumnMetaPB {
optional uint64 none_null_size = 23 [default = 0];
}
+// External column meta entry describing one top-level column's externalized
+// ColumnMetaPB in the Column Meta Region.
+message ColumnMetaEntryPB {
+ // unique column id (top-level only, including variant roots)
+ optional int32 unique_id = 1;
+ // serialized length in bytes of this column's ColumnMetaPB in the
+ // external Column Meta Region.
+ optional uint32 length = 2;
+}
+
message PrimaryKeyIndexMetaPB {
// primary key index
optional IndexedColumnMetaPB primary_key_index = 1;
@@ -235,6 +245,14 @@ message SegmentFooterPB {
// Primary key index meta
optional PrimaryKeyIndexMetaPB primary_key_index_meta = 10;
+
+ // Externalized ColumnMetaPB layout (optional; presence indicates v2 meta layout available)
+ // Start offset of the contiguous per-column ColumnMetaPB region (absolute file offset).
+ optional uint64 col_meta_region_start = 11;
+ // Per-column external meta entries, in col_id order. Each entry contains the top-level
+ // unique id and the serialized length (in bytes) of its ColumnMetaPB in the external
+ // Column Meta Region.
+ repeated ColumnMetaEntryPB column_meta_entries = 12;
}
message BTreeMetaPB {
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 91658f929ea..d1130637846 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -57,9 +57,11 @@ struct TTabletSchema {
// V1 for Segment-V1
// V2 for Segment-V2
enum TStorageFormat {
- DEFAULT,
- V1,
- V2
+ DEFAULT = 0,
+ V1 = 1,
+ V2 = 2,
+ // V3 stands externalized column meta (CMO)
+ V3 = 3
}
enum TEncryptionAlgorithm {
diff --git a/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out b/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out
new file mode 100644
index 00000000000..a04a08cbf67
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_storage_format_v2_1.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+1 1
+2 10
+
+-- !sql2 --
+1 2
+2 20
+
+-- !sql3 --
+2 40
+
+-- !sql4 --
+3 500
+
+-- !sql5 --
+1 100
+
+-- !sql6 --
+1 value
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out
new file mode 100644
index 00000000000..293c7c5371d
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_concurrent.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !concurrent_1 --
+20
+
+-- !concurrent_2 --
+20
+
+-- !write_read_1 --
+16
+
+-- !write_read_2 --
+10
+
+-- !write_read_3 --
+6
+
+-- !stress_1 --
+15
+
+-- !stress_2 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !cache_1 --
+5
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out
new file mode 100644
index 00000000000..b95f6f8f269
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_edge_cases.out
@@ -0,0 +1,174 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !empty_1 --
+1 \N
+2 \N
+3 \N
+
+-- !empty_2 --
+
+-- !empty_3 --
+1
+2
+3
+
+-- !many_1 --
+0 0
+1 0
+2 0
+3 0
+4 0
+5 0
+6 0
+7 0
+8 0
+9 0
+
+-- !many_2 --
+5 50
+6 50
+7 50
+8 50
+9 50
+
+-- !many_3 --
+9 99
+
+-- !many_4 --
+10
+
+-- !mixed_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !mixed_2 --
+1 2
+
+-- !mixed_3 --
+2 3
+
+-- !mixed_4 --
+3 4
+
+-- !mixed_5 --
+4 5
+
+-- !mixed_after_compact_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !mixed_after_compact_2 --
+4
+
+-- !nested_1 --
+1 value
+2 value2
+3 \N
+4 \N
+
+-- !nested_2 --
+2 123
+
+-- !nested_3 --
+1
+2
+
+-- !nested_4 --
+1
+2
+
+-- !array_1 --
+1 [1, 2, 3]
+2 [10, 20, 30, 40]
+3 \N
+4 \N
+
+-- !array_2 --
+1
+2
+
+-- !evolution_1 --
+1 1
+2 2
+3 3
+4 \N
+
+-- !evolution_2 --
+1 \N
+2 2
+3 3
+4 \N
+
+-- !evolution_3 --
+1 \N
+2 \N
+3 3
+4 \N
+
+-- !evolution_4 --
+1 \N
+2 \N
+3 \N
+4 10
+
+-- !evolution_5 --
+1 \N
+2 \N
+3 \N
+4 20
+
+-- !special_1 --
+1 1
+
+-- !special_2 --
+2 2
+
+-- !special_3 --
+3 3
+
+-- !null_1 --
+1 \N
+2 1
+3 \N
+
+-- !null_2 --
+3
+
+-- !null_3 --
+1
+3
+
+-- !null_4 --
+2
+
+-- !large_1 --
+1 10000
+2 5
+
+-- !large_2 --
+1
+
+-- !toggle_1 --
+1 1
+2 2
+3 3
+
+-- !toggle_2 --
+2 2
+3 3
+
+-- !toggle_3 --
+3 3
+
+-- !toggle_after_compact_1 --
+1 1
+2 2
+3 3
+
+-- !toggle_after_compact_2 --
+2 2
+3 3
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out
new file mode 100644
index 00000000000..0926c7eb167
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_integration.out
@@ -0,0 +1,120 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !inverted_1 --
+1 Alice Smith
+3 Charlie Smith
+
+-- !inverted_2 --
+1 Alice Smith
+
+-- !inverted_3 --
+2 30
+3 35
+
+-- !inverted_after_compact_1 --
+1 Alice Smith
+3 Charlie Smith
+
+-- !inverted_after_compact_2 --
+1
+
+-- !parent_idx_1 --
+1 Hello World
+3 World News
+
+-- !parent_idx_2 --
+1 This is a test
+2 Another test here
+
+-- !parent_idx_3 --
+1
+
+-- !multi_var_1 --
+1 1 10
+2 100 1000
+3 \N \N
+
+-- !multi_var_2 --
+1 2
+
+-- !multi_var_3 --
+1 20
+
+-- !multi_var_4 --
+3 4 40
+
+-- !multi_var_after_compact_1 --
+1 1 10
+2 100 1000
+3 \N \N
+
+-- !multi_var_after_compact_2 --
+3
+
+-- !agg_1 --
+A 300
+B 400
+
+-- !agg_2 --
+A 1.5
+B 2
+
+-- !agg_3 --
+2
+
+-- !join_1 --
+1 Alice Engineering
+2 Bob Sales
+3 Charlie Engineering
+
+-- !join_2 --
+2
+
+-- !union_1 --
+1 table1 100
+2 table1 200
+3 table2 150
+4 table2 250
+
+-- !union_2 --
+table1 2
+table2 2
+
+-- !subquery_1 --
+6 60 A
+7 70 A
+8 80 A
+9 90 A
+10 100 A
+
+-- !subquery_2 --
+A 5
+B 1
+
+-- !having_1 --
+A 600
+
+-- !order_limit_1 --
+1 19 item_1
+2 18 item_2
+3 17 item_3
+4 16 item_4
+5 15 item_5
+
+-- !order_limit_2 --
+9 item_9
+8 item_8
+7 item_7
+
+-- !distinct_1 --
+A
+B
+C
+
+-- !distinct_2 --
+100
+150
+200
+
+-- !distinct_3 --
+3
+
diff --git a/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out
new file mode 100644
index 00000000000..d5f580cabc9
--- /dev/null
+++ b/regression-test/data/variant_p0/ext_meta/test_variant_external_meta_with_sparse.out
@@ -0,0 +1,197 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sparse_1 --
+1 1
+2 10
+3 100
+
+-- !sparse_2 --
+1 2
+2 20
+
+-- !sparse_3 --
+1 3
+2 30
+
+-- !sparse_4 --
+2 40
+
+-- !sparse_5 --
+3 500
+
+-- !sparse_6 --
+3 600
+
+-- !sparse_7 --
+3 700
+
+-- !sparse_after_compact_1 --
+1 1
+2 10
+3 100
+
+-- !sparse_after_compact_2 --
+2 40
+
+-- !sparse_after_compact_3 --
+3 700
+
+-- !bucket_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+
+-- !bucket_2 --
+0 0
+1 10
+2 20
+3 30
+4 40
+5 50
+6 60
+7 70
+8 80
+9 90
+
+-- !bucket_3 --
+0 0
+
+-- !bucket_4 --
+5 503
+
+-- !bucket_5 --
+10
+
+-- !bucket_after_compact_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+
+-- !bucket_after_compact_2 --
+0 0
+
+-- !mixed_1 --
+1 1
+2 10
+3 100
+
+-- !mixed_2 --
+1 5
+2 50
+
+-- !mixed_3 --
+2 60
+
+-- !mixed_4 --
+3 200
+
+-- !mixed_5 --
+1 \N
+2 \N
+3 \N
+
+-- !threshold_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !threshold_2 --
+1 2
+2 20
+3 200
+
+-- !threshold_3 --
+2 30
+3 300
+
+-- !threshold_4 --
+3 400
+
+-- !threshold_5 --
+4 5000
+
+-- !threshold_after_compact_1 --
+1 1
+2 10
+3 100
+4 1000
+
+-- !threshold_after_compact_2 --
+3 400
+
+-- !threshold_after_compact_3 --
+4 5000
+
+-- !toggle_sparse_1 --
+1 1
+2 10
+3 100
+
+-- !toggle_sparse_2 --
+1 2
+2 20
+
+-- !toggle_sparse_3 --
+1 3
+2 30
+
+-- !toggle_sparse_4 --
+1 4
+
+-- !toggle_sparse_5 --
+2 50
+
+-- !toggle_sparse_6 --
+3 600
+
+-- !toggle_sparse_after_compact_1 --
+1 1
+2 10
+3 100
+
+-- !toggle_sparse_after_compact_2 --
+1 4
+
+-- !toggle_sparse_after_compact_3 --
+3
+
+-- !high_cardinality_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !high_cardinality_2 --
+0 0
+
+-- !high_cardinality_3 --
+10 1005
+
+-- !high_cardinality_4 --
+20
+
+-- !high_cardinality_after_compact_1 --
+0 0
+1 1
+2 2
+3 3
+4 4
+
+-- !high_cardinality_after_compact_2 --
+20
+
diff --git a/regression-test/data/variant_p0/query_subcolumns.out b/regression-test/data/variant_p0/query_subcolumns.out
new file mode 100644
index 00000000000..1abc0a027c9
--- /dev/null
+++ b/regression-test/data/variant_p0/query_subcolumns.out
@@ -0,0 +1,211 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
+-- !sql --
+1
+1
+1
+1
+
+-- !sql --
+2
+2
+2
+
+-- !sql --
+3
+3
+
+-- !sql --
+4
+
+-- !sql --
+hello, world
+
diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
index 09cb8b8a43e..3d21a05a9ab 100644
--- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
+++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy
@@ -152,6 +152,20 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){
qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where v["repo"]["name"] match 'xpressengine' order by 1;"""
qt_sql """select count() from github_events where v["repo"]["name"] match 'apache' order by 1;"""
+ sql """ALTER TABLE github_events SET("bloom_filter_columns" = "k,v")"""
+
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='github_events' ORDER BY createtime DESC LIMIT 1"""
+ time 600
+ }
+
+ sql """ALTER TABLE github_events2 SET("bloom_filter_columns" = "k,v")"""
+
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='github_events' ORDER BY createtime DESC LIMIT 1"""
+ time 600
+ }
+
// specify schema
// sql "alter table github_events2 modify column v variant<`payload.comment.id`:int,`payload.commits.url`:text,`payload.forkee.has_pages`:tinyint>"
// load_json_data.call("github_events2", """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""")
diff --git a/regression-test/suites/variant_p0/agg.groovy b/regression-test/suites/variant_p0/agg.groovy
index 97999a122ff..ef9420c84d1 100644
--- a/regression-test/suites/variant_p0/agg.groovy
+++ b/regression-test/suites/variant_p0/agg.groovy
@@ -16,23 +16,24 @@
// under the License.
suite("regression_test_variant_agg"){
- sql """DROP TABLE IF EXISTS var_agg"""
+ // sql """DROP TABLE IF EXISTS var_agg"""
- int max_subcolumns_count = Math.floor(Math.random() * 10) + 1
- def var = "variant<properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")> replace"
- if (max_subcolumns_count % 2 == 0) {
- var = "variant <'d' : int, 'b.f' : int, 'xxxx' : string, 'point' : bigint, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")> replace"
- }
- sql """
- CREATE TABLE IF NOT EXISTS var_agg (
- k bigint,
- v ${var},
- s bigint sum
- )
- AGGREGATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 4
- properties("replication_num" = "1", "disable_auto_compaction" = "true");
- """
+ // int max_subcolumns_count = Math.floor(Math.random() * 10) + 1
+ // def var = "variant<properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")> replace"
+ // if (max_subcolumns_count % 2 == 0) {
+ // var = "variant <'d' : int, 'b.f' : int, 'xxxx' : string, 'point' : bigint, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")> replace"
+ // }
+ // sql """
+ // CREATE TABLE IF NOT EXISTS var_agg (
+ // k bigint,
+ // v ${var},
+ // s bigint sum
+ // )
+ // AGGREGATE KEY(`k`)
+ // DISTRIBUTED BY HASH(k) BUCKETS 4
+ // properties("replication_num" = "1", "disable_auto_compaction" = "true");
+ // """
+sql "alter table var_agg add column s bigint sum;"
sql """insert into var_agg values (1, '[1]', 1),(1, '{"a" : 1}', 1);"""
sql """insert into var_agg values (2, '[2]', 2),(1, '{"a" : [[[1]]]}', 2);"""
sql """insert into var_agg values (3, '3', 3),(1, '{"a" : 1}', 3), (1, '{"a" : [1]}', 3);"""
diff --git a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
new file mode 100644
index 00000000000..d7fe1e48e3a
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_storage_format_v2_1") {
+ def tableName = "test_storage_format_v2_1_table"
+
+ // Test 1: Create table with storage_format = V2.1
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_format" = "V3"
+ );
+ """
+
+ // Insert some data
+ sql """insert into ${tableName} values (1, '{"a": 1, "b": 2, "c": 3}')"""
+ sql """insert into ${tableName} values (2, '{"a": 10, "b": 20, "d": 40}')"""
+ sql """insert into ${tableName} values (3, '{"e": 500, "f": 600}')"""
+
+ // Query to verify data is correct
+ qt_sql1 "select k, v['a'] from ${tableName} where cast(v['a'] as int) is not null order by k"
+ qt_sql2 "select k, v['b'] from ${tableName} where cast(v['b'] as int) is not null order by k"
+ qt_sql3 "select k, v['d'] from ${tableName} where cast(v['d'] as int) is not null order by k"
+ qt_sql4 "select k, v['e'] from ${tableName} where cast(v['e'] as int) is not null order by k"
+
+ // Verify table properties
+ def result = sql "SHOW CREATE TABLE ${tableName}"
+ logger.info("Show create table result: ${result}")
+ assertTrue(result[0][1].contains("V3"),
+ "Table should be created with storage_format V3")
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS ${tableName}"
+
+ // Test 2: Create table without specifying storage_format (should default to V2)
+ def tableName2 = "test_storage_format_default"
+ sql "DROP TABLE IF EXISTS ${tableName2}"
+ sql """
+ CREATE TABLE ${tableName2} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """insert into ${tableName2} values (1, '{"x": 100}')"""
+ qt_sql5 "select k, v['x'] from ${tableName2} order by k"
+
+ sql "DROP TABLE IF EXISTS ${tableName2}"
+
+ // Test 3: Verify that storage_format is case-insensitive
+ def tableName3 = "test_storage_format_case_insensitive"
+ sql "DROP TABLE IF EXISTS ${tableName3}"
+ sql """
+ CREATE TABLE ${tableName3} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_format" = "V3"
+ );
+ """
+
+ sql """insert into ${tableName3} values (1, '{"test": "value"}')"""
+ qt_sql6 "select k, v['test'] from ${tableName3} order by k"
+
+ sql "DROP TABLE IF EXISTS ${tableName3}"
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy
new file mode 100644
index 00000000000..8ba7c427d35
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_concurrent.groovy
@@ -0,0 +1,307 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_concurrent", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Concurrent reads of external meta
+ sql "DROP TABLE IF EXISTS test_concurrent_read"
+ sql """
+ CREATE TABLE test_concurrent_read (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 4
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data with many subcolumns to stress test external meta loading
+ for (int i = 0; i < 20; i++) {
+ def fields = []
+ for (int j = 0; j < 50; j++) {
+ fields.add("\"field_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_concurrent_read values (${i}, '${json}')"""
+ }
+
+ // Run concurrent queries that will trigger external meta loading
+ def threads = []
+ def results = [].asSynchronized()
+
+ for (int t = 0; t < 5; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ // Each thread queries different fields
+ def fieldIdx = threadId * 10
+ def result = sql """
+ select count(*) from test_concurrent_read
+ where cast(v['field_${fieldIdx}'] as int) is not null
+ """
+ results.add([threadId: threadId, success: true, result: result])
+ } catch (Exception e) {
+ results.add([threadId: threadId, success: false, error: e.message])
+ logger.error("Thread ${threadId} failed: ${e.message}")
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ // Verify all threads succeeded
+ def failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ logger.error("Concurrent read test failed: ${failures}")
+ throw new Exception("Concurrent read test had failures: ${failures}")
+ }
+
+ qt_concurrent_1 "select count(*) from test_concurrent_read"
+ qt_concurrent_2 "select count(distinct k) from test_concurrent_read"
+
+ // Test 2: Multiple tables with external meta
+ for (int tableIdx = 0; tableIdx < 5; tableIdx++) {
+ def tableName = "test_multi_table_${tableIdx}"
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into ${tableName} values (1, '{"table_${tableIdx}": ${tableIdx}}')"""
+ sql """insert into ${tableName} values (2, '{"table_${tableIdx}": ${tableIdx * 10}}')"""
+ }
+
+ // Query all tables concurrently
+ threads = []
+ results = [].asSynchronized()
+
+ for (int t = 0; t < 5; t++) {
+ def tableIdx = t
+ def tableName = "test_multi_table_${tableIdx}"
+ threads.add(Thread.start {
+ try {
+ def result = sql """
+ select k, v['table_${tableIdx}'] from ${tableName} order by k
+ """
+ results.add([tableIdx: tableIdx, success: true, rowCount: result.size()])
+ } catch (Exception e) {
+ results.add([tableIdx: tableIdx, success: false, error: e.message])
+ logger.error("Table ${tableIdx} query failed: ${e.message}")
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Multi-table concurrent test had failures: ${failures}")
+ }
+
+ // Test 3: Concurrent writes and reads
+ sql "DROP TABLE IF EXISTS test_concurrent_write_read"
+ sql """
+ CREATE TABLE test_concurrent_write_read (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 4
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Initial data
+ for (int i = 0; i < 10; i++) {
+ sql """insert into test_concurrent_write_read values (${i}, '{"initial": ${i}}')"""
+ }
+
+ threads = []
+ results = [].asSynchronized()
+
+ // Start reader threads
+ for (int t = 0; t < 3; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 5; i++) {
+ def result = sql """select count(*) from test_concurrent_write_read"""
+ Thread.sleep(100) // Small delay between reads
+ }
+ results.add([type: 'reader', threadId: threadId, success: true])
+ } catch (Exception e) {
+ results.add([type: 'reader', threadId: threadId, success: false, error: e.message])
+ }
+ })
+ }
+
+ // Start writer threads
+ for (int t = 0; t < 2; t++) {
+ def threadId = t
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 3; i++) {
+ def k = 100 + threadId * 10 + i
+ sql """insert into test_concurrent_write_read values (${k}, '{"writer": ${threadId}}')"""
+ Thread.sleep(150)
+ }
+ results.add([type: 'writer', threadId: threadId, success: true])
+ } catch (Exception e) {
+ results.add([type: 'writer', threadId: threadId, success: false, error: e.message])
+ }
+ })
+ }
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Concurrent write-read test had failures: ${failures}")
+ }
+
+ qt_write_read_1 "select count(*) from test_concurrent_write_read"
+ qt_write_read_2 "select count(*) from test_concurrent_write_read where cast(v['initial'] as int) is not null"
+ qt_write_read_3 "select count(*) from test_concurrent_write_read where cast(v['writer'] as int) is not null"
+
+ // Test 4: Stress test with frequent compaction triggers
+ sql "DROP TABLE IF EXISTS test_compaction_stress"
+ sql """
+ CREATE TABLE test_compaction_stress (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 2
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert multiple small segments
+ for (int i = 0; i < 15; i++) {
+ sql """insert into test_compaction_stress values (${i}, '{"segment": ${i}, "value": ${i * 100}}')"""
+ }
+
+ // Trigger compaction while querying
+ threads = []
+ results = [].asSynchronized()
+
+ // Query thread
+ threads.add(Thread.start {
+ try {
+ for (int i = 0; i < 10; i++) {
+ def result = sql """select count(*) from test_compaction_stress"""
+ Thread.sleep(200)
+ }
+ results.add([type: 'query', success: true])
+ } catch (Exception e) {
+ results.add([type: 'query', success: false, error: e.message])
+ }
+ })
+
+ // Give query thread a head start
+ Thread.sleep(100)
+
+ // Compaction thread
+ threads.add(Thread.start {
+ try {
+ trigger_and_wait_compaction("test_compaction_stress", "full")
+ results.add([type: 'compaction', success: true])
+ } catch (Exception e) {
+ results.add([type: 'compaction', success: false, error: e.message])
+ }
+ })
+
+ threads.each { it.join() }
+
+ failures = results.findAll { !it.success }
+ if (!failures.isEmpty()) {
+ throw new Exception("Compaction stress test had failures: ${failures}")
+ }
+
+ qt_stress_1 "select count(*) from test_compaction_stress"
+ qt_stress_2 "select k, v['segment'] from test_compaction_stress order by k limit 5"
+
+ // Test 5: Cache behavior - repeated queries should hit cache
+ sql "DROP TABLE IF EXISTS test_cache_behavior"
+ sql """
+ CREATE TABLE test_cache_behavior (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data with many subcolumns
+ def fields = []
+ for (int i = 0; i < 100; i++) {
+ fields.add("\"cache_field_${i}\": ${i}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+
+ for (int i = 0; i < 5; i++) {
+ sql """insert into test_cache_behavior values (${i}, '${json}')"""
+ }
+
+ // First query - should load external meta
+ def startTime = System.currentTimeMillis()
+ sql """select k, v['cache_field_0'], v['cache_field_50'], v['cache_field_99']
+ from test_cache_behavior order by k"""
+ def firstQueryTime = System.currentTimeMillis() - startTime
+
+ // Subsequent queries - should use cached meta (should be faster)
+ startTime = System.currentTimeMillis()
+ for (int i = 0; i < 5; i++) {
+ sql """select k, v['cache_field_0'], v['cache_field_50'], v['cache_field_99']
+ from test_cache_behavior order by k"""
+ }
+ def cachedQueriesTime = System.currentTimeMillis() - startTime
+ def avgCachedTime = cachedQueriesTime / 5
+
+ logger.info("First query time: ${firstQueryTime}ms, Avg cached query time: ${avgCachedTime}ms")
+
+ qt_cache_1 "select count(*) from test_cache_behavior"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_concurrent_read"
+ for (int i = 0; i < 5; i++) {
+ sql "DROP TABLE IF EXISTS test_multi_table_${i}"
+ }
+ sql "DROP TABLE IF EXISTS test_concurrent_write_read"
+ sql "DROP TABLE IF EXISTS test_compaction_stress"
+ sql "DROP TABLE IF EXISTS test_cache_behavior"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy
new file mode 100644
index 00000000000..029200519e9
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_edge_cases.groovy
@@ -0,0 +1,296 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_edge_cases", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Empty subcolumns (only root variant column)
+ sql "DROP TABLE IF EXISTS test_empty_subcolumns"
+ sql """
+ CREATE TABLE test_empty_subcolumns (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_empty_subcolumns values (1, '{}')"""
+ sql """insert into test_empty_subcolumns values (2, 'null')"""
+ sql """insert into test_empty_subcolumns values (3, '[]')"""
+
+ qt_empty_1 "select k, cast(v as string) from test_empty_subcolumns order by k"
+ qt_empty_2 "select k from test_empty_subcolumns where v is null order by k"
+ qt_empty_3 "select k from test_empty_subcolumns where v is not null order by k"
+
+ // Test 2: Many subcolumns (test performance and correctness)
+ sql "DROP TABLE IF EXISTS test_many_subcolumns"
+ sql """
+ CREATE TABLE test_many_subcolumns (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Generate JSON with 100 subcolumns
+ def generateLargeJson = { int numFields ->
+ def fields = []
+ for (int i = 0; i < numFields; i++) {
+ fields.add("\"field_${i}\": ${i}")
+ }
+ return "{" + fields.join(", ") + "}"
+ }
+
+ // external meta enabled by table property
+ // Insert rows with different numbers of subcolumns
+ for (int row = 0; row < 10; row++) {
+ int numFields = (row + 1) * 10 // 10, 20, 30, ..., 100 fields
+ def json = generateLargeJson(numFields)
+ sql """insert into test_many_subcolumns values (${row}, '${json}')"""
+ }
+
+ qt_many_1 "select k, v['field_0'] from test_many_subcolumns order by k"
+ qt_many_2 "select k, v['field_50'] from test_many_subcolumns where cast(v['field_50'] as int) is not null order by k"
+ qt_many_3 "select k, v['field_99'] from test_many_subcolumns where cast(v['field_99'] as int) is not null order by k"
+ qt_many_4 "select count(*) from test_many_subcolumns"
+
+ // Test 3: Mixed old and new format segments
+ sql "DROP TABLE IF EXISTS test_mixed_format"
+ sql """
+ CREATE TABLE test_mixed_format (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Write with old format (V2)
+ sql """insert into test_mixed_format values (1, '{"a": 1, "b": 2}')"""
+ sql """insert into test_mixed_format values (2, '{"a": 10, "c": 3}')"""
+
+ // Switch to new format (V2.1) and write more rows
+ sql """insert into test_mixed_format values (3, '{"a": 100, "d": 4}')"""
+ sql """insert into test_mixed_format values (4, '{"a": 1000, "e": 5}')"""
+
+ // Query should work across all segments
+ qt_mixed_1 "select k, v['a'] from test_mixed_format order by k"
+ qt_mixed_2 "select k, v['b'] from test_mixed_format where cast(v['b'] as int) is not null order by k"
+ qt_mixed_3 "select k, v['c'] from test_mixed_format where cast(v['c'] as int) is not null order by k"
+ qt_mixed_4 "select k, v['d'] from test_mixed_format where cast(v['d'] as int) is not null order by k"
+ qt_mixed_5 "select k, v['e'] from test_mixed_format where cast(v['e'] as int) is not null order by k"
+
+ // Trigger compaction and verify
+ trigger_and_wait_compaction("test_mixed_format", "full")
+
+ qt_mixed_after_compact_1 "select k, v['a'] from test_mixed_format order by k"
+ qt_mixed_after_compact_2 "select count(distinct k) from test_mixed_format"
+
+ // Test 4: Nested structures with external meta
+ sql "DROP TABLE IF EXISTS test_nested_external"
+ sql """
+ CREATE TABLE test_nested_external (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_nested_external values (1, '{"nested": {"level1": {"level2": "value"}}}')"""
+ sql """insert into test_nested_external values (2, '{"nested": {"level1": {"level2": "value2", "level2_b": 123}}}')"""
+ sql """insert into test_nested_external values (3, '{"nested": {"level1": null}}')"""
+ sql """insert into test_nested_external values (4, '{"nested": null}')"""
+
+ qt_nested_1 "select k, v['nested']['level1']['level2'] from test_nested_external order by k"
+ qt_nested_2 "select k, v['nested']['level1']['level2_b'] from test_nested_external where cast(v['nested']['level1']['level2_b'] as int) is not null order by k"
+ qt_nested_3 "select k from test_nested_external where v['nested'] is not null order by k"
+ qt_nested_4 "select k from test_nested_external where v['nested']['level1'] is not null order by k"
+
+ // Test 5: Array types with external meta
+ sql "DROP TABLE IF EXISTS test_array_external"
+ sql """
+ CREATE TABLE test_array_external (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_array_external values (1, '{"arr": [1, 2, 3]}')"""
+ sql """insert into test_array_external values (2, '{"arr": [10, 20, 30, 40]}')"""
+ sql """insert into test_array_external values (3, '{"arr": []}')"""
+ sql """insert into test_array_external values (4, '{"arr": null}')"""
+
+ qt_array_1 "select k, v['arr'] from test_array_external order by k"
+ qt_array_2 "select k from test_array_external where v['arr'] is not null order by k"
+
+ // Test 6: Schema evolution - adding subcolumns over time
+ sql "DROP TABLE IF EXISTS test_schema_evolution"
+ sql """
+ CREATE TABLE test_schema_evolution (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_schema_evolution values (1, '{"a": 1}')"""
+
+ // Segment 2: add field 'b'
+ sql """insert into test_schema_evolution values (2, '{"a": 2, "b": 2}')"""
+
+ // Segment 3: add field 'c'
+ sql """insert into test_schema_evolution values (3, '{"a": 3, "b": 3, "c": 3}')"""
+
+ // Segment 4: completely different schema
+ sql """insert into test_schema_evolution values (4, '{"x": 10, "y": 20}')"""
+
+ qt_evolution_1 "select k, v['a'] from test_schema_evolution order by k"
+ qt_evolution_2 "select k, v['b'] from test_schema_evolution order by k"
+ qt_evolution_3 "select k, v['c'] from test_schema_evolution order by k"
+ qt_evolution_4 "select k, v['x'] from test_schema_evolution order by k"
+ qt_evolution_5 "select k, v['y'] from test_schema_evolution order by k"
+
+ // Test 7: Special characters in field names
+ sql "DROP TABLE IF EXISTS test_special_chars"
+ sql """
+ CREATE TABLE test_special_chars (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_special_chars values (1, '{"field-with-dash": 1}')"""
+ sql """insert into test_special_chars values (2, '{"field.with.dot": 2}')"""
+ sql """insert into test_special_chars values (3, '{"field_with_underscore": 3}')"""
+
+ qt_special_1 "select k, v['field-with-dash'] from test_special_chars where cast(v['field-with-dash'] as int) is not null order by k"
+ qt_special_2 "select k, v['field.with.dot'] from test_special_chars where cast(v['field.with.dot'] as int) is not null order by k"
+ qt_special_3 "select k, v['field_with_underscore'] from test_special_chars where cast(v['field_with_underscore'] as int) is not null order by k"
+
+ // Test 8: NULL handling
+ sql "DROP TABLE IF EXISTS test_null_handling"
+ sql """
+ CREATE TABLE test_null_handling (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_null_handling values (1, '{"a": null}')"""
+ sql """insert into test_null_handling values (2, '{"a": 1}')"""
+ sql """insert into test_null_handling values (3, null)"""
+
+ qt_null_1 "select k, v['a'] from test_null_handling order by k"
+ qt_null_2 "select k from test_null_handling where v is null order by k"
+ qt_null_3 "select k from test_null_handling where v['a'] is null order by k"
+ qt_null_4 "select k from test_null_handling where v['a'] is not null order by k"
+
+ // Test 9: Large string values
+ sql "DROP TABLE IF EXISTS test_large_strings"
+ sql """
+ CREATE TABLE test_large_strings (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ def largeString = "x" * 10000 // 10KB string
+ sql """insert into test_large_strings values (1, '{"large_field": "${largeString}"}')"""
+ sql """insert into test_large_strings values (2, '{"large_field": "small"}')"""
+
+ qt_large_1 "select k, length(cast(v['large_field'] as string)) from test_large_strings order by k"
+ qt_large_2 "select k from test_large_strings where length(cast(v['large_field'] as string)) > 100 order by k"
+
+ // Test 10: Config toggle during operation
+ sql "DROP TABLE IF EXISTS test_config_toggle"
+ sql """
+ CREATE TABLE test_config_toggle (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Start with new format (V2.1)
+ sql """insert into test_config_toggle values (1, '{"a": 1}')"""
+
+ // Switch to legacy (V2)
+ sql """insert into test_config_toggle values (2, '{"a": 2, "b": 2}')"""
+
+ // Enable again (V2.1)
+ sql """insert into test_config_toggle values (3, '{"a": 3, "b": 3, "c": 3}')"""
+
+ // All queries should work regardless of config state
+ qt_toggle_1 "select k, v['a'] from test_config_toggle order by k"
+ qt_toggle_2 "select k, v['b'] from test_config_toggle where cast(v['b'] as int) is not null order by k"
+ qt_toggle_3 "select k, v['c'] from test_config_toggle where cast(v['c'] as int) is not null order by k"
+
+ // Compact and verify
+ trigger_and_wait_compaction("test_config_toggle", "full")
+
+ qt_toggle_after_compact_1 "select k, v['a'] from test_config_toggle order by k"
+ qt_toggle_after_compact_2 "select k, v['b'] from test_config_toggle where cast(v['b'] as int) is not null order by k"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_empty_subcolumns"
+ sql "DROP TABLE IF EXISTS test_many_subcolumns"
+ sql "DROP TABLE IF EXISTS test_mixed_format"
+ sql "DROP TABLE IF EXISTS test_nested_external"
+ sql "DROP TABLE IF EXISTS test_array_external"
+ sql "DROP TABLE IF EXISTS test_schema_evolution"
+ sql "DROP TABLE IF EXISTS test_special_chars"
+ sql "DROP TABLE IF EXISTS test_null_handling"
+ sql "DROP TABLE IF EXISTS test_large_strings"
+ sql "DROP TABLE IF EXISTS test_config_toggle"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy
new file mode 100644
index 00000000000..fa68799b041
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_integration.groovy
@@ -0,0 +1,355 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_integration", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ sql "set enable_match_without_inverted_index = false"
+ sql "set enable_common_expr_pushdown_for_inverted_index = true"
+ sql "set enable_common_expr_pushdown = true"
+
+ // Test 1: External meta with inverted index on extracted columns
+ sql "DROP TABLE IF EXISTS test_inverted_index_extracted"
+ sql """
+ CREATE TABLE test_inverted_index_extracted (
+ k bigint,
+ v variant<'name':string, 'age':int>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english")
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_inverted_index_extracted values (1, '{"name": "Alice Smith", "age": 25}')"""
+ sql """insert into test_inverted_index_extracted values (2, '{"name": "Bob Johnson", "age": 30}')"""
+ sql """insert into test_inverted_index_extracted values (3, '{"name": "Charlie Smith", "age": 35}')"""
+
+ // Test inverted index on name field (should work with external meta)
+ qt_inverted_1 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Smith' order by k"
+ qt_inverted_2 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Alice' order by k"
+ qt_inverted_3 "select k, v['age'] from test_inverted_index_extracted where cast(v['age'] as int) > 25 order by k"
+
+ // After compaction, inverted index should still work
+ trigger_and_wait_compaction("test_inverted_index_extracted", "full")
+
+ qt_inverted_after_compact_1 "select k, v['name'] from test_inverted_index_extracted where v['name'] match 'Smith' order by k"
+ qt_inverted_after_compact_2 "select count(*) from test_inverted_index_extracted where v['name'] match 'Johnson'"
+
+ // Test 2: External meta with parent column inverted index
+ sql "DROP TABLE IF EXISTS test_parent_index"
+ sql """
+ CREATE TABLE test_parent_index (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "5")>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english")
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_parent_index values (1, '{"title": "Hello World", "content": "This is a test"}')"""
+ sql """insert into test_parent_index values (2, '{"title": "Test Document", "content": "Another test here"}')"""
+ sql """insert into test_parent_index values (3, '{"title": "World News", "content": "Breaking news today"}')"""
+
+ // Query with inherited inverted index
+ qt_parent_idx_1 "select k, v['title'] from test_parent_index where v['title'] match 'World' order by k"
+ qt_parent_idx_2 "select k, v['content'] from test_parent_index where v['content'] match 'test' order by k"
+ qt_parent_idx_3 "select count(*) from test_parent_index where v['title'] match 'Document'"
+
+ // Test 3: Multiple variant columns with external meta
+ sql "DROP TABLE IF EXISTS test_multiple_variants"
+ sql """
+ CREATE TABLE test_multiple_variants (
+ k bigint,
+ v1 variant,
+ v2 variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ sql """insert into test_multiple_variants values (1, '{"a": 1, "b": 2}', '{"x": 10, "y": 20}')"""
+ sql """insert into test_multiple_variants values (2, '{"a": 100, "c": 300}', '{"x": 1000, "z": 3000}')"""
+ sql """insert into test_multiple_variants values (3, '{"d": 4}', '{"w": 40}')"""
+
+ // Both variant columns should use external meta independently
+ qt_multi_var_1 "select k, v1['a'], v2['x'] from test_multiple_variants order by k"
+ qt_multi_var_2 "select k, v1['b'] from test_multiple_variants where cast(v1['b'] as int) is not null order by k"
+ qt_multi_var_3 "select k, v2['y'] from test_multiple_variants where cast(v2['y'] as int) is not null order by k"
+ qt_multi_var_4 "select k, v1['d'], v2['w'] from test_multiple_variants where cast(v1['d'] as int) is not null order by k"
+
+ trigger_and_wait_compaction("test_multiple_variants", "full")
+
+ qt_multi_var_after_compact_1 "select k, v1['a'], v2['x'] from test_multiple_variants order by k"
+ qt_multi_var_after_compact_2 "select count(*) from test_multiple_variants"
+
+ // Test 4: External meta with aggregation functions
+ sql "DROP TABLE IF EXISTS test_aggregation"
+ sql """
+ CREATE TABLE test_aggregation (
+ k bigint,
+ category varchar(30),
+ v variant
+ )
+ DUPLICATE KEY(`k`, `category`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_aggregation values (1, 'A', '{"value": 100, "count": 1}')"""
+ sql """insert into test_aggregation values (2, 'A', '{"value": 200, "count": 2}')"""
+ sql """insert into test_aggregation values (3, 'B', '{"value": 150, "count": 1}')"""
+ sql """insert into test_aggregation values (4, 'B', '{"value": 250, "count": 3}')"""
+
+ qt_agg_1 "select category, sum(cast(v['value'] as int)) from test_aggregation group by category order by category"
+ qt_agg_2 "select category, avg(cast(v['count'] as double)) from test_aggregation group by category order by category"
+ qt_agg_3 "select count(distinct category) from test_aggregation where cast(v['value'] as int) > 100"
+
+ // Test 5: External meta with JOIN operations
+ sql "DROP TABLE IF EXISTS test_join_left"
+ sql "DROP TABLE IF EXISTS test_join_right"
+
+ sql """
+ CREATE TABLE test_join_left (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """
+ CREATE TABLE test_join_right (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_join_left values (1, '{"name": "Alice", "dept_id": 10}')"""
+ sql """insert into test_join_left values (2, '{"name": "Bob", "dept_id": 20}')"""
+ sql """insert into test_join_left values (3, '{"name": "Charlie", "dept_id": 10}')"""
+
+ sql """insert into test_join_right values (10, '{"dept_name": "Engineering", "location": "Building A"}')"""
+ sql """insert into test_join_right values (20, '{"dept_name": "Sales", "location": "Building B"}')"""
+
+ qt_join_1 """
+ select l.k, l.v['name'], r.v['dept_name']
+ from test_join_left l
+ join test_join_right r on cast(l.v['dept_id'] as int) = r.k
+ order by l.k
+ """
+
+ qt_join_2 """
+ select count(*)
+ from test_join_left l
+ join test_join_right r on cast(l.v['dept_id'] as int) = r.k
+ where r.v['location'] = 'Building A'
+ """
+
+ // Test 6: External meta with UNION operations
+ sql "DROP TABLE IF EXISTS test_union_1"
+ sql "DROP TABLE IF EXISTS test_union_2"
+
+ sql """
+ CREATE TABLE test_union_1 (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """
+ CREATE TABLE test_union_2 (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_union_1 values (1, '{"source": "table1", "value": 100}')"""
+ sql """insert into test_union_1 values (2, '{"source": "table1", "value": 200}')"""
+
+ sql """insert into test_union_2 values (3, '{"source": "table2", "value": 150}')"""
+ sql """insert into test_union_2 values (4, '{"source": "table2", "value": 250}')"""
+
+ qt_union_1 """
+ select k, v['source'], v['value'] from test_union_1
+ union all
+ select k, v['source'], v['value'] from test_union_2
+ order by k
+ """
+
+ qt_union_2 """
+ select cast(v['source'] as string) as src, count(*)
+ from (
+ select v from test_union_1
+ union all
+ select v from test_union_2
+ ) t
+ group by src
+ order by src
+ """
+
+ // Test 7: External meta with subquery and WHERE clause
+ sql "DROP TABLE IF EXISTS test_subquery"
+ sql """
+ CREATE TABLE test_subquery (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ for (int i = 1; i <= 10; i++) {
+ sql """insert into test_subquery values (${i}, '{"score": ${i * 10}, "grade": "${i > 5 ? 'A' : 'B'}"}')"""
+ }
+
+ qt_subquery_1 """
+ select k, v['score'], v['grade']
+ from test_subquery
+ where cast(v['score'] as int) > (select avg(cast(v['score'] as int)) from test_subquery)
+ order by k
+ """
+
+ qt_subquery_2 """
+ select cast(v['grade'] as string) as grade, count(*) as cnt
+ from test_subquery
+ where k in (select k from test_subquery where cast(v['score'] as int) >= 50)
+ group by grade
+ order by grade
+ """
+
+ // Test 8: External meta with HAVING clause
+ sql "DROP TABLE IF EXISTS test_having"
+ sql """
+ CREATE TABLE test_having (
+ k bigint,
+ category varchar(30),
+ v variant
+ )
+ DUPLICATE KEY(`k`, `category`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_having values (1, 'A', '{"amount": 100}')"""
+ sql """insert into test_having values (2, 'A', '{"amount": 200}')"""
+ sql """insert into test_having values (3, 'A', '{"amount": 300}')"""
+ sql """insert into test_having values (4, 'B', '{"amount": 50}')"""
+ sql """insert into test_having values (5, 'B', '{"amount": 60}')"""
+
+ qt_having_1 """
+ select category, sum(cast(v['amount'] as int)) as total
+ from test_having
+ group by category
+ having total > 150
+ order by category
+ """
+
+ // Test 9: External meta with ORDER BY and LIMIT on variant fields
+ sql "DROP TABLE IF EXISTS test_order_limit"
+ sql """
+ CREATE TABLE test_order_limit (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ for (int i = 1; i <= 20; i++) {
+ sql """insert into test_order_limit values (${i}, '{"priority": ${20 - i}, "name": "item_${i}"}')"""
+ }
+
+ qt_order_limit_1 """
+ select k, v['priority'], v['name']
+ from test_order_limit
+ order by cast(v['priority'] as int) desc
+ limit 5
+ """
+
+ qt_order_limit_2 """
+ select k, v['name']
+ from test_order_limit
+ where cast(v['priority'] as int) > 10
+ order by cast(v['priority'] as int)
+ limit 3
+ """
+
+ // Test 10: External meta with DISTINCT on variant fields
+ sql "DROP TABLE IF EXISTS test_distinct"
+ sql """
+ CREATE TABLE test_distinct (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "false", "storage_format" = "V3");
+ """
+
+ sql """insert into test_distinct values (1, '{"category": "A", "value": 100}')"""
+ sql """insert into test_distinct values (2, '{"category": "B", "value": 200}')"""
+ sql """insert into test_distinct values (3, '{"category": "A", "value": 150}')"""
+ sql """insert into test_distinct values (4, '{"category": "C", "value": 100}')"""
+
+ qt_distinct_1 "select distinct cast(v['category'] as string) from test_distinct order by cast(v['category'] as string)"
+ qt_distinct_2 "select distinct cast(v['value'] as int) from test_distinct order by cast(v['value'] as int)"
+ qt_distinct_3 "select count(distinct cast(v['category'] as string)) from test_distinct"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_inverted_index_extracted"
+ sql "DROP TABLE IF EXISTS test_parent_index"
+ sql "DROP TABLE IF EXISTS test_multiple_variants"
+ sql "DROP TABLE IF EXISTS test_aggregation"
+ sql "DROP TABLE IF EXISTS test_join_left"
+ sql "DROP TABLE IF EXISTS test_join_right"
+ sql "DROP TABLE IF EXISTS test_union_1"
+ sql "DROP TABLE IF EXISTS test_union_2"
+ sql "DROP TABLE IF EXISTS test_subquery"
+ sql "DROP TABLE IF EXISTS test_having"
+ sql "DROP TABLE IF EXISTS test_order_limit"
+ sql "DROP TABLE IF EXISTS test_distinct"
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy
new file mode 100644
index 00000000000..676366758b1
--- /dev/null
+++ b/regression-test/suites/variant_p0/ext_meta/test_variant_external_meta_with_sparse.groovy
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_external_meta_with_sparse", "nonConcurrent") {
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
+ key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+
+ // Test 1: Sparse columns should be embedded in root's children_columns
+ sql "DROP TABLE IF EXISTS test_sparse_embedded"
+ sql """
+ CREATE TABLE test_sparse_embedded (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "3")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Insert data that will exceed subcolumn limit, creating sparse columns
+ sql """insert into test_sparse_embedded values (1, '{"a": 1, "b": 2, "c": 3}')"""
+ sql """insert into test_sparse_embedded values (2, '{"a": 10, "b": 20, "c": 30, "d": 40}')"""
+ sql """insert into test_sparse_embedded values (3, '{"a": 100, "e": 500, "f": 600, "g": 700}')"""
+
+ // Query extracted columns (should be in external meta)
+ qt_sparse_1 "select k, v['a'] from test_sparse_embedded order by k"
+ qt_sparse_2 "select k, v['b'] from test_sparse_embedded where cast(v['b'] as int) is not null order by k"
+ qt_sparse_3 "select k, v['c'] from test_sparse_embedded where cast(v['c'] as int) is not null order by k"
+
+ // Query sparse columns (should be embedded in root's children_columns)
+ qt_sparse_4 "select k, v['d'] from test_sparse_embedded where cast(v['d'] as int) is not null order by k"
+ qt_sparse_5 "select k, v['e'] from test_sparse_embedded where cast(v['e'] as int) is not null order by k"
+ qt_sparse_6 "select k, v['f'] from test_sparse_embedded where cast(v['f'] as int) is not null order by k"
+ qt_sparse_7 "select k, v['g'] from test_sparse_embedded where cast(v['g'] as int) is not null order by k"
+
+ // Compaction should handle both extracted and sparse columns correctly
+ trigger_and_wait_compaction("test_sparse_embedded", "full")
+
+ qt_sparse_after_compact_1 "select k, v['a'] from test_sparse_embedded order by k"
+ qt_sparse_after_compact_2 "select k, v['d'] from test_sparse_embedded where cast(v['d'] as int) is not null order by k"
+ qt_sparse_after_compact_3 "select k, v['g'] from test_sparse_embedded where cast(v['g'] as int) is not null order by k"
+
+ // Test 2: Bucketed sparse columns with external meta
+ sql "DROP TABLE IF EXISTS test_bucket_sparse"
+ sql """
+ CREATE TABLE test_bucket_sparse (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "2")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Create multiple rows that will generate bucketed sparse columns
+ for (int i = 0; i < 10; i++) {
+ def fields = ["\"extracted_1\": ${i}", "\"extracted_2\": ${i * 10}"]
+ // Add varying sparse fields
+ for (int j = 0; j < 5; j++) {
+ fields.add("\"sparse_${i}_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_bucket_sparse values (${i}, '${json}')"""
+ }
+
+ // Query extracted columns
+ qt_bucket_1 "select k, v['extracted_1'] from test_bucket_sparse order by k"
+ qt_bucket_2 "select k, v['extracted_2'] from test_bucket_sparse order by k"
+
+ // Query sparse columns from different rows
+ qt_bucket_3 "select k, v['sparse_0_0'] from test_bucket_sparse where cast(v['sparse_0_0'] as int) is not null order by k"
+ qt_bucket_4 "select k, v['sparse_5_3'] from test_bucket_sparse where cast(v['sparse_5_3'] as int) is not null order by k"
+ qt_bucket_5 "select count(*) from test_bucket_sparse"
+
+ // Compact and verify buckets are handled correctly
+ trigger_and_wait_compaction("test_bucket_sparse", "full")
+
+ qt_bucket_after_compact_1 "select k, v['extracted_1'] from test_bucket_sparse order by k"
+ qt_bucket_after_compact_2 "select k, v['sparse_0_0'] from test_bucket_sparse where cast(v['sparse_0_0'] as int) is not null order by k"
+
+ // Test 3: Mixed extracted, sparse, and non-existent columns
+ sql "DROP TABLE IF EXISTS test_mixed_column_types"
+ sql """
+ CREATE TABLE test_mixed_column_types (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "5")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Row 1: Only extracted columns
+ sql """insert into test_mixed_column_types values (1, '{"e1": 1, "e2": 2, "e3": 3, "e4": 4, "e5": 5}')"""
+
+ // Row 2: Extracted + sparse
+ sql """insert into test_mixed_column_types values (2, '{"e1": 10, "e2": 20, "e3": 30, "e4": 40, "e5": 50, "s1": 60, "s2": 70}')"""
+
+ // Row 3: Different set of fields
+ sql """insert into test_mixed_column_types values (3, '{"e1": 100, "new1": 200, "new2": 300}')"""
+
+ // Query all types
+ qt_mixed_1 "select k, v['e1'] from test_mixed_column_types order by k" // Exists in all rows (extracted)
+ qt_mixed_2 "select k, v['e5'] from test_mixed_column_types where cast(v['e5'] as int) is not null order by k" // Exists in row 1,2 (extracted)
+ qt_mixed_3 "select k, v['s1'] from test_mixed_column_types where cast(v['s1'] as int) is not null order by k" // Exists in row 2 (sparse)
+ qt_mixed_4 "select k, v['new1'] from test_mixed_column_types where cast(v['new1'] as int) is not null order by k" // Exists in row 3 (sparse or extracted)
+ qt_mixed_5 "select k, v['non_existent'] from test_mixed_column_types order by k" // Doesn't exist anywhere
+
+ // Test 4: Transition from extracted to sparse
+ sql "DROP TABLE IF EXISTS test_extraction_threshold"
+ sql """
+ CREATE TABLE test_extraction_threshold (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "3")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Segment 1: Within extraction limit
+ sql """insert into test_extraction_threshold values (1, '{"field_a": 1, "field_b": 2}')"""
+ sql """insert into test_extraction_threshold values (2, '{"field_a": 10, "field_b": 20, "field_c": 30}')"""
+
+ // Segment 2: Exceeds limit, triggers sparse columns
+ sql """insert into test_extraction_threshold values (3, '{"field_a": 100, "field_b": 200, "field_c": 300, "field_d": 400}')"""
+ sql """insert into test_extraction_threshold values (4, '{"field_a": 1000, "field_e": 5000}')"""
+
+ // All fields should be queryable
+ qt_threshold_1 "select k, v['field_a'] from test_extraction_threshold order by k"
+ qt_threshold_2 "select k, v['field_b'] from test_extraction_threshold where cast(v['field_b'] as int) is not null order by k"
+ qt_threshold_3 "select k, v['field_c'] from test_extraction_threshold where cast(v['field_c'] as int) is not null order by k"
+ qt_threshold_4 "select k, v['field_d'] from test_extraction_threshold where cast(v['field_d'] as int) is not null order by k"
+ qt_threshold_5 "select k, v['field_e'] from test_extraction_threshold where cast(v['field_e'] as int) is not null order by k"
+
+ // After compaction, schema should be merged correctly
+ trigger_and_wait_compaction("test_extraction_threshold", "full")
+
+ qt_threshold_after_compact_1 "select k, v['field_a'] from test_extraction_threshold order by k"
+ qt_threshold_after_compact_2 "select k, v['field_d'] from test_extraction_threshold where cast(v['field_d'] as int) is not null order by k"
+ qt_threshold_after_compact_3 "select k, v['field_e'] from test_extraction_threshold where cast(v['field_e'] as int) is not null order by k"
+
+ // Test 5: Sparse columns with config toggle
+ sql "DROP TABLE IF EXISTS test_sparse_config_toggle"
+ sql """
+ CREATE TABLE test_sparse_config_toggle (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "2")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Write with external meta enabled (V2.1)
+ sql """insert into test_sparse_config_toggle values (1, '{"a": 1, "b": 2, "c": 3, "d": 4}')"""
+
+ // Switch to legacy V2
+ sql """insert into test_sparse_config_toggle values (2, '{"a": 10, "b": 20, "c": 30, "e": 50}')"""
+
+ // Enable external meta again (V2.1)
+ sql """insert into test_sparse_config_toggle values (3, '{"a": 100, "f": 600, "g": 700}')"""
+
+ // All queries should work regardless of which format was used
+ qt_toggle_sparse_1 "select k, v['a'] from test_sparse_config_toggle order by k"
+ qt_toggle_sparse_2 "select k, v['b'] from test_sparse_config_toggle where cast(v['b'] as int) is not null order by k"
+ qt_toggle_sparse_3 "select k, v['c'] from test_sparse_config_toggle where cast(v['c'] as int) is not null order by k"
+ qt_toggle_sparse_4 "select k, v['d'] from test_sparse_config_toggle where cast(v['d'] as int) is not null order by k"
+ qt_toggle_sparse_5 "select k, v['e'] from test_sparse_config_toggle where cast(v['e'] as int) is not null order by k"
+ qt_toggle_sparse_6 "select k, v['f'] from test_sparse_config_toggle where cast(v['f'] as int) is not null order by k"
+
+ // Compact
+ trigger_and_wait_compaction("test_sparse_config_toggle", "full")
+
+ qt_toggle_sparse_after_compact_1 "select k, v['a'] from test_sparse_config_toggle order by k"
+ qt_toggle_sparse_after_compact_2 "select k, v['d'] from test_sparse_config_toggle where cast(v['d'] as int) is not null order by k"
+ qt_toggle_sparse_after_compact_3 "select count(*) from test_sparse_config_toggle"
+
+ // Test 6: Very high cardinality sparse columns
+ sql "DROP TABLE IF EXISTS test_high_cardinality_sparse"
+ sql """
+ CREATE TABLE test_high_cardinality_sparse (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "1")>
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+
+ // Each row has one extracted column and many unique sparse columns
+ for (int i = 0; i < 20; i++) {
+ def fields = ["\"common\": ${i}"]
+ // Each row has unique sparse fields
+ for (int j = 0; j < 10; j++) {
+ fields.add("\"unique_${i}_${j}\": ${i * 100 + j}")
+ }
+ def json = "{" + fields.join(", ") + "}"
+ sql """insert into test_high_cardinality_sparse values (${i}, '${json}')"""
+ }
+
+ // Query common extracted field
+ qt_high_cardinality_1 "select k, v['common'] from test_high_cardinality_sparse order by k limit 5"
+
+ // Query sparse fields (each exists in only one row)
+ qt_high_cardinality_2 "select k, v['unique_0_0'] from test_high_cardinality_sparse where cast(v['unique_0_0'] as int) is not null order by k"
+ qt_high_cardinality_3 "select k, v['unique_10_5'] from test_high_cardinality_sparse where cast(v['unique_10_5'] as int) is not null order by k"
+ qt_high_cardinality_4 "select count(*) from test_high_cardinality_sparse"
+
+ // Compaction with high cardinality sparse columns
+ trigger_and_wait_compaction("test_high_cardinality_sparse", "full")
+
+ qt_high_cardinality_after_compact_1 "select k, v['common'] from test_high_cardinality_sparse order by k limit 5"
+ qt_high_cardinality_after_compact_2 "select count(*) from test_high_cardinality_sparse"
+
+ // Cleanup
+ sql "DROP TABLE IF EXISTS test_sparse_embedded"
+ sql "DROP TABLE IF EXISTS test_bucket_sparse"
+ sql "DROP TABLE IF EXISTS test_mixed_column_types"
+ sql "DROP TABLE IF EXISTS test_extraction_threshold"
+ sql "DROP TABLE IF EXISTS test_sparse_config_toggle"
+ sql "DROP TABLE IF EXISTS test_high_cardinality_sparse"
+
+
+}
+
+
diff --git a/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy b/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
index d47c486047e..5b129ac1363 100644
--- a/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
+++ b/regression-test/suites/variant_p0/predefine/test_variant_compaction_with_sparse_limit.groovy
@@ -130,8 +130,8 @@ suite("test_compaction_variant_predefine_with_sparse_limit", "nonConcurrent") {
order_qt_select "select * from ${tableName} order by k, cast(v as string) limit 5;"
}
} finally {
- sql "DROP TABLE IF EXISTS simple_variant_DUPLICATE"
- sql "DROP TABLE IF EXISTS simple_variant_UNIQUE"
- sql "DROP TABLE IF EXISTS simple_variant_AGGREGATE"
+ // sql "DROP TABLE IF EXISTS simple_variant_DUPLICATE"
+ // sql "DROP TABLE IF EXISTS simple_variant_UNIQUE"
+ // sql "DROP TABLE IF EXISTS simple_variant_AGGREGATE"
}
}
diff --git a/regression-test/suites/variant_p0/query_subcolumns.groovy b/regression-test/suites/variant_p0/query_subcolumns.groovy
new file mode 100644
index 00000000000..5590bcaab6a
--- /dev/null
+++ b/regression-test/suites/variant_p0/query_subcolumns.groovy
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("regression_test_query_subcolumns", "nonConcurrent"){
+
+ def set_be_config = { key, value ->
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+ }
+ sql """DROP TABLE IF EXISTS query_subcolumns"""
+
+ // query must use inverted index for match operator
+ sql """ set enable_common_expr_pushdown = true; """
+ sql "set enable_match_without_inverted_index = false"
+ for (int i = 0; i < 5; i++) {
+ int max_subcolumns_count = Math.floor(Math.random() * 10)
+ // int max_subcolumns_count =1
+ def var = """variant<properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")>,
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") """
+ if (max_subcolumns_count % 2 == 0) {
+ var = """variant <'d' : int, 'b' : int, 'e' : string, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")>,
+ INDEX idx_v_e (v) USING INVERTED PROPERTIES("field_pattern" = "e", "parser" = "english", "support_phrase" = "true"),
+ INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") """
+ }
+ sql "DROP TABLE IF EXISTS query_subcolumns"
+ sql """
+ CREATE TABLE IF NOT EXISTS query_subcolumns (
+ k bigint,
+ v ${var}
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3");
+ """
+ sql """insert into query_subcolumns values (1, '{"a" : 1, "b" : "2"}')"""
+ // legacy V2 write
+ sql """insert into query_subcolumns values (2, '{"a" : 1, "b" : "2", "c" : 3}')"""
+ // switch to V2.1
+ sql """insert into query_subcolumns values (3, '{"a" : 1, "b" : "2", "c" : 3, "d" : 4}')"""
+ // keep V2.1
+ sql """insert into query_subcolumns values (4, '{"a" : 1}')"""
+ // switch back to V2
+ sql """insert into query_subcolumns values (5, '{"e" : "hello, world"}')"""
+ // and again V2.1
+ sql """insert into query_subcolumns values (6, '{"f" : "make it work"}')"""
+
+ qt_sql "select v['a'] from query_subcolumns where cast(v['a'] as int) is not null order by k"
+ qt_sql "select v['b'] from query_subcolumns where cast(v['b'] as int) is not null order by k"
+ qt_sql "select v['c'] from query_subcolumns where cast(v['c'] as int) is not null order by k"
+ qt_sql "select v['d'] from query_subcolumns where cast(v['d'] as int) is not null order by k"
+ qt_sql "select v['e'] from query_subcolumns where v['e'] like 'hello%' order by k"
+ if (max_subcolumns_count >= 6) {
+ sql "select v['f'] from query_subcolumns where v['f'] match 'make' order by k"
+ }
+
+ // if (max_subcolumns_count % 2) {
+ // // sql """alter table query_subcolumns set ("storage_format" = "V3")"""
+ // } else {
+ // // sql """alter table query_subcolumns set ("storage_format" = "V2")"""
+ // }
+
+ // triger compaction
+ trigger_and_wait_compaction("query_subcolumns", "full")
+
+ qt_sql "select v['a'] from query_subcolumns where cast(v['a'] as int) is not null order by k"
+ qt_sql "select v['b'] from query_subcolumns where cast(v['b'] as int) is not null order by k"
+ qt_sql "select v['c'] from query_subcolumns where cast(v['c'] as int) is not null order by k"
+ qt_sql "select v['d'] from query_subcolumns where cast(v['d'] as int) is not null order by k"
+ qt_sql "select v['e'] from query_subcolumns where v['e'] like 'hello%' order by k"
+ if (max_subcolumns_count >= 6) {
+ sql "select v['f'] from query_subcolumns where v['f'] match 'make' order by k"
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org