You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/29 02:48:40 UTC
(doris) branch master updated: [refactor](BE) split EncodingInfo defaults into 4 explicit maps (#63622)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4f1dcdf3395 [refactor](BE) split EncodingInfo defaults into 4 explicit maps (#63622)
4f1dcdf3395 is described below
commit 4f1dcdf33956a8adaf956ebca46da2820abf3ffb
Author: Chenyang Sun <su...@selectdb.com>
AuthorDate: Fri May 29 10:48:33 2026 +0800
[refactor](BE) split EncodingInfo defaults into 4 explicit maps (#63622)
Replace the EncodingPreference + runtime hook machinery in
EncodingInfoResolver with four explicit maps and four matching get
methods:
- _v2_default_map -> get_v2_default_encoding(type)
- _v3_default_map -> get_v3_default_encoding(type)
- _index_column_default_map -> get_index_column_encoding(type)
- _encoding_map -> get(type, encoding, out)
No on-disk format change; the resolved encodings written into
ColumnMetaPB match the pre-refactor outputs for both v2 and V3 tablets.
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [x] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---------
Co-authored-by: Claude Opus 4.7 (1M context) <no...@anthropic.com>
---
be/src/cloud/pb_convert.cpp | 12 +
be/src/storage/index/indexed_column_reader.cpp | 2 +-
be/src/storage/index/indexed_column_writer.cpp | 13 +-
be/src/storage/index/primary_key_index.cpp | 2 +-
be/src/storage/index/zone_map/zone_map_index.cpp | 3 +-
be/src/storage/segment/binary_dict_page.cpp | 19 +-
be/src/storage/segment/binary_dict_page.h | 7 +-
be/src/storage/segment/column_reader.cpp | 10 +-
be/src/storage/segment/column_reader.h | 2 +-
be/src/storage/segment/column_writer.cpp | 41 +-
be/src/storage/segment/column_writer.h | 9 +-
be/src/storage/segment/encoding_info.cpp | 443 +++++++++++----------
be/src/storage/segment/encoding_info.h | 129 ++++--
be/src/storage/segment/options.h | 11 +-
be/src/storage/segment/page_io.cpp | 8 +-
be/src/storage/segment/page_io.h | 5 -
be/src/storage/segment/segment_writer.cpp | 26 +-
be/src/storage/segment/segment_writer.h | 2 +-
.../segment/variant/variant_column_writer_impl.cpp | 24 +-
.../segment/variant/variant_column_writer_impl.h | 2 +-
be/src/storage/segment/vertical_segment_writer.cpp | 26 +-
be/src/storage/segment/vertical_segment_writer.h | 3 +-
be/src/storage/tablet/tablet_meta.cpp | 18 +-
be/src/storage/tablet/tablet_schema.cpp | 43 +-
be/src/storage/tablet/tablet_schema.h | 36 +-
be/test/exec/common/schema_util_rowset_test.cpp | 28 +-
be/test/storage/segment/binary_dict_page_test.cpp | 68 ++--
.../storage/segment/column_meta_accessor_test.cpp | 17 +-
.../storage/segment/column_reader_cache_test.cpp | 39 +-
be/test/storage/segment/encoding_info_test.cpp | 433 +++++++++++++++-----
.../segment/external_col_meta_util_test.cpp | 2 +-
.../segment/variant_column_writer_reader_test.cpp | 288 +++++++++++---
be/test/storage/tablet/tablet_schema_test.cpp | 66 +++
.../cloud/datasource/CloudInternalCatalog.java | 7 +-
gensrc/proto/olap_file.proto | 20 +
.../test_storage_format_controls_encoding.groovy | 19 +-
36 files changed, 1238 insertions(+), 645 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 108d61fa18f..913f7633781 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -480,6 +480,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
+ if (in.has_storage_format()) {
+ out->set_storage_format(in.storage_format());
+ }
if (in.has_seq_map()) {
out->mutable_seq_map()->CopyFrom(in.seq_map());
}
@@ -527,6 +530,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
+ if (in.has_storage_format()) {
+ out->set_storage_format(in.storage_format());
+ }
if (in.has_seq_map()) {
out->mutable_seq_map()->CopyFrom(in.seq_map());
}
@@ -587,6 +593,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
+ if (in.has_storage_format()) {
+ out->set_storage_format(in.storage_format());
+ }
if (in.has_seq_map()) {
out->mutable_seq_map()->CopyFrom(in.seq_map());
}
@@ -635,6 +644,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
+ if (in.has_storage_format()) {
+ out->set_storage_format(in.storage_format());
+ }
if (in.has_seq_map()) {
out->mutable_seq_map()->CopyFrom(in.seq_map());
}
diff --git a/be/src/storage/index/indexed_column_reader.cpp b/be/src/storage/index/indexed_column_reader.cpp
index b8fe9a57541..d30c2c82dbc 100644
--- a/be/src/storage/index/indexed_column_reader.cpp
+++ b/be/src/storage/index/indexed_column_reader.cpp
@@ -68,7 +68,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
if (!is_scalar_type(_type)) {
return Status::NotSupported("unsupported typeinfo, type={}", _meta.data_type());
}
- RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), {}, &_encoding_info));
+ RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), &_encoding_info));
_value_key_coder = get_key_coder(_type);
// read and parse ordinal index page when exists
diff --git a/be/src/storage/index/indexed_column_writer.cpp b/be/src/storage/index/indexed_column_writer.cpp
index 87a05579295..b9ce702f7e1 100644
--- a/be/src/storage/index/indexed_column_writer.cpp
+++ b/be/src/storage/index/indexed_column_writer.cpp
@@ -53,12 +53,15 @@ IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& optio
IndexedColumnWriter::~IndexedColumnWriter() = default;
Status IndexedColumnWriter::init() {
+ // Caller must set _options.encoding to a concrete value before calling init.
+ if (_options.encoding == DEFAULT_ENCODING) {
+ return Status::InternalError(
+ "IndexedColumnWriterOptions::encoding is DEFAULT_ENCODING for type={}; caller must "
+ "resolve to a concrete encoding before IndexedColumnWriter::init",
+ _type);
+ }
const EncodingInfo* encoding_info;
- RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, {}, &encoding_info));
- _options.encoding = encoding_info->encoding();
- // should store more concrete encoding type instead of DEFAULT_ENCODING
- // because the default encoding of a data type can be changed in the future
- DCHECK_NE(_options.encoding, DEFAULT_ENCODING);
+ RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, &encoding_info));
PageBuilder* data_page_builder = nullptr;
PageBuilderOptions builder_option;
diff --git a/be/src/storage/index/primary_key_index.cpp b/be/src/storage/index/primary_key_index.cpp
index d91f9e8f586..654584948e7 100644
--- a/be/src/storage/index/primary_key_index.cpp
+++ b/be/src/storage/index/primary_key_index.cpp
@@ -42,7 +42,7 @@ Status PrimaryKeyIndexBuilder::init() {
options.write_ordinal_index = true;
options.write_value_index = true;
options.data_page_size = config::primary_key_data_page_size;
- options.encoding = segment_v2::EncodingInfo::get_default_encoding(type, {}, true);
+ options.encoding = segment_v2::EncodingInfo::get_index_column_encoding(type);
options.compression = segment_v2::ZSTD;
_primary_key_index_builder.reset(
new segment_v2::IndexedColumnWriter(options, type, _file_writer));
diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp
index 632936a2021..b12de9d03b6 100644
--- a/be/src/storage/index/zone_map/zone_map_index.cpp
+++ b/be/src/storage/index/zone_map/zone_map_index.cpp
@@ -282,7 +282,8 @@ Status TypedZoneMapIndexWriter<Type>::finish(io::FileWriter* file_writer,
IndexedColumnWriterOptions options;
options.write_ordinal_index = true;
options.write_value_index = false;
- options.encoding = EncodingInfo::get_default_encoding(type, {}, false);
+ // Zone map page always uses PLAIN_ENCODING. Do not change.
+ options.encoding = PLAIN_ENCODING;
options.compression = NO_COMPRESSION; // currently not compressed
IndexedColumnWriter writer(options, type, file_writer);
diff --git a/be/src/storage/segment/binary_dict_page.cpp b/be/src/storage/segment/binary_dict_page.cpp
index 2a4438550ea..b3af337714b 100644
--- a/be/src/storage/segment/binary_dict_page.cpp
+++ b/be/src/storage/segment/binary_dict_page.cpp
@@ -47,13 +47,8 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options)
_data_page_builder(nullptr),
_dict_builder(nullptr),
_encoding_type(DICT_ENCODING),
- _dict_word_page_encoding_type(
- options.encoding_preference.binary_plain_encoding_default_impl ==
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
- ? PLAIN_ENCODING_V2
- : PLAIN_ENCODING),
- _fallback_binary_encoding_type(
- options.encoding_preference.binary_plain_encoding_default_impl ==
+ _binary_plain_encoding_type(
+ options.dict_binary_plain_encoding ==
BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
? PLAIN_ENCODING_V2
: PLAIN_ENCODING) {}
@@ -74,7 +69,7 @@ Status BinaryDictPageBuilder::init() {
const EncodingInfo* encoding_info;
RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR,
- _dict_word_page_encoding_type, {}, &encoding_info));
+ _binary_plain_encoding_type, &encoding_info));
RETURN_IF_ERROR(encoding_info->create_page_builder(dict_builder_options, _dict_builder));
return reset();
}
@@ -183,13 +178,11 @@ Status BinaryDictPageBuilder::reset() {
_buffer.resize(BINARY_DICT_PAGE_HEADER_SIZE);
if (_encoding_type == DICT_ENCODING && _dict_builder->is_page_full()) {
- DCHECK(_fallback_binary_encoding_type == PLAIN_ENCODING ||
- _fallback_binary_encoding_type == PLAIN_ENCODING_V2);
const EncodingInfo* encoding_info;
RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR,
- _fallback_binary_encoding_type, {}, &encoding_info));
+ _binary_plain_encoding_type, &encoding_info));
RETURN_IF_ERROR(encoding_info->create_page_builder(_options, _data_page_builder));
- _encoding_type = _fallback_binary_encoding_type;
+ _encoding_type = _binary_plain_encoding_type;
} else {
RETURN_IF_ERROR(_data_page_builder->reset());
}
@@ -210,7 +203,7 @@ Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page) {
}
Status BinaryDictPageBuilder::get_dictionary_page_encoding(EncodingTypePB* encoding) const {
- *encoding = _dict_word_page_encoding_type;
+ *encoding = _binary_plain_encoding_type;
return Status::OK();
}
diff --git a/be/src/storage/segment/binary_dict_page.h b/be/src/storage/segment/binary_dict_page.h
index 861b57f7763..c5367628561 100644
--- a/be/src/storage/segment/binary_dict_page.h
+++ b/be/src/storage/segment/binary_dict_page.h
@@ -106,10 +106,9 @@ private:
EncodingTypePB _encoding_type;
- EncodingTypePB
- _dict_word_page_encoding_type; // currently only support PLAIN_ENCODING and PLAIN_ENCODING_V2
- EncodingTypePB
- _fallback_binary_encoding_type; // currently only support PLAIN_ENCODING and PLAIN_ENCODING_V2
+ // Binary-plain flavor (V1 or V2) used both for the dictionary word page and for the data
+ // page when the dictionary overflows. Resolved from PageBuilderOptions::dict_binary_plain_encoding.
+ const EncodingTypePB _binary_plain_encoding_type;
struct HashOfSlice {
size_t operator()(const Slice& slice) const { return crc32_hash(slice.data, slice.size); }
diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp
index 7fc5d5e1466..e1f9a8bdf7a 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -356,7 +356,7 @@ Status ColumnReader::init(const ColumnMetaPB* meta) {
if (_type == FieldType::OLAP_FIELD_TYPE_NONE || _type == FieldType::OLAP_FIELD_TYPE_UNKNOWN) {
return Status::NotSupported("unsupported typeinfo, type={}", meta->type());
}
- RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), {}, &_encoding_info));
+ RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), &_encoding_info));
for (int i = 0; i < meta->indexes_size(); i++) {
const auto& index_meta = meta->indexes(i);
@@ -416,7 +416,7 @@ Status ColumnReader::new_index_iterator(const std::shared_ptr<IndexFileReader>&
Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp,
PageHandle* handle, Slice* page_body, PageFooterPB* footer,
- BlockCompressionCodec* codec, bool is_dict_page) const {
+ BlockCompressionCodec* codec) const {
SCOPED_CONCURRENCY_COUNT(ConcurrencyStatsManager::instance().column_reader_read_page);
iter_opts.sanity_check();
PageReadOptions opts(iter_opts.io_ctx);
@@ -429,7 +429,6 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag
opts.codec = codec;
opts.stats = iter_opts.stats;
opts.encoding_info = _encoding_info;
- opts.is_dict_page = is_dict_page;
return PageIO::read_and_decompress_page(opts, handle, page_body, footer);
}
@@ -2441,11 +2440,10 @@ Status FileColumnIterator::_read_dict_data() {
_opts.type = INDEX_PAGE;
RETURN_IF_ERROR(_reader->read_page(_opts, _reader->get_dict_page_pointer(), &_dict_page_handle,
- &dict_data, &dict_footer, _compress_codec, true));
+ &dict_data, &dict_footer, _compress_codec));
const EncodingInfo* encoding_info;
RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR,
- dict_footer.dict_page_footer().encoding(), {},
- &encoding_info));
+ dict_footer.dict_page_footer().encoding(), &encoding_info));
RETURN_IF_ERROR(encoding_info->create_page_decoder(dict_data, {}, _dict_decoder));
RETURN_IF_ERROR(_dict_decoder->init());
diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h
index dcee88dcbdf..64c60008dd5 100644
--- a/be/src/storage/segment/column_reader.h
+++ b/be/src/storage/segment/column_reader.h
@@ -175,7 +175,7 @@ public:
// read a page from file into a page handle
Status read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp,
PageHandle* handle, Slice* page_body, PageFooterPB* footer,
- BlockCompressionCodec* codec, bool is_dict_page = false) const;
+ BlockCompressionCodec* codec) const;
bool is_nullable() const { return _meta_is_nullable; }
diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp
index 15d4f558f9b..5dd14d0c980 100644
--- a/be/src/storage/segment/column_writer.cpp
+++ b/be/src/storage/segment/column_writer.cpp
@@ -130,18 +130,19 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts,
null_options.meta->set_is_nullable(false);
null_options.meta->set_length(
cast_set<int32_t>(field_type_size(FieldType::OLAP_FIELD_TYPE_TINYINT)));
- null_options.meta->set_encoding(DEFAULT_ENCODING);
null_options.meta->set_compression(opts.meta->compression());
null_options.need_zone_map = false;
null_options.need_bloom_filter = false;
- null_options.encoding_preference = opts.encoding_preference;
+ null_options.storage_format = opts.storage_format;
auto null_column_ptr = std::make_shared<TabletColumn>(
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false,
null_options.meta->unique_id(), null_options.meta->length());
null_column_ptr->set_name("nullable");
null_column_ptr->set_index_length(-1); // no short key index
+ null_options.meta->set_encoding(
+ EncodingInfo::resolve_default_encoding(opts.storage_format, *null_column_ptr));
return new ScalarColumnWriter(null_options, std::move(null_column_ptr), file_writer);
}
@@ -165,7 +166,7 @@ Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts,
column_options.meta = opts.meta->mutable_children_columns(i);
column_options.need_zone_map = false;
column_options.need_bloom_filter = sub_column.is_bf_column();
- column_options.encoding_preference = opts.encoding_preference;
+ column_options.storage_format = opts.storage_format;
std::unique_ptr<ColumnWriter> sub_column_writer;
RETURN_IF_ERROR(
ColumnWriter::create(column_options, &sub_column, file_writer, &sub_column_writer));
@@ -192,7 +193,7 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
item_options.meta = opts.meta->mutable_children_columns(0);
item_options.need_zone_map = false;
item_options.need_bloom_filter = item_column.is_bf_column();
- item_options.encoding_preference = opts.encoding_preference;
+ item_options.storage_format = opts.storage_format;
std::unique_ptr<ColumnWriter> item_writer;
RETURN_IF_ERROR(ColumnWriter::create(item_options, &item_column, file_writer, &item_writer));
@@ -207,12 +208,11 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
length_options.meta->set_is_nullable(false);
length_options.meta->set_length(
cast_set<int32_t>(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT)));
- length_options.meta->set_encoding(DEFAULT_ENCODING);
length_options.meta->set_compression(opts.meta->compression());
length_options.need_zone_map = false;
length_options.need_bloom_filter = false;
- length_options.encoding_preference = opts.encoding_preference;
+ length_options.storage_format = opts.storage_format;
auto length_column_ptr = std::make_shared<TabletColumn>(
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type,
@@ -220,6 +220,8 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
length_options.meta->length());
length_column_ptr->set_name("length");
length_column_ptr->set_index_length(-1); // no short key index
+ length_options.meta->set_encoding(
+ EncodingInfo::resolve_default_encoding(opts.storage_format, *length_column_ptr));
auto* length_writer =
new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer);
@@ -251,7 +253,7 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta
item_options.meta = opts.meta->mutable_children_columns(i);
item_options.need_zone_map = false;
item_options.need_bloom_filter = item_column.is_bf_column();
- item_options.encoding_preference = opts.encoding_preference;
+ item_options.storage_format = opts.storage_format;
std::unique_ptr<ColumnWriter> item_writer;
RETURN_IF_ERROR(
ColumnWriter::create(item_options, &item_column, file_writer, &item_writer));
@@ -270,12 +272,11 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta
length_options.meta->set_is_nullable(false);
length_options.meta->set_length(
cast_set<int32_t>(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT)));
- length_options.meta->set_encoding(DEFAULT_ENCODING);
length_options.meta->set_compression(opts.meta->compression());
length_options.need_zone_map = false;
length_options.need_bloom_filter = false;
- length_options.encoding_preference = opts.encoding_preference;
+ length_options.storage_format = opts.storage_format;
auto length_column_ptr = std::make_shared<TabletColumn>(
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type,
@@ -283,6 +284,8 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta
length_options.meta->length());
length_column_ptr->set_name("length");
length_column_ptr->set_index_length(-1); // no short key index
+ length_options.meta->set_encoding(
+ EncodingInfo::resolve_default_encoding(opts.storage_format, *length_column_ptr));
auto* length_writer =
new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer);
@@ -487,22 +490,28 @@ Status ScalarColumnWriter::init() {
PageBuilder* page_builder = nullptr;
- RETURN_IF_ERROR(EncodingInfo::get(get_column()->type(), _opts.meta->encoding(),
- _opts.encoding_preference, &_encoding_info));
- _opts.meta->set_encoding(_encoding_info->encoding());
+ // Caller must set a concrete (non-DEFAULT) encoding on the meta before init.
+ if (_opts.meta->encoding() == DEFAULT_ENCODING) {
+ return Status::InternalError(
+ "ColumnMetaPB encoding is DEFAULT_ENCODING for column_id={}, type={}; caller must "
+ "resolve to a concrete encoding before ScalarColumnWriter::init",
+ _opts.meta->column_id(), get_column()->type());
+ }
+ RETURN_IF_ERROR(
+ EncodingInfo::get(get_column()->type(), _opts.meta->encoding(), &_encoding_info));
// create page builder
PageBuilderOptions opts;
opts.data_page_size = _opts.data_page_size;
opts.dict_page_size = _opts.dict_page_size;
- opts.encoding_preference = _opts.encoding_preference;
+ opts.dict_binary_plain_encoding =
+ (_opts.storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3)
+ ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
+ : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder));
if (page_builder == nullptr) {
return Status::NotSupported("Failed to create page builder for type {} and encoding {}",
get_column()->type(), _opts.meta->encoding());
}
- // should store more concrete encoding type instead of DEFAULT_ENCODING
- // because the default encoding of a data type can be changed in the future
- DCHECK_NE(_opts.meta->encoding(), DEFAULT_ENCODING);
VLOG_DEBUG << fmt::format(
"[verbose] scalar column writer init, column_id={}, type={}, encoding={}, "
"is_nullable={}",
diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h
index 44567c2d8d3..50822c945c1 100644
--- a/be/src/storage/segment/column_writer.h
+++ b/be/src/storage/segment/column_writer.h
@@ -17,6 +17,8 @@
#pragma once
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>
#include <stddef.h>
#include <stdint.h>
@@ -85,7 +87,12 @@ struct ColumnWriterOptions {
std::vector<RowsetReaderSharedPtr> input_rs_readers;
const TabletIndex* ann_index = nullptr;
- EncodingPreference encoding_preference {};
+ // Storage format of the owning tablet (V2 or V3). Set once by the segment writer
+ // (from TabletMeta::storage_format()) and propagated down to aux child writers
+ // (null / array-length / map-length), struct subcolumn writers and variant subcolumn
+ // writers. All encoding-default decisions consult this via resolve_default_encoding().
+ // Also forwarded to BinaryDictPageBuilder via PageBuilderOptions::binary_plain_encoding.
+ TabletStorageFormatPB storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
std::string to_string() const {
std::stringstream ss;
diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp
index 9fdad850406..30700fe0f41 100644
--- a/be/src/storage/segment/encoding_info.cpp
+++ b/be/src/storage/segment/encoding_info.cpp
@@ -27,6 +27,7 @@
#include <utility>
#include "common/config.h"
+#include "common/exception.h"
#include "runtime/exec_env.h"
#include "storage/olap_common.h"
#include "storage/segment/binary_dict_page.h"
@@ -41,6 +42,7 @@
#include "storage/segment/options.h"
#include "storage/segment/plain_page.h"
#include "storage/segment/rle_page.h"
+#include "storage/tablet/tablet_schema.h"
#include "storage/types.h"
namespace doris {
@@ -214,124 +216,181 @@ struct TypeEncodingTraits<type, PREFIX_ENCODING, Slice> {
};
EncodingInfoResolver::EncodingInfoResolver() {
- _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, FOR_ENCODING, true>();
- _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, FOR_ENCODING, true>();
- _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_INT, FOR_ENCODING, true>();
- _add_map<FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, FOR_ENCODING, true>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V2>();
- _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PREFIX_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V2>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V2>();
- _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PREFIX_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V2>();
- _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PREFIX_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V2>();
- _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PREFIX_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, RLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, true>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2>();
-
- _add_map<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING>();
- _add_map<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2>();
+ // ===== Phase 1: register every supported (type, encoding) combination exactly once =====
+ // _register_supported_encoding CHECKs against duplicates; the Phase 2 calls below do not insert into _encoding_map,
+ // so every (type, encoding) used as a default must appear here first.
+
+ // signed integers
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TINYINT, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_SMALLINT, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_INT, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BIGINT, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_LARGEINT, FOR_ENCODING>();
+
+ // unsigned integers
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();
+
+ // FLOAT / DOUBLE
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();
+
+ // binary types (CHAR/VARCHAR/STRING/JSONB/VARIANT)
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_CHAR, PREFIX_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_STRING, PREFIX_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_JSONB, PREFIX_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_VARIANT, PREFIX_ENCODING>();
+
+ // BOOL
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BOOL, RLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING>();
+
+ // date / datetime
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING>();
+
+ // decimal
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL32, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL64, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING>();
+
+ // ip
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_IPV4, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING>();
+
+ // aggregate / binary-flavored types
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2>();
+
+ // ===== Phase 2a: V2 defaults (write path, V1/V2 segments) =====
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_BOOL, RLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING>();
+
+ // ===== Phase 2b: V3 defaults (write path, V3 segments) =====
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_BOOL, RLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2>();
+
+ // ===== Phase 2c: IndexedColumn (value-seek) defaults =====
+ // Only the PrimaryKeyIndexBuilder consults this map, and it hardcodes VARCHAR.
+ // Other types were registered historically (since #2308, 2019) for a generic
+ // "any IndexedColumn value-seek caller" use case that never materialized; they
+ // were removed to keep this map honest about what production actually needs.
+ _set_index_column_encoding<FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING>();
}
EncodingInfoResolver::~EncodingInfoResolver() {
@@ -341,90 +400,20 @@ EncodingInfoResolver::~EncodingInfoResolver() {
_encoding_map.clear();
}
-namespace {
-bool is_integer_type(FieldType type) {
- return type == FieldType::OLAP_FIELD_TYPE_TINYINT ||
- type == FieldType::OLAP_FIELD_TYPE_SMALLINT || type == FieldType::OLAP_FIELD_TYPE_INT ||
- type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT;
+EncodingTypePB EncodingInfoResolver::get_v2_default_encoding(FieldType type) const {
+ return _lookup(_v2_default_map, type);
}
-bool is_binary_type(FieldType type) {
- return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
- type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB ||
- type == FieldType::OLAP_FIELD_TYPE_VARIANT || type == FieldType::OLAP_FIELD_TYPE_HLL ||
- type == FieldType::OLAP_FIELD_TYPE_BITMAP ||
- type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
- type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
+EncodingTypePB EncodingInfoResolver::get_v3_default_encoding(FieldType type) const {
+ return _lookup(_v3_default_map, type);
}
-} // namespace
-
-EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type,
- EncodingPreference encoding_preference,
- bool optimize_value_seek) const {
- // Predicate for default encoding transformation
- // Parameters: (type, current_default_encoding, optimize_value_seek)
- // Returns: true if the transformation should be applied
- using Predicate = std::function<bool(FieldType, EncodingTypePB,
- EncodingPreference encoding_preference, bool)>;
-
- // Hook for transforming default encoding: predicate -> target encoding
- struct EncodingTransform {
- Predicate predicate;
- EncodingTypePB target_encoding;
- };
-
- // Static array of hooks for default encoding transformations
- static const std::vector<EncodingTransform> hooks = {
- // Hook 1: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V2
- // Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v2
- EncodingTransform {
- .predicate =
- [](FieldType type, EncodingTypePB encoding,
- EncodingPreference encoding_preference, bool optimize_value_seek) {
- return encoding == PLAIN_ENCODING && is_binary_type(type) &&
- encoding_preference.binary_plain_encoding_default_impl ==
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
- },
- .target_encoding = PLAIN_ENCODING_V2},
-
- // Hook 2: Integer types - any encoding -> PLAIN_ENCODING
- // Applies when: type is integer and config enables plain encoding for integers
- EncodingTransform {
- .predicate =
- [](FieldType type, EncodingTypePB encoding,
- EncodingPreference encoding_preference, bool optimize_value_seek) {
- return is_integer_type(type) &&
- encoding_preference.integer_type_default_use_plain_encoding;
- },
- .target_encoding = PLAIN_ENCODING}};
-
- auto& encoding_map =
- optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map;
- auto it = encoding_map.find(type);
- if (it != encoding_map.end()) {
- EncodingTypePB encoding = it->second;
-
- // Apply hooks in order to transform the default encoding
- for (const auto& hook : hooks) {
- if (hook.predicate(type, encoding, encoding_preference, optimize_value_seek)) {
- // Verify target encoding is available for this type
- if (_encoding_map.contains(std::make_pair(type, hook.target_encoding))) {
- encoding = hook.target_encoding;
- break; // Apply only the first matching hook
- }
- }
- }
- return encoding;
- }
- return UNKNOWN_ENCODING;
+EncodingTypePB EncodingInfoResolver::get_index_column_encoding(FieldType type) const {
+ return _lookup(_index_column_encoding_map, type);
}
Status EncodingInfoResolver::get(FieldType data_type, EncodingTypePB encoding_type,
- EncodingPreference encoding_preference, const EncodingInfo** out) {
- if (encoding_type == DEFAULT_ENCODING) {
- encoding_type = get_default_encoding(data_type, encoding_preference, false);
- }
+ const EncodingInfo** out) {
auto key = std::make_pair(data_type, encoding_type);
auto it = _encoding_map.find(key);
if (it == std::end(_encoding_map)) {
@@ -446,10 +435,18 @@ EncodingInfo::EncodingInfo(TraitsClass traits)
} else if (_encoding == DICT_ENCODING) {
_data_page_pre_decoder = std::make_unique<BinaryDictPagePreDecoder>();
} else if (_encoding == PLAIN_ENCODING_V2) {
- // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2
- // to convert varint-encoded lengths to offset array format
+ // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2 — it converts
+ // varint-encoded lengths to an offset-array format that downstream Slice decoders expect.
+ // All current (type, PLAIN_ENCODING_V2) registrations are Slice (CHAR/VARCHAR/STRING/
+ // JSONB/VARIANT/HLL/BITMAP/QUANTILE_STATE/AGG_STATE per storage/types.h). The else throws
+ // at construction time to fail loudly if a future non-Slice registration is added.
if constexpr (std::is_same_v<typename TraitsClass::CppType, Slice>) {
_data_page_pre_decoder = std::make_unique<BinaryPlainPageV2PreDecoder>();
+ } else {
+ throw Exception(Status::FatalError(
+ "PLAIN_ENCODING_V2 is only supported for Slice (binary) types, but got "
+ "non-Slice type {}",
+ int(TraitsClass::type)));
}
}
}
@@ -458,34 +455,66 @@ EncodingInfo::EncodingInfo(TraitsClass traits)
static EncodingInfoResolver s_encoding_info_resolver;
#endif
-Status EncodingInfo::get(FieldType type, EncodingTypePB encoding_type,
- EncodingPreference encoding_preference, const EncodingInfo** out) {
+Status EncodingInfo::get(FieldType type, EncodingTypePB encoding_type, const EncodingInfo** out) {
#ifdef BE_TEST
- return s_encoding_info_resolver.get(type, encoding_type, encoding_preference, out);
+ return s_encoding_info_resolver.get(type, encoding_type, out);
#else
auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
if (resolver == nullptr) {
return Status::InternalError("EncodingInfoResolver not initialized");
}
- return resolver->get(type, encoding_type, encoding_preference, out);
+ return resolver->get(type, encoding_type, out);
#endif
}
-EncodingTypePB EncodingInfo::get_default_encoding(FieldType type,
- EncodingPreference encoding_preference,
- bool optimize_value_seek) {
+EncodingTypePB EncodingInfo::get_v2_default_encoding(FieldType type) {
#ifdef BE_TEST
- return s_encoding_info_resolver.get_default_encoding(type, encoding_preference,
- optimize_value_seek);
+ return s_encoding_info_resolver.get_v2_default_encoding(type);
#else
auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
if (resolver == nullptr) {
return UNKNOWN_ENCODING;
}
- return resolver->get_default_encoding(type, encoding_preference, optimize_value_seek);
+ return resolver->get_v2_default_encoding(type);
#endif
}
+EncodingTypePB EncodingInfo::get_v3_default_encoding(FieldType type) {
+#ifdef BE_TEST
+ return s_encoding_info_resolver.get_v3_default_encoding(type);
+#else
+ auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
+ if (resolver == nullptr) {
+ return UNKNOWN_ENCODING;
+ }
+ return resolver->get_v3_default_encoding(type);
+#endif
+}
+
+EncodingTypePB EncodingInfo::get_index_column_encoding(FieldType type) {
+#ifdef BE_TEST
+ return s_encoding_info_resolver.get_index_column_encoding(type);
+#else
+ auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
+ if (resolver == nullptr) {
+ return UNKNOWN_ENCODING;
+ }
+ return resolver->get_index_column_encoding(type);
+#endif
+}
+
+EncodingTypePB EncodingInfo::resolve_default_encoding(TabletStorageFormatPB storage_format,
+ const TabletColumn& column) {
+ const bool is_v3 = (storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
+
+ // Row store data is already serialized as a single blob. Keep it on plain pages to
+ // avoid introducing dictionary pages for the hidden row store column.
+ if (column.is_row_store_column()) {
+ return is_v3 ? PLAIN_ENCODING_V2 : PLAIN_ENCODING;
+ }
+ return is_v3 ? get_v3_default_encoding(column.type()) : get_v2_default_encoding(column.type());
+}
+
Status EncodingInfo::create_page_builder(const PageBuilderOptions& opts,
std::unique_ptr<PageBuilder>& builder) const {
PageBuilder* raw_builder = nullptr;
diff --git a/be/src/storage/segment/encoding_info.h b/be/src/storage/segment/encoding_info.h
index 3ecf817a42c..de670da0964 100644
--- a/be/src/storage/segment/encoding_info.h
+++ b/be/src/storage/segment/encoding_info.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/AgentService_types.h>
#include <stddef.h>
#include <functional>
@@ -31,6 +32,7 @@
namespace doris {
enum class FieldType;
+class TabletColumn;
namespace segment_v2 {
@@ -51,15 +53,22 @@ public:
class EncodingInfo {
public:
- // Get EncodingInfo for TypeInfo and EncodingTypePB
- static Status get(FieldType type, EncodingTypePB encoding_type,
- EncodingPreference encoding_preference, const EncodingInfo** encoding);
-
- // optimize_value_search: whether the encoding scheme should optimize for ordered data
- // and support fast value seek operation
- static EncodingTypePB get_default_encoding(FieldType type,
- EncodingPreference encoding_preference,
- bool optimize_value_seek);
+ // Look up the EncodingInfo for an already-resolved (type, encoding) pair.
+ // Read paths use this directly; write paths first resolve a default via the
+ // get_*_default_encoding helpers below.
+ static Status get(FieldType type, EncodingTypePB encoding_type, const EncodingInfo** encoding);
+
+ // Default encoding for IndexedColumn writers (PK index, variant ext meta key writer)
+ // that need fast value-seek via BinaryPrefixPage.
+ static EncodingTypePB get_index_column_encoding(FieldType type);
+
+ // Resolve the default encoding for one column when populating a fresh ColumnMetaPB.
+ // All write paths (top-level segment writer, aux child writers for null bitmap /
+ // array length / map length, struct subcolumns, variant subcolumns) should route the
+ // encoding decision through this helper. Centralizing it here means future
+ // per-storage-format or per-column encoding policy lives in one place.
+ static EncodingTypePB resolve_default_encoding(TabletStorageFormatPB storage_format,
+ const TabletColumn& column);
Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) const {
return _create_builder_func(opts, builder);
@@ -79,6 +88,14 @@ public:
private:
friend class EncodingInfoResolver;
+ friend class EncodingInfoTest;
+ friend class ColumnReaderCacheTest;
+
+ // Per-storage-format defaults are an internal lookup table. Production write paths
+ // go through resolve_default_encoding(); other callers (e.g., zone map index) hardcode
+ // the encoding they want. Tests use the friend declarations above to read these tables.
+ static EncodingTypePB get_v2_default_encoding(FieldType type);
+ static EncodingTypePB get_v3_default_encoding(FieldType type);
template <typename TypeEncodingTraits>
explicit EncodingInfo(TypeEncodingTraits traits);
@@ -107,21 +124,43 @@ public:
EncodingInfoResolver();
~EncodingInfoResolver();
- EncodingTypePB get_default_encoding(FieldType type, EncodingPreference encoding_preference,
- bool optimize_value_seek) const;
+ EncodingTypePB get_v2_default_encoding(FieldType type) const;
+ EncodingTypePB get_v3_default_encoding(FieldType type) const;
+ EncodingTypePB get_index_column_encoding(FieldType type) const;
- Status get(FieldType data_type, EncodingTypePB encoding_type,
- EncodingPreference encoding_preference, const EncodingInfo** out);
+ Status get(FieldType data_type, EncodingTypePB encoding_type, const EncodingInfo** out);
private:
- // Not thread-safe
- template <FieldType type, EncodingTypePB encoding_type, bool optimize_value_seek = false>
- void _add_map();
-
- std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash> _default_encoding_type_map;
+ // Registration helpers used by the constructor. Not thread-safe.
+ //
+ // _register_supported_encoding: declare that this (type, encoding) is a supported combination,
+ // inserting one EncodingInfo* into _encoding_map. CHECK-fails on
+ // duplicate registration of the same key.
+ // _set_v2_default: mark this (type, encoding) as the default for the V2 (V1/V2)
+ // write path. Only writes _v2_default_map; the caller must have
+ // already _register_supported_encoding'd the same key, otherwise
+ // EncodingInfo::get will return InternalError when first asked
+ // about this combination.
+ // _set_v3_default: same, for the V3 write path.
+ // _set_index_column_encoding: same, for IndexedColumn value-seek writers (PK index).
+ template <FieldType type, EncodingTypePB encoding>
+ void _register_supported_encoding();
+ template <FieldType type, EncodingTypePB encoding>
+ void _set_v2_default();
+ template <FieldType type, EncodingTypePB encoding>
+ void _set_v3_default();
+ template <FieldType type, EncodingTypePB encoding>
+ void _set_index_column_encoding();
+
+ static EncodingTypePB _lookup(
+ const std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash>& m, FieldType t) {
+ auto it = m.find(t);
+ return it != m.end() ? it->second : UNKNOWN_ENCODING;
+ }
- // default encoding for each type which optimizes value seek
- std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash> _value_seek_encoding_map;
+ std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash> _v2_default_map;
+ std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash> _v3_default_map;
+ std::unordered_map<FieldType, EncodingTypePB, EncodingMapHash> _index_column_encoding_map;
std::unordered_map<std::pair<FieldType, EncodingTypePB>, EncodingInfo*, EncodingMapHash>
_encoding_map;
@@ -138,23 +177,39 @@ struct EncodingTraits : TypeEncodingTraits<field_type, encoding_type,
static const EncodingTypePB encoding = encoding_type;
};
-template <FieldType type, EncodingTypePB encoding_type, bool optimize_value_seek>
-void EncodingInfoResolver::_add_map() {
- EncodingTraits<type, encoding_type> traits;
- std::unique_ptr<EncodingInfo> encoding(new EncodingInfo(traits));
- if (_default_encoding_type_map.find(type) == std::end(_default_encoding_type_map)) {
- _default_encoding_type_map[type] = encoding_type;
- }
- if (optimize_value_seek &&
- _value_seek_encoding_map.find(type) == _value_seek_encoding_map.end()) {
- _value_seek_encoding_map[type] = encoding_type;
- }
- auto key = std::make_pair(type, encoding_type);
- auto it = _encoding_map.find(key);
- if (it != _encoding_map.end()) {
- return;
- }
- _encoding_map.emplace(key, encoding.release());
+template <FieldType type, EncodingTypePB encoding>
+void EncodingInfoResolver::_register_supported_encoding() {
+ auto key = std::make_pair(type, encoding);
+ CHECK(_encoding_map.find(key) == _encoding_map.end())
+ << "duplicate _register_supported_encoding for (type=" << int(type)
+ << ", encoding=" << encoding << ")";
+ EncodingTraits<type, encoding> traits;
+ _encoding_map.emplace(key, new EncodingInfo(traits));
+}
+
+// The _set_*_default helpers only write to the corresponding default map. The caller must
+// _register_supported_encoding the same (type, encoding) separately; if not, EncodingInfo::get
+// will return InternalError when that combination is first looked up.
+
+template <FieldType type, EncodingTypePB encoding>
+void EncodingInfoResolver::_set_v2_default() {
+ DCHECK(_v2_default_map.find(type) == _v2_default_map.end())
+ << "duplicate v2 default for type " << int(type);
+ _v2_default_map[type] = encoding;
+}
+
+template <FieldType type, EncodingTypePB encoding>
+void EncodingInfoResolver::_set_v3_default() {
+ DCHECK(_v3_default_map.find(type) == _v3_default_map.end())
+ << "duplicate v3 default for type " << int(type);
+ _v3_default_map[type] = encoding;
+}
+
+template <FieldType type, EncodingTypePB encoding>
+void EncodingInfoResolver::_set_index_column_encoding() {
+ DCHECK(_index_column_encoding_map.find(type) == _index_column_encoding_map.end())
+ << "duplicate index_column_encoding for type " << int(type);
+ _index_column_encoding_map[type] = encoding;
}
} // namespace segment_v2
diff --git a/be/src/storage/segment/options.h b/be/src/storage/segment/options.h
index 4ff94c0a8e3..a7c1ef0fa64 100644
--- a/be/src/storage/segment/options.h
+++ b/be/src/storage/segment/options.h
@@ -29,12 +29,6 @@ static constexpr size_t STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 256 * 1024l;
constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store page size: 16KB
-struct EncodingPreference {
- bool integer_type_default_use_plain_encoding {false};
- BinaryPlainEncodingTypePB binary_plain_encoding_default_impl {
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1};
-};
-
struct PageBuilderOptions {
size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
@@ -44,7 +38,10 @@ struct PageBuilderOptions {
bool is_dict_page = false; // page used for saving dictionary
- EncodingPreference encoding_preference {};
+ // BinaryPlain variant used by BinaryDictPageBuilder for its dict word page and
+ // dict-overflow fallback. Consumed only by BinaryDictPageBuilder.
+ BinaryPlainEncodingTypePB dict_binary_plain_encoding =
+ BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
};
struct PageDecoderOptions {
diff --git a/be/src/storage/segment/page_io.cpp b/be/src/storage/segment/page_io.cpp
index 2324f29efc9..cd415d93d5d 100644
--- a/be/src/storage/segment/page_io.cpp
+++ b/be/src/storage/segment/page_io.cpp
@@ -231,11 +231,11 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle
if (opts.pre_decode) {
const auto* encoding_info = opts.encoding_info;
- if (opts.is_dict_page) {
- // for dict page, we need to use encoding_info based on footer->dict_page_footer().encoding()
- // to get its pre_decoder
+ if (footer->type() == DICTIONARY_PAGE) {
+ // dict page uses its own encoding from footer->dict_page_footer().encoding()
+ // to look up the pre_decoder
RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR,
- footer->dict_page_footer().encoding(), {},
+ footer->dict_page_footer().encoding(),
&encoding_info));
}
if (encoding_info) {
diff --git a/be/src/storage/segment/page_io.h b/be/src/storage/segment/page_io.h
index 2a1b4539d8d..4da3f53530d 100644
--- a/be/src/storage/segment/page_io.h
+++ b/be/src/storage/segment/page_io.h
@@ -72,10 +72,6 @@ struct PageReadOptions {
const io::IOContext io_ctx;
- // for dict page, we need to use encoding_info based on footer->dict_page_footer().encoding()
- // to get its pre_decoder
- bool is_dict_page {false};
-
void sanity_check() const {
CHECK_NOTNULL(file_reader);
CHECK_NOTNULL(stats);
@@ -93,7 +89,6 @@ struct PageReadOptions {
type = old.type;
encoding_info = old.encoding_info;
pre_decode = old.pre_decode;
- is_dict_page = old.is_dict_page;
}
};
diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp
index 917777f24eb..5db2561b9d8 100644
--- a/be/src/storage/segment/segment_writer.cpp
+++ b/be/src/storage/segment/segment_writer.cpp
@@ -62,6 +62,7 @@
#include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "storage/rowset/segment_creator.h"
#include "storage/segment/column_writer.h" // ColumnWriter
+#include "storage/segment/encoding_info.h"
#include "storage/segment/external_col_meta_util.h"
#include "storage/segment/historical_row_retriever.h"
#include "storage/segment/page_io.h"
@@ -146,11 +147,11 @@ SegmentWriter::~SegmentWriter() {
}
void SegmentWriter::init_column_meta(ColumnMetaPB* meta, uint32_t column_id,
- const TabletColumn& column, TabletSchemaSPtr tablet_schema) {
+ const TabletColumn& column, const ColumnWriterOptions& opts) {
meta->set_column_id(column_id);
meta->set_type(int(column.type()));
meta->set_length(column.length());
- meta->set_encoding(DEFAULT_ENCODING);
+ meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column));
meta->set_compression(_opts.compression_type);
meta->set_is_nullable(column.is_nullable());
meta->set_default_value(column.default_value());
@@ -162,8 +163,7 @@ void SegmentWriter::init_column_meta(ColumnMetaPB* meta, uint32_t column_id,
}
meta->set_unique_id(column.unique_id());
for (uint32_t i = 0; i < column.get_subtype_count(); ++i) {
- init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i),
- tablet_schema);
+ init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts);
}
meta->set_result_is_nullable(column.get_result_is_nullable());
meta->set_function_name(column.get_aggregation_name());
@@ -187,8 +187,9 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
const TabletSchemaSPtr& schema) {
ColumnWriterOptions opts;
opts.meta = _footer.add_columns();
+ opts.storage_format = schema->storage_format();
- init_column_meta(opts.meta, cid, column, schema);
+ init_column_meta(opts.meta, cid, column, opts);
// now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS
// except for columns whose type don't support zone map.
@@ -291,16 +292,11 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
}
})
if (column.is_row_store_column()) {
- // smaller page size for row store column
+ // smaller page size for row store column; encoding is already set to PLAIN /
+ // PLAIN_V2 by init_column_meta via resolve_default_encoding().
auto page_size = _tablet_schema->row_store_page_size();
opts.data_page_size =
(page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
- // Row store data is already serialized as a single blob. Keep it on plain pages
- // to avoid introducing dictionary pages for the hidden row store column.
- opts.meta->set_encoding(_tablet_schema->binary_plain_encoding_default_impl() ==
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
- ? PLAIN_ENCODING_V2
- : PLAIN_ENCODING);
}
opts.rowset_ctx = _opts.rowset_ctx;
@@ -310,10 +306,6 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
if (_opts.rowset_ctx != nullptr) {
opts.input_rs_readers = _opts.rowset_ctx->input_rs_readers;
}
- opts.encoding_preference = {.integer_type_default_use_plain_encoding =
- _tablet_schema->integer_type_default_use_plain_encoding(),
- .binary_plain_encoding_default_impl =
- _tablet_schema->binary_plain_encoding_default_impl()};
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer));
@@ -1124,7 +1116,7 @@ Status SegmentWriter::_write_primary_key_index() {
Status SegmentWriter::_write_footer() {
_footer.set_num_rows(_row_count);
// Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
- if (_tablet_schema->is_external_segment_column_meta_used()) {
+ if (_tablet_schema->storage_format() == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) {
_footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
VLOG_DEBUG << "use external column meta";
// External ColumnMetaPB writing (optional)
diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h
index 5623a06a70c..c98a6cead20 100644
--- a/be/src/storage/segment/segment_writer.h
+++ b/be/src/storage/segment/segment_writer.h
@@ -125,7 +125,7 @@ public:
Status finalize_footer(uint64_t* segment_file_size);
void init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column,
- TabletSchemaSPtr tablet_schema);
+ const ColumnWriterOptions& opts);
Slice min_encoded_key();
Slice max_encoded_key();
diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
index 4d9fd6a9966..3b4982cb7c6 100644
--- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp
+++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
@@ -50,6 +50,7 @@
#include "storage/olap_define.h"
#include "storage/rowset/rowset_writer_context.h"
#include "storage/segment/column_writer.h"
+#include "storage/segment/encoding_info.h"
#include "storage/segment/variant/nested_group_path.h"
#include "storage/segment/variant/nested_group_routing_plan.h"
#include "storage/segment/variant/variant_writer_helpers.h"
@@ -60,13 +61,13 @@
namespace doris::segment_v2 {
void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column,
- CompressionTypePB compression_type) {
+ const ColumnWriterOptions& opts) {
meta->Clear();
meta->set_column_id(column_id);
meta->set_type(int(column.type()));
meta->set_length(column.length());
- meta->set_encoding(DEFAULT_ENCODING);
- meta->set_compression(compression_type);
+ meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column));
+ meta->set_compression(opts.compression_type);
meta->set_is_nullable(column.is_nullable());
meta->set_default_value(column.default_value());
meta->set_precision(column.precision());
@@ -77,8 +78,7 @@ void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColum
}
meta->set_unique_id(column.unique_id());
for (uint32_t i = 0; i < column.get_subtype_count(); ++i) {
- _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i),
- compression_type);
+ _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts);
}
if (column.is_variant_type()) {
meta->set_variant_max_subcolumns_count(column.variant_max_subcolumns_count());
@@ -92,7 +92,7 @@ Status _create_column_writer(uint32_t cid, const TabletColumn& column,
std::unique_ptr<ColumnWriter>* writer,
TabletIndexes& subcolumn_indexes, ColumnWriterOptions* opt,
int64_t none_null_value_size, bool need_record_none_null_value_size) {
- _init_column_meta(opt->meta, cid, column, opt->compression_type);
+ _init_column_meta(opt->meta, cid, column, *opt);
// no need to record none null value size for typed column or nested column, since it's compaction stage
// will directly pick it as sub column
if (need_record_none_null_value_size) {
@@ -378,6 +378,7 @@ Status prepare_materialized_subcolumn_writer(
opts.compression_type = base_opts.compression_type;
opts.rowset_ctx = base_opts.rowset_ctx;
opts.file_writer = base_opts.file_writer;
+ opts.storage_format = base_opts.storage_format;
std::unique_ptr<ColumnWriter> writer;
variant_util::inherit_column_attributes(parent_column, tablet_column);
@@ -596,7 +597,7 @@ Status UnifiedSparseColumnWriter::init_single(const TabletColumn& sparse_column,
SegmentFooterPB* footer) {
_single_opts = base_opts;
_single_opts.meta = footer->add_columns();
- _init_column_meta(_single_opts.meta, column_id, sparse_column, base_opts.compression_type);
+ _init_column_meta(_single_opts.meta, column_id, sparse_column, base_opts);
RETURN_IF_ERROR(ColumnWriter::create_map_writer(_single_opts, &sparse_column,
base_opts.file_writer, &_single_writer));
RETURN_IF_ERROR(_single_writer->init());
@@ -616,7 +617,7 @@ Status UnifiedSparseColumnWriter::init_buckets(int bucket_num, const TabletColum
TabletColumn bucket_col = variant_util::create_sparse_shard_column(parent_column, b);
_bucket_opts[b] = base_opts;
_bucket_opts[b].meta = footer->add_columns();
- _init_column_meta(_bucket_opts[b].meta, column_id, bucket_col, base_opts.compression_type);
+ _init_column_meta(_bucket_opts[b].meta, column_id, bucket_col, base_opts);
RETURN_IF_ERROR(ColumnWriter::create_map_writer(
_bucket_opts[b], &bucket_col, base_opts.file_writer, &_bucket_writers[b]));
RETURN_IF_ERROR(_bucket_writers[b]->init());
@@ -839,8 +840,7 @@ Status VariantDocWriter::init(const TabletColumn* parent_column, int bucket_num,
variant_util::create_doc_value_column(*parent_column, b);
_doc_value_column_opts[b] = opts;
_doc_value_column_opts[b].meta = footer->add_columns();
- _init_column_meta(_doc_value_column_opts[b].meta, column_id, bucket_column,
- opts.compression_type);
+ _init_column_meta(_doc_value_column_opts[b].meta, column_id, bucket_column, opts);
RETURN_IF_ERROR(ColumnWriter::create_map_writer(_doc_value_column_opts[b], &bucket_column,
opts.file_writer,
&_doc_value_column_writers[b]));
@@ -1138,7 +1138,7 @@ Status prepare_subcolumn_writer_target(
opts.compression_type = base_opts.compression_type;
opts.rowset_ctx = base_opts.rowset_ctx;
opts.file_writer = base_opts.file_writer;
- opts.encoding_preference = base_opts.encoding_preference;
+ opts.storage_format = base_opts.storage_format;
variant_util::inherit_column_attributes(parent_column, tablet_column);
bool need_record_none_null_value_size =
@@ -1851,7 +1851,7 @@ Status VariantDocCompactWriter::_write_doc_value_column(const TabletColumn& pare
int bucket_value = std::stoi(doc_value_column_path.substr(pos + 1));
TabletColumn doc_value_column =
variant_util::create_doc_value_column(parent_column, bucket_value);
- _init_column_meta(_opts.meta, column_id, doc_value_column, _opts.compression_type);
+ _init_column_meta(_opts.meta, column_id, doc_value_column, _opts);
RETURN_IF_ERROR(ColumnWriter::create_map_writer(_opts, &doc_value_column, _opts.file_writer,
&_doc_value_column_writer));
RETURN_IF_ERROR(_doc_value_column_writer->init());
diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h b/be/src/storage/segment/variant/variant_column_writer_impl.h
index 639e58bb65f..578365dc80e 100644
--- a/be/src/storage/segment/variant/variant_column_writer_impl.h
+++ b/be/src/storage/segment/variant/variant_column_writer_impl.h
@@ -288,7 +288,7 @@ private:
};
void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column,
- CompressionTypePB compression_type);
+ const ColumnWriterOptions& opts);
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp
index 97821a7d1a4..fed06831fbd 100644
--- a/be/src/storage/segment/vertical_segment_writer.cpp
+++ b/be/src/storage/segment/vertical_segment_writer.cpp
@@ -68,6 +68,7 @@
#include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext
#include "storage/rowset/segment_creator.h"
#include "storage/segment/column_writer.h" // ColumnWriter
+#include "storage/segment/encoding_info.h"
#include "storage/segment/external_col_meta_util.h"
#include "storage/segment/historical_row_retriever.h"
#include "storage/segment/page_io.h"
@@ -162,11 +163,12 @@ VerticalSegmentWriter::~VerticalSegmentWriter() {
}
void VerticalSegmentWriter::_init_column_meta(ColumnMetaPB* meta, uint32_t column_id,
- const TabletColumn& column) {
+ const TabletColumn& column,
+ const ColumnWriterOptions& opts) {
meta->set_column_id(column_id);
meta->set_type(int(column.type()));
meta->set_length(cast_set<int32_t>(column.length()));
- meta->set_encoding(DEFAULT_ENCODING);
+ meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column));
meta->set_compression(_opts.compression_type);
meta->set_is_nullable(column.is_nullable());
meta->set_default_value(column.default_value());
@@ -178,7 +180,7 @@ void VerticalSegmentWriter::_init_column_meta(ColumnMetaPB* meta, uint32_t colum
}
meta->set_unique_id(column.unique_id());
for (uint32_t i = 0; i < column.get_subtype_count(); ++i) {
- _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i));
+ _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts);
}
if (column.is_variant_type()) {
meta->set_variant_max_subcolumns_count(column.variant_max_subcolumns_count());
@@ -193,8 +195,9 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
const TabletSchemaSPtr& tablet_schema) {
ColumnWriterOptions opts;
opts.meta = _footer.add_columns();
+ opts.storage_format = tablet_schema->storage_format();
- _init_column_meta(opts.meta, cid, column);
+ _init_column_meta(opts.meta, cid, column, opts);
// now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS
// except for columns whose type don't support zone map.
@@ -300,16 +303,11 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
}
})
if (column.is_row_store_column()) {
- // smaller page size for row store column
+ // smaller page size for row store column; encoding is already set to PLAIN /
+ // PLAIN_V2 by _init_column_meta via resolve_default_encoding().
auto page_size = _tablet_schema->row_store_page_size();
opts.data_page_size =
(page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
- // Row store data is already serialized as a single blob. Keep it on plain pages
- // to avoid introducing dictionary pages for the hidden row store column.
- opts.meta->set_encoding(_tablet_schema->binary_plain_encoding_default_impl() ==
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
- ? PLAIN_ENCODING_V2
- : PLAIN_ENCODING);
}
opts.rowset_ctx = _opts.rowset_ctx;
@@ -318,10 +316,6 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
opts.footer = &_footer;
opts.input_rs_readers = _opts.rowset_ctx->input_rs_readers;
- opts.encoding_preference = {.integer_type_default_use_plain_encoding =
- _tablet_schema->integer_type_default_use_plain_encoding(),
- .binary_plain_encoding_default_impl =
- _tablet_schema->binary_plain_encoding_default_impl()};
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer));
RETURN_IF_ERROR(writer->init());
@@ -1455,7 +1449,7 @@ Status VerticalSegmentWriter::_write_footer() {
// Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version
- if (_tablet_schema->is_external_segment_column_meta_used()) {
+ if (_tablet_schema->storage_format() == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) {
_footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META);
VLOG_DEBUG << "use external column meta";
// External ColumnMetaPB writing (optional)
diff --git a/be/src/storage/segment/vertical_segment_writer.h b/be/src/storage/segment/vertical_segment_writer.h
index 8aa8a24a7c4..e6ee4933dd6 100644
--- a/be/src/storage/segment/vertical_segment_writer.h
+++ b/be/src/storage/segment/vertical_segment_writer.h
@@ -129,7 +129,8 @@ public:
}
private:
- void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column);
+ void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column,
+ const ColumnWriterOptions& opts);
Status _create_column_writer(uint32_t cid, const TabletColumn& column,
const TabletSchemaSPtr& schema);
uint64_t _estimated_remaining_size();
diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp
index d11b69f59f3..e7d7742c1b4 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -264,21 +264,11 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
case TStorageFormat::V1:
break;
case TStorageFormat::V3:
- schema_pb_for_data->set_is_external_segment_column_meta_used(true);
- _schema->set_external_segment_meta_used_default(true);
-
- schema_pb_for_data->set_integer_type_default_use_plain_encoding(true);
- _schema->set_integer_type_default_use_plain_encoding(true);
- schema_pb_for_data->set_binary_plain_encoding_default_impl(
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
- _schema->set_binary_plain_encoding_default_impl(
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
-
+ schema_pb_for_data->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
+ _schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
if (schema_pb_for_row_binlog != nullptr) {
- schema_pb_for_row_binlog->set_is_external_segment_column_meta_used(true);
- schema_pb_for_row_binlog->set_integer_type_default_use_plain_encoding(true);
- schema_pb_for_row_binlog->set_binary_plain_encoding_default_impl(
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
+ schema_pb_for_row_binlog->set_storage_format(
+ TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
}
break;
default:
diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp
index ed729dfec4c..8397e6d619a 100644
--- a/be/src/storage/tablet/tablet_schema.cpp
+++ b/be/src/storage/tablet/tablet_schema.cpp
@@ -1289,16 +1289,16 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
_row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
schema.row_store_column_unique_ids().end());
_deprecated_enable_variant_flatten_nested = schema.enable_variant_flatten_nested();
- if (schema.has_is_external_segment_column_meta_used()) {
- _is_external_segment_column_meta_used = schema.is_external_segment_column_meta_used();
+ if (schema.has_storage_format()) {
+ _storage_format = schema.storage_format();
+ } else if (schema.is_external_segment_column_meta_used() ||
+ schema.integer_type_default_use_plain_encoding() ||
+ schema.binary_plain_encoding_default_impl() ==
+ BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2) {
+ // Old PB without storage_format: any of the three legacy V3-flavor flags implies V3.
+ _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3;
} else {
- _is_external_segment_column_meta_used = false;
- }
- if (schema.has_integer_type_default_use_plain_encoding()) {
- _integer_type_default_use_plain_encoding = schema.integer_type_default_use_plain_encoding();
- }
- if (schema.has_binary_plain_encoding_default_impl()) {
- _binary_plain_encoding_default_impl = schema.binary_plain_encoding_default_impl();
+ _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
}
update_metadata_size();
}
@@ -1571,11 +1571,19 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign(
_row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end());
tablet_schema_pb->set_enable_variant_flatten_nested(_deprecated_enable_variant_flatten_nested);
- tablet_schema_pb->set_is_external_segment_column_meta_used(
- _is_external_segment_column_meta_used);
- tablet_schema_pb->set_integer_type_default_use_plain_encoding(
- _integer_type_default_use_plain_encoding);
- tablet_schema_pb->set_binary_plain_encoding_default_impl(_binary_plain_encoding_default_impl);
+ tablet_schema_pb->set_storage_format(_storage_format);
+ // Backward downgrade safety: if a new BE rewrites tablet_meta.json carrying only
+ // storage_format and the deployment is then rolled back to an old BE, the old BE
+ // does not know the new field and would default-derive V2 for a V3 tablet, causing
+ // it to write V2-encoded segments into a V3 tablet. Redundantly emit the three
+ // legacy V3-flavor flags so old BEs can recover the format via the prior "any of
+ // these implies V3" rule. ~3 bytes per schema PB; only paid for V3 tablets.
+ if (_storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) {
+ tablet_schema_pb->set_is_external_segment_column_meta_used(true);
+ tablet_schema_pb->set_integer_type_default_use_plain_encoding(true);
+ tablet_schema_pb->set_binary_plain_encoding_default_impl(
+ BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
+ }
auto column_groups_pb = tablet_schema_pb->mutable_seq_map();
for (const auto& it : _seq_col_uid_to_value_cols_uid) {
uint32_t key = it.first;
@@ -1968,12 +1976,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
b._deprecated_enable_variant_flatten_nested) {
return false;
}
- if (a._is_external_segment_column_meta_used != b._is_external_segment_column_meta_used)
- return false;
- if (a._integer_type_default_use_plain_encoding != b._integer_type_default_use_plain_encoding)
- return false;
- if (a._binary_plain_encoding_default_impl != b._binary_plain_encoding_default_impl)
- return false;
+ if (a._storage_format != b._storage_format) return false;
return true;
}
diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h
index dec976d26e0..300b291bfb9 100644
--- a/be/src/storage/tablet/tablet_schema.h
+++ b/be/src/storage/tablet/tablet_schema.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/olap_common.pb.h>
#include <gen_cpp/olap_file.pb.h>
@@ -753,30 +754,8 @@ public:
bool has_pruned_columns() const { return !_pruned_columns_data_type.empty(); }
- // Whether new segments use externalized ColumnMetaPB layout (CMO) by default
- bool is_external_segment_column_meta_used() const {
- return _is_external_segment_column_meta_used;
- }
-
- void set_external_segment_meta_used_default(bool v) {
- _is_external_segment_column_meta_used = v;
- }
-
- bool integer_type_default_use_plain_encoding() const {
- return _integer_type_default_use_plain_encoding;
- }
-
- void set_integer_type_default_use_plain_encoding(bool v) {
- _integer_type_default_use_plain_encoding = v;
- }
-
- BinaryPlainEncodingTypePB binary_plain_encoding_default_impl() const {
- return _binary_plain_encoding_default_impl;
- }
-
- void set_binary_plain_encoding_default_impl(BinaryPlainEncodingTypePB impl) {
- _binary_plain_encoding_default_impl = impl;
- }
+ TabletStorageFormatPB storage_format() const { return _storage_format; }
+ void set_storage_format(TabletStorageFormatPB v) { _storage_format = v; }
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
@@ -858,11 +837,10 @@ private:
std::unordered_map<int32_t, PatternToIndex> _index_by_unique_id_with_pattern;
// Default behavior for new segments: use external ColumnMeta region + CMO table if true
- bool _is_external_segment_column_meta_used = false;
-
- bool _integer_type_default_use_plain_encoding {false};
- BinaryPlainEncodingTypePB _binary_plain_encoding_default_impl {
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1};
+ // Persisted tablet storage format. Authoritative source for "is this tablet V3?"
+ // decisions in the segment write paths. Old PBs without this field are upgraded in
+ // init_from_pb() by deriving V3 from any of the three legacy V3-flavor flags.
+ TabletStorageFormatPB _storage_format {TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2};
// Sequence column unique id mapping to value columns unique id
std::unordered_map<uint32_t, std::vector<uint32_t>> _seq_col_uid_to_value_cols_uid;
// Value column unique id mapping to sequence column unique id(also map sequence column it self)
diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp
index cdfb84431d3..2f3f32226ed 100644
--- a/be/test/exec/common/schema_util_rowset_test.cpp
+++ b/be/test/exec/common/schema_util_rowset_test.cpp
@@ -229,7 +229,9 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
@@ -279,7 +281,9 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows1");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
@@ -329,7 +333,7 @@ TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) {
// 2. create tablet and data dir
TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema));
// First write a few rowsets with external_segment_meta_used_default = false (old format)
- tablet_schema->set_external_segment_meta_used_default(false);
+ tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows_mixed");
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok());
@@ -362,7 +366,7 @@ TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) {
}
// 3.2 flip tablet default to enable external column meta for subsequent segments
- tablet_schema->set_external_segment_meta_used_default(true);
+ tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
// 3.3 write a few new-format rowsets (external meta enabled)
for (int i = 0; i < 3; ++i) {
@@ -395,7 +399,9 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 12345;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -602,7 +608,9 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref,
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
std::unique_ptr<DataDir> _data_dir = std::make_unique<DataDir>(*_engine_ref, _absolute_dir);
@@ -695,7 +703,9 @@ TEST_F(SchemaUtilRowsetTest, some_test_for_subcolumn_writer) {
std::cout << compaction_schema->dump_structure() << std::endl;
// this is v1.key1
TabletColumn column = compaction_schema->column(2);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(
ColumnWriter::create_variant_writer(opts, &column, file_writer.get(), &writer).ok());
@@ -736,7 +746,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
diff --git a/be/test/storage/segment/binary_dict_page_test.cpp b/be/test/storage/segment/binary_dict_page_test.cpp
index 70f3b1ada15..36d455c72a5 100644
--- a/be/test/storage/segment/binary_dict_page_test.cpp
+++ b/be/test/storage/segment/binary_dict_page_test.cpp
@@ -143,7 +143,7 @@ public:
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 256 * 1024;
- options.encoding_preference.binary_plain_encoding_default_impl =
+ options.dict_binary_plain_encoding =
use_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
: BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
@@ -163,13 +163,14 @@ public:
<< "Expected encoding type does not match when use_v2=" << use_v2;
}
- void test_by_small_data_size(const std::vector<Slice>& slices,
- EncodingPreference encoding_preference = EncodingPreference()) {
+ void test_by_small_data_size(const std::vector<Slice>& slices, bool use_plain_v2 = false) {
// Encode
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 256 * 1024;
- options.encoding_preference = encoding_preference;
+ options.dict_binary_plain_encoding =
+ use_plain_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
+ : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
PageBuilder* builder_ptr = nullptr;
Status ret0 = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -285,15 +286,16 @@ public:
}
}
- void test_with_large_data_size(const std::vector<Slice>& contents,
- EncodingPreference encoding_preference = EncodingPreference()) {
+ void test_with_large_data_size(const std::vector<Slice>& contents, bool use_plain_v2 = false) {
// Encode
PageBuilderOptions options;
// Use smaller page sizes to ensure we trigger fallback scenario
// where dictionary gets full and we switch to plain encoding
options.data_page_size = 64 * 1024; // 64KB data page
options.dict_page_size = 1024; // 1KB dict page to trigger fallback
- options.encoding_preference = encoding_preference;
+ options.dict_binary_plain_encoding =
+ use_plain_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2
+ : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
PageBuilder* builder_ptr = nullptr;
Status ret0 = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -603,8 +605,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsDictionaryPageEncoding) {
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 256 * 1024;
- options.encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
+ options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
PageBuilder* builder_ptr = nullptr;
Status status = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -644,8 +645,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsDictionaryPageEncoding) {
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 256 * 1024;
- options.encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
+ options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
PageBuilder* builder_ptr = nullptr;
Status status = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -707,8 +707,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) {
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 128; // Small dict size to force fallback
- options.encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
+ options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
PageBuilder* builder_ptr = nullptr;
Status status = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -739,13 +738,9 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) {
status = page_builder->reset();
EXPECT_TRUE(status.ok());
- // Access private member _fallback_binary_encoding_type to verify
- EXPECT_EQ(PLAIN_ENCODING, page_builder->_fallback_binary_encoding_type)
- << "Fallback encoding should be PLAIN_ENCODING with V1 preference";
-
- // Also check the dict word page encoding type
- EXPECT_EQ(PLAIN_ENCODING, page_builder->_dict_word_page_encoding_type)
- << "Dict word page encoding should be PLAIN_ENCODING with V1 preference";
+ // Verify the binary-plain flavor (used for both dict word page and fallback data page).
+ EXPECT_EQ(PLAIN_ENCODING, page_builder->_binary_plain_encoding_type)
+ << "Binary plain encoding should be PLAIN_ENCODING with V1 preference";
// Check the actual encoding type used (should have fallen back)
EXPECT_EQ(PLAIN_ENCODING, page_builder->_encoding_type)
@@ -757,8 +752,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) {
PageBuilderOptions options;
options.data_page_size = 256 * 1024;
options.dict_page_size = 128; // Small dict size to force fallback
- options.encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
+ options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
PageBuilder* builder_ptr = nullptr;
Status status = BinaryDictPageBuilder::create(&builder_ptr, options);
@@ -789,13 +783,9 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) {
status = page_builder->reset();
EXPECT_TRUE(status.ok());
- // Access private member _fallback_binary_encoding_type to verify
- EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_fallback_binary_encoding_type)
- << "Fallback encoding should be PLAIN_ENCODING_V2 with V2 preference";
-
- // Also check the dict word page encoding type
- EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_dict_word_page_encoding_type)
- << "Dict word page encoding should be PLAIN_ENCODING_V2 with V2 preference";
+ // Verify the binary-plain flavor (used for both dict word page and fallback data page).
+ EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_binary_plain_encoding_type)
+ << "Binary plain encoding should be PLAIN_ENCODING_V2 with V2 preference";
// Check the actual encoding type used (should have fallen back)
EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_encoding_type)
@@ -824,10 +814,7 @@ TEST_F(BinaryDictPageTest, TestSmallDataWithConfigFalse) {
slices.emplace_back(str);
}
- EncodingPreference encoding_preference;
- encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
- test_by_small_data_size(slices, encoding_preference);
+ test_by_small_data_size(slices, /*use_plain_v2=*/false);
}
TEST_F(BinaryDictPageTest, TestSmallDataWithConfigTrue) {
@@ -837,10 +824,7 @@ TEST_F(BinaryDictPageTest, TestSmallDataWithConfigTrue) {
slices.emplace_back(str);
}
- EncodingPreference encoding_preference;
- encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
- test_by_small_data_size(slices, encoding_preference);
+ test_by_small_data_size(slices, /*use_plain_v2=*/true);
}
TEST_F(BinaryDictPageTest, TestLargeDataWithConfigFalse) {
@@ -860,11 +844,8 @@ TEST_F(BinaryDictPageTest, TestLargeDataWithConfigFalse) {
slices.push_back(str);
}
- EncodingPreference encoding_preference;
- encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
LOG(INFO) << "Testing large data with V1 preference, entry count: " << slices.size();
- test_with_large_data_size(slices, encoding_preference);
+ test_with_large_data_size(slices, /*use_plain_v2=*/false);
}
TEST_F(BinaryDictPageTest, TestLargeDataWithConfigTrue) {
@@ -884,11 +865,8 @@ TEST_F(BinaryDictPageTest, TestLargeDataWithConfigTrue) {
slices.push_back(str);
}
- EncodingPreference encoding_preference;
- encoding_preference.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
LOG(INFO) << "Testing large data with V2 preference, entry count: " << slices.size();
- test_with_large_data_size(slices, encoding_preference);
+ test_with_large_data_size(slices, /*use_plain_v2=*/true);
}
} // namespace segment_v2
diff --git a/be/test/storage/segment/column_meta_accessor_test.cpp b/be/test/storage/segment/column_meta_accessor_test.cpp
index bdcdd39d8c8..48c2581b841 100644
--- a/be/test/storage/segment/column_meta_accessor_test.cpp
+++ b/be/test/storage/segment/column_meta_accessor_test.cpp
@@ -27,6 +27,7 @@
#include "common/consts.h"
#include "core/field.h"
#include "io/fs/local_file_system.h"
+#include "storage/segment/external_col_meta_util.h"
#include "storage/segment/segment.h"
#include "storage/segment/segment_writer.h"
#include "util/coding.h"
@@ -639,7 +640,7 @@ TEST(ColumnMetaAccessorTest, FooterSizeWithManyColumnsExternalVsInline) {
TabletSchemaSPtr external_schema = create_schema(columns, UNIQUE_KEYS);
// Enable external ColumnMetaPB for the second schema so that SegmentWriter
// produces a V3 footer with externalized column meta region.
- external_schema->set_external_segment_meta_used_default(true);
+ external_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
// 2. Common SegmentWriter options and row generator.
SegmentWriterOptions opts;
@@ -712,8 +713,7 @@ TEST(ColumnMetaAccessorTest, RowStoreColumnDoesNotUseDictEncoding) {
columns.emplace_back(create_row_store_test_column(kRowStoreUid));
auto tablet_schema = create_schema(columns, UNIQUE_KEYS);
- tablet_schema->set_binary_plain_encoding_default_impl(
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
+ tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
SegmentWriterOptions opts;
opts.enable_unique_key_merge_on_write = false;
@@ -739,9 +739,14 @@ TEST(ColumnMetaAccessorTest, RowStoreColumnDoesNotUseDictEncoding) {
SegmentFooterPB footer;
ASSERT_TRUE(read_footer_from_file(reader, &footer).ok());
- ASSERT_EQ(2, footer.columns_size());
-
- const auto& row_store_meta = footer.columns(1);
+ // V3 schemas externalize column meta -- read row_store col (col_id=1) from the
+ // external region instead of the inline footer.columns().
+ ExternalColMetaUtil::ExternalMetaPointers ptrs;
+ ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok());
+ ColumnMetaPB row_store_meta;
+ ASSERT_TRUE(
+ ExternalColMetaUtil::read_col_meta(reader, footer, ptrs, /*col_id=*/1, &row_store_meta)
+ .ok());
EXPECT_EQ(kRowStoreUid, row_store_meta.unique_id());
EXPECT_EQ(static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRING), row_store_meta.type());
EXPECT_EQ(PLAIN_ENCODING_V2, row_store_meta.encoding());
diff --git a/be/test/storage/segment/column_reader_cache_test.cpp b/be/test/storage/segment/column_reader_cache_test.cpp
index 6a11b77f32f..da8aa13d3bf 100644
--- a/be/test/storage/segment/column_reader_cache_test.cpp
+++ b/be/test/storage/segment/column_reader_cache_test.cpp
@@ -29,6 +29,7 @@
#include "io/fs/file_reader.h"
#include "storage/segment/column_meta_accessor.h"
#include "storage/segment/column_reader.h"
+#include "storage/segment/encoding_info.h"
#include "storage/segment/mock/mock_segment.h"
#include "storage/segment/segment.h"
#include "storage/segment/variant/variant_column_reader.h"
@@ -57,6 +58,14 @@ private:
class ColumnReaderCacheTest : public ::testing::Test {
protected:
+ // EncodingInfo friended ColumnReaderCacheTest so the fixture can pick a valid V2
+ // default encoding for the synthetic ColumnMetaPBs constructed by these tests.
+ // TEST_F bodies subclass this fixture and call the helper (friendship doesn't carry
+ // to subclasses).
+ static segment_v2::EncodingTypePB get_v2_default_encoding(FieldType t) {
+ return segment_v2::EncodingInfo::get_v2_default_encoding(t);
+ }
+
void SetUp() override {
// Set up test configuration
config::max_segment_partial_column_cache_size = 3;
@@ -141,7 +150,7 @@ TEST_F(ColumnReaderCacheTest, BasicCacheOperations) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta.set_unique_id(1);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
@@ -174,7 +183,7 @@ TEST_F(ColumnReaderCacheTest, LRUEviction) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta.set_unique_id(i);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
@@ -202,11 +211,11 @@ TEST_F(ColumnReaderCacheTest, LRUOrderMaintenance) {
ColumnMetaPB col_meta1, col_meta2;
col_meta1.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta1.set_unique_id(1);
- col_meta1.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta1.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta1.type())));
col_meta1.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
col_meta2.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta2.set_unique_id(2);
- col_meta2.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta2.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta2.type())));
col_meta2.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta1, col_meta2});
@@ -230,7 +239,7 @@ TEST_F(ColumnReaderCacheTest, LRUOrderMaintenance) {
ColumnMetaPB col_meta3;
col_meta3.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta3.set_unique_id(3);
- col_meta3.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta3.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta3.type())));
col_meta3.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta1, col_meta2, col_meta3});
@@ -254,14 +263,14 @@ TEST_F(ColumnReaderCacheTest, VariantColumnPathReading) {
ColumnMetaPB variant_meta;
variant_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_VARIANT));
variant_meta.set_unique_id(1);
- variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ variant_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(variant_meta.type())));
variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
// Create subcolumn meta
ColumnMetaPB subcol_meta;
subcol_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_STRING));
subcol_meta.set_unique_id(2);
- subcol_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ subcol_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(subcol_meta.type())));
subcol_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({variant_meta, subcol_meta});
@@ -289,7 +298,7 @@ TEST_F(ColumnReaderCacheTest, NonExistentVariantPath) {
ColumnMetaPB variant_meta;
variant_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_VARIANT));
variant_meta.set_unique_id(1);
- variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ variant_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(variant_meta.type())));
variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({variant_meta});
@@ -307,7 +316,7 @@ TEST_F(ColumnReaderCacheTest, ConcurrentAccess) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta.set_unique_id(1);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
@@ -342,7 +351,7 @@ TEST_F(ColumnReaderCacheTest, CacheStatistics) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta.set_unique_id(1);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
@@ -372,7 +381,7 @@ TEST_F(ColumnReaderCacheTest, DifferentColumnTypes) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(types[i]));
col_meta.set_unique_id(static_cast<int32_t>(i + 1));
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
@@ -393,11 +402,11 @@ TEST_F(ColumnReaderCacheTest, NodeHintParameter) {
ColumnMetaPB variant_meta, subcol_meta;
variant_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_VARIANT));
variant_meta.set_unique_id(1);
- variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ variant_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(variant_meta.type())));
variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
subcol_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_STRING));
subcol_meta.set_unique_id(2);
- subcol_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ subcol_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(subcol_meta.type())));
subcol_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({variant_meta, subcol_meta});
@@ -431,7 +440,7 @@ TEST_F(ColumnReaderCacheTest, PerformanceUnderLoad) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_INT));
col_meta.set_unique_id(i);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
all_columns.push_back(col_meta);
}
@@ -471,7 +480,7 @@ TEST_F(ColumnReaderCacheTest, EmptyPath) {
ColumnMetaPB col_meta;
col_meta.set_type(static_cast<int32_t>(FieldType::OLAP_FIELD_TYPE_VARIANT));
col_meta.set_unique_id(1);
- col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING);
+ col_meta.set_encoding(get_v2_default_encoding(static_cast<FieldType>(col_meta.type())));
col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX);
setup_segment_footer({col_meta});
diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp
index 666363c9566..641e84e0759 100644
--- a/be/test/storage/segment/encoding_info_test.cpp
+++ b/be/test/storage/segment/encoding_info_test.cpp
@@ -39,13 +39,23 @@ class EncodingInfoTest : public testing::Test {
public:
EncodingInfoTest() {}
virtual ~EncodingInfoTest() {}
+
+protected:
+ // EncodingInfo has friended EncodingInfoTest so this fixture can poke the private
+ // get_v2/v3_default_encoding lookup tables. TEST_F bodies subclass this fixture and
+ // call these helpers (they don't get the friend permission directly).
+ static EncodingTypePB get_v2_default_encoding(FieldType t) {
+ return EncodingInfo::get_v2_default_encoding(t);
+ }
+ static EncodingTypePB get_v3_default_encoding(FieldType t) {
+ return EncodingInfo::get_v3_default_encoding(t);
+ }
};
TEST_F(EncodingInfoTest, normal) {
constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT;
const EncodingInfo* encoding_info = nullptr;
- EncodingPreference encoding_preference;
- auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, PLAIN_ENCODING, &encoding_info);
EXPECT_TRUE(status.ok());
EXPECT_NE(nullptr, encoding_info);
}
@@ -53,98 +63,45 @@ TEST_F(EncodingInfoTest, normal) {
TEST_F(EncodingInfoTest, no_encoding) {
constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT;
const EncodingInfo* encoding_info = nullptr;
- EncodingPreference encoding_preference;
- auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, DICT_ENCODING, &encoding_info);
EXPECT_FALSE(status.ok());
}
-TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) {
- // Helper lambda to test string/JSON types with DICT_ENCODING as default
- auto test_dict_type_encoding = [](FieldType type, const std::string& type_name) {
- // Test with BINARY_PLAIN_ENCODING_V1 (default)
- // String and JSON types default to DICT_ENCODING
- EncodingPreference pref_v1;
- pref_v1.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
- EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false);
- EXPECT_EQ(DICT_ENCODING, encoding_type)
- << "Type " << type_name << " should use DICT_ENCODING with V1 preference";
-
- // Test with BINARY_PLAIN_ENCODING_V2
- // Preference doesn't affect DICT_ENCODING types
- EncodingPreference pref_v2;
- pref_v2.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
- encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false);
- EXPECT_EQ(DICT_ENCODING, encoding_type)
- << "Type " << type_name << " should still use DICT_ENCODING with V2 preference";
+TEST_F(EncodingInfoTest, v2_vs_v3_defaults) {
+ // String / JSON / variant: default is DICT_ENCODING for both V2 and V3.
+ auto check_same = [](FieldType type, const std::string& name, EncodingTypePB expected) {
+ EXPECT_EQ(expected, get_v2_default_encoding(type)) << name << " v2 default";
+ EXPECT_EQ(expected, get_v3_default_encoding(type)) << name << " v3 default";
};
-
- // Helper lambda to test aggregate state types with PLAIN_ENCODING as default
- auto test_plain_type_encoding = [](FieldType type, const std::string& type_name) {
- // Test with BINARY_PLAIN_ENCODING_V1 (default)
- EncodingPreference pref_v1;
- pref_v1.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
- EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false);
- EXPECT_EQ(PLAIN_ENCODING, encoding_type)
- << "Type " << type_name << " should use PLAIN_ENCODING with V1 preference";
-
- // Test with BINARY_PLAIN_ENCODING_V2
- EncodingPreference pref_v2;
- pref_v2.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
- encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false);
- EXPECT_EQ(PLAIN_ENCODING_V2, encoding_type)
- << "Type " << type_name << " should use PLAIN_ENCODING_V2 with V2 preference";
+ check_same(FieldType::OLAP_FIELD_TYPE_VARCHAR, "VARCHAR", DICT_ENCODING);
+ check_same(FieldType::OLAP_FIELD_TYPE_STRING, "STRING", DICT_ENCODING);
+ check_same(FieldType::OLAP_FIELD_TYPE_CHAR, "CHAR", DICT_ENCODING);
+ check_same(FieldType::OLAP_FIELD_TYPE_JSONB, "JSONB", DICT_ENCODING);
+ check_same(FieldType::OLAP_FIELD_TYPE_VARIANT, "VARIANT", DICT_ENCODING);
+
+ // Aggregate/binary-flavored types: V2=PLAIN, V3=PLAIN_V2.
+ auto check_split = [](FieldType type, const std::string& name) {
+ EXPECT_EQ(PLAIN_ENCODING, get_v2_default_encoding(type)) << name << " v2 default";
+ EXPECT_EQ(PLAIN_ENCODING_V2, get_v3_default_encoding(type)) << name << " v3 default";
};
+ check_split(FieldType::OLAP_FIELD_TYPE_HLL, "HLL");
+ check_split(FieldType::OLAP_FIELD_TYPE_BITMAP, "BITMAP");
+ check_split(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE");
+ check_split(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE");
- // Test string types (default to DICT_ENCODING, not affected by preference)
- test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_VARCHAR, "VARCHAR");
- test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_STRING, "STRING");
- test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_CHAR, "CHAR");
-
- // Test JSON/variant types (default to DICT_ENCODING, not affected by preference)
- test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_JSONB, "JSONB");
- test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_VARIANT, "VARIANT");
-
- // Test aggregate state types (default to PLAIN_ENCODING, affected by preference)
- test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_HLL, "HLL");
- test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_BITMAP, "BITMAP");
- test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE");
- test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE");
-
- // Test non-binary type (BIGINT) - should not be affected by binary preference
+ // Signed integers: V2=BIT_SHUFFLE, V3=PLAIN.
constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT;
+ EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(bigint_type));
+ EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(bigint_type));
- // Test with plain encoding disabled for integers (default)
- EncodingPreference pref_plain_disabled;
- pref_plain_disabled.integer_type_default_use_plain_encoding = false;
- pref_plain_disabled.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
- EncodingTypePB encoding_type =
- EncodingInfo::get_default_encoding(bigint_type, pref_plain_disabled, false);
- EXPECT_EQ(BIT_SHUFFLE, encoding_type);
-
- // Test with plain encoding enabled for integers
- EncodingPreference pref_plain_enabled;
- pref_plain_enabled.integer_type_default_use_plain_encoding = true;
- pref_plain_enabled.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1;
- encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false);
- EXPECT_EQ(PLAIN_ENCODING, encoding_type);
-
- // Verify binary preference doesn't affect integer types
- pref_plain_enabled.binary_plain_encoding_default_impl =
- BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
- encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false);
- EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING
+ // Value-seek default is only registered for VARCHAR (the only production caller).
+ EXPECT_EQ(PREFIX_ENCODING,
+ EncodingInfo::get_index_column_encoding(FieldType::OLAP_FIELD_TYPE_VARCHAR));
+ EXPECT_EQ(UNKNOWN_ENCODING, EncodingInfo::get_index_column_encoding(bigint_type));
}
// Comprehensive test for _data_page_pre_decoder for all encoding types
TEST_F(EncodingInfoTest, test_all_pre_decoders) {
- EncodingPreference encoding_preference;
-
// Test BIT_SHUFFLE encoding - should have BitShufflePagePreDecoder
// Test various integer types
std::vector<FieldType> bitshuffle_types = {
@@ -163,7 +120,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : bitshuffle_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(type, BIT_SHUFFLE, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, BIT_SHUFFLE, &encoding_info);
if (status.ok()) {
ASSERT_NE(nullptr, encoding_info);
auto* pre_decoder = encoding_info->get_data_page_pre_decoder();
@@ -185,7 +142,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : dict_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, DICT_ENCODING, &encoding_info);
ASSERT_TRUE(status.ok()) << "Type " << static_cast<int>(type)
<< " should support DICT_ENCODING";
ASSERT_NE(nullptr, encoding_info);
@@ -209,8 +166,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : plain_v2_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status =
- EncodingInfo::get(type, PLAIN_ENCODING_V2, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, PLAIN_ENCODING_V2, &encoding_info);
ASSERT_TRUE(status.ok()) << "Type " << static_cast<int>(type)
<< " should support PLAIN_ENCODING_V2";
ASSERT_NE(nullptr, encoding_info);
@@ -257,7 +213,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : plain_encoding_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, PLAIN_ENCODING, &encoding_info);
if (status.ok() && encoding_info != nullptr) {
auto* pre_decoder = encoding_info->get_data_page_pre_decoder();
EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast<int>(type)
@@ -276,7 +232,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : for_encoding_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(type, FOR_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, FOR_ENCODING, &encoding_info);
if (status.ok() && encoding_info != nullptr) {
auto* pre_decoder = encoding_info->get_data_page_pre_decoder();
EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast<int>(type)
@@ -293,7 +249,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
for (auto type : prefix_encoding_types) {
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(type, PREFIX_ENCODING, encoding_preference, &encoding_info);
+ auto status = EncodingInfo::get(type, PREFIX_ENCODING, &encoding_info);
if (status.ok() && encoding_info != nullptr) {
auto* pre_decoder = encoding_info->get_data_page_pre_decoder();
EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast<int>(type)
@@ -304,8 +260,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
// Test RLE - should NOT have pre_decoder (only for BOOL)
{
const EncodingInfo* encoding_info = nullptr;
- auto status = EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_BOOL, RLE, encoding_preference,
- &encoding_info);
+ auto status = EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_BOOL, RLE, &encoding_info);
if (status.ok() && encoding_info != nullptr) {
auto* pre_decoder = encoding_info->get_data_page_pre_decoder();
EXPECT_EQ(nullptr, pre_decoder) << "BOOL with RLE should NOT have pre_decoder";
@@ -313,5 +268,301 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) {
}
}
+// ============================================================================
+// Behavior-locking tests. The expectation tables below are the authoritative
+// contract for what each get_*_encoding API returns per FieldType. Changing
+// a value here is a deliberate behavior change and must be justified.
+// ============================================================================
+
+namespace {
+
+// (type, expected_encoding) row for a default-encoding test.
+struct DefaultExpectation {
+ FieldType type;
+ EncodingTypePB expected;
+ const char* name;
+};
+
+// (type, encoding, should_exist) row for the global encoding_map completeness test.
+struct EncodingMapEntry {
+ FieldType type;
+ EncodingTypePB encoding;
+ bool should_exist;
+ const char* name;
+};
+
+// Expected V3 default per type. Differs from V2 default in two type families:
+// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V2 (vs V2's PLAIN)
+// - signed integers (TINYINT..LARGEINT): PLAIN_ENCODING (vs V2's BIT_SHUFFLE)
+const std::vector<DefaultExpectation> kV3DefaultExpect = {
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, "TINYINT"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, "SMALLINT"},
+ {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING, "INT"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING, "BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, "LARGEINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, "UNSIGNED_BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, "JSONB"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, "VARIANT"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, "BOOL"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, "DATE"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, "DATEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, "DATETIMEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, "DATETIME"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, "TIMESTAMPTZ"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, "DECIMAL"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, "DECIMAL32"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, "DECIMAL64"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, "DECIMAL128I"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, "DECIMAL256"},
+ {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, "IPV4"},
+ {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, "IPV6"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2, "HLL"},
+ {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2, "BITMAP"},
+ {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2, "QUANTILE_STATE"},
+ {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2, "AGG_STATE"},
+};
+
+// Expected V2 (non-V3) default per type.
+const std::vector<DefaultExpectation> kV2DefaultExpect = {
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE, "TINYINT"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE, "SMALLINT"},
+ {FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, "INT"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE, "BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE, "LARGEINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, "UNSIGNED_BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, "JSONB"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, "VARIANT"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, "BOOL"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, "DATE"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, "DATEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, "DATETIMEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, "DATETIME"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, "TIMESTAMPTZ"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, "DECIMAL"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, "DECIMAL32"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, "DECIMAL64"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, "DECIMAL128I"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, "DECIMAL256"},
+ {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, "IPV4"},
+ {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, "IPV6"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING, "HLL"},
+ {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING, "BITMAP"},
+ {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING, "QUANTILE_STATE"},
+ {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING, "AGG_STATE"},
+};
+
+// Expected index_column_encoding per type. Only VARCHAR is consulted in production
+// (PrimaryKeyIndexBuilder::init hardcodes VARCHAR); all other entries return
+// UNKNOWN_ENCODING because they were never queried and have been removed.
+const std::vector<DefaultExpectation> kIndexColumnEncodingExpect = {
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, UNKNOWN_ENCODING, "TINYINT"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, UNKNOWN_ENCODING, "SMALLINT"},
+ {FieldType::OLAP_FIELD_TYPE_INT, UNKNOWN_ENCODING, "INT"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, UNKNOWN_ENCODING, "BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, UNKNOWN_ENCODING, "LARGEINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, UNKNOWN_ENCODING, "UNSIGNED_BIGINT"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, UNKNOWN_ENCODING, "UNSIGNED_INT"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, UNKNOWN_ENCODING, "FLOAT"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, UNKNOWN_ENCODING, "DOUBLE"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, UNKNOWN_ENCODING, "CHAR"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, "VARCHAR"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, UNKNOWN_ENCODING, "STRING"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, UNKNOWN_ENCODING, "JSONB"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, UNKNOWN_ENCODING, "VARIANT"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, UNKNOWN_ENCODING, "BOOL"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, UNKNOWN_ENCODING, "DATE"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, UNKNOWN_ENCODING, "DATEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, UNKNOWN_ENCODING, "DATETIMEV2"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, UNKNOWN_ENCODING, "DATETIME"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, UNKNOWN_ENCODING, "TIMESTAMPTZ"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, UNKNOWN_ENCODING, "DECIMAL"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL32, UNKNOWN_ENCODING, "DECIMAL32"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL64, UNKNOWN_ENCODING, "DECIMAL64"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, UNKNOWN_ENCODING, "DECIMAL128I"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL256, UNKNOWN_ENCODING, "DECIMAL256"},
+ {FieldType::OLAP_FIELD_TYPE_IPV4, UNKNOWN_ENCODING, "IPV4"},
+ {FieldType::OLAP_FIELD_TYPE_IPV6, UNKNOWN_ENCODING, "IPV6"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, UNKNOWN_ENCODING, "HLL"},
+ {FieldType::OLAP_FIELD_TYPE_BITMAP, UNKNOWN_ENCODING, "BITMAP"},
+ {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, UNKNOWN_ENCODING, "QUANTILE_STATE"},
+ {FieldType::OLAP_FIELD_TYPE_AGG_STATE, UNKNOWN_ENCODING, "AGG_STATE"},
+};
+
+// Full enumeration of (type, encoding) combinations that EncodingInfo::get should
+// find or reject. should_exist=true means EncodingInfo::get must succeed.
+const std::vector<EncodingMapEntry> kEncodingMapEntries = {
+ // signed integers: BIT_SHUFFLE, PLAIN_ENCODING, FOR_ENCODING
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE, true, "TINYINT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, true, "TINYINT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_TINYINT, FOR_ENCODING, true, "TINYINT+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE, true, "SMALLINT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, true, "SMALLINT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_SMALLINT, FOR_ENCODING, true, "SMALLINT+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, true, "INT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING, true, "INT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_INT, FOR_ENCODING, true, "INT+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE, true, "BIGINT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING, true, "BIGINT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, FOR_ENCODING, true, "BIGINT+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE, true, "LARGEINT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, true, "LARGEINT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_LARGEINT, FOR_ENCODING, true, "LARGEINT+FOR"},
+ // unsigned integers: only BIT_SHUFFLE
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, true,
+ "UNSIGNED_BIGINT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, true, "UNSIGNED_INT+BIT_SHUFFLE"},
+ // FLOAT/DOUBLE: BIT_SHUFFLE, PLAIN_ENCODING
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, true, "FLOAT+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING, true, "FLOAT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, true, "DOUBLE+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING, true, "DOUBLE+PLAIN"},
+ // binary types: DICT, PLAIN, PLAIN_V2, PREFIX
+ {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, true, "CHAR+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING, true, "CHAR+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V2, true, "CHAR+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_CHAR, PREFIX_ENCODING, true, "CHAR+PREFIX"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, true, "VARCHAR+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING, true, "VARCHAR+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V2, true, "VARCHAR+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, true, "VARCHAR+PREFIX"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, true, "STRING+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING, true, "STRING+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V2, true, "STRING+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_STRING, PREFIX_ENCODING, true, "STRING+PREFIX"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, true, "JSONB+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING, true, "JSONB+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V2, true, "JSONB+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_JSONB, PREFIX_ENCODING, true, "JSONB+PREFIX"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, true, "VARIANT+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING, true, "VARIANT+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V2, true, "VARIANT+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_VARIANT, PREFIX_ENCODING, true, "VARIANT+PREFIX"},
+ // BOOL: RLE, BIT_SHUFFLE, PLAIN
+ {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, true, "BOOL+RLE"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE, true, "BOOL+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING, true, "BOOL+PLAIN"},
+ // date / datetime: BIT_SHUFFLE, PLAIN, FOR
+ {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, true, "DATE+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING, true, "DATE+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING, true, "DATE+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, true, "DATEV2+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING, true, "DATEV2+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING, true, "DATEV2+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, true, "DATETIMEV2+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING, true, "DATETIMEV2+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING, true, "DATETIMEV2+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, true, "DATETIME+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING, true, "DATETIME+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING, true, "DATETIME+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, true, "TIMESTAMPTZ+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, PLAIN_ENCODING, true, "TIMESTAMPTZ+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING, true, "TIMESTAMPTZ+FOR"},
+ // decimal: BIT_SHUFFLE, PLAIN
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, true, "DECIMAL+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING, true, "DECIMAL+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, true, "DECIMAL32+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL32, PLAIN_ENCODING, true, "DECIMAL32+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, true, "DECIMAL64+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL64, PLAIN_ENCODING, true, "DECIMAL64+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, true, "DECIMAL128I+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING, true, "DECIMAL128I+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true, "DECIMAL256+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING, true, "DECIMAL256+PLAIN"},
+ // ip
+ {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, true, "IPV4+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_IPV4, PLAIN_ENCODING, true, "IPV4+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, true, "IPV6+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING, true, "IPV6+PLAIN"},
+ // aggregate-flavored binary
+ {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING, true, "HLL+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2, true, "HLL+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING, true, "BITMAP+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2, true, "BITMAP+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING, true, "QUANTILE_STATE+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2, true,
+ "QUANTILE_STATE+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING, true, "AGG_STATE+PLAIN"},
+ {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2, true, "AGG_STATE+PLAIN_V2"},
+ // Negative samples: combinations that should NOT be registered.
+ {FieldType::OLAP_FIELD_TYPE_INT, DICT_ENCODING, false, "INT+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_INT, RLE, false, "INT+RLE"},
+ {FieldType::OLAP_FIELD_TYPE_INT, PREFIX_ENCODING, false, "INT+PREFIX"},
+ {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING_V2, false, "INT+PLAIN_V2"},
+ {FieldType::OLAP_FIELD_TYPE_BIGINT, RLE, false, "BIGINT+RLE"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, DICT_ENCODING, false, "FLOAT+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, FOR_ENCODING, false, "FLOAT+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, BIT_SHUFFLE, false, "VARCHAR+BIT_SHUFFLE"},
+ {FieldType::OLAP_FIELD_TYPE_VARCHAR, RLE, false, "VARCHAR+RLE"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, DICT_ENCODING, false, "BOOL+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_BOOL, FOR_ENCODING, false, "BOOL+FOR"},
+ {FieldType::OLAP_FIELD_TYPE_DATE, DICT_ENCODING, false, "DATE+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_DECIMAL, DICT_ENCODING, false, "DECIMAL+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, DICT_ENCODING, false, "HLL+DICT"},
+ {FieldType::OLAP_FIELD_TYPE_HLL, BIT_SHUFFLE, false, "HLL+BIT_SHUFFLE"},
+};
+
+} // namespace
+
+// case 1: V3 default encoding for every type.
+TEST_F(EncodingInfoTest, locked_v3_default_per_type) {
+ for (const auto& row : kV3DefaultExpect) {
+ auto got = get_v3_default_encoding(row.type);
+ EXPECT_EQ(row.expected, got)
+ << "V3 default mismatch for " << row.name << " (type=" << int(row.type)
+ << "): expected " << row.expected << ", got " << got;
+ }
+}
+
+// case 2: V2 (non-V3) default encoding for every type.
+TEST_F(EncodingInfoTest, locked_v2_default_per_type) {
+ for (const auto& row : kV2DefaultExpect) {
+ auto got = get_v2_default_encoding(row.type);
+ EXPECT_EQ(row.expected, got)
+ << "Legacy default mismatch for " << row.name << " (type=" << int(row.type)
+ << "): expected " << row.expected << ", got " << got;
+ }
+}
+
+// case 3: index_column_encoding for every type. UNKNOWN_ENCODING means no
+// index_column_encoding was registered for that type.
+TEST_F(EncodingInfoTest, locked_index_column_encoding_per_type) {
+ for (const auto& row : kIndexColumnEncodingExpect) {
+ auto got = EncodingInfo::get_index_column_encoding(row.type);
+ EXPECT_EQ(row.expected, got)
+ << "Value-seek default mismatch for " << row.name << " (type=" << int(row.type)
+ << "): expected " << row.expected << ", got " << got;
+ }
+}
+
+// case 4: every (type, encoding) combination has the expected presence in the
+// global encoding map.
+TEST_F(EncodingInfoTest, locked_encoding_map_completeness) {
+ for (const auto& row : kEncodingMapEntries) {
+ const EncodingInfo* info = nullptr;
+ auto status = EncodingInfo::get(row.type, row.encoding, &info);
+ if (row.should_exist) {
+ EXPECT_TRUE(status.ok())
+ << "Expected entry missing: " << row.name << ": " << status.to_string();
+ EXPECT_NE(nullptr, info) << "Entry " << row.name << " is null";
+ } else {
+ EXPECT_FALSE(status.ok())
+ << "Unexpected entry present: " << row.name << " (type=" << int(row.type)
+ << ", encoding=" << row.encoding << ")";
+ }
+ }
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/test/storage/segment/external_col_meta_util_test.cpp b/be/test/storage/segment/external_col_meta_util_test.cpp
index e2c2f285b45..b8aed9bfa09 100644
--- a/be/test/storage/segment/external_col_meta_util_test.cpp
+++ b/be/test/storage/segment/external_col_meta_util_test.cpp
@@ -494,7 +494,7 @@ TEST(ExternalColMetaUtilTest, BuildSegmentAndVerifyDataAndFooterMeta) {
TabletSchemaSPtr tablet_schema = create_schema(columns, UNIQUE_KEYS);
// Enable external ColumnMetaPB so that SegmentWriter produces a V3 footer with
// externalized column meta region + column_meta_entries.
- tablet_schema->set_external_segment_meta_used_default(true);
+ tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
// 2. Use existing build_segment helper (SegmentWriter-based) to write a segment file.
SegmentWriterOptions opts;
diff --git a/be/test/storage/segment/variant_column_writer_reader_test.cpp b/be/test/storage/segment/variant_column_writer_reader_test.cpp
index 517469331c3..2e3788a7701 100644
--- a/be/test/storage/segment/variant_column_writer_reader_test.cpp
+++ b/be/test/storage/segment/variant_column_writer_reader_test.cpp
@@ -231,7 +231,7 @@ protected:
_tablet_schema->init_from_pb(schema_pb);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(true);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
tablet_meta->_tablet_id = tablet_id;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -510,7 +510,7 @@ TEST_F(VariantColumnWriterReaderTest, test_legacy_flat_dot_key_reader_init) {
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 20000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -535,7 +535,9 @@ TEST_F(VariantColumnWriterReaderTest, test_legacy_flat_dot_key_reader_init) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -663,7 +665,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -689,7 +693,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -1220,7 +1226,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_and_read_hierarchical_doc)
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
bool external_segment_meta_used_default = false;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 31000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1245,7 +1253,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_and_read_hierarchical_doc)
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn parent_column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, parent_column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok());
@@ -1355,7 +1365,7 @@ TEST_F(VariantColumnWriterReaderTest,
_tablet_schema->init_from_pb(schema_pb);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 31002;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1378,7 +1388,9 @@ TEST_F(VariantColumnWriterReaderTest,
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn parent_column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, parent_column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok());
@@ -1466,6 +1478,116 @@ TEST_F(VariantColumnWriterReaderTest,
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
}
+// Regression: materialized subcolumns in V3 doc-mode tablets must inherit the parent's
+// storage_format and resolve V3 default encodings (e.g. integer family = PLAIN, not BIT_SHUFFLE).
+// Without propagating base_opts.storage_format into the per-subcolumn ColumnWriterOptions,
+// `_init_column_meta` falls back to the V2 default map and writes V2 encodings even for V3
+// tablets, defeating the storage-format-based encoding policy.
+TEST_F(VariantColumnWriterReaderTest, test_write_doc_materialized_v3_uses_v3_encoding) {
+ constexpr int kRows = 200;
+ constexpr int kDocBuckets = 2;
+
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "V1", 3, false, false,
+ /*variant_sparse_hash_shard_count=*/0,
+ /*variant_enable_doc_mode=*/true,
+ /*variant_doc_materialization_min_rows=*/0,
+ /*variant_doc_hash_shard_count=*/kDocBuckets);
+ _tablet_schema = std::make_shared<TabletSchema>();
+ _tablet_schema->init_from_pb(schema_pb);
+
+ TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
+ tablet_meta->_tablet_id = 31003;
+ _tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
+ EXPECT_TRUE(_tablet->init().ok());
+ EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+ EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+
+ io::FileWriterPtr file_writer;
+ auto file_path = local_segment_path(_tablet->tablet_path(), "0", 0);
+ auto st = io::global_local_filesystem()->create_file(file_path, &file_writer);
+ EXPECT_TRUE(st.ok()) << st.msg();
+
+ SegmentFooterPB footer;
+ ColumnWriterOptions opts;
+ opts.meta = footer.add_columns();
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.file_writer = file_writer.get();
+ opts.footer = &footer;
+ RowsetWriterContext rowset_ctx;
+ rowset_ctx.write_type = DataWriteType::TYPE_DIRECT;
+ opts.rowset_ctx = &rowset_ctx;
+ opts.rowset_ctx->tablet_schema = _tablet_schema;
+ TabletColumn parent_column = _tablet_schema->column(0);
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3;
+ _init_column_meta(opts.meta, 0, parent_column, opts);
+
+ std::unique_ptr<ColumnWriter> writer;
+ EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok());
+ EXPECT_TRUE(writer->init().ok());
+
+ auto olap_data_convertor = std::make_unique<OlapBlockDataConvertor>();
+ auto block = _tablet_schema->create_block();
+ auto column_object = (*std::move(block.get_by_position(0).column)).mutate();
+ std::unordered_map<int, std::string> inserted_jsonstr;
+ fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr);
+ olap_data_convertor->add_column_data_convertor(parent_column);
+ olap_data_convertor->set_source_content(&block, 0, kRows);
+ auto [result, accessor] = olap_data_convertor->convert_column_data(0);
+ EXPECT_TRUE(result.ok());
+ EXPECT_TRUE(accessor != nullptr);
+ EXPECT_TRUE(writer->append(accessor->get_nullmap(), accessor->get_data(), kRows).ok());
+ EXPECT_TRUE(writer->finish().ok());
+ EXPECT_TRUE(writer->write_data().ok());
+ EXPECT_TRUE(writer->write_ordinal_index().ok());
+ EXPECT_TRUE(file_writer->close().ok());
+
+ // Materialization must have produced extra subcolumns beyond the doc-bucket columns.
+ EXPECT_GT(footer.columns_size(), 1 + kDocBuckets) << "no subcolumns were materialized";
+
+ // Locate materialized subcolumns. Doc bucket columns have DOC_VALUE_COLUMN_PATH in their
+ // path; everything else (other than the root variant at index 0) is a materialized subcolumn.
+ int integer_subcolumns_checked = 0;
+ int string_subcolumns_checked = 0;
+ for (int i = 1; i < footer.columns_size(); ++i) {
+ const auto& col = footer.columns(i);
+ if (!col.has_column_path_info()) continue;
+ PathInData path;
+ path.from_protobuf(col.column_path_info());
+ std::string rel = path.copy_pop_front().get_path();
+ if (rel.find(DOC_VALUE_COLUMN_PATH) != std::string::npos) continue;
+ const auto field_type = static_cast<FieldType>(col.type());
+ switch (field_type) {
+ case FieldType::OLAP_FIELD_TYPE_TINYINT:
+ case FieldType::OLAP_FIELD_TYPE_SMALLINT:
+ case FieldType::OLAP_FIELD_TYPE_INT:
+ case FieldType::OLAP_FIELD_TYPE_BIGINT:
+ case FieldType::OLAP_FIELD_TYPE_LARGEINT:
+ EXPECT_EQ(col.encoding(), EncodingTypePB::PLAIN_ENCODING)
+ << "V3 integer subcolumn '" << rel << "' got "
+ << EncodingTypePB_Name(col.encoding()) << " instead of PLAIN_ENCODING";
+ ++integer_subcolumns_checked;
+ break;
+ case FieldType::OLAP_FIELD_TYPE_CHAR:
+ case FieldType::OLAP_FIELD_TYPE_VARCHAR:
+ case FieldType::OLAP_FIELD_TYPE_STRING:
+ EXPECT_EQ(col.encoding(), EncodingTypePB::DICT_ENCODING)
+ << "V3 string subcolumn '" << rel << "' got "
+ << EncodingTypePB_Name(col.encoding()) << " instead of DICT_ENCODING";
+ ++string_subcolumns_checked;
+ break;
+ default:
+ break;
+ }
+ }
+ EXPECT_GT(integer_subcolumns_checked + string_subcolumns_checked, 0)
+ << "no scalar materialized subcolumns were found to verify";
+
+ EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+}
+
TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucket) {
constexpr int kRows = 200;
constexpr int kDocBuckets = 4;
@@ -1483,7 +1605,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 32000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1507,7 +1629,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn parent_column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, parent_column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok());
@@ -1633,7 +1757,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc
// 2. create tablet
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 33000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1659,7 +1783,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc
root_opts.file_writer = file_writer.get();
root_opts.footer = &footer;
root_opts.rowset_ctx = &rowset_ctx;
- _init_column_meta(root_opts.meta, 0, parent_column, CompressionTypePB::LZ4);
+ root_opts.compression_type = CompressionTypePB::LZ4;
+ root_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(root_opts.meta, 0, parent_column, root_opts);
std::unique_ptr<ColumnWriter> root_writer;
EXPECT_TRUE(
@@ -1669,7 +1795,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc
TabletColumn extracted_doc_bucket_col = _tablet_schema->column(1);
ColumnWriterOptions doc_compact_opts = root_opts;
doc_compact_opts.meta = footer.add_columns();
- _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, CompressionTypePB::LZ4);
+ doc_compact_opts.compression_type = CompressionTypePB::LZ4;
+ doc_compact_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, doc_compact_opts);
std::unique_ptr<ColumnWriter> doc_compact_writer;
EXPECT_TRUE(ColumnWriter::create(doc_compact_opts, &extracted_doc_bucket_col, file_writer.get(),
&doc_compact_writer)
@@ -1831,7 +1959,7 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_compact_sparse_write_array_gap) {
_tablet_schema->append_column(extracted_doc_bucket);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 33001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1855,7 +1983,9 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_compact_sparse_write_array_gap) {
doc_compact_opts.file_writer = file_writer.get();
doc_compact_opts.footer = &footer;
doc_compact_opts.rowset_ctx = &rowset_ctx;
- _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, CompressionTypePB::LZ4);
+ doc_compact_opts.compression_type = CompressionTypePB::LZ4;
+ doc_compact_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, doc_compact_opts);
std::unique_ptr<ColumnWriter> doc_compact_writer;
EXPECT_TRUE(ColumnWriter::create(doc_compact_opts, &extracted_doc_bucket_col, file_writer.get(),
@@ -1933,7 +2063,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_sparse_write_array_gap_and_
_tablet_schema->init_from_pb(schema_pb);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 33002;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -1957,7 +2087,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_sparse_write_array_gap_and_
opts.file_writer = file_writer.get();
opts.footer = &footer;
opts.rowset_ctx = &rowset_ctx;
- _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, parent_column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok());
@@ -2077,7 +2209,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2102,7 +2236,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2278,7 +2414,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2303,7 +2441,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2349,7 +2489,7 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_inherit
_tablet_schema->init_from_pb(schema_pb);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
ASSERT_TRUE(_tablet->init().ok());
@@ -2372,7 +2512,9 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_inherit
rowset_ctx.tablet_schema = _tablet_schema;
opts.rowset_ctx = &rowset_ctx;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2436,7 +2578,7 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_branch_
_tablet_schema->init_from_pb(schema_pb);
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
- _tablet_schema->set_external_segment_meta_used_default(false);
+ _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10002;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
ASSERT_TRUE(_tablet->init().ok());
@@ -2459,7 +2601,9 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_branch_
rowset_ctx.tablet_schema = _tablet_schema;
opts.rowset_ctx = &rowset_ctx;
TabletColumn root_column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, root_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, root_column, opts);
std::unique_ptr<ColumnWriter> writer;
ASSERT_TRUE(ColumnWriter::create(opts, &root_column, file_writer.get(), &writer).ok());
@@ -2622,7 +2766,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2648,7 +2794,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) {
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2756,7 +2904,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize)
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2782,7 +2932,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize)
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2832,7 +2984,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2858,7 +3012,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) {
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2908,7 +3064,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -2934,7 +3092,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) {
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -2986,7 +3146,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -3012,7 +3174,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) {
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -3064,7 +3228,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -3090,7 +3256,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) {
opts.rowset_ctx->tablet_schema = _tablet_schema;
// nullable variant column
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -3141,7 +3309,9 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -3166,7 +3336,9 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -3274,7 +3446,9 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10001;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
EXPECT_TRUE(_tablet->init().ok());
@@ -3299,7 +3473,9 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -3421,7 +3597,9 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
@@ -3452,7 +3630,9 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn tablet_column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, tablet_column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, tablet_column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &tablet_column, file_writer.get(), &writer).ok());
@@ -3839,7 +4019,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
bool external_segment_meta_used_default = rand() % 2 == 0;
std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default
<< std::endl;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 10000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -3865,7 +4047,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) {
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -3994,7 +4178,9 @@ TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get
// VariantColumnReader builds a VariantExternalMetaReader.
TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
bool external_segment_meta_used_default = true;
- _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default);
+ _tablet_schema->set_storage_format(external_segment_meta_used_default
+ ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3
+ : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
tablet_meta->_tablet_id = 20000;
_tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
@@ -4020,7 +4206,9 @@ TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get
opts.rowset_ctx = &rowset_ctx;
opts.rowset_ctx->tablet_schema = _tablet_schema;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
@@ -4155,7 +4343,9 @@ TEST_F(VariantColumnWriterReaderTest,
opts.rowset_ctx = &rowset_ctx;
TabletColumn column = _tablet_schema->column(0);
- _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4);
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
std::unique_ptr<ColumnWriter> writer;
ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok());
diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp
index 9753bad2f67..e2b8424e8d6 100644
--- a/be/test/storage/tablet/tablet_schema_test.cpp
+++ b/be/test/storage/tablet/tablet_schema_test.cpp
@@ -864,4 +864,70 @@ TEST_F(TabletSchemaTest, test_tablet_schema_get_index) {
EXPECT_EQ(14002, ann_col_ids[0]);
}
+// Rolling-upgrade compat: a TabletSchemaPB persisted by an old BE has the three
+// legacy V3-flavor flags but no storage_format. init_from_pb must derive V3.
+TEST_F(TabletSchemaTest, init_from_pb_legacy_flags_derive_v3) {
+ TabletSchemaPB pb;
+ pb.set_keys_type(DUP_KEYS);
+ pb.set_is_external_segment_column_meta_used(true);
+ pb.set_integer_type_default_use_plain_encoding(true);
+ pb.set_binary_plain_encoding_default_impl(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
+ // storage_format is intentionally not set
+ ASSERT_FALSE(pb.has_storage_format());
+
+ TabletSchema schema;
+ schema.init_from_pb(pb);
+ EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, schema.storage_format());
+}
+
+// PB with neither storage_format nor any of the legacy flags falls back to V2.
+TEST_F(TabletSchemaTest, init_from_pb_no_flags_defaults_v2) {
+ TabletSchemaPB pb;
+ pb.set_keys_type(DUP_KEYS);
+ ASSERT_FALSE(pb.has_storage_format());
+
+ TabletSchema schema;
+ schema.init_from_pb(pb);
+ EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, schema.storage_format());
+}
+
+// Rolling-downgrade compat: a V3 TabletSchema must redundantly emit the three
+// legacy flags so an old BE rolled back from a new one can still recognize V3.
+TEST_F(TabletSchemaTest, to_schema_pb_v3_emits_legacy_flags) {
+ TabletSchemaPB in;
+ in.set_keys_type(DUP_KEYS);
+ in.set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3);
+
+ TabletSchema schema;
+ schema.init_from_pb(in);
+ ASSERT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, schema.storage_format());
+
+ TabletSchemaPB out;
+ schema.to_schema_pb(&out);
+ EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, out.storage_format());
+ EXPECT_TRUE(out.is_external_segment_column_meta_used());
+ EXPECT_TRUE(out.integer_type_default_use_plain_encoding());
+ EXPECT_EQ(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2,
+ out.binary_plain_encoding_default_impl());
+}
+
+// V2 schemas must NOT emit the V3-flavor legacy flags.
+TEST_F(TabletSchemaTest, to_schema_pb_v2_skips_legacy_flags) {
+ TabletSchemaPB in;
+ in.set_keys_type(DUP_KEYS);
+ in.set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2);
+
+ TabletSchema schema;
+ schema.init_from_pb(in);
+ ASSERT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, schema.storage_format());
+
+ TabletSchemaPB out;
+ schema.to_schema_pb(&out);
+ EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, out.storage_format());
+ EXPECT_FALSE(out.is_external_segment_column_meta_used());
+ EXPECT_FALSE(out.integer_type_default_use_plain_encoding());
+ EXPECT_NE(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2,
+ out.binary_plain_encoding_default_impl());
+}
+
} // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index cabc936772d..b208e712c27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -333,15 +333,20 @@ public class CloudInternalCatalog extends InternalCatalog {
break;
}
- // Enable external column meta layout when storage_format is V3 (Cloud mode).
+ // Persist the storage format directly on the schema so the BE doesn't have to
+ // derive it from the three legacy flags below on the way back from MS. The flags
+ // are still written for backward-compat with BEs that predate the storage_format
+ // schema field; both representations agree on every V3 tablet.
switch (storageFormat) {
case V3:
+ schemaBuilder.setStorageFormat(OlapFile.TabletStorageFormatPB.TABLET_STORAGE_FORMAT_V3);
schemaBuilder.setIsExternalSegmentColumnMetaUsed(true);
schemaBuilder.setIntegerTypeDefaultUsePlainEncoding(true);
schemaBuilder.setBinaryPlainEncodingDefaultImpl(
OlapFile.BinaryPlainEncodingTypePB.BINARY_PLAIN_ENCODING_V2);
break;
default:
+ schemaBuilder.setStorageFormat(OlapFile.TabletStorageFormatPB.TABLET_STORAGE_FORMAT_V2);
break;
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 4b70c65ff66..e440124122b 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -448,6 +448,15 @@ enum InvertedIndexStorageFormatPB {
V3 = 2;
}
+// Tablet-level storage format. Values match TStorageFormat (Thrift) integer values so
+// the C++ side can cast between TStorageFormat::type and this enum 1:1.
+enum TabletStorageFormatPB {
+ TABLET_STORAGE_FORMAT_DEFAULT = 0;
+ TABLET_STORAGE_FORMAT_V1 = 1;
+ TABLET_STORAGE_FORMAT_V2 = 2;
+ TABLET_STORAGE_FORMAT_V3 = 3;
+}
+
message TabletIndexPB {
optional int64 index_id = 1;
optional string index_name = 2;
@@ -523,6 +532,14 @@ message TabletSchemaPB {
// mapping from seq column unique id to value column unique id
optional ColumnGroupsPB seq_map = 35;
+ // Persisted TStorageFormat (V2=2, V3=3). This is the new authoritative field.
+ // The legacy is_external_segment_column_meta_used, integer_type_default_use_plain_encoding
+ // and binary_plain_encoding_default_impl fields above are still emitted redundantly when
+ // storage_format == V3 so that an old BE rolled back from a new one can still recover
+ // the format via the prior "any of those three implies V3" rule. New code should read
+ // storage_format; the fallback path lives in TabletSchema::init_from_pb.
+ optional TabletStorageFormatPB storage_format = 36;
+
optional SplitSchemaPB __split_schema = 1000; // A special field, DO NOT change it.
}
@@ -567,6 +584,9 @@ message TabletSchemaCloudPB {
// mapping from seq column unique id to value column unique id
optional ColumnGroupsPB seq_map = 36;
+ // Persisted TStorageFormat (V2=2, V3=3); see TabletSchemaPB::storage_format.
+ optional TabletStorageFormatPB storage_format = 37;
+
optional bool is_dynamic_schema = 100 [default=false];
// FIXME(gavin): deprecate and remove in the future
diff --git a/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy b/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy
index d3aebdb5ec5..54d19117262 100644
--- a/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy
+++ b/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy
@@ -38,14 +38,12 @@ suite('test_storage_format_controls_encoding') {
logger.info("begin curl ${metaUrl}")
def jsonMeta = Http.GET(metaUrl, true, false)
+ assert jsonMeta.schema.storage_format == "TABLET_STORAGE_FORMAT_V3"
+ // V3 tablets redundantly emit the three legacy V3-flavor flags so that an old BE
+ // rolled back from a new deployment can still recognize the tablet as V3.
assert jsonMeta.schema.integer_type_default_use_plain_encoding == true
assert jsonMeta.schema.binary_plain_encoding_default_impl == "BINARY_PLAIN_ENCODING_V2"
-
- def res = sql """show variables like "%use_v3_storage_format%";""";
- logger.info("session var use_v3_storage_format: ${res}")
- if (res[0][1] == "true") return
-
tableName = "test_storage_format_controls_encoding2"
sql """drop table if exists `${tableName}` force; """
@@ -53,12 +51,12 @@ suite('test_storage_format_controls_encoding') {
CREATE TABLE ${tableName}
(k int, v1 int, v2 varchar(100))
duplicate KEY(k)
- DISTRIBUTED BY HASH (k)
+ DISTRIBUTED BY HASH (k)
BUCKETS 1 PROPERTIES(
"replication_num" = "1",
"storage_format" = "V2");
"""
-
+
sql "insert into ${tableName} values(1, 1, 'aaa');"
sql "select * from ${tableName};"
@@ -66,6 +64,9 @@ suite('test_storage_format_controls_encoding') {
logger.info("begin curl ${metaUrl}")
jsonMeta = Http.GET(metaUrl, true, false)
- assert jsonMeta.schema.integer_type_default_use_plain_encoding == false
- assert jsonMeta.schema.binary_plain_encoding_default_impl == "BINARY_PLAIN_ENCODING_V1"
+ assert jsonMeta.schema.storage_format == "TABLET_STORAGE_FORMAT_V2"
+ // V2 tablets intentionally omit the three legacy V3-flavor flags; absent fields are
+ // semantically equivalent to false / V1 for an old BE reading this PB.
+ assert (jsonMeta.schema.integer_type_default_use_plain_encoding ?: false) == false
+ assert (jsonMeta.schema.binary_plain_encoding_default_impl ?: "BINARY_PLAIN_ENCODING_V1") == "BINARY_PLAIN_ENCODING_V1"
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org