You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/10 11:18:50 UTC
(doris) branch branch-4.0 updated: [branch-4.0](variant) Skip full footer scan when constructing VariantStatsCaculator (#62819) (#63113)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 97dc5d7be85 [branch-4.0](variant) Skip full footer scan when constructing VariantStatsCaculator (#62819) (#63113)
97dc5d7be85 is described below
commit 97dc5d7be8589768f85f6540b716a01e14ec47f4
Author: Chenyang Sun <su...@selectdb.com>
AuthorDate: Sun May 10 19:18:42 2026 +0800
[branch-4.0](variant) Skip full footer scan when constructing VariantStatsCaculator (#62819) (#63113)
pick from master #62819
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 7 +++--
.../rowset/segment_v2/variant_stats_calculator.cpp | 10 +++---
.../rowset/segment_v2/variant_stats_calculator.h | 8 +++--
.../segment_v2/variant_stats_calculator_test.cpp | 36 ++++++++++++++++++++++
4 files changed, 53 insertions(+), 8 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index ec2e12cd7e4..7cffa35cc0f 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -349,11 +349,14 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) {
_opts.compression_type = _tablet_schema->compression_type();
}
+ // Vertical compaction calls init() multiple times against the same writer; the footer accumulates entries
+ // across calls, so this init()'s slice of footer columns starts at the current size.
+ const int variant_stats_footer_offset = _footer.columns_size();
RETURN_IF_ERROR(_create_writers(_tablet_schema, col_ids));
// Initialize variant statistics calculator
- _variant_stats_calculator =
- std::make_unique<VariantStatsCaculator>(&_footer, _tablet_schema, col_ids);
+ _variant_stats_calculator = std::make_unique<VariantStatsCaculator>(
+ &_footer, _tablet_schema, col_ids, variant_stats_footer_offset);
// we don't need the short key index for unique key merge on write table.
if (_has_key) {
diff --git a/be/src/olap/rowset/segment_v2/variant_stats_calculator.cpp b/be/src/olap/rowset/segment_v2/variant_stats_calculator.cpp
index 70722ca8748..4d230139a6f 100644
--- a/be/src/olap/rowset/segment_v2/variant_stats_calculator.cpp
+++ b/be/src/olap/rowset/segment_v2/variant_stats_calculator.cpp
@@ -30,10 +30,12 @@ namespace doris::segment_v2 {
VariantStatsCaculator::VariantStatsCaculator(SegmentFooterPB* footer,
TabletSchemaSPtr tablet_schema,
- const std::vector<uint32_t>& column_ids)
+ const std::vector<uint32_t>& column_ids,
+ int footer_column_offset)
: _footer(footer), _tablet_schema(tablet_schema), _column_ids(column_ids) {
- // Build the path to footer index mapping during initialization
- for (int i = 0; i < _footer->columns_size(); ++i) {
+ // Only walk this init()'s slice of footer entries; earlier init() calls (vertical compaction's previous
+ // column groups) are not addressable via `column_ids` and would only inflate this scan.
+ for (int i = footer_column_offset; i < _footer->columns_size(); ++i) {
const auto& column = _footer->columns(i);
// path that need to record stats
if (column.has_column_path_info() &&
@@ -119,4 +121,4 @@ void VariantStatsCaculator::_calculate_sub_column_stats(const vectorized::IColum
#include "common/compile_check_end.h"
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant_stats_calculator.h b/be/src/olap/rowset/segment_v2/variant_stats_calculator.h
index 44885e3adc5..85c8975cca4 100644
--- a/be/src/olap/rowset/segment_v2/variant_stats_calculator.h
+++ b/be/src/olap/rowset/segment_v2/variant_stats_calculator.h
@@ -28,8 +28,12 @@ namespace doris::segment_v2 {
class VariantStatsCaculator {
public:
+ // `footer_column_offset` is the index of the first footer entry that belongs to this init()'s `column_ids`.
+ // Required because SegmentWriter::init() can be invoked multiple times (vertical compaction) against
+ // an ever-growing footer; without the offset every additional init() would re-scan the whole footer.
explicit VariantStatsCaculator(SegmentFooterPB* footer, TabletSchemaSPtr tablet_schema,
- const std::vector<uint32_t>& column_ids);
+ const std::vector<uint32_t>& column_ids,
+ int footer_column_offset = 0);
// Calculate variant statistics for the given column and block
Status calculate_variant_stats(const vectorized::Block* block, size_t row_pos, size_t num_rows);
@@ -54,4 +58,4 @@ private:
size_t row_pos, size_t num_rows);
};
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/test/olap/rowset/segment_v2/variant_stats_calculator_test.cpp b/be/test/olap/rowset/segment_v2/variant_stats_calculator_test.cpp
index 31f694536ad..3063f9efef8 100644
--- a/be/test/olap/rowset/segment_v2/variant_stats_calculator_test.cpp
+++ b/be/test/olap/rowset/segment_v2/variant_stats_calculator_test.cpp
@@ -464,4 +464,40 @@ TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithExtendedSchema) {
EXPECT_TRUE(status.ok());
}
+TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithFooterOffset) {
+ // Vertical compaction calls SegmentWriter::init() multiple times against
+ // the same writer (key columns first, then each value-column group). The
+ // footer accumulates entries across calls, so the calculator built for the
+ // second init() must only index its own slice — not the leftover entries
+ // from the first init(). The offset tells the constructor where its slice
+ // starts and also where stats results should land.
+ add_footer_column_with_path(1, "stale_from_prev_init"); // pre-existing
+ add_footer_column_with_path(1, "another_stale_entry"); // pre-existing
+ const int footer_offset = _footer->columns_size();
+ add_footer_column_with_path(1, "sub_column"); // belongs to this init()
+
+ TabletColumn sub_column = create_variant_column(2, "variant.sub_column", 1, "sub_column");
+ _tablet_schema->append_column(sub_column);
+
+ std::vector<uint32_t> column_ids = {0};
+ VariantStatsCaculator calculator(_footer.get(), _tablet_schema, column_ids, footer_offset);
+
+ vectorized::Block block;
+ auto nullable_column = create_nullable_column({false, true, false}, {"a", "", "c"});
+ block.insert({std::move(nullable_column),
+ std::make_shared<vectorized::DataTypeNullable>(
+ std::make_shared<vectorized::DataTypeString>()),
+ "sub_column"});
+
+ auto status = calculator.calculate_variant_stats(&block, 0, 3);
+ EXPECT_TRUE(status.ok());
+
+ // Stats land on this init()'s slice, not the pre-existing entries — proves
+ // we ignored the stale entries even though they share parent_unique_id=1
+ // and the same path keys would otherwise collide in the index map.
+ EXPECT_EQ(_footer->columns(0).none_null_size(), 0);
+ EXPECT_EQ(_footer->columns(1).none_null_size(), 0);
+ EXPECT_EQ(_footer->columns(footer_offset).none_null_size(), 2);
+}
+
} // namespace doris::segment_v2
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org