You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/07 07:48:15 UTC
(doris) branch branch-4.0 updated: branch-4.0: [fix](variant) preserve subcolumns after row-store partial update #62067 (#62075)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new ce8c4652322 branch-4.0: [fix](variant) preserve subcolumns after row-store partial update #62067 (#62075)
ce8c4652322 is described below
commit ce8c4652322bb8e85000eb0bf77ae2a6b9df10e1
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu May 7 15:48:05 2026 +0800
branch-4.0: [fix](variant) preserve subcolumns after row-store partial update #62067 (#62075)
Cherry-picked from #62067
---------
Co-authored-by: lihangyu <li...@selectdb.com>
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 25 ++++++++++++
.../rowset/segment_v2/vertical_segment_writer.cpp | 25 ++++++++++++
be/test/vec/common/schema_util_test.cpp | 47 ++++++++++++++++++++++
.../suites/variant_p0/delete_update.groovy | 40 ++++++++++++++++++
4 files changed, 137 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 2118b949144..ec2e12cd7e4 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -72,6 +72,7 @@
#include "vec/core/block.h"
#include "vec/core/column_with_type_and_name.h"
#include "vec/core/types.h"
+#include "vec/json/json_parser.h"
#include "vec/jsonb/serialize.h"
#include "vec/olap/olap_data_convertor.h"
#include "vec/runtime/vdatetime_value.h"
@@ -88,6 +89,26 @@ inline std::string segment_mem_tracker_name(uint32_t segment_id) {
return "SegmentWriter:Segment-" + std::to_string(segment_id);
}
+namespace {
+
+Status parse_variant_columns_in_block(vectorized::Block& block, const TabletSchema& tablet_schema) {
+ std::vector<int> variant_column_pos;
+ variant_column_pos.reserve(block.columns());
+ for (int i = 0; i < block.columns(); ++i) {
+ if (block.get_by_position(i).type->get_primitive_type() == TYPE_VARIANT) {
+ variant_column_pos.push_back(i);
+ }
+ }
+ if (variant_column_pos.empty()) {
+ return Status::OK();
+ }
+ vectorized::ParseConfig config;
+ config.enable_flatten_nested = tablet_schema.variant_flatten_nested();
+ return vectorized::schema_util::parse_variant_columns(block, variant_column_pos, config);
+}
+
+} // namespace
+
SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id,
TabletSchemaSPtr tablet_schema, BaseTabletSPtr tablet,
DataDir* data_dir, const SegmentWriterOptions& opts,
@@ -624,6 +645,10 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
use_default_or_null_flag, has_default_or_nullable,
cast_set<uint32_t>(segment_start_pos), block));
+ if (_tablet_schema->num_variant_columns() > 0) {
+ RETURN_IF_ERROR(parse_variant_columns_in_block(full_block, *_tablet_schema));
+ }
+
// convert block to row store format
_serialize_block_to_row_column(full_block);
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index c50a03cb845..52a7cde37f9 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -77,6 +77,7 @@
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_factory.hpp"
#include "vec/data_types/data_type_number.h" // IWYU pragma: keep
+#include "vec/json/json_parser.h"
#include "vec/json/path_in_data.h"
#include "vec/jsonb/serialize.h"
#include "vec/olap/olap_data_convertor.h"
@@ -93,6 +94,26 @@ inline std::string vertical_segment_writer_mem_tracker_name(uint32_t segment_id)
return "VerticalSegmentWriter:Segment-" + std::to_string(segment_id);
}
+namespace {
+
+Status parse_variant_columns_in_block(vectorized::Block& block, const TabletSchema& tablet_schema) {
+ std::vector<int> variant_column_pos;
+ variant_column_pos.reserve(block.columns());
+ for (int i = 0; i < block.columns(); ++i) {
+ if (block.get_by_position(i).type->get_primitive_type() == TYPE_VARIANT) {
+ variant_column_pos.push_back(i);
+ }
+ }
+ if (variant_column_pos.empty()) {
+ return Status::OK();
+ }
+ vectorized::ParseConfig config;
+ config.enable_flatten_nested = tablet_schema.variant_flatten_nested();
+ return vectorized::schema_util::parse_variant_columns(block, variant_column_pos, config);
+}
+
+} // namespace
+
VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32_t segment_id,
TabletSchemaSPtr tablet_schema, BaseTabletSPtr tablet,
DataDir* data_dir,
@@ -632,6 +653,10 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
_opts.rowset_ctx, _rsid_to_rowset, *_tablet_schema, full_block,
use_default_or_null_flag, has_default_or_nullable, segment_start_pos, data.block));
+ if (_tablet_schema->num_variant_columns() > 0) {
+ RETURN_IF_ERROR(parse_variant_columns_in_block(full_block, *_tablet_schema));
+ }
+
// row column should be filled here
// convert block to row store format
_serialize_block_to_row_column(full_block);
diff --git a/be/test/vec/common/schema_util_test.cpp b/be/test/vec/common/schema_util_test.cpp
index 92f052c1d54..6339372ca1f 100644
--- a/be/test/vec/common/schema_util_test.cpp
+++ b/be/test/vec/common/schema_util_test.cpp
@@ -1288,6 +1288,53 @@ TEST_F(SchemaUtilTest, TestParseVariantColumnsWithNulls) {
EXPECT_TRUE(result_column->is_nullable());
}
+TEST_F(SchemaUtilTest, TestParseVariantColumnsAcrossMixedBlock) {
+ Block block;
+ auto variant_type = std::make_shared<DataTypeVariant>(10);
+
+ auto variant_column1 = ColumnVariant::create(10);
+ auto root_column1 = ColumnString::create();
+ root_column1->insert(vectorized::Field::create_field<PrimitiveType::TYPE_STRING>(
+ "{\"user\":{\"name\":\"alice\"},\"score\":7}"));
+ variant_column1->create_root(std::make_shared<DataTypeString>(), root_column1->get_ptr());
+ block.insert({variant_column1->get_ptr(), variant_type, "variant_col_1"});
+
+ auto int_column = ColumnInt32::create();
+ int_column->insert_value(42);
+ block.insert({int_column->get_ptr(), std::make_shared<DataTypeInt32>(), "plain_int_col"});
+
+ auto variant_column2 = ColumnVariant::create(10);
+ auto root_column2 = ColumnString::create();
+ root_column2->insert(vectorized::Field::create_field<PrimitiveType::TYPE_STRING>(
+ "{\"event\":{\"type\":\"click\"},\"flag\":true}"));
+ variant_column2->create_root(std::make_shared<DataTypeString>(), root_column2->get_ptr());
+ block.insert({variant_column2->get_ptr(), variant_type, "variant_col_2"});
+
+ std::vector<int> variant_pos;
+ for (int i = 0; i < block.columns(); ++i) {
+ if (block.get_by_position(i).type->get_primitive_type() == TYPE_VARIANT) {
+ variant_pos.push_back(i);
+ }
+ }
+
+ ASSERT_THAT(variant_pos, ::testing::ElementsAre(0, 2));
+
+ ParseConfig config;
+ config.enable_flatten_nested = true;
+ auto status = schema_util::parse_variant_columns(block, variant_pos, config);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& result_column1 =
+ assert_cast<const ColumnVariant&>(*block.get_by_position(0).column);
+ EXPECT_NE(result_column1.get_subcolumn(PathInData("user.name")), nullptr);
+ EXPECT_NE(result_column1.get_subcolumn(PathInData("score")), nullptr);
+
+ const auto& result_column2 =
+ assert_cast<const ColumnVariant&>(*block.get_by_position(2).column);
+ EXPECT_NE(result_column2.get_subcolumn(PathInData("event.type")), nullptr);
+ EXPECT_NE(result_column2.get_subcolumn(PathInData("flag")), nullptr);
+}
+
TEST_F(SchemaUtilTest, get_compaction_typed_columns) {
TabletColumn variant;
variant.set_unique_id(10);
diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy
index f7c27b55dc8..7c8f77b46ec 100644
--- a/regression-test/suites/variant_p0/delete_update.groovy
+++ b/regression-test/suites/variant_p0/delete_update.groovy
@@ -73,6 +73,46 @@ suite("regression_test_variant_delete_and_update", "variant_type"){
sql """delete from ${table_name} where vs = 'xxx' or vs = 'yyy'"""
qt_sql "select * from ${table_name} order by k"
+ def partialUpdateVariantTable = "var_partial_update_keep_subcolumn"
+ sql "DROP TABLE IF EXISTS ${partialUpdateVariantTable}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${partialUpdateVariantTable} (
+ event_id bigint,
+ oneid string,
+ event_property_variant variant
+ )
+ UNIQUE KEY(`event_id`)
+ DISTRIBUTED BY HASH(event_id) BUCKETS 1
+ properties(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "true"
+ );
+ """
+ sql """insert into ${partialUpdateVariantTable} values
+ (1, 'old-oneid', '{"base_id":"100000000009523363","other_key":"abc"}')"""
+ sql "sync"
+
+ def beforeUpdate = sql """select cast(event_property_variant['base_id'] as string)
+ from ${partialUpdateVariantTable}
+ where event_id = 1"""
+ assertEquals("100000000009523363", beforeUpdate[0][0])
+
+ sql """update ${partialUpdateVariantTable}
+ set oneid = 'new-oneid'
+ where event_id = 1"""
+ sql "sync"
+
+ def afterUpdate = sql """select cast(event_property_variant['base_id'] as string)
+ from ${partialUpdateVariantTable}
+ where event_id = 1"""
+ assertEquals("100000000009523363", afterUpdate[0][0])
+
+ def wholeVariant = sql """select cast(event_property_variant as string)
+ from ${partialUpdateVariantTable}
+ where event_id = 1"""
+ assertEquals('{"base_id":"100000000009523363","other_key":"abc"}', wholeVariant[0][0])
+
// delete & insert concurrently
sql "set enable_unique_key_partial_update=true;"
sql "sync"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org