You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2025/04/14 02:47:03 UTC
(doris) branch variant-sparse updated: [ut](column_variant)update ut cases and some fix (#49776)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch variant-sparse
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/variant-sparse by this push:
new c2d0c667271 [ut](column_variant)update ut cases and some fix (#49776)
c2d0c667271 is described below
commit c2d0c66727142afb7bc0cffb19165ca8888301c5
Author: amory <wa...@selectdb.com>
AuthorDate: Mon Apr 14 10:46:55 2025 +0800
[ut](column_variant)update ut cases and some fix (#49776)
---
be/src/vec/columns/column_object.cpp | 34 +-
be/src/vec/columns/column_object.h | 80 -
be/src/vec/core/field.h | 3 +
be/src/vec/data_types/data_type_factory.cpp | 3 +
.../data_types/serde/data_type_object_serde.cpp | 18 +
.../vec/data_types/serde/data_type_object_serde.h | 11 +-
.../vec/columns/column_variant_allocated_bytes.out | Bin 0 -> 55 bytes
.../vec/columns/column_variant_byte_size.out | Bin 0 -> 51 bytes
.../column_variant_update_crc_with_value.out | Bin 0 -> 54 bytes
..._variant_update_crc_with_value.out_with_nullmap | 1 +
.../column_variant_update_crcs_with_value.out | Bin 0 -> 23836 bytes
...variant_update_crcs_with_value.out_with_nullmap | 1 +
.../column_variant_update_hashes_with_value.out | Bin 0 -> 45343 bytes
...riant_update_hashes_with_value.out_with_nullmap | 1 +
.../column_variant_update_sip_hash_with_value.out | Bin 0 -> 64 bytes
.../column_variant_update_xxHash_with_value.out | Bin 0 -> 63 bytes
...riant_update_xxHash_with_value.out_with_nullmap | 1 +
be/test/testutil/test_util.cpp | 135 ++
be/test/testutil/test_util.h | 13 +
be/test/testutil/variant_util.h | 170 ++
be/test/vec/columns/column_array_test.cpp | 2 +-
be/test/vec/columns/column_object_test.cpp | 371 +--
be/test/vec/columns/column_variant_test.cpp | 2420 ++++++++++++++++++++
be/test/vec/columns/common_column_test.h | 667 +++++-
run-be-ut.sh | 13 +-
25 files changed, 3636 insertions(+), 308 deletions(-)
diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index fd71c0ff967..8662ba5b6cb 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -1280,6 +1280,22 @@ void ColumnObject::insert_from_sparse_column_and_fill_remaing_dense_column(
}
ColumnPtr ColumnObject::permute(const Permutation& perm, size_t limit) const {
+ if (limit == 0)
+ limit = num_rows;
+ else
+ limit = std::min(num_rows, limit);
+
+ if (perm.size() < limit) {
+ throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+ "Size of permutation ({}) is less than required ({})", perm.size(),
+ limit);
+ __builtin_unreachable();
+ }
+
+ if (limit == 0) {
+ return ColumnObject::create(_max_subcolumns_count);
+ }
+
return apply_for_columns([&](const ColumnPtr column) { return column->permute(perm, limit); });
}
@@ -1558,6 +1574,7 @@ bool ColumnObject::Subcolumn::is_empty_nested(size_t row) const {
DCHECK(type->equals(*ColumnObject::NESTED_TYPE));
Field field;
get(row, field);
+ field = get_field_from_variant_field(field);
if (field.get_type() == Field::Types::Array) {
const auto& array = field.get<Array>();
bool only_nulls_inside = true;
@@ -1943,20 +1960,13 @@ bool ColumnObject::empty() const {
return subcolumns.empty() || subcolumns.begin()->get()->path.get_path() == COLUMN_NAME_DUMMY;
}
-ColumnPtr get_base_column_of_array(const ColumnPtr& column) {
- if (const auto* column_array = check_and_get_column<ColumnArray>(column.get())) {
- return column_array->get_data_ptr();
- }
- return column;
-}
-
ColumnPtr ColumnObject::filter(const Filter& filter, ssize_t count) const {
if (!is_finalized()) {
auto finalized = clone_finalized();
auto& finalized_object = assert_cast<ColumnObject&>(*finalized);
return finalized_object.filter(filter, count);
}
- if (subcolumns.empty()) {
+ if (num_rows == 0) {
auto res = ColumnObject::create(_max_subcolumns_count, count_bytes_in_filter(filter));
ENABLE_CHECK_CONSISTENCY(res.get());
return res;
@@ -1979,6 +1989,9 @@ ColumnPtr ColumnObject::filter(const Filter& filter, ssize_t count) const {
ColumnPtr ColumnObject::replicate(const IColumn::Offsets& offsets) const {
column_match_offsets_size(num_rows, offsets.size());
+ if (num_rows == 0) {
+ return ColumnObject::create(_max_subcolumns_count);
+ }
return apply_for_columns([&](const ColumnPtr column) { return column->replicate(offsets); });
}
@@ -2162,8 +2175,9 @@ Status ColumnObject::sanitize() const {
for (const auto& subcolumn : subcolumns) {
if (subcolumn->data.is_finalized()) {
auto column = subcolumn->data.get_least_common_type()->create_column();
- std::string original = subcolumn->data.get_finalized_column().get_name();
- std::string expected = column->get_name();
+ std::string original =
+ remove_nullable(subcolumn->data.get_finalized_column().get_ptr())->get_name();
+ std::string expected = remove_nullable(column->get_ptr())->get_name();
if (original != expected) {
return Status::InternalError("Incompatible type between {} and {}, debug_info:",
original, expected, debug_string());
diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
index 2747cbc89d7..cbc2bb78c2b 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -510,86 +510,6 @@ public:
void update_crc_with_value(size_t start, size_t end, uint32_t& hash,
const uint8_t* __restrict null_data) const override;
- Int64 get_int(size_t /*n*/) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "get_int " + get_name());
- }
-
- bool get_bool(size_t /*n*/) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "get_bool " + get_name());
- }
-
- void insert_many_fix_len_data(const char* pos, size_t num) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_fix_len_data " + get_name());
- }
-
- void insert_many_dict_data(const int32_t* data_array, size_t start_index, const StringRef* dict,
- size_t data_num, uint32_t dict_num = 0) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_dict_data " + get_name());
- }
-
- void insert_many_continuous_binary_data(const char* data, const uint32_t* offsets,
- const size_t num) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_continuous_binary_data " + get_name());
- }
-
- void insert_many_strings(const StringRef* strings, size_t num) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_strings " + get_name());
- }
-
- void insert_many_strings_overflow(const StringRef* strings, size_t num,
- size_t max_length) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_strings_overflow " + get_name());
- }
-
- void insert_many_raw_data(const char* pos, size_t num) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "insert_many_raw_data " + get_name());
- }
-
- size_t get_max_row_byte_size() const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "get_max_row_byte_size " + get_name());
- }
-
- void serialize_vec(StringRef* keys, size_t num_rows, size_t max_row_byte_size) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "serialize_vec" + std::string(get_name()));
- }
-
- void serialize_vec_with_null_map(StringRef* keys, size_t num_rows,
- const uint8_t* null_map) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "serialize_vec_with_null_map " + get_name());
- }
-
- void deserialize_vec(StringRef* keys, const size_t num_rows) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "deserialize_vec" + std::string(get_name()));
- }
-
- void deserialize_vec_with_null_map(StringRef* keys, const size_t num_rows,
- const uint8_t* null_map) override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
- "deserialize_vec_with_null_map " + get_name());
- }
-
- bool structure_equals(const IColumn&) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "structure_equals " + get_name());
- }
-
- StringRef get_raw_data() const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "get_raw_data " + get_name());
- }
-
- StringRef get_data_at(size_t) const override {
- throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "get_data_at " + get_name());
- }
-
StringRef serialize_value_into_arena(size_t n, Arena& arena,
char const*& begin) const override {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
index ab06bed08d8..246bf551469 100644
--- a/be/src/vec/core/field.h
+++ b/be/src/vec/core/field.h
@@ -184,6 +184,9 @@ public:
}
JsonbField& operator=(const JsonbField& x) {
+ if (data) {
+ delete[] data;
+ }
data = new char[size];
if (!data) {
throw Exception(Status::FatalError("new data buffer failed, size: {}", size));
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 1c1e83851f5..e3c47381e2c 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -420,6 +420,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
case FieldType::OLAP_FIELD_TYPE_STRING:
result = std::make_shared<vectorized::DataTypeString>();
break;
+ case FieldType::OLAP_FIELD_TYPE_VARIANT:
+ result = std::make_shared<vectorized::DataTypeObject>();
+ break;
case FieldType::OLAP_FIELD_TYPE_JSONB:
result = std::make_shared<vectorized::DataTypeJsonb>();
break;
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.cpp b/be/src/vec/data_types/serde/data_type_object_serde.cpp
index ecc8cea0d2b..c4a1dc51a51 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_object_serde.cpp
@@ -36,6 +36,8 @@
#else
#include "util/jsonb_parser.h"
#endif
+#include "vec/json/json_parser.h"
+#include "vec/json/parse2column.cpp"
namespace doris {
@@ -124,6 +126,22 @@ Status DataTypeObjectSerDe::serialize_one_cell_to_json(const IColumn& column, in
return Status::OK();
}
+Status DataTypeObjectSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice,
+ const FormatOptions& options) const {
+ vectorized::ParseConfig config;
+ auto parser = parsers_pool.get([] { return new JsonParser(); });
+ RETURN_IF_CATCH_EXCEPTION(
+ parse_json_to_variant(column, slice.data, slice.size, parser.get(), config));
+ return Status::OK();
+}
+
+Status DataTypeObjectSerDe::deserialize_column_from_json_vector(
+ IColumn& column, std::vector<Slice>& slices, uint64_t* num_deserialized,
+ const FormatOptions& options) const {
+ DESERIALIZE_COLUMN_FROM_JSON_VECTOR()
+ return Status::OK();
+}
+
void DataTypeObjectSerDe::write_column_to_arrow(const IColumn& column, const NullMap* null_map,
arrow::ArrayBuilder* array_builder, int64_t start,
int64_t end, const cctz::time_zone& ctz) const {
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.h b/be/src/vec/data_types/serde/data_type_object_serde.h
index 4ba144760fb..11ac1eea10c 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.h
+++ b/be/src/vec/data_types/serde/data_type_object_serde.h
@@ -46,16 +46,11 @@ public:
return Status::NotSupported("serialize_column_to_json with type [{}]", column.get_name());
}
Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
- const FormatOptions& options) const override {
- return Status::NotSupported("deserialize_one_cell_from_text with type " +
- column.get_name());
- }
+ const FormatOptions& options) const override;
+
Status deserialize_column_from_json_vector(IColumn& column, std::vector<Slice>& slices,
uint64_t* num_deserialized,
- const FormatOptions& options) const override {
- return Status::NotSupported("deserialize_column_from_text_vector with type " +
- column.get_name());
- }
+ const FormatOptions& options) const override;
Status write_column_to_pb(const IColumn& column, PValues& result, int64_t start,
int64_t end) const override {
diff --git a/be/test/expected_result/vec/columns/column_variant_allocated_bytes.out b/be/test/expected_result/vec/columns/column_variant_allocated_bytes.out
new file mode 100644
index 00000000000..fd8ee03f235
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_allocated_bytes.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_byte_size.out b/be/test/expected_result/vec/columns/column_variant_byte_size.out
new file mode 100644
index 00000000000..716643527f5
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_byte_size.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out b/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out
new file mode 100644
index 00000000000..4344346ede9
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out_with_nullmap b/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out_with_nullmap
new file mode 100644
index 00000000000..4344346ede9
--- /dev/null
+++ b/be/test/expected_result/vec/columns/column_variant_update_crc_with_value.out_with_nullmap
@@ -0,0 +1 @@
+column: variant with hashes: 2476392122 with ptr: 2225
\ No newline at end of file
diff --git a/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out b/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out
new file mode 100644
index 00000000000..9f9d94d10b9
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out_with_nullmap b/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out_with_nullmap
new file mode 100644
index 00000000000..9f9d94d10b9
--- /dev/null
+++ b/be/test/expected_result/vec/columns/column_variant_update_crcs_with_value.out_with_nullmap
@@ -0,0 +1 @@
+column: variant with hashes: [2554212050,875308879,875308879,531770243,2554212050,292727423,531770243,56215268,531770243,3870712440,2800491184,2110884430,875308879,2800491184,292727423,2110884430,292727423,2110884430,2800491184,292727423,875308879,531770243,3151753056,228149016,2554212050,228149016,531770243,3151753056,2800491184,531770243,292727423,2554212050,3151753056,2110884430,531770243,56215268,2554212050,56215268,875308879,2554212050,2110884430,875308879,292727423,3151753056,87530 [...]
\ No newline at end of file
diff --git a/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out b/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out
new file mode 100644
index 00000000000..ce7abe610e3
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out_with_nullmap b/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out_with_nullmap
new file mode 100644
index 00000000000..ce7abe610e3
--- /dev/null
+++ b/be/test/expected_result/vec/columns/column_variant_update_hashes_with_value.out_with_nullmap
@@ -0,0 +1 @@
+column: variant with hashes: [15051269310814893253,9207780176212329026,9207780176212329026,8769178907691657826,15051269310814893253,14523939085247816014,8769178907691657826,9403745004122609994,8769178907691657826,3326292242930667833,18294539114392923310,5059852396507440228,9207780176212329026,18294539114392923310,14523939085247816014,5059852396507440228,14523939085247816014,5059852396507440228,18294539114392923310,14523939085247816014,9207780176212329026,8769178907691657826,1599870548429 [...]
\ No newline at end of file
diff --git a/be/test/expected_result/vec/columns/column_variant_update_sip_hash_with_value.out b/be/test/expected_result/vec/columns/column_variant_update_sip_hash_with_value.out
new file mode 100644
index 00000000000..04334fe112c
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_update_sip_hash_with_value.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out b/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out
new file mode 100644
index 00000000000..5bc4d8f8e5c
Binary files /dev/null and b/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out differ
diff --git a/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out_with_nullmap b/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out_with_nullmap
new file mode 100644
index 00000000000..5bc4d8f8e5c
--- /dev/null
+++ b/be/test/expected_result/vec/columns/column_variant_update_xxHash_with_value.out_with_nullmap
@@ -0,0 +1 @@
+column: variant with hashes: 9301615508800336962 with ptr: 2225
\ No newline at end of file
diff --git a/be/test/testutil/test_util.cpp b/be/test/testutil/test_util.cpp
index d24dc2dc22d..2326509659a 100644
--- a/be/test/testutil/test_util.cpp
+++ b/be/test/testutil/test_util.cpp
@@ -33,16 +33,23 @@
#include <unistd.h>
#include <algorithm>
+#include <string>
// IWYU pragma: no_include <bits/chrono.h>
#include <chrono> // IWYU pragma: keep
+#include <filesystem>
+#include <fstream>
+#include <iostream>
#include <random>
+#include "gflags/gflags.h"
#include "gutil/strings/numbers.h"
#include "gutil/strings/substitute.h"
#include "olap/olap_common.h"
using strings::Substitute;
+DEFINE_bool(gen_out, true, "generate expected check data for test");
+
namespace doris {
static const char* const kSlowTestsEnvVar = "DORIS_ALLOW_SLOW_TESTS";
@@ -130,4 +137,132 @@ std::string rand_rng_by_type(FieldType fieldType) {
}
}
+void load_columns_data_from_file(vectorized::MutableColumns& columns,
+ vectorized::DataTypeSerDeSPtrs serders, char col_spliter,
+ std::set<int> idxes, const std::string& column_data_file) {
+ ASSERT_EQ(serders.size(), columns.size());
+ // Load column data and expected data from CSV files
+ std::vector<std::vector<std::string>> res;
+ struct stat buff;
+ if (stat(column_data_file.c_str(), &buff) == 0) {
+ if (S_ISREG(buff.st_mode)) {
+ // file
+ load_data_from_csv(serders, columns, column_data_file, col_spliter, idxes);
+ } else if (S_ISDIR(buff.st_mode)) {
+ // dir
+ std::filesystem::path fs_path(column_data_file);
+ for (const auto& entry : std::filesystem::directory_iterator(fs_path)) {
+ std::string file_path = entry.path().string();
+ std::cout << "load data from file: " << file_path << std::endl;
+ load_data_from_csv(serders, columns, file_path, col_spliter, idxes);
+ }
+ }
+ }
+}
+
+// Helper function to load data from CSV, with index which splited by spliter and load to columns
+void load_data_from_csv(const vectorized::DataTypeSerDeSPtrs serders,
+ vectorized::MutableColumns& columns, const std::string& file_path,
+ const char spliter, const std::set<int> idxes) {
+ ASSERT_EQ(serders.size(), columns.size())
+ << "serder size: " << serders.size() << " column size: " << columns.size();
+ ASSERT_EQ(serders.size(), idxes.size())
+ << "serder size: " << serders.size() << " idxes size: " << idxes.size();
+ ASSERT_EQ(serders.size(), *idxes.end())
+ << "serder size: " << serders.size() << " idxes size: " << *idxes.end();
+ std::ifstream file(file_path);
+ if (!file) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+ file_path);
+ }
+
+ std::string line;
+ vectorized::DataTypeSerDe::FormatOptions options;
+ while (std::getline(file, line)) {
+ std::stringstream lineStream(line);
+ std::string value;
+ int l_idx = 0;
+ int c_idx = 0;
+ while (std::getline(lineStream, value, spliter)) {
+ if (!value.starts_with("//") && idxes.contains(l_idx)) {
+ Slice string_slice(value.data(), value.size());
+ if (auto st = serders[c_idx]->deserialize_one_cell_from_json(*columns[c_idx],
+ string_slice, options);
+ !st.ok()) {
+ LOG(INFO) << "error in deserialize but continue: " << st.to_string();
+ }
+ ++c_idx;
+ }
+ ++l_idx;
+ }
+ }
+ std::cout << "loading data done, file: " << file_path << ", row count: " << columns[0]->size()
+ << std::endl;
+}
+// res_columns[i] is the i-th column of the data
+void check_or_generate_res_file(const std::string& res_file_path,
+ const std::vector<std::vector<std::string>>& res_columns) {
+ if (res_columns.empty()) {
+ return;
+ }
+ auto row_count = res_columns[0].size();
+ if (FLAGS_gen_out) {
+ std::ofstream res_file(res_file_path);
+ LOG(INFO) << "gen check data with file: " << res_file_path;
+ if (!res_file.is_open()) {
+ throw std::ios_base::failure("Failed to open file: " + res_file_path);
+ }
+
+ for (size_t i = 1; i < res_columns.size(); ++i) {
+ EXPECT_EQ(res_columns[i].size(), row_count);
+ }
+ auto column_count = res_columns.size();
+ for (size_t i = 0; i < row_count; ++i) {
+ // output one row
+ for (size_t j = 0; j < column_count; ++j) {
+ const auto& column = res_columns[j];
+ // output one column
+ res_file << column[i];
+ if (j < column_count - 1) {
+ res_file << ";"; // Add semicolon between columns
+ }
+ }
+ if (i < row_count - 1) {
+ res_file << "\n";
+ }
+ }
+ res_file.close();
+ } else {
+ // we read generate file to check result
+ LOG(INFO) << "check data with file: " << res_file_path;
+ std::ifstream file(res_file_path);
+ if (!file) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+ res_file_path);
+ }
+
+ size_t file_line_count = 0;
+ std::string line;
+ while (getline(file, line)) {
+ file_line_count++;
+ }
+ EXPECT_EQ(file_line_count, row_count);
+
+ file.clear();
+ file.seekg(0, std::ios::beg);
+
+ size_t line_idx = 0;
+ while (std::getline(file, line)) {
+ std::stringstream line_stream(line);
+ std::string value;
+ size_t col_idx = 0;
+ while (std::getline(line_stream, value, ';')) {
+ EXPECT_EQ(value, res_columns[col_idx][line_idx]);
+ col_idx++;
+ }
+ line_idx++;
+ }
+ }
+}
+
} // namespace doris
diff --git a/be/test/testutil/test_util.h b/be/test/testutil/test_util.h
index 6bd6b4c7192..6e936e2a829 100644
--- a/be/test/testutil/test_util.h
+++ b/be/test/testutil/test_util.h
@@ -20,7 +20,11 @@
#include <cstdlib>
#include <string>
+#include "gflags/gflags.h"
#include "gtest/gtest.h"
+#include "vec/data_types/serde/data_type_serde.h"
+
+DECLARE_bool(gen_out);
struct TestCaseInfo {
inline static int arg_const_info {};
@@ -76,4 +80,13 @@ char rand_rng_char();
std::string rand_rng_string(size_t length = 8);
std::string rand_rng_by_type(FieldType fieldType);
+void load_columns_data_from_file(vectorized::MutableColumns& columns,
+ vectorized::DataTypeSerDeSPtrs serders, char col_spliter,
+ std::set<int> idxes, const std::string& column_data_file);
+void load_data_from_csv(const vectorized::DataTypeSerDeSPtrs serders,
+ vectorized::MutableColumns& columns, const std::string& file_path,
+ const char spliter = ';', const std::set<int> idxes = {0});
+void check_or_generate_res_file(const std::string& res_file_path,
+ const std::vector<std::vector<std::string>>& res_columns);
+
} // namespace doris
diff --git a/be/test/testutil/variant_util.h b/be/test/testutil/variant_util.h
index 27b9000ce55..c5cb1759b00 100644
--- a/be/test/testutil/variant_util.h
+++ b/be/test/testutil/variant_util.h
@@ -29,6 +29,176 @@ using namespace vectorized;
class VariantUtil {
public:
+ static doris::vectorized::Field get_field(std::string_view type) {
+ static std::unordered_map<std::string_view, doris::vectorized::Field> field_map;
+ if (field_map.empty()) {
+ doris::vectorized::Field int_field = 20;
+ doris::vectorized::Field str_field(String("str", 3));
+ doris::vectorized::Field arr_int_field = Array();
+ auto& array1 = arr_int_field.get<Array>();
+ array1.emplace_back(int_field);
+ array1.emplace_back(int_field);
+ doris::vectorized::Field arr_str_field = Array();
+ auto& array2 = arr_str_field.get<Array>();
+ array2.emplace_back(str_field);
+ array2.emplace_back(str_field);
+ field_map["int"] = int_field;
+ field_map["string"] = str_field;
+ field_map["array_int"] = arr_int_field;
+ field_map["array_str"] = arr_str_field;
+
+ // add other int value
+ field_map["int_16"] = std::numeric_limits<Int16>::max();
+ field_map["int_32"] = std::numeric_limits<Int32>::max();
+ field_map["int_64"] = Int64(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1);
+ }
+ return field_map[type];
+ }
+
+ static doris::vectorized::Field construct_variant_map(
+ const std::vector<std::pair<std::string, doris::vectorized::Field>>& key_and_values) {
+ doris::vectorized::Field res = VariantMap();
+ auto& object = res.get<VariantMap&>();
+ for (const auto& [k, v] : key_and_values) {
+ object.try_emplace(k, v);
+ }
+ return res;
+ }
+
+ static auto construct_basic_varint_column() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(5);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. sparse column path
+ data.emplace_back("v.d.d", "50");
+ data.emplace_back("v.c.d", 30);
+ data.emplace_back("v.b.d", 30);
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ return variant;
+ }
+
+ static auto construct_dst_varint_column() {
+ // 1. create an empty variant column
+ vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+ dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0, true, true /*root*/));
+ dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.e"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ return ColumnObject::create(5, std::move(dynamic_subcolumns));
+ }
+
+ static auto construct_advanced_varint_column() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(5);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", get_field("int"));
+ data.emplace_back("v.b", get_field("string"));
+ data.emplace_back("v.c", get_field("array_int"));
+ data.emplace_back("v.f", get_field("array_str"));
+ data.emplace_back("v.e", get_field("string"));
+
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. sparse column path
+ data.emplace_back("v.d.d", get_field("array_int"));
+ data.emplace_back("v.c.d", get_field("string"));
+ data.emplace_back("v.b.d", get_field("array_int"));
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ data.clear();
+ data.emplace_back("v.a", get_field("int"));
+ data.emplace_back("v.b", get_field("int"));
+ data.emplace_back("v.c", get_field("array_int"));
+ data.emplace_back("v.f", get_field("array_str"));
+ data.emplace_back("v.e", get_field("string"));
+ data.emplace_back("v.d.d", get_field("array_str"));
+ data.emplace_back("v.c.d", get_field("int"));
+ data.emplace_back("v.b.d", get_field("array_str"));
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ return variant;
+ }
+
+ static auto construct_varint_column_only_subcolumns() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(5);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. root
+ variant->try_insert(doris::vectorized::Field(20));
+ return variant;
+ }
+
+ static auto construct_varint_column_more_subcolumns() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(5);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ data.emplace_back("v.s", "str");
+ data.emplace_back("v.x", get_field("int_16"));
+ data.emplace_back("v.y", get_field("int_32"));
+ data.emplace_back("v.z", get_field("int_64"));
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ return variant;
+ }
+
static schema_util::PathToNoneNullValues fill_string_column_with_test_data(
auto& column_string, int size, std::unordered_map<int, std::string>* inserted_jsonstr) {
schema_util::PathToNoneNullValues all_path_stats;
diff --git a/be/test/vec/columns/column_array_test.cpp b/be/test/vec/columns/column_array_test.cpp
index 2961e72b6cc..76d6d071579 100644
--- a/be/test/vec/columns/column_array_test.cpp
+++ b/be/test/vec/columns/column_array_test.cpp
@@ -599,7 +599,7 @@ TEST_F(ColumnArrayTest, ConvertToPredicateColumnIfDictionaryTest) {
// test assert_convert_dict_codes_if_necessary_callback
TEST_F(ColumnArrayTest, ConvertDictCodesIfNecessaryTest) {
auto callback = [&](IColumn* col, size_t index) {
- checkColumn(*col->get_ptr(), *array_columns[index], *array_types[index], col->size());
+ checkColumn(*col->get_ptr(), *array_columns[index], col->size());
};
assert_convert_dict_codes_if_necessary_callback(array_columns, callback);
}
diff --git a/be/test/vec/columns/column_object_test.cpp b/be/test/vec/columns/column_object_test.cpp
index e4c9a6f09af..99130c739a5 100644
--- a/be/test/vec/columns/column_object_test.cpp
+++ b/be/test/vec/columns/column_object_test.cpp
@@ -23,7 +23,10 @@
#include <stdio.h>
#include "runtime/jsonb_value.h"
+#include "testutil/variant_util.h"
#include "vec/common/string_ref.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
#include "vec/data_types/data_type_array.h"
#include "vec/data_types/data_type_factory.hpp"
@@ -33,63 +36,115 @@ using namespace doris;
// #define ADD_SUB_COLUMN(key) \
// varaint->add_sub_column(PathInData(std::string_view(key)), 0);
-doris::vectorized::Field construct_variant_map(
- const std::vector<std::pair<std::string, doris::vectorized::Field>>& key_and_values) {
- doris::vectorized::Field res = VariantMap();
- auto& object = res.get<VariantMap&>();
- for (const auto& [k, v] : key_and_values) {
- object.try_emplace(k, v);
+void convert_field_to_rapidjson(const vectorized::Field& field, rapidjson::Value& target,
+ rapidjson::Document::AllocatorType& allocator) {
+ switch (field.get_type()) {
+ case vectorized::Field::Types::Null:
+ target.SetNull();
+ break;
+ case vectorized::Field::Types::Int64:
+ target.SetInt64(field.get<Int64>());
+ break;
+ case vectorized::Field::Types::Float64:
+ target.SetDouble(field.get<Float64>());
+ break;
+ case vectorized::Field::Types::JSONB: {
+ const auto& val = field.get<JsonbField>();
+ JsonbValue* json_val = JsonbDocument::createValue(val.get_value(), val.get_size());
+ convert_jsonb_to_rapidjson(*json_val, target, allocator);
+ break;
+ }
+ case vectorized::Field::Types::String: {
+ const String& val = field.get<String>();
+ target.SetString(val.data(), cast_set<rapidjson::SizeType>(val.size()));
+ break;
+ }
+ case vectorized::Field::Types::Array: {
+ const vectorized::Array& array = field.get<Array>();
+ target.SetArray();
+ for (const vectorized::Field& item : array) {
+ rapidjson::Value val;
+ convert_field_to_rapidjson(item, val, allocator);
+ target.PushBack(val, allocator);
+ }
+ break;
+ }
+ case vectorized::Field::Types::VariantMap: {
+ const vectorized::VariantMap& map = field.get<VariantMap>();
+ target.SetObject();
+ for (const auto& item : map) {
+ if (item.second.is_null()) {
+ continue;
+ }
+ rapidjson::Value key;
+ key.SetString(item.first.data(), cast_set<rapidjson::SizeType>(item.first.size()));
+ rapidjson::Value val;
+ convert_field_to_rapidjson(item.second, val, allocator);
+ if (val.IsNull() && item.first.empty()) {
+ // skip null value with empty key, indicate the null json value of root in variant map,
+ // usally padding in nested arrays
+ continue;
+ }
+ target.AddMember(key, val, allocator);
+ }
+ break;
+ }
+ default:
+ throw doris::Exception(ErrorCode::INTERNAL_ERROR, "unkown field type: {}",
+ field.get_type_name());
+ break;
}
- return res;
}
-auto construct_basic_varint_column() {
- // 1. create an empty variant column
- auto variant = ColumnObject::create(5);
-
- std::vector<std::pair<std::string, doris::vectorized::Field>> data;
-
- // 2. subcolumn path
- data.emplace_back("v.a", 20);
- data.emplace_back("v.b", "20");
- data.emplace_back("v.c", 20);
- data.emplace_back("v.f", 20);
- data.emplace_back("v.e", "50");
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
+void convert_variant_map_to_rapidjson(const vectorized::VariantMap& map, rapidjson::Value& target,
+ rapidjson::Document::AllocatorType& allocator) {
+ target.SetObject();
+ for (const auto& item : map) {
+ if (item.second.is_null()) {
+ continue;
+ }
+ rapidjson::Value key;
+ key.SetString(item.first.data(), cast_set<rapidjson::SizeType>(item.first.size()));
+ rapidjson::Value val;
+ convert_field_to_rapidjson(item.second, val, allocator);
+ if (val.IsNull() && item.first.empty()) {
+ // skip null value with empty key, indicate the null json value of root in variant map,
+ // usally padding in nested arrays
+ continue;
+ }
+ target.AddMember(key, val, allocator);
}
+}
- // 3. sparse column path
- data.emplace_back("v.d.d", "50");
- data.emplace_back("v.c.d", 30);
- data.emplace_back("v.b.d", 30);
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
+void convert_array_to_rapidjson(const vectorized::Array& array, rapidjson::Value& target,
+ rapidjson::Document::AllocatorType& allocator) {
+ target.SetArray();
+ for (const vectorized::Field& item : array) {
+ rapidjson::Value val;
+ convert_field_to_rapidjson(item, val, allocator);
+ target.PushBack(val, allocator);
}
- return variant;
}
-auto construct_dst_varint_column() {
- // 1. create an empty variant column
- vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
- dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0, true, true /*root*/));
- dynamic_subcolumns.add(vectorized::PathInData("v.f"),
- vectorized::ColumnObject::Subcolumn {0, true});
- dynamic_subcolumns.add(vectorized::PathInData("v.e"),
- vectorized::ColumnObject::Subcolumn {0, true});
- dynamic_subcolumns.add(vectorized::PathInData("v.b"),
- vectorized::ColumnObject::Subcolumn {0, true});
- dynamic_subcolumns.add(vectorized::PathInData("v.b.d"),
- vectorized::ColumnObject::Subcolumn {0, true});
- dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
- vectorized::ColumnObject::Subcolumn {0, true});
- return ColumnObject::create(5, std::move(dynamic_subcolumns));
+TEST(ColumnVariantTest, insert_try_insert) {
+ auto v = VariantUtil::construct_dst_varint_column();
+ FieldInfo info;
+ info.scalar_type_id = TypeIndex::Nothing;
+ info.num_dimensions = 0;
+ PathInData path("v.f");
+ auto sub = v->get_subcolumn(path);
+ Int64 value = 43;
+ sub->insert(value, info);
+
+ info.num_dimensions = 1;
+ sub->insert(value, info);
+
+ info.num_dimensions = 2;
+ sub->insert(value, info);
}
TEST(ColumnVariantTest, basic_finalize) {
- auto variant = construct_basic_varint_column();
+ auto variant = VariantUtil::construct_basic_varint_column();
// 4. finalize
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
EXPECT_EQ(variant->size(), 10);
@@ -115,7 +170,7 @@ TEST(ColumnVariantTest, basic_finalize) {
}
TEST(ColumnVariantTest, basic_deserialize) {
- auto variant = construct_basic_varint_column();
+ auto variant = VariantUtil::construct_basic_varint_column();
// 4. finalize
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
@@ -146,12 +201,12 @@ TEST(ColumnVariantTest, basic_deserialize) {
}
TEST(ColumnVariantTest, basic_inset_range_from) {
- auto src = construct_basic_varint_column();
+ auto src = VariantUtil::construct_basic_varint_column();
EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
EXPECT_EQ(src->size(), 10);
// dst is an empty column, has 5 subcolumn + 1 root
- auto dst = construct_dst_varint_column();
+ auto dst = VariantUtil::construct_dst_varint_column();
// subcolumn->subcolumn v.b v.f v.e
// subcolumn->sparse_column v.a v.c
@@ -234,27 +289,6 @@ TEST(ColumnVariantTest, basic_inset_range_from) {
}
}
-doris::vectorized::Field get_field(std::string_view type) {
- static std::unordered_map<std::string_view, doris::vectorized::Field> field_map;
- if (field_map.empty()) {
- doris::vectorized::Field int_field = 20;
- doris::vectorized::Field str_field(String("str", 3));
- doris::vectorized::Field arr_int_field = Array();
- auto& array1 = arr_int_field.get<Array>();
- array1.emplace_back(int_field);
- array1.emplace_back(int_field);
- doris::vectorized::Field arr_str_field = Array();
- auto& array2 = arr_str_field.get<Array>();
- array2.emplace_back(str_field);
- array2.emplace_back(str_field);
- field_map["int"] = int_field;
- field_map["string"] = str_field;
- field_map["array_int"] = arr_int_field;
- field_map["array_str"] = arr_str_field;
- }
- return field_map[type];
-}
-
auto convert_to_jsonb_field(auto serde, auto& column) {
vectorized::DataTypeSerDe::FormatOptions options;
options.escape_char = '\\';
@@ -305,7 +339,7 @@ doris::vectorized::Field get_jsonb_field(std::string_view type) {
DataTypePtr data_type_array_int =
std::make_shared<doris::vectorized::DataTypeArray>(data_type_int);
auto array_column_int = data_type_array_int->create_column();
- array_column_int->insert(get_field("array_int"));
+ array_column_int->insert(VariantUtil::get_field("array_int"));
auto array_serde_int = data_type_array_int->get_serde();
field_map["array_int"] = convert_to_jsonb_field(array_serde_int, *array_column_int);
@@ -314,65 +348,22 @@ doris::vectorized::Field get_jsonb_field(std::string_view type) {
DataTypePtr data_type_array_str =
std::make_shared<doris::vectorized::DataTypeArray>(data_type_str);
auto array_column_str = data_type_array_str->create_column();
- array_column_str->insert(get_field("array_str"));
+ array_column_str->insert(VariantUtil::get_field("array_str"));
auto array_serde_str = data_type_array_str->get_serde();
field_map["array_str"] = convert_to_jsonb_field(array_serde_str, *array_column_str);
auto column_int = data_type_int->create_column();
- column_int->insert(get_field("int"));
+ column_int->insert(VariantUtil::get_field("int"));
auto serde_int = data_type_int->get_serde();
field_map["int"] = convert_to_jsonb_field(serde_int, *column_int);
// auto column_str = data_type_str->create_column();
- // column_str->insert(get_field("string"));
+ // column_str->insert(VariantUtil::get_field("string"));
// field_map["string"] = convert_string_to_jsonb_field(*column_str);
}
return field_map[type];
}
-auto construct_advanced_varint_column() {
- // 1. create an empty variant column
- auto variant = ColumnObject::create(5);
-
- std::vector<std::pair<std::string, doris::vectorized::Field>> data;
-
- // 2. subcolumn path
- data.emplace_back("v.a", get_field("int"));
- data.emplace_back("v.b", get_field("string"));
- data.emplace_back("v.c", get_field("array_int"));
- data.emplace_back("v.f", get_field("array_str"));
- data.emplace_back("v.e", get_field("string"));
-
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
- }
-
- // 3. sparse column path
- data.emplace_back("v.d.d", get_field("array_int"));
- data.emplace_back("v.c.d", get_field("string"));
- data.emplace_back("v.b.d", get_field("array_int"));
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
- }
-
- data.clear();
- data.emplace_back("v.a", get_field("int"));
- data.emplace_back("v.b", get_field("int"));
- data.emplace_back("v.c", get_field("array_int"));
- data.emplace_back("v.f", get_field("array_str"));
- data.emplace_back("v.e", get_field("string"));
- data.emplace_back("v.d.d", get_field("array_str"));
- data.emplace_back("v.c.d", get_field("int"));
- data.emplace_back("v.b.d", get_field("array_str"));
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
- }
- return variant;
-}
-
// std::string convert_jsonb_field_to_string(doris::vectorized::Field jsonb) {
// const auto& val = jsonb.get<JsonbField>();
// const JsonbValue* json_val = JsonbDocument::createValue(val.get_value(), val.get_size());
@@ -394,7 +385,7 @@ std::string convert_field_to_string(doris::vectorized::Field array) {
doc.SetObject();
rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
rapidjson::Value json_value;
- DataTypeSerDe::convert_field_to_rapidjson(array, json_value, allocator);
+ convert_field_to_rapidjson(array, json_value, allocator);
doc.AddMember("value", json_value, allocator);
rapidjson::StringBuffer buffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
@@ -402,8 +393,33 @@ std::string convert_field_to_string(doris::vectorized::Field array) {
return std::string(buffer.GetString());
}
+TEST(ColumnVariantTest, is_null_at) {
+ auto v = VariantUtil::construct_dst_varint_column();
+ PathInData path("v.f");
+ auto sub = v->get_subcolumn(path);
+ std::cout << sub->get_least_common_typeBase()->get_name() << std::endl;
+ EXPECT_TRUE(sub->is_null_at(0));
+
+ auto v1 = VariantUtil::construct_advanced_varint_column();
+ PathInData path1("v.b.d");
+ auto sub1 = v1->get_subcolumn(path1);
+ EXPECT_TRUE(sub1->is_null_at(2));
+ EXPECT_ANY_THROW(sub1->is_null_at(16));
+ vectorized::Field f;
+ EXPECT_ANY_THROW(sub1->get(16, f));
+ std::cout << sub1->num_rows << std::endl;
+ EXPECT_NO_THROW(sub1->resize(sub1->num_rows));
+
+ auto [sparse_column_keys, sparse_column_values] = v1->get_sparse_data_paths_and_values();
+ std::string_view pa("v.a");
+ EXPECT_NO_THROW(
+ sub1->serialize_to_sparse_column(sparse_column_keys, pa, sparse_column_values, 2));
+ EXPECT_ANY_THROW(
+ sub1->serialize_to_sparse_column(sparse_column_keys, pa, sparse_column_values, 16));
+}
+
TEST(ColumnVariantTest, advanced_finalize) {
- auto variant = construct_advanced_varint_column();
+ auto variant = VariantUtil::construct_advanced_varint_column();
// 4. finalize
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
@@ -427,10 +443,21 @@ TEST(ColumnVariantTest, advanced_finalize) {
for (int row = 5; row < 15; ++row) {
EXPECT_EQ(offsets[row] - offsets[row - 1], 3);
}
+
+ {
+ // Test fill_path_column_from_sparse_data
+ auto map = std::make_unique<NullMap>(15, 0);
+ vectorized::ColumnObject::fill_path_column_from_sparse_data(
+ *variant->get_subcolumn({}) /*root*/, map.get(), StringRef {"array"},
+ variant->get_sparse_column(), 0, 5);
+ vectorized::ColumnObject::fill_path_column_from_sparse_data(
+ *variant->get_subcolumn({}) /*root*/, map.get(), StringRef {"array"},
+ variant->get_sparse_column(), 5, 15);
+ }
}
TEST(ColumnVariantTest, advanced_deserialize) {
- auto variant = construct_advanced_varint_column();
+ auto variant = VariantUtil::construct_advanced_varint_column();
// 4. finalize
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
@@ -452,7 +479,7 @@ TEST(ColumnVariantTest, advanced_deserialize) {
auto pair2 = variant->deserialize_from_sparse_column(value, start++);
EXPECT_EQ(data2, StringRef("v.c.d", 5));
EXPECT_EQ(convert_field_to_string(pair2.first),
- convert_field_to_string(get_field("string")));
+ convert_field_to_string(VariantUtil::get_field("string")));
auto data3 = path->get_data_at(start);
auto pair3 = variant->deserialize_from_sparse_column(value, start++);
@@ -488,11 +515,11 @@ TEST(ColumnVariantTest, advanced_deserialize) {
}
TEST(ColumnVariantTest, advanced_insert_range_from) {
- auto src = construct_advanced_varint_column();
+ auto src = VariantUtil::construct_advanced_varint_column();
EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
EXPECT_EQ(src->size(), 15);
- auto dst = construct_dst_varint_column();
+ auto dst = VariantUtil::construct_dst_varint_column();
// subcolumn->subcolumn v.b v.f v.e
// subcolumn->sparse_column v.a v.c
@@ -555,7 +582,7 @@ TEST(ColumnVariantTest, advanced_insert_range_from) {
EXPECT_EQ(data2, StringRef("v.c", 3));
auto pair2 = dst->deserialize_from_sparse_column(value, start++);
EXPECT_EQ(convert_field_to_string(pair2.first),
- convert_field_to_string(get_field("array_int")));
+ convert_field_to_string(VariantUtil::get_field("array_int")));
EXPECT_EQ(start, end);
}
@@ -573,7 +600,7 @@ TEST(ColumnVariantTest, advanced_insert_range_from) {
auto pair2 = dst->deserialize_from_sparse_column(value, start++);
EXPECT_EQ(data2, StringRef("v.c", 3));
EXPECT_EQ(convert_field_to_string(pair2.first),
- convert_field_to_string(get_field("array_int")));
+ convert_field_to_string(VariantUtil::get_field("array_int")));
auto data3 = path->get_data_at(start);
auto pair3 = dst->deserialize_from_sparse_column(value, start++);
@@ -597,7 +624,7 @@ TEST(ColumnVariantTest, advanced_insert_range_from) {
auto pair2 = dst->deserialize_from_sparse_column(value, start++);
EXPECT_EQ(data2, StringRef("v.c", 3));
EXPECT_EQ(convert_field_to_string(pair2.first),
- convert_field_to_string(get_field("array_int")));
+ convert_field_to_string(VariantUtil::get_field("array_int")));
auto data3 = path->get_data_at(start);
auto pair3 = dst->deserialize_from_sparse_column(value, start++);
@@ -609,52 +636,8 @@ TEST(ColumnVariantTest, advanced_insert_range_from) {
}
}
-auto construct_varint_column_only_subcolumns() {
- // 1. create an empty variant column
- auto variant = ColumnObject::create(5);
-
- std::vector<std::pair<std::string, doris::vectorized::Field>> data;
-
- // 2. subcolumn path
- data.emplace_back("v.a", 20);
- data.emplace_back("v.b", "20");
- data.emplace_back("v.c", 20);
- data.emplace_back("v.f", 20);
- data.emplace_back("v.e", "50");
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
- }
-
- // 3. root
- variant->try_insert(doris::vectorized::Field(20));
- return variant;
-}
-
-auto construct_varint_column_more_subcolumns() {
- // 1. create an empty variant column
- auto variant = ColumnObject::create(5);
-
- std::vector<std::pair<std::string, doris::vectorized::Field>> data;
-
- // 2. subcolumn path
- data.emplace_back("v.a", 20);
- data.emplace_back("v.b", "20");
- data.emplace_back("v.c", 20);
- data.emplace_back("v.f", 20);
- data.emplace_back("v.e", "50");
- data.emplace_back("v.x", "str");
- data.emplace_back("v.y", 20);
- for (int i = 0; i < 5; ++i) {
- auto field = construct_variant_map(data);
- variant->try_insert(field);
- }
-
- return variant;
-}
-
TEST(ColumnVariantTest, empty_inset_range_from) {
- auto src = construct_varint_column_only_subcolumns();
+ auto src = VariantUtil::construct_varint_column_only_subcolumns();
EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
EXPECT_EQ(src->size(), 6);
@@ -686,14 +669,14 @@ TEST(ColumnVariantTest, empty_inset_range_from) {
EXPECT_EQ(offsets[4], offsets[-1]);
EXPECT_EQ(path->size(), value->size());
- auto src_contains_seven_subcolumns = construct_varint_column_more_subcolumns();
+ auto src_contains_seven_subcolumns = VariantUtil::construct_varint_column_more_subcolumns();
EXPECT_TRUE(
src_contains_seven_subcolumns->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
EXPECT_EQ(src_contains_seven_subcolumns->size(), 5);
// subcolumn->subcolumn v.a v.b v.c v.f v.e
- // add sprase columns v.x v.y
+ // add sprase columns v.s v.x v.y v.z
dst->insert_range_from(*src_contains_seven_subcolumns, 0, 5);
EXPECT_EQ(dst->size(), 11);
@@ -707,33 +690,44 @@ TEST(ColumnVariantTest, empty_inset_range_from) {
EXPECT_EQ(start, end);
}
- // v.x v.y
+ // v.s v.x v.y v.z
for (int row = 6; row < 11; ++row) {
size_t start = offsets[row - 1];
size_t end = offsets[row];
+ auto data0 = path->get_data_at(start);
+ EXPECT_EQ(data0, StringRef("v.s", 3));
+ auto pair0 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(convert_field_to_string(pair0.first),
+ convert_field_to_string(VariantUtil::get_field("string")));
+
auto data = path->get_data_at(start);
EXPECT_EQ(data, StringRef("v.x", 3));
auto pair = dst->deserialize_from_sparse_column(value, start++);
- EXPECT_EQ(convert_field_to_string(pair.first),
- convert_field_to_string(get_field("string")));
+ EXPECT_EQ(pair.first.get<Int16>(), std::numeric_limits<Int16>::max());
auto data2 = path->get_data_at(start);
EXPECT_EQ(data2, StringRef("v.y", 3));
auto pair2 = dst->deserialize_from_sparse_column(value, start++);
- EXPECT_EQ(pair2.first.get<Int64>(), 20);
+ EXPECT_EQ(pair2.first.get<Int32>(), std::numeric_limits<Int32>::max());
+
+ auto data3 = path->get_data_at(start);
+ EXPECT_EQ(data3, StringRef("v.z", 3));
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair3.first.get<Int64>(),
+ Int64(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1));
EXPECT_EQ(start, end);
}
- auto src_contains_subcoumns_and_sparse_columns = construct_basic_varint_column();
+ auto src_contains_subcoumns_and_sparse_columns = VariantUtil::construct_basic_varint_column();
EXPECT_TRUE(src_contains_subcoumns_and_sparse_columns
->finalize(ColumnObject::FinalizeMode::WRITE_MODE)
.ok());
EXPECT_EQ(src_contains_subcoumns_and_sparse_columns->size(), 10);
// subcolumn->subcolumn v.a v.b v.c v.f v.e
- // add sprase columns v.x v.y v.b.d v.c.d v.d.d
+ // add sprase columns v.s v.x v.y v.b.d v.c.d v.d.d
dst->insert_range_from(*src_contains_subcoumns_and_sparse_columns, 0, 10);
EXPECT_EQ(dst->size(), 21);
@@ -752,16 +746,27 @@ TEST(ColumnVariantTest, empty_inset_range_from) {
size_t start = offsets[row - 1];
size_t end = offsets[row];
+ auto data0 = path->get_data_at(start);
+ EXPECT_EQ(data0, StringRef("v.s", 3));
+ auto pair0 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(convert_field_to_string(pair0.first),
+ convert_field_to_string(VariantUtil::get_field("string")));
+
auto data = path->get_data_at(start);
EXPECT_EQ(data, StringRef("v.x", 3));
auto pair = dst->deserialize_from_sparse_column(value, start++);
- EXPECT_EQ(convert_field_to_string(pair.first),
- convert_field_to_string(get_field("string")));
+ EXPECT_EQ(pair.first.get<Int16>(), std::numeric_limits<Int16>::max());
auto data2 = path->get_data_at(start);
EXPECT_EQ(data2, StringRef("v.y", 3));
auto pair2 = dst->deserialize_from_sparse_column(value, start++);
- EXPECT_EQ(pair2.first.get<Int64>(), 20);
+ EXPECT_EQ(pair2.first.get<Int32>(), std::numeric_limits<Int32>::max());
+
+ auto data3 = path->get_data_at(start);
+ EXPECT_EQ(data3, StringRef("v.z", 3));
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair3.first.get<Int64>(),
+ Int64(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1));
EXPECT_EQ(start, end);
}
@@ -794,4 +799,4 @@ TEST(ColumnVariantTest, empty_inset_range_from) {
EXPECT_EQ(pair3.first.get<String>(), "50");
EXPECT_EQ(start, end);
}
-}
+}
\ No newline at end of file
diff --git a/be/test/vec/columns/column_variant_test.cpp b/be/test/vec/columns/column_variant_test.cpp
new file mode 100644
index 00000000000..b3584e7f2c5
--- /dev/null
+++ b/be/test/vec/columns/column_variant_test.cpp
@@ -0,0 +1,2420 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gmock/gmock-more-matchers.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+
+#include "testutil/test_util.h"
+#include "testutil/variant_util.h"
+#include "vec/columns/column_object.cpp"
+#include "vec/columns/column_object.h"
+#include "vec/columns/common_column_test.h"
+#include "vec/columns/subcolumn_tree.h"
+#include "vec/common/schema_util.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+using namespace doris;
+namespace doris::vectorized {
+static std::string root_dir;
+static std::string test_data_dir;
+static std::string test_result_dir;
+static std::string test_data_dir_json;
+static DataTypePtr dt_variant =
+ DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_VARIANT, 0, 0);
+
+DataTypeSerDeSPtrs serde;
+
+static ColumnObject::MutablePtr column_variant;
+
+class ColumnObjectTest : public CommonColumnTest {
+protected:
+ static void SetUpTestSuite() {
+ root_dir = std::string(getenv("ROOT"));
+ std::cout << "root_dir: " << root_dir << std::endl;
+ test_data_dir = root_dir + "/be/test/data/vec/columns";
+ test_result_dir = root_dir + "/be/test/expected_result/vec/columns";
+
+ column_variant = ColumnObject::create(true);
+ std::cout << dt_variant->get_name() << std::endl;
+
+ load_json_columns_data();
+ }
+
+ static void load_json_columns_data() {
+ std::cout << "loading json dataset : " << FLAGS_gen_out << std::endl;
+ {
+ MutableColumns columns;
+ columns.push_back(column_variant->get_ptr());
+ serde = {dt_variant->get_serde()};
+ test_data_dir_json = root_dir + "/regression-test/data/nereids_function_p0/";
+ std::vector<string> json_files = {
+ test_data_dir_json + "json_variant/boolean_boundary.jsonl",
+ test_data_dir_json + "json_variant/null_boundary.jsonl",
+ test_data_dir_json + "json_variant/number_boundary.jsonl",
+ test_data_dir_json + "json_variant/string_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_boolean_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_nullable_null_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_number_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_string_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_object_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_nullable_boolean_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_nullable_number_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_nullable_string_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_nullable_object_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_array_boolean_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_array_number_boundary.jsonl",
+ test_data_dir_json + "json_variant/array_array_string_boundary.jsonl",
+ test_data_dir_json +
+ "json_variant/array_nullable_array_nullable_boolean_boundary.jsonl",
+ test_data_dir_json +
+ "json_variant/array_nullable_array_nullable_null_boundary.jsonl",
+ test_data_dir_json +
+ "json_variant/array_nullable_array_nullable_number_boundary.jsonl",
+ test_data_dir_json +
+ "json_variant/array_nullable_array_nullable_string_boundary.jsonl",
+ test_data_dir_json + "json_variant/object_boundary.jsonl",
+ test_data_dir_json + "json_variant/object_nested_1025.jsonl"};
+
+ for (const auto& json_file : json_files) {
+ load_columns_data_from_file(columns, serde, '\n', {0}, json_file);
+ EXPECT_TRUE(!column_variant->empty());
+ column_variant->insert_default();
+ std::cout << "column variant size: " << column_variant->size() << std::endl;
+ }
+ column_variant->finalize();
+ std::cout << "column variant finalize size: " << column_variant->size() << std::endl;
+ }
+ }
+
+ template <typename T>
+ void column_common_test(T callback) {
+ callback(ColumnObject(true), column_variant->get_ptr());
+ }
+
+ void hash_common_test(
+ const std::string& function_name,
+ std::function<void(const MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
+ const std::string& res_file_name)>
+ assert_callback) {
+ {
+ MutableColumns columns;
+ columns.push_back(column_variant->get_ptr());
+ DataTypeSerDeSPtrs serdes = {dt_variant->get_serde()};
+ assert_callback(columns, serdes,
+ test_result_dir + "/column_variant_" + function_name + ".out");
+ }
+ }
+};
+
+TEST_F(ColumnObjectTest, is_variable_length) {
+ EXPECT_TRUE(column_variant->is_variable_length());
+}
+
+TEST_F(ColumnObjectTest, byte_size) {
+ hash_common_test("byte_size", assert_byte_size_with_file_callback);
+}
+
+TEST_F(ColumnObjectTest, has_enough_capacity) {
+ auto test_func = [](const auto& src_col) {
+ auto src_size = src_col->size();
+ // variant always return fasle
+ auto assert_col = src_col->clone_empty();
+ ASSERT_FALSE(assert_col->has_enough_capacity(*src_col));
+ assert_col->reserve(src_size);
+ ASSERT_FALSE(assert_col->has_enough_capacity(*src_col));
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, allocated_bytes) {
+ hash_common_test("allocated_bytes", assert_allocated_bytes_with_file_callback);
+}
+
+TEST_F(ColumnObjectTest, clone_resized) {
+ auto src_size = column_variant->size();
+ auto test_func = [&](size_t clone_count) {
+ auto target_column = column_variant->clone_resized(clone_count);
+ EXPECT_NE(target_column.get(), column_variant.get());
+ EXPECT_EQ(target_column->size(), clone_count);
+ size_t same_count = std::min(clone_count, src_size);
+ size_t i = 0;
+ for (; i < same_count; ++i) {
+ checkField(*target_column, *column_variant, i, i);
+ }
+ for (; i < clone_count; ++i) {
+ // more than source size
+ Field target_field;
+ Field source_field = column_variant->get_root_type()->get_default();
+ target_column->get(i, target_field);
+ EXPECT_EQ(target_field, source_field)
+ << "target_field: " << target_field.get_type_name()
+ << ", source_field: " << source_field.get_type_name();
+ }
+ };
+ test_func(0);
+ test_func(3);
+ test_func(src_size);
+ test_func(src_size + 10);
+ // test clone_empty
+
+ auto target_column = column_variant->clone_empty();
+ EXPECT_NE(target_column.get(), column_variant.get());
+ // assert subcolumns
+ auto target_subcolumns = assert_cast<ColumnObject*>(target_column.get())->get_subcolumns();
+ // always has root for ColumnObject(0)
+ EXPECT_EQ(target_subcolumns.size(), 1);
+}
+TEST_F(ColumnObjectTest, field_test) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ {
+ auto assert_col = source_column->clone();
+ for (size_t i = 0; i != src_size; ++i) {
+ Field f;
+ source_column->get(i, f);
+ assert_col->insert(f);
+ }
+ for (size_t i = 0; i != src_size; ++i) {
+ Field assert_field;
+ assert_col->get(i, assert_field);
+ Field source_field;
+ source_column->get(i, source_field);
+ ASSERT_EQ(assert_field, source_field);
+ }
+ }
+ {
+ auto assert_col = source_column->clone();
+ std::cout << source_column->size() << std::endl;
+ for (size_t i = 0; i != src_size; ++i) {
+ VariantMap jsonbf;
+ Field f(std::move(jsonbf));
+ source_column->get(i, f);
+ assert_col->insert(f);
+ }
+ for (size_t i = 0; i != src_size; ++i) {
+ VariantMap jsonbf;
+ Field f(std::move(jsonbf));
+ assert_col->get(i, f);
+ const auto& real_field = vectorized::get<const VariantMap&>(f);
+ Field source_field;
+ source_column->get(i, source_field);
+ ASSERT_EQ(real_field, source_field);
+ }
+ }
+ };
+ ColumnObject::MutablePtr obj;
+ obj = ColumnObject::create(1);
+ MutableColumns cols;
+ cols.push_back(obj->get_ptr());
+ const auto& json_file_obj = test_data_dir_json + "json_variant/object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_obj);
+ EXPECT_TRUE(!obj->empty());
+ test_func(obj);
+}
+
+// is seri
+TEST_F(ColumnObjectTest, is_column_string64) {
+ EXPECT_FALSE(column_variant->is_column_string64());
+}
+
+TEST_F(ColumnObjectTest, is_column_string) {
+ EXPECT_FALSE(column_variant->is_column_string());
+}
+
+TEST_F(ColumnObjectTest, serialize_one_row_to_string) {
+ {
+ const auto* variant = assert_cast<const ColumnObject*>(column_variant.get());
+ // Serialize hierarchy types to json format
+ std::string buffer;
+ for (size_t row_idx = 2000; row_idx < variant->size(); ++row_idx) {
+ Status st = variant->serialize_one_row_to_string(row_idx, &buffer);
+ EXPECT_TRUE(st.ok());
+ }
+ {
+ // TEST buffer
+ auto tmp_col = ColumnString::create();
+ VectorBufferWriter write_buffer(*tmp_col.get());
+ for (size_t row_idx = 2000; row_idx < variant->size(); ++row_idx) {
+ Status st = variant->serialize_one_row_to_string(row_idx, write_buffer);
+ EXPECT_TRUE(st.ok());
+ }
+ }
+ }
+ {
+ // TEST SCALA_VARAINT
+ // 1. create an empty variant column
+ auto v = ColumnObject::create(true);
+ auto dt = DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_STRING, 0,
+ 0);
+ auto cs = dt->create_column();
+ cs->insert(Field("amory"));
+ cs->insert(Field("doris"));
+ v->create_root(dt, std::move(cs));
+ EXPECT_TRUE(v->is_scalar_variant());
+
+ // 3. serialize
+ std::string buf2;
+ for (size_t row_idx = 0; row_idx < v->size(); ++row_idx) {
+ Status st = v->serialize_one_row_to_string(row_idx, &buf2);
+ EXPECT_TRUE(st.ok());
+ }
+ auto tmp_col = ColumnString::create();
+ VectorBufferWriter write_buffer(*tmp_col.get());
+ for (size_t row_idx = 0; row_idx < v->size(); ++row_idx) {
+ Status st = v->serialize_one_row_to_string(row_idx, write_buffer);
+ EXPECT_TRUE(st.ok());
+ }
+ }
+}
+// insert interface
+// not implemented: insert_many_fix_len_data, insert_many_dict_data, insert_many_continuous_binary_data, insert_from_multi_column
+// insert_many_strings, insert_many_strings_overflow, insert_range_from_ignore_overflow, insert_many_raw_data, insert_data, get_data_at, replace_column_data,
+// serialize_value_into_arena, deserialize_and_insert_from_arena
+TEST_F(ColumnObjectTest, insert_many_fix_len_data) {
+ EXPECT_ANY_THROW(column_variant->insert_many_fix_len_data(nullptr, 0));
+}
+
+TEST_F(ColumnObjectTest, insert_many_dict_data) {
+ EXPECT_ANY_THROW(column_variant->insert_many_dict_data(nullptr, 0, nullptr, 0, 0));
+}
+
+TEST_F(ColumnObjectTest, insert_many_continuous_binary_data) {
+ EXPECT_ANY_THROW(column_variant->insert_many_continuous_binary_data(nullptr, 0, 0));
+}
+
+//TEST_F(ColumnObjectTest, insert_from_multi_column) {
+// EXPECT_ANY_THROW(column_variant->insert_from_multi_column({column_variant.get()}, {0}));
+//}
+
+TEST_F(ColumnObjectTest, insert_many_strings) {
+ EXPECT_ANY_THROW(column_variant->insert_many_strings(nullptr, 0));
+}
+
+TEST_F(ColumnObjectTest, insert_many_strings_overflow) {
+ EXPECT_ANY_THROW(column_variant->insert_many_strings_overflow(nullptr, 0, 0));
+}
+
+TEST_F(ColumnObjectTest, insert_many_raw_data) {
+ EXPECT_ANY_THROW(column_variant->insert_many_raw_data(nullptr, 0));
+}
+
+TEST_F(ColumnObjectTest, insert_data) {
+ EXPECT_ANY_THROW(column_variant->insert_data(nullptr, 0));
+}
+
+TEST_F(ColumnObjectTest, get_data_at) {
+ EXPECT_ANY_THROW(column_variant->get_data_at(0));
+}
+
+TEST_F(ColumnObjectTest, replace_column_data) {
+ EXPECT_ANY_THROW(
+ column_variant->replace_column_data(column_variant->assume_mutable_ref(), 0, 0));
+}
+
+TEST_F(ColumnObjectTest, serialize_value_into_arena) {
+ Arena a;
+ const char* begin = nullptr;
+ EXPECT_ANY_THROW(column_variant->serialize_value_into_arena(0, a, begin));
+}
+
+TEST_F(ColumnObjectTest, deserialize_and_insert_from_arena) {
+ EXPECT_ANY_THROW(column_variant->deserialize_and_insert_from_arena(nullptr));
+}
+
+// insert series:
+// insert_from, insert_many_from, insert_range_from, insert_range_from_ignore_overflow, insert_indices_from
+// insert_default, insert_many_defaults
+TEST_F(ColumnObjectTest, insert_many_from) {
+ assert_insert_many_from_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, insert_from) {
+ assert_insert_from_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, insert_range_from) {
+ // insert_range_from_ignore_overflow call insert_range_from
+ assert_insert_range_from_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, insert_indices_from) {
+ assert_insert_indices_from_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, insert_default_insert_many_defaults) {
+ assert_insert_default_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, get_name) {
+ EXPECT_TRUE(column_variant->get_name().find("variant") != std::string::npos);
+}
+
+// pop_back interface
+TEST_F(ColumnObjectTest, pop_back_test) {
+ assert_pop_back_with_field_callback(column_variant->get_ptr());
+}
+
+// serialize and deserialize is not implemented
+// serialize_vec, deserialize_vec, serialize_vec_with_null_map, deserialize_vec_with_null_map, get_max_row_byte_size
+TEST_F(ColumnObjectTest, ser_deser_test) {
+ EXPECT_ANY_THROW(column_variant->get_max_row_byte_size());
+ EXPECT_ANY_THROW(column_variant->serialize_vec(nullptr, 0, 0));
+ EXPECT_ANY_THROW(column_variant->deserialize_vec(nullptr, 0));
+ EXPECT_ANY_THROW(column_variant->serialize_vec_with_null_map(nullptr, 0, nullptr));
+ EXPECT_ANY_THROW(column_variant->deserialize_vec_with_null_map(nullptr, 0, nullptr));
+}
+
+// hash interface
+TEST_F(ColumnObjectTest, update_xxHash_with_value) {
+ hash_common_test("update_xxHash_with_value", assert_update_xxHash_with_value_callback);
+}
+
+// hang
+//TEST_F(ColumnObjectTest, update_sip_hash_with_value_test) {
+// hash_common_test("update_sip_hash_with_value",
+// assert_column_vector_update_siphashes_with_value_callback);
+//}
+TEST_F(ColumnObjectTest, update_hashes_with_value_test) {
+ hash_common_test("update_hashes_with_value",
+ assert_column_vector_update_hashes_with_value_callback);
+}
+TEST_F(ColumnObjectTest, update_crc_with_value_test) {
+ hash_common_test("update_crc_with_value", assert_update_crc_with_value_callback);
+}
+TEST_F(ColumnObjectTest, update_crcs_with_value_test) {
+ std::string function_name = "update_crcs_with_value";
+ MutableColumns columns;
+ columns.push_back(column_variant->get_ptr());
+ DataTypeSerDeSPtrs serdes = {dt_variant->get_serde()};
+ std::vector<PrimitiveType> pts(columns.size(), PrimitiveType::TYPE_VARIANT);
+ assert_column_vector_update_crc_hashes_callback(
+ columns, serdes, pts, test_result_dir + "/column_variant_" + function_name + ".out");
+}
+
+// filter interface
+TEST_F(ColumnObjectTest, filter) {
+ assert_filter_with_field_callback(column_variant->get_ptr());
+}
+
+TEST_F(ColumnObjectTest, filter_by_selector) {
+ auto test_func = [&](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size <= UINT16_MAX);
+
+ auto target_column = source_column->clone_empty();
+
+ std::vector<uint16_t> indices(src_size);
+ std::iota(indices.begin(), indices.end(), 0);
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(indices.begin(), indices.end(), g);
+ size_t sel_size = src_size / 2;
+ indices.resize(sel_size);
+ std::sort(indices.begin(), indices.end());
+
+ EXPECT_ANY_THROW(Status st = source_column->filter_by_selector(indices.data(), 0,
+ target_column.get()));
+ };
+ test_func(column_variant);
+}
+TEST_F(ColumnObjectTest, permute) {
+ {
+ // test empty column and limit == 0
+ IColumn::Permutation permutation(0);
+ auto col = column_variant->clone_empty();
+ col->permute(permutation, 0);
+ EXPECT_EQ(col->size(), 0);
+ }
+ {
+ IColumn::Permutation permutation(0);
+ EXPECT_THROW(column_variant->permute(permutation, 10), Exception);
+ }
+
+ MutableColumns columns;
+ columns.push_back(column_variant->get_ptr());
+ assert_column_vector_permute(columns, 0, false);
+ assert_column_vector_permute(columns, 1, false);
+ assert_column_vector_permute(columns, column_variant->size(), false);
+ assert_column_vector_permute(columns, UINT64_MAX, false);
+}
+
+// not support
+TEST_F(ColumnObjectTest, get_permutation) {
+ EXPECT_ANY_THROW(assert_column_permutations2(*column_variant, dt_variant));
+}
+TEST_F(ColumnObjectTest, structure_equals) {
+ auto cl = column_variant->clone_empty();
+ EXPECT_ANY_THROW(column_variant->structure_equals(*cl));
+}
+
+TEST_F(ColumnObjectTest, replicate) {
+ assert_replicate_with_field(column_variant->get_ptr());
+}
+
+// Compare Interface not implement: compare_at, compare_internal
+TEST_F(ColumnObjectTest, compare_at) {
+ EXPECT_ANY_THROW(column_variant->compare_at(0, 0, *column_variant, -1));
+ std::vector<uint8> com_res(column_variant->size());
+ EXPECT_ANY_THROW(column_variant->compare_internal(0, *column_variant, 0, 0, com_res, nullptr));
+}
+
+TEST_F(ColumnObjectTest, clear) {
+ auto tmp_col = column_variant->clone();
+ EXPECT_EQ(tmp_col->size(), column_variant->size());
+
+ tmp_col->clear();
+ EXPECT_EQ(tmp_col->size(), 0);
+}
+
+TEST_F(ColumnObjectTest, convert_column_if_overflow) {
+ // convert_column_if_overflow may need impl in ColumnObject, like ColumnArray?
+ auto ret = column_variant->convert_column_if_overflow();
+ EXPECT_EQ(ret.get(), column_variant.get());
+}
+
+TEST_F(ColumnObjectTest, resize) {
+ auto test_func = [](const auto& source_column, size_t add_count) {
+ {
+ auto source_size = source_column->size();
+ auto tmp_col = source_column->clone();
+ auto default_col = source_column->clone_empty();
+ default_col->insert_default();
+ tmp_col->resize(source_size + add_count);
+ EXPECT_EQ(tmp_col->size(), source_size + add_count);
+ for (size_t i = 0; i != source_size; ++i) {
+ checkField(*tmp_col, *source_column, i, i);
+ }
+ for (size_t i = 0; i != add_count; ++i) {
+ checkField(*tmp_col, *default_col, source_size + i, 0);
+ }
+ }
+ {
+ // resize in self
+ auto ptr = source_column.get();
+ source_column->resize(add_count);
+ EXPECT_EQ(source_column.get(), ptr);
+ EXPECT_EQ(source_column->size(), add_count);
+ }
+ };
+ test_func(column_variant, 0);
+ test_func(column_variant, 10);
+}
+
+// ================= variant specific interface =================
+// meta info related interface
+TEST_F(ColumnObjectTest, get_least_common_type) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_least_common_type for root column
+ const auto& root = source_column->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ EXPECT_TRUE(root->data.get_least_common_type() != nullptr);
+
+ // Test get_least_common_type for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ EXPECT_TRUE(subcolumn->data.get_least_common_type() != nullptr);
+ }
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, get_dimensions) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_dimensions for root column
+ const auto& root = source_column->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ EXPECT_GE(root->data.get_dimensions(), 0);
+
+ // Test get_dimensions for subcolumns
+ for (auto& entry : source_column->get_subcolumns()) {
+ EXPECT_TRUE(entry != nullptr);
+ EXPECT_GE(entry->data.get_dimensions(), 0);
+ }
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, get_last_field) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_last_field for root column
+ const auto& root = source_column->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ Field last_field;
+ root->data.get_last_field();
+
+ // Test get_last_field for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ subcolumn->data.get_last_field();
+ }
+ };
+ test_func(column_variant);
+}
+
+// sub column op related interface
+TEST_F(ColumnObjectTest, get_finalized_column) {
+ auto test_func = [](const auto& source_column) {
+ // do not clone and then get , will case heap-after-use-free cause of defined in COW as temporary Ptr
+ // auto source_column = assert_cast<ColumnObject*>(var_column->clone_resized(var_column->size()).get());
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+ // Test get_finalized_column for root column
+ auto root = source_column->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ source_column->finalize();
+ root = source_column->get_subcolumns().get_root();
+ const auto& finalized_col = root->data.get_finalized_column();
+ EXPECT_TRUE(source_column->is_finalized());
+ Field rf;
+ finalized_col.get(0, rf);
+ EXPECT_TRUE(strlen(rf.get_type_name()) > 0);
+
+ // Test get_finalized_column for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ EXPECT_TRUE(subcolumn->data.is_finalized());
+ const auto& subcolumn_finalized = subcolumn->data.get_finalized_column();
+
+ // Verify finalized column data
+ Field field;
+ subcolumn_finalized.get(0, field);
+ EXPECT_TRUE(strlen(field.get_type_name()) > 0);
+ // Verify column size
+ EXPECT_EQ(subcolumn_finalized.size(), src_size);
+ }
+ };
+
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ EXPECT_NE(cloned_object, column_variant.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, get_finalized_column_ptr) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+ // Test get_finalized_column_ptr for root column
+ auto root = source_column->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ source_column->finalize();
+ // when finalized , the root will be changed
+ root = source_column->get_subcolumns().get_root();
+ const auto& finalized_col_ptr = root->data.get_finalized_column_ptr();
+ EXPECT_TRUE(finalized_col_ptr.get() != nullptr);
+
+ // Test get_finalized_column_ptr for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ const auto& subcolumn_finalized_ptr = subcolumn->data.get_finalized_column_ptr();
+ EXPECT_TRUE(subcolumn_finalized_ptr.get() != nullptr);
+ EXPECT_TRUE(subcolumn->data.is_finalized());
+
+ // Verify finalized column data
+ Field field;
+ subcolumn_finalized_ptr->get(0, field);
+ EXPECT_TRUE(strlen(field.get_type_name()) > 0);
+ // Verify column size
+ EXPECT_EQ(subcolumn_finalized_ptr->size(), src_size);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, remove_nullable) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test remove_nullable for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ subcolumn->data.finalize();
+ auto subcolumn_type_before = subcolumn->data.get_least_common_type();
+ subcolumn->data.remove_nullable();
+ auto subcolumn_type_after = subcolumn->data.get_least_common_type();
+ EXPECT_TRUE(remove_nullable(subcolumn_type_before)->equals(*subcolumn_type_after));
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, add_new_column_part) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test add_new_column_part for subcolumns
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+
+ // Store original type before adding new part
+ auto original_type = subcolumn->data.get_least_common_type();
+
+ // The add_new_column_part interface must be added to the minimum common type of the data type vector in the current subcolumn,
+ // otherwise an error will be reported: [E3] Not implemeted
+ subcolumn->data.add_new_column_part(original_type);
+ // Verify the type is updated
+ auto updated_type = subcolumn->data.get_least_common_type();
+ EXPECT_TRUE(updated_type != nullptr);
+ // Verify column size
+ EXPECT_EQ(subcolumn->data.size(), src_size);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, get_subcolumn) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_subcolumn
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+
+ // Verify subcolumn properties
+ EXPECT_TRUE(subcolumn->data.get_least_common_type() != nullptr);
+ EXPECT_GE(subcolumn->data.get_dimensions(), 0);
+
+ // Verify subcolumn data
+ Field field;
+ subcolumn->data.get(0, field);
+ EXPECT_TRUE(strlen(field.get_type_name()) > 0);
+ EXPECT_EQ(subcolumn->data.size(), src_size);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, ensure_root_node_type) {
+ ColumnObject::MutablePtr obj;
+ obj = ColumnObject::create(1);
+ MutableColumns cols;
+ cols.push_back(obj->get_ptr());
+ const auto& json_file_obj = test_data_dir_json + "json_variant/object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_obj);
+ EXPECT_TRUE(!obj->empty());
+ // Store original root type
+ auto root = obj->get_subcolumns().get_root();
+ auto original_root_type = root->data.get_least_common_type();
+ obj->finalize();
+
+ // Test ensure_root_node_type
+ auto new_type =
+ DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_STRING, 0, 0);
+ obj->ensure_root_node_type(new_type);
+
+ // Verify root type is updated
+ root = obj->get_subcolumns().get_root();
+ auto updated_root_type = root->data.get_least_common_type();
+ EXPECT_TRUE(updated_root_type->equals(*new_type));
+};
+
+TEST_F(ColumnObjectTest, create_root) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test case 1: Create root with string type
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+ auto type = DataTypeFactory::instance().create_data_type(
+ FieldType::OLAP_FIELD_TYPE_STRING, 0, 0);
+ auto column = type->create_column();
+ obj->create_root(type, std::move(column));
+
+ // Verify root is created with correct type
+ const auto& root = obj->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ EXPECT_TRUE(root->data.get_least_common_type()->equals(*type));
+ }
+
+ // Test case 2: Create root with int type
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+ auto type = DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_INT,
+ 0, 0);
+ auto column = type->create_column();
+ obj->create_root(type, std::move(column));
+
+ // Verify root is created with correct type
+ const auto& root = obj->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ EXPECT_TRUE(root->data.get_least_common_type()->equals(*type));
+ }
+
+ // Test case 3: Create root on existing column
+ {
+ auto col = source_column->clone();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+ auto original_root = obj->get_subcolumns().get_root();
+ EXPECT_TRUE(original_root != nullptr);
+
+ // Create root with new type
+ auto type = DataTypeFactory::instance().create_data_type(
+ FieldType::OLAP_FIELD_TYPE_STRING, 0, 0);
+ auto column = type->create_column();
+ obj->create_root(type, std::move(column));
+
+ // Verify root is replaced with new type
+ const auto& new_root = obj->get_subcolumns().get_root();
+ EXPECT_TRUE(new_root != nullptr);
+ EXPECT_EQ(new_root, original_root);
+ }
+
+ // Test case 4: Create root and verify data operations
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+ auto type = DataTypeFactory::instance().create_data_type(
+ FieldType::OLAP_FIELD_TYPE_STRING, 0, 0);
+ auto column = type->create_column();
+ obj->create_root(type, std::move(column));
+
+ // Insert some data
+ Field field;
+ source_column->get(0, field);
+ obj->insert(field);
+
+ // Verify data is inserted correctly
+ Field inserted_field;
+ obj->get(0, inserted_field);
+ }
+
+ // Test case 5: Create root with nullable type
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+ auto type = DataTypeFactory::instance().create_data_type(
+ FieldType::OLAP_FIELD_TYPE_STRING, 0, 0);
+ auto nullable_type = make_nullable(type);
+ auto column = nullable_type->create_column();
+ obj->create_root(nullable_type, std::move(column));
+
+ // Verify root is created with nullable type
+ const auto& root = obj->get_subcolumns().get_root();
+ EXPECT_TRUE(root != nullptr);
+ EXPECT_TRUE(root->data.get_least_common_type()->equals(*nullable_type));
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+TEST_F(ColumnObjectTest, get_most_common_type) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_most_common_type
+ DataTypePtr most_common_type = source_column->get_most_common_type();
+ EXPECT_TRUE(most_common_type != nullptr);
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, is_null_root) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test is_null_root
+ bool is_null = source_column->is_null_root();
+ EXPECT_FALSE(is_null); // Since we have data, root should not be null
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, is_scalar_variant) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test is_scalar_variant
+ bool is_scalar = source_column->is_scalar_variant();
+ // The result depends on the actual data structure
+ EXPECT_FALSE(is_scalar);
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, is_exclusive) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test is_exclusive
+ bool is_exclusive = source_column->is_exclusive();
+ // The result depends on the actual data structure
+ EXPECT_TRUE(is_exclusive);
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, get_root_type) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test get_root_type
+ DataTypePtr root_type = source_column->get_root_type();
+ EXPECT_TRUE(root_type != nullptr);
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, has_subcolumn) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test has_subcolumn
+ for (const auto& subcolumn : source_column->get_subcolumns()) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ bool has_subcolumn = source_column->has_subcolumn(subcolumn->path);
+ EXPECT_TRUE(has_subcolumn);
+ }
+ };
+ test_func(column_variant);
+}
+
+TEST_F(ColumnObjectTest, finalize) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test case 1: Test finalize with READ_MODE
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Insert data from source column
+ for (size_t i = 0; i < src_size; ++i) {
+ Field field;
+ source_column->get(i, field);
+ obj->insert(field);
+ }
+
+ // Verify initial state
+ EXPECT_FALSE(obj->is_finalized());
+
+ // Finalize in READ_MODE
+ Status st = obj->finalize(ColumnObject::FinalizeMode::READ_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Verify data integrity
+ for (size_t i = 0; i < src_size; ++i) {
+ Field original_field, finalized_field;
+ source_column->get(i, original_field);
+ obj->get(i, finalized_field);
+ EXPECT_EQ(original_field, finalized_field);
+ }
+ }
+
+ // Test case 2: Test finalize with WRITE_MODE
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Insert data from source column
+ for (size_t i = 0; i < src_size; ++i) {
+ Field field;
+ source_column->get(i, field);
+ obj->insert(field);
+ }
+
+ // Verify initial state
+ EXPECT_FALSE(obj->is_finalized());
+
+ // Finalize in WRITE_MODE
+ Status st = obj->finalize(ColumnObject::FinalizeMode::WRITE_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Verify data integrity
+ for (size_t i = 0; i < src_size; ++i) {
+ Field original_field, finalized_field;
+ source_column->get(i, original_field);
+ obj->get(i, finalized_field);
+ EXPECT_EQ(original_field, finalized_field);
+ }
+ }
+
+ // Test case 3: Test finalize without mode (default READ_MODE)
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Insert data from source column
+ for (size_t i = 0; i < src_size; ++i) {
+ Field field;
+ source_column->get(i, field);
+ obj->insert(field);
+ }
+
+ // Verify initial state
+ EXPECT_FALSE(obj->is_finalized());
+
+ // Finalize without mode
+ obj->finalize();
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Verify data integrity
+ for (size_t i = 0; i < src_size; ++i) {
+ Field original_field, finalized_field;
+ source_column->get(i, original_field);
+ obj->get(i, finalized_field);
+ EXPECT_EQ(original_field, finalized_field);
+ }
+ }
+
+ // Test case 4: Test finalize on empty column
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Empty column always finalized
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Finalize empty column
+ Status st = obj->finalize(ColumnObject::FinalizeMode::READ_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(obj->is_finalized());
+ EXPECT_EQ(obj->size(), 0);
+ }
+
+ // Test case 5: Test finalize preserves column structure
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Insert data from source column
+ for (size_t i = 0; i < src_size; ++i) {
+ Field field;
+ source_column->get(i, field);
+ obj->insert(field);
+ }
+
+ // Verify initial state
+ EXPECT_FALSE(obj->is_finalized());
+
+ // Store original structure
+ auto original_subcolumns = obj->get_subcolumns();
+
+ // Finalize
+ Status st = obj->finalize(ColumnObject::FinalizeMode::READ_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Verify structure is preserved
+ auto final_subcolumns = obj->get_subcolumns();
+ EXPECT_EQ(final_subcolumns.size(), original_subcolumns.size());
+
+ // Verify each subcolumn is finalized
+ for (const auto& subcolumn : final_subcolumns) {
+ EXPECT_TRUE(subcolumn->data.is_finalized());
+ }
+ }
+
+ // Test case 6: Test finalize with WRITE_MODE on sparse columns
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Insert data from source column
+ for (size_t i = 0; i < src_size; ++i) {
+ Field field;
+ source_column->get(i, field);
+ obj->insert(field);
+ }
+
+ // Verify initial state
+ EXPECT_FALSE(obj->is_finalized());
+
+ // Finalize in WRITE_MODE
+ Status st = obj->finalize(ColumnObject::FinalizeMode::WRITE_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(obj->is_finalized());
+
+ // Verify sparse columns are handled
+ auto sparse_column = obj->get_sparse_column().get();
+ EXPECT_TRUE(sparse_column != nullptr);
+
+ // Verify data integrity
+ for (size_t i = 0; i < src_size; ++i) {
+ Field original_field, finalized_field;
+ source_column->get(i, original_field);
+ obj->get(i, finalized_field);
+ EXPECT_EQ(original_field, finalized_field);
+ }
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, clone_finalized) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Store original data for comparison
+ auto original_subcolumns = source_column->get_subcolumns();
+
+ // Test clone_finalized
+ auto cloned = source_column->clone_finalized();
+ EXPECT_TRUE(cloned.get() != nullptr);
+ EXPECT_EQ(cloned->size(), src_size);
+
+ // Verify cloned column has same subcolumns
+ auto cloned_subcolumns = assert_cast<ColumnObject*>(cloned.get())->get_subcolumns();
+ EXPECT_EQ(cloned_subcolumns.size(), original_subcolumns.size());
+
+ // Verify data integrity
+ for (size_t i = 0; i < src_size; ++i) {
+ Field original_field, cloned_field;
+ source_column->get(i, original_field);
+ cloned->get(i, cloned_field);
+ EXPECT_EQ(original_field, cloned_field);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, sanitize) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Store original data for comparison
+ auto original_subcolumns = source_column->get_subcolumns();
+
+ // Test sanitize
+ Status status = source_column->sanitize();
+ EXPECT_TRUE(status.ok());
+
+ // Verify data integrity after sanitization
+ auto subcolumns_after = source_column->get_subcolumns();
+ EXPECT_EQ(subcolumns_after.size(), original_subcolumns.size());
+
+ // Verify all subcolumns are valid
+ for (const auto& subcolumn : subcolumns_after) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, debug_string) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test debug_string
+ std::string debug = source_column->debug_string();
+ EXPECT_FALSE(debug.empty());
+ };
+ test_func(column_variant);
+}
+
+// used in function_element_at for variant
+TEST_F(ColumnObjectTest, find_path_lower_bound_in_sparse_data) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+ auto* mutable_ptr = assert_cast<ColumnObject*>(source_column.get());
+ // auto [sparse_data_paths, sparse_data_values] = mutable_ptr->get_sparse_data_paths_and_values();
+ // forloop
+ PathInData pat("object.array");
+ StringRef prefix_ref(pat.get_path());
+ std::string_view path_prefix(prefix_ref.data, prefix_ref.size);
+ const auto& sparse_data_map =
+ assert_cast<const ColumnMap&>(*mutable_ptr->get_sparse_column());
+ const auto& src_sparse_data_offsets = sparse_data_map.get_offsets();
+ const auto& src_sparse_data_paths =
+ assert_cast<const ColumnString&>(sparse_data_map.get_keys());
+
+ for (size_t i = 0; i != src_sparse_data_offsets.size(); ++i) {
+ size_t start = src_sparse_data_offsets[ssize_t(i) - 1];
+ size_t end = src_sparse_data_offsets[ssize_t(i)];
+ size_t lower_bound_index =
+ vectorized::ColumnObject::find_path_lower_bound_in_sparse_data(
+ prefix_ref, src_sparse_data_paths, start, end);
+ for (; lower_bound_index != end; ++lower_bound_index) {
+ auto path_ref = src_sparse_data_paths.get_data_at(lower_bound_index);
+ std::string_view path(path_ref.data, path_ref.size);
+ std::cout << "path : " << path << std::endl;
+ }
+ }
+ };
+ ColumnObject::MutablePtr obj;
+ obj = ColumnObject::create(1);
+ MutableColumns cols;
+ cols.push_back(obj->get_ptr());
+ const auto& json_file_obj = test_data_dir_json + "json_variant/object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_obj);
+ EXPECT_TRUE(!obj->empty());
+ std::cout << "column variant size: " << obj->size() << std::endl;
+ test_func(obj);
+}
+
+// used in SparseColumnExtractReader::_fill_path_column
+TEST_F(ColumnObjectTest, fill_path_column_from_sparse_data) {
+ ColumnObject::MutablePtr obj;
+ obj = ColumnObject::create(1);
+ MutableColumns cols;
+ cols.push_back(obj->get_ptr());
+ const auto& json_file_obj = test_data_dir_json + "json_variant/object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_obj);
+ EXPECT_TRUE(!obj->empty());
+ auto sparse_col = obj->get_sparse_column();
+ auto cloned_sparse = sparse_col->clone_empty();
+ auto& offsets = obj->serialized_sparse_column_offsets();
+ for (size_t i = 0; i != offsets.size(); ++i) {
+ auto start = offsets[i - 1];
+ auto end = offsets[i];
+ vectorized::ColumnObject::fill_path_column_from_sparse_data(
+ *obj->get_subcolumn({}) /*root*/, nullptr, StringRef {"array"},
+ cloned_sparse->get_ptr(), start, end);
+ }
+
+ EXPECT_NE(cloned_sparse->size(), sparse_col->size());
+
+ vectorized::ColumnObject::fill_path_column_from_sparse_data(
+ *obj->get_subcolumn({}) /*root*/, nullptr, StringRef {"array"}, sparse_col->get_ptr(),
+ 0, sparse_col->size());
+ EXPECT_ANY_THROW(obj->check_consistency());
+}
+
+TEST_F(ColumnObjectTest, not_finalized) {
+ ColumnObject::MutablePtr obj;
+ obj = ColumnObject::create(1);
+ MutableColumns cols;
+ cols.push_back(obj->get_ptr());
+ const auto& json_file_obj = test_data_dir_json + "json_variant/object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_obj);
+ const auto& json_file_arr = test_data_dir_json + "json_variant/array_object_boundary.jsonl";
+ load_columns_data_from_file(cols, serde, '\n', {0}, json_file_arr);
+ EXPECT_TRUE(obj->size() == 200);
+ EXPECT_FALSE(obj->is_finalized());
+ // test get_finalized_column_ptr/ get_finalized_column for subColumn
+ auto subcolumns = obj->get_subcolumns();
+ for (const auto& subcolumn : subcolumns) {
+ EXPECT_TRUE(subcolumn != nullptr);
+ EXPECT_FALSE(subcolumn->data.is_finalized());
+ EXPECT_ANY_THROW(subcolumn->data.get_finalized_column_ptr());
+ EXPECT_ANY_THROW(subcolumn->data.get_finalized_column());
+ }
+}
+
+doris::vectorized::Field get_field_v2(std::string_view type, size_t array_element_cnt = 0) {
+ static std::unordered_map<std::string_view, doris::vectorized::Field> field_map;
+ if (field_map.empty()) {
+ doris::vectorized::Field int_field = 20;
+ doris::vectorized::Field str_field(String("str", 3));
+ doris::vectorized::Field arr_int_field = Array();
+ doris::vectorized::Field arr_str_field = Array();
+ auto& array1 = arr_int_field.get<Array>();
+ auto& array2 = arr_str_field.get<Array>();
+ for (size_t i = 0; i < array_element_cnt; ++i) {
+ array1.emplace_back(int_field);
+ array2.emplace_back(str_field);
+ }
+ field_map["int"] = int_field;
+ field_map["string"] = str_field;
+ field_map["ai"] = arr_int_field;
+ field_map["as"] = arr_str_field;
+ }
+ return field_map[type];
+}
+
+TEST_F(ColumnObjectTest, array_field_operations) {
+ auto test_func = [](const auto& source_column) {
+ auto src_size = source_column->size();
+ EXPECT_TRUE(src_size > 0);
+
+ // Test case 1: Test create_empty_array_field
+ {
+ EXPECT_ANY_THROW(create_empty_array_field(0));
+ // Test with different dimensions
+ for (size_t dim = 1; dim <= 3; ++dim) {
+ Field array_field = create_empty_array_field(dim);
+ EXPECT_TRUE(array_field.get_type() == Field::Types::Array);
+ const Array& array = array_field.get<Array>();
+ if (dim > 1) {
+ EXPECT_FALSE(array.empty());
+ } else {
+ EXPECT_TRUE(array.empty());
+ }
+ }
+ }
+
+ // Test case 2: Test create_array
+ {
+ // Test with different types
+ std::vector<TypeIndex> types = {TypeIndex::Int8, TypeIndex::String, TypeIndex::Float64};
+ for (const auto& type : types) {
+ for (size_t dim = 1; dim <= 3; ++dim) {
+ DataTypePtr array_type = create_array(type, dim);
+ EXPECT_TRUE(array_type != nullptr);
+ }
+ }
+ // Test create_array_of_type with TypeIndex::Nothing
+ auto dt_ptr = create_array_of_type(TypeIndex::Nothing, 0, false);
+ EXPECT_TRUE(dt_ptr->get_type_id() == TypeIndex::Nothing);
+ }
+
+ // Test case 3: Test recreate_column_with_default_values
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Create a subcolumn with array type
+ PathInData path("array_field");
+ auto array_type = create_array(TypeIndex::Int8, 2);
+ auto column = array_type->create_column();
+ auto column_a = array_type->create_column();
+ column_a->insert(Array(1));
+ obj->add_sub_column(path, std::move(column));
+
+ // Get the subcolumn
+ const auto* subcolumn = obj->get_subcolumn(path);
+ EXPECT_TRUE(subcolumn != nullptr);
+
+ EXPECT_ANY_THROW(subcolumn->get_finalized_column_ptr());
+
+ // Recreate column with default values
+ auto new_column = recreate_column_with_default_values(
+ column_a->convert_to_full_column_if_const(), TypeIndex::Int8, 2);
+ EXPECT_TRUE(new_column->get_name().find("Array") != std::string::npos);
+ EXPECT_EQ(new_column->size(), subcolumn->size());
+ }
+
+ // Test case 4: Test clone_with_default_values
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Create a subcolumn with array type
+ PathInData path("array_field");
+ auto array_type = create_array(TypeIndex::Int8, 1);
+ auto column = array_type->create_column();
+ Array array1 = {1, 2, 3};
+ column->insert(array1);
+ obj->add_sub_column(path, std::move(column), array_type);
+
+ // Get the subcolumn
+ const auto* subcolumn = obj->get_subcolumn(path);
+ EXPECT_TRUE(subcolumn != nullptr);
+ EXPECT_TRUE(subcolumn->size() > 0);
+ std::cout << "subcolumn size: " << subcolumn->size() << std::endl;
+ Field f = subcolumn->get_last_field();
+ EXPECT_TRUE(f.get_type() == Field::Types::Array);
+
+ // Create field info
+ FieldInfo info;
+ info.scalar_type_id = TypeIndex::Int8;
+ info.num_dimensions = 1;
+ info.have_nulls = false;
+ info.need_convert = false;
+
+ // Clone with default values
+ auto cloned = subcolumn->clone_with_default_values(info);
+ std::cout << "cloned size: " << cloned.size() << std::endl;
+ EXPECT_TRUE(cloned.size() == subcolumn->size());
+ }
+
+ // Test case 5: Test Subcolumn::resize
+ {
+ auto col = source_column->clone_empty();
+ auto obj = assert_cast<ColumnObject*>(col.get());
+
+ // Create a subcolumn
+ PathInData path("test_field");
+ obj->add_sub_column(path, src_size);
+
+ // Get the subcolumn
+ auto* subcolumn = obj->get_subcolumn(path);
+ EXPECT_TRUE(subcolumn != nullptr);
+
+ // Test resize to larger size
+ size_t new_size = src_size + 10;
+ subcolumn->resize(new_size);
+ EXPECT_EQ(subcolumn->size(), new_size);
+
+ // Test resize to smaller size
+ new_size = src_size / 2;
+ subcolumn->resize(new_size);
+ EXPECT_EQ(subcolumn->size(), new_size);
+
+ // Test resize to zero
+ subcolumn->resize(0);
+ EXPECT_EQ(subcolumn->size(), 0);
+ }
+ {
+ // Test wrapp_array_nullable
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(2);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.ai", get_field_v2("ai", 1));
+ data.emplace_back("v.as", get_field_v2("as", 1));
+
+ for (int i = 0; i < 2; ++i) {
+ auto field = VariantUtil::construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ EXPECT_FALSE(variant->is_finalized());
+ Status st = variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(variant->is_finalized());
+ std::cout << "sub: " << variant->get_subcolumns().size() << std::endl;
+ for (auto& entry : variant->get_subcolumns()) {
+ std::cout << "entry path: " << entry->path.get_path() << std::endl;
+ std::cout << "entry type: " << entry->data.get_least_common_typeBase()->get_name()
+ << std::endl;
+ std::cout << "entry dimension " << entry->data.get_dimensions() << std::endl;
+ }
+
+ // then clear
+ variant->clear_column_data();
+ EXPECT_TRUE(variant->size() == 0);
+ }
+ };
+ auto temp = column_variant->clone();
+ auto cloned_object = assert_cast<ColumnObject*>(temp.get());
+ test_func(std::move(cloned_object));
+}
+
+TEST_F(ColumnObjectTest, assert_exception_happen) {
+ // Test case 1: Test assert_exception_happen
+ {
+ // 1. create an empty variant column
+ vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+ dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0, true, true /*root*/));
+ dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.e"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ std::cout << "dynamic_subcolumns size: " << dynamic_subcolumns.size() << std::endl;
+ EXPECT_ANY_THROW(ColumnObject::create(2, std::move(dynamic_subcolumns)));
+ }
+
+ {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create(5);
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", get_field_v2("int"));
+ data.emplace_back("v.b", get_field_v2("string"));
+ data.emplace_back("v.c", get_field_v2("ai", 2));
+ data.emplace_back("v.f", get_field_v2("as", 2));
+ data.emplace_back("v.e", get_field_v2("string"));
+
+ for (int i = 0; i < 5; ++i) {
+ auto field = VariantUtil::construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. sparse column path
+ data.emplace_back("v.d.d", get_field_v2("ai", 2));
+ data.emplace_back("v.c.d", get_field_v2("string"));
+ data.emplace_back("v.b.d", get_field_v2("ai", 2));
+ for (int i = 0; i < 5; ++i) {
+ auto field = VariantUtil::construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ data.clear();
+ data.emplace_back("v.a", get_field_v2("int"));
+ data.emplace_back("v.b", get_field_v2("int"));
+ data.emplace_back("v.c", get_field_v2("ai", 2));
+ data.emplace_back("v.f", get_field_v2("as", 2));
+ data.emplace_back("v.e", get_field_v2("string"));
+ data.emplace_back("v.d.d", get_field_v2("as", 2));
+ data.emplace_back("v.c.d", get_field_v2("int"));
+ data.emplace_back("v.b.d", get_field_v2("as", 2));
+ for (int i = 0; i < 5; ++i) {
+ auto field = VariantUtil::construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ EXPECT_FALSE(variant->is_finalized());
+ for (const auto& column : variant->get_subcolumns()) {
+ if (!column->data.is_finalized()) {
+ EXPECT_ANY_THROW(column->data.remove_nullable());
+ EXPECT_ANY_THROW(column->data.get_finalized_column());
+ } else {
+ std::cout << "column path: " << column->path.get_path() << std::endl;
+ EXPECT_NO_THROW(column->data.remove_nullable());
+ EXPECT_NO_THROW(column->data.get_finalized_column());
+ }
+ }
+ }
+}
+
+TEST_F(ColumnObjectTest, try_insert_default_from_nested) {
+ // 1. create an empty variant column
+ vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+ auto array_type = create_array(TypeIndex::String, 1);
+ auto column = array_type->create_column();
+ Array array1 = {"amory", "commit"};
+ Array array2 = {"amory", "doris"};
+ column->insert(array1);
+ column->insert(array2);
+
+ auto array_type2 = create_array(TypeIndex::String, 2);
+ auto column2 = array_type2->create_column();
+ Array array22, array23;
+ array22.push_back(array1);
+ array22.push_back(array2);
+ array23.push_back(array2);
+ array23.push_back(array1);
+ column2->insert(array22);
+ column2->insert(array23);
+
+ dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0, true, true /*root*/));
+ dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(
+ vectorized::PathInData("v.a"),
+ vectorized::ColumnObject::Subcolumn {std::move(column2), array_type2, false, false});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(
+ vectorized::PathInData("v.b.a"),
+ vectorized::ColumnObject::Subcolumn {std::move(column), array_type, false, false});
+ dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ std::cout << "dynamic_subcolumns size: " << dynamic_subcolumns.size() << std::endl;
+ auto obj = ColumnObject::create(5, std::move(dynamic_subcolumns));
+
+ for (auto& entry : obj->get_subcolumns()) {
+ std::cout << "entry path: " << entry->path.get_path() << std::endl;
+ std::cout << "entry type: " << entry->data.get_least_common_typeBase()->get_name()
+ << std::endl;
+ std::cout << "entry dimension " << entry->data.get_dimensions() << std::endl;
+ bool inserted = obj->try_insert_default_from_nested(entry);
+ if (!inserted) {
+ entry->data.insert_default();
+ }
+ }
+}
+
+// unnest, clear_column_data
+TEST_F(ColumnObjectTest, unnest) {
+ // 1. create an empty variant column
+ vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+ auto nested_col = ColumnObject::NESTED_TYPE->create_column();
+ Array array1 = {"amory", "commit"};
+ Array array2 = {"amory", "doris"};
+ std::cout << "array: " << array1.size() << std::endl;
+ nested_col->insert(array1);
+ nested_col->insert(array2);
+ std::cout << nested_col->size() << std::endl;
+
+ // 2. subcolumn path
+ dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(2, true, true /*root*/));
+ dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+ vectorized::ColumnObject::Subcolumn {2, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.a"),
+ vectorized::ColumnObject::Subcolumn {
+ std::move(nested_col), ColumnObject::NESTED_TYPE, true, false});
+ std::cout << "dynamic_subcolumns size: " << dynamic_subcolumns.size() << std::endl;
+ auto obj = ColumnObject::create(2, std::move(dynamic_subcolumns));
+ obj->set_num_rows(2);
+ EXPECT_TRUE(!obj->empty());
+ std::cout << obj->size() << std::endl;
+ Status st = obj->finalize(ColumnObject::FinalizeMode::WRITE_MODE);
+ EXPECT_TRUE(st.ok());
+}
+
+TEST_F(ColumnObjectTest, path_in_data_builder_test) {
+ // Create a ColumnObject with nested subcolumns
+ auto variant = ColumnObject::create(5);
+
+ // Test case 1: Build a nested path with PathInDataBuilder
+ {
+ PathInDataBuilder builder;
+ builder.append("v", false); // First part is not array
+ builder.append("a", true); // Second part is array
+ builder.append("b", true); // Third part is array
+ builder.append("c", false); // Fourth part is not array
+
+ PathInData path = builder.build();
+ EXPECT_TRUE(path.has_nested_part());
+
+ // Create field info for nested type
+ FieldInfo field_info;
+ field_info.scalar_type_id = TypeIndex::Int8;
+ field_info.have_nulls = true;
+ field_info.need_convert = false;
+ field_info.num_dimensions = 2; // Array of Array
+
+ // Test add_nested_subcolumn
+ variant->add_nested_subcolumn(path, field_info, 5);
+
+ // Verify the subcolumn was added correctly
+ const auto* subcolumn = variant->get_subcolumn(path);
+ EXPECT_TRUE(subcolumn != nullptr);
+
+ // then clear
+ variant->clear_column_data();
+ EXPECT_TRUE(variant->size() == 0);
+ }
+}
+
+TEST_F(ColumnObjectTest, get_field_info_all_types) {
+ // Test Int32
+ {
+ Int32 field(42);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test Int64
+ {
+ Int64 field(42);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test UInt64
+ {
+ Field field(UInt64(42));
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test Int64 with different ranges
+ // Test Int64 with different ranges
+ {
+ // Test Int64 <= Int8::max()
+ Int64 field1(std::numeric_limits<Int8>::max());
+ FieldInfo info1;
+ schema_util::get_field_info(field1, &info1);
+ EXPECT_EQ(info1.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info1.have_nulls);
+ EXPECT_FALSE(info1.need_convert);
+ EXPECT_EQ(info1.num_dimensions, 0);
+
+ // Test Int64 <= Int16::max()
+ Int64 field2(std::numeric_limits<Int16>::max());
+ FieldInfo info2;
+ schema_util::get_field_info(field2, &info2);
+ EXPECT_EQ(info2.scalar_type_id, TypeIndex::Int16);
+ EXPECT_FALSE(info2.have_nulls);
+ EXPECT_FALSE(info2.need_convert);
+ EXPECT_EQ(info2.num_dimensions, 0);
+
+ // Test Int64 <= Int32::max()
+ Int64 field3(std::numeric_limits<Int32>::max());
+ FieldInfo info3;
+ schema_util::get_field_info(field3, &info3);
+ EXPECT_EQ(info3.scalar_type_id, TypeIndex::Int32);
+ EXPECT_FALSE(info3.have_nulls);
+ EXPECT_FALSE(info3.need_convert);
+ EXPECT_EQ(info3.num_dimensions, 0);
+
+ // Test Int64 > Int32::max()
+ Int64 field4(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1);
+ FieldInfo info4;
+ schema_util::get_field_info(field4, &info4);
+ EXPECT_EQ(info4.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info4.have_nulls);
+ EXPECT_FALSE(info4.need_convert);
+ EXPECT_EQ(info4.num_dimensions, 0);
+
+ // Test Int64 <= Int8::min()
+ Int64 field5(std::numeric_limits<Int8>::min());
+ FieldInfo info5;
+ schema_util::get_field_info(field5, &info5);
+ EXPECT_EQ(info5.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info5.have_nulls);
+ EXPECT_FALSE(info5.need_convert);
+ EXPECT_EQ(info5.num_dimensions, 0);
+
+ // Test Int64 <= Int16::min()
+ Int64 field6(std::numeric_limits<Int16>::min());
+ FieldInfo info6;
+ schema_util::get_field_info(field6, &info6);
+ EXPECT_EQ(info6.scalar_type_id, TypeIndex::Int16);
+ EXPECT_FALSE(info6.have_nulls);
+ EXPECT_FALSE(info6.need_convert);
+ EXPECT_EQ(info6.num_dimensions, 0);
+
+ // Test Int64 <= Int32::min()
+ Int64 field7(std::numeric_limits<Int32>::min());
+ FieldInfo info7;
+ schema_util::get_field_info(field7, &info7);
+ EXPECT_EQ(info7.scalar_type_id, TypeIndex::Int32);
+ EXPECT_FALSE(info7.have_nulls);
+ EXPECT_FALSE(info7.need_convert);
+ EXPECT_EQ(info7.num_dimensions, 0);
+
+ // Test Int64 < Int32::min()
+ Int64 field8(static_cast<Int64>(std::numeric_limits<Int32>::min()) - 1);
+ FieldInfo info8;
+ schema_util::get_field_info(field8, &info8);
+ EXPECT_EQ(info8.scalar_type_id, TypeIndex::Int64);
+ }
+
+ // Test UInt64 with different ranges
+ {
+ // Test UInt64 <= UInt8::max()
+ UInt64 field1(std::numeric_limits<UInt8>::max());
+ FieldInfo info1;
+ schema_util::get_field_info(field1, &info1);
+ EXPECT_EQ(info1.scalar_type_id, TypeIndex::Int16);
+ EXPECT_FALSE(info1.have_nulls);
+ EXPECT_FALSE(info1.need_convert);
+ EXPECT_EQ(info1.num_dimensions, 0);
+
+ // Test UInt64 <= UInt16::max()
+ UInt64 field2(std::numeric_limits<UInt16>::max());
+ FieldInfo info2;
+ schema_util::get_field_info(field2, &info2);
+ EXPECT_EQ(info2.scalar_type_id, TypeIndex::Int32);
+ EXPECT_FALSE(info2.have_nulls);
+ EXPECT_FALSE(info2.need_convert);
+ EXPECT_EQ(info2.num_dimensions, 0);
+
+ // Test UInt64 <= UInt32::max()
+ UInt64 field3(std::numeric_limits<UInt32>::max());
+ FieldInfo info3;
+ schema_util::get_field_info(field3, &info3);
+ EXPECT_EQ(info3.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info3.have_nulls);
+ EXPECT_FALSE(info3.need_convert);
+ EXPECT_EQ(info3.num_dimensions, 0);
+
+ // Test UInt64 > UInt32::max()
+ UInt64 field4(static_cast<UInt64>(std::numeric_limits<UInt32>::max()) + 1);
+ FieldInfo info4;
+ schema_util::get_field_info(field4, &info4);
+ EXPECT_EQ(info4.scalar_type_id, TypeIndex::Int64);
+ }
+
+ // Test Float32
+ {
+ Field field(Float32(42.0f));
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Float64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test Float64
+ {
+ Field field(Float64(42.0));
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Float64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test String
+ {
+ Field field(String("test"));
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::String);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ {
+ Slice slice("\"amory is cute\"");
+ JsonBinaryValue value;
+ Status st = value.from_json_string(slice.data, slice.size);
+ EXPECT_TRUE(st.ok()) << st.to_string();
+ JsonbField field(value.value(), value.size());
+
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::JSONB);
+ }
+
+ // Test Array
+ {
+ Array array;
+ array.push_back(Int64(1));
+ array.push_back(Int64(2));
+ Field field(array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 1);
+ }
+
+ // Test nested Array
+ {
+ Array inner_array;
+ inner_array.push_back(Int64(1));
+ inner_array.push_back(Int64(2));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+ }
+
+ // Test Tuple
+ {
+ Tuple t1;
+ t1.push_back(Field("amory cute"));
+ t1.push_back(__int128_t(37));
+ t1.push_back(true);
+ FieldInfo info;
+ schema_util::get_field_info(t1, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::JSONB)
+ << "info.scalar_type_id: " << getTypeName(info.scalar_type_id);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test Map will throw exception: Bad type of Field 25
+ {
+ Array k1 = {"a", "b", "c"};
+ Array v1 = {1, 2, 3};
+ Map map;
+ map.push_back(k1);
+ map.push_back(v1);
+ FieldInfo info;
+ EXPECT_ANY_THROW(schema_util::get_field_info(map, &info));
+ }
+
+ // Test VariantMap
+ {
+ VariantMap variant_map;
+ variant_map["key1"] = Int64(1);
+ variant_map["key2"] = String("value");
+ Field field(variant_map);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::VARIANT);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 0);
+ }
+
+ // Test Array with different types
+ {
+ Array array;
+ array.push_back(Int64(1));
+ Field field(array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8)
+ << "info.scalar_type_id: " << getTypeName(info.scalar_type_id);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 1);
+ }
+
+ // Test Array with nulls
+ {
+ Array array;
+ array.push_back(Int64(1));
+ array.push_back(Null());
+ Field field(array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_TRUE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 1);
+ }
+
+ // Test nested Array with Int64 in different ranges
+ {// Test nested Array with Int64 <= Int8::max()
+ {Array inner_array;
+ inner_array.push_back(Int64(std::numeric_limits<Int8>::max()));
+ inner_array.push_back(Int64(std::numeric_limits<Int8>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with Int64 <= Int16::max()
+{
+ Array inner_array;
+ inner_array.push_back(Int64(std::numeric_limits<Int16>::max()));
+ inner_array.push_back(Int64(std::numeric_limits<Int16>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int16);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with Int64 <= Int32::max()
+{
+ Array inner_array;
+ inner_array.push_back(Int64(std::numeric_limits<Int32>::max()));
+ inner_array.push_back(Int64(std::numeric_limits<Int32>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int32);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with Int64 > Int32::max()
+{
+ Array inner_array;
+ inner_array.push_back(Int64(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1));
+ inner_array.push_back(Int64(static_cast<Int64>(std::numeric_limits<Int32>::max()) + 1));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+} // namespace doris::vectorized
+
+// Test nested Array with UInt64 in different ranges
+{// Test nested Array with UInt64 <= UInt8::max()
+ {Array inner_array;
+inner_array.push_back(UInt64(std::numeric_limits<UInt8>::max()));
+inner_array.push_back(UInt64(std::numeric_limits<UInt8>::max()));
+
+Array outer_array;
+outer_array.push_back(inner_array);
+outer_array.push_back(inner_array);
+
+Field field(outer_array);
+FieldInfo info;
+schema_util::get_field_info(field, &info);
+EXPECT_EQ(info.scalar_type_id, TypeIndex::Int16);
+EXPECT_FALSE(info.have_nulls);
+EXPECT_FALSE(info.need_convert);
+EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with UInt64 <= UInt16::max()
+{
+ Array inner_array;
+ inner_array.push_back(UInt64(std::numeric_limits<UInt16>::max()));
+ inner_array.push_back(UInt64(std::numeric_limits<UInt16>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int32);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with UInt64 <= UInt32::max()
+{
+ Array inner_array;
+ inner_array.push_back(UInt64(std::numeric_limits<UInt32>::max()));
+ inner_array.push_back(UInt64(std::numeric_limits<UInt32>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with UInt64 > UInt32::max()
+{
+ Array inner_array;
+ inner_array.push_back(UInt64(static_cast<UInt64>(std::numeric_limits<UInt32>::max()) + 1));
+ inner_array.push_back(UInt64(static_cast<UInt64>(std::numeric_limits<UInt32>::max()) + 1));
+
+ Array outer_array;
+ outer_array.push_back(inner_array);
+ outer_array.push_back(inner_array);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+}
+
+// Test nested Array with mixed Int64 and UInt64
+{
+ Array inner_array1;
+ inner_array1.push_back(Int64(std::numeric_limits<Int32>::max()));
+ inner_array1.push_back(Int64(std::numeric_limits<Int32>::max()));
+
+ Array inner_array2;
+ inner_array2.push_back(UInt64(std::numeric_limits<UInt32>::max()));
+ inner_array2.push_back(UInt64(std::numeric_limits<UInt32>::max()));
+
+ Array outer_array;
+ outer_array.push_back(inner_array1);
+ outer_array.push_back(inner_array2);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int64);
+ EXPECT_FALSE(info.have_nulls);
+ EXPECT_TRUE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test nested Array with nulls
+{
+ Array inner_array1;
+ inner_array1.push_back(Int64(1));
+ inner_array1.push_back(Int64(2));
+
+ Array inner_array2;
+ inner_array2.push_back(Int64(3));
+ inner_array2.push_back(Null());
+
+ Array outer_array;
+ outer_array.push_back(inner_array1);
+ outer_array.push_back(inner_array2);
+
+ Field field(outer_array);
+ FieldInfo info;
+ schema_util::get_field_info(field, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::Int8);
+ EXPECT_TRUE(info.have_nulls);
+ EXPECT_FALSE(info.need_convert);
+ EXPECT_EQ(info.num_dimensions, 2);
+}
+
+// Test Array with JsonbField
+{
+ Slice slice("\"amory is cute\"");
+ JsonBinaryValue value;
+ Status st = value.from_json_string(slice.data, slice.size);
+ EXPECT_TRUE(st.ok()) << st.to_string();
+ JsonbField field(value.value(), value.size());
+
+ Array array;
+ array.push_back(field);
+ array.push_back(field);
+ FieldInfo info;
+ schema_util::get_field_info(array, &info);
+ EXPECT_EQ(info.scalar_type_id, TypeIndex::JSONB);
+}
+}
+
+TEST_F(ColumnObjectTest, field_visitor) {
+ // Test replacing scalar values in a flat array
+ {
+ Array array;
+ array.push_back(Int64(1));
+ array.push_back(Int64(2));
+ array.push_back(Int64(3));
+
+ Field field(array);
+ Field replacement(Int64(42));
+ Field result = apply_visitor(FieldVisitorReplaceScalars(replacement, 0), field);
+
+ EXPECT_EQ(result.get<Int64>(), 42);
+
+ Field replacement1(Int64(42));
+ Field result1 = apply_visitor(FieldVisitorReplaceScalars(replacement, 1), field);
+
+ EXPECT_EQ(result1.get<Array>().size(), 3);
+ EXPECT_EQ(result1.get<Array>()[0].get<Int64>(), 42);
+ EXPECT_EQ(result1.get<Array>()[1].get<Int64>(), 42);
+ EXPECT_EQ(result1.get<Array>()[2].get<Int64>(), 42);
+ }
+}
+
+TEST_F(ColumnObjectTest, subcolumn_operations_coverage) {
+ // Test insert_range_from
+ {
+ auto src_column = VariantUtil::construct_basic_varint_column();
+ auto dst_column = VariantUtil::construct_dst_varint_column();
+
+ // Test normal case
+ auto* dst_subcolumn = const_cast<ColumnObject::Subcolumn*>(
+ &dst_column->get_subcolumns().get_root()->data);
+ dst_subcolumn->insert_range_from(src_column->get_subcolumns().get_root()->data, 0, 2);
+
+ // Test empty range
+ dst_subcolumn->insert_range_from(src_column->get_subcolumns().get_root()->data, 0, 0);
+
+ // Test with different types
+ auto src_column2 = VariantUtil::construct_advanced_varint_column();
+ dst_subcolumn->insert_range_from(src_column2->get_subcolumns().get_root()->data, 0, 1);
+ }
+
+ // Test parse_binary_from_sparse_column
+ {
+ auto column = VariantUtil::construct_basic_varint_column();
+ vectorized::Field res;
+ FieldInfo field_info;
+
+ // Test String type
+ {
+ std::string test_str = "test_data";
+ std::vector<char> binary_data;
+ size_t str_size = test_str.size();
+ binary_data.resize(sizeof(size_t) + test_str.size());
+ memcpy(binary_data.data(), &str_size, sizeof(size_t));
+ memcpy(binary_data.data() + sizeof(size_t), test_str.data(), test_str.size());
+ const char* data = binary_data.data();
+ parse_binary_from_sparse_column(TypeIndex::String, data, res, field_info);
+ EXPECT_EQ(res.get<String>(), "test_data");
+ }
+
+ // Test integer types
+ {
+ Int8 int8_val = 42;
+ const char* data = reinterpret_cast<const char*>(&int8_val);
+ parse_binary_from_sparse_column(TypeIndex::Int8, data, res, field_info);
+ EXPECT_EQ(res.get<Int8>(), 42);
+ }
+
+ {
+ Int16 int16_val = 12345;
+ const char* data = reinterpret_cast<const char*>(&int16_val);
+ parse_binary_from_sparse_column(TypeIndex::Int16, data, res, field_info);
+ EXPECT_EQ(res.get<Int16>(), 12345);
+ }
+
+ {
+ Int32 int32_val = 123456789;
+ const char* data = reinterpret_cast<const char*>(&int32_val);
+ parse_binary_from_sparse_column(TypeIndex::Int32, data, res, field_info);
+ EXPECT_EQ(res.get<Int32>(), 123456789);
+ }
+
+ {
+ Int64 int64_val = 1234567890123456789LL;
+ const char* data = reinterpret_cast<const char*>(&int64_val);
+ parse_binary_from_sparse_column(TypeIndex::Int64, data, res, field_info);
+ EXPECT_EQ(res.get<Int64>(), 1234567890123456789LL);
+ }
+
+ // Test floating point types
+ {
+ Float32 float32_val = 3.1415901f;
+ const char* data = reinterpret_cast<const char*>(&float32_val);
+ parse_binary_from_sparse_column(TypeIndex::Float32, data, res, field_info);
+ EXPECT_FLOAT_EQ(res.get<Float32>(), 0);
+ }
+
+ {
+ Float64 float64_val = 3.141592653589793;
+ const char* data = reinterpret_cast<const char*>(&float64_val);
+ parse_binary_from_sparse_column(TypeIndex::Float64, data, res, field_info);
+ EXPECT_DOUBLE_EQ(res.get<Float64>(), 3.141592653589793);
+ }
+
+ // Test JSONB type
+ {
+ std::string json_str = "{\"key\": \"value\"}";
+ std::vector<char> binary_data;
+ size_t json_size = json_str.size();
+ binary_data.resize(sizeof(size_t) + json_str.size());
+ memcpy(binary_data.data(), &json_size, sizeof(size_t));
+ memcpy(binary_data.data() + sizeof(size_t), json_str.data(), json_str.size());
+ const char* data = binary_data.data();
+ parse_binary_from_sparse_column(TypeIndex::JSONB, data, res, field_info);
+ }
+
+ // Test Nothing type
+ {
+ const char* data = nullptr;
+ parse_binary_from_sparse_column(TypeIndex::Nothing, data, res, field_info);
+ EXPECT_TRUE(res.is_null());
+ }
+
+ // Test Array type
+ {
+ std::vector<char> binary_data;
+ size_t array_size = 2;
+ binary_data.resize(sizeof(size_t) + 2 * (sizeof(uint8_t) + sizeof(Int32)));
+ char* data_ptr = binary_data.data();
+
+ // Write array size
+ memcpy(data_ptr, &array_size, sizeof(size_t));
+ data_ptr += sizeof(size_t);
+
+ // Write first element (Int32)
+ *data_ptr++ = static_cast<uint8_t>(TypeIndex::Int32);
+ Int32 val1 = 42;
+ memcpy(data_ptr, &val1, sizeof(Int32));
+ data_ptr += sizeof(Int32);
+
+ // Write second element (Int32)
+ *data_ptr++ = static_cast<uint8_t>(TypeIndex::Int32);
+ Int32 val2 = 43;
+ memcpy(data_ptr, &val2, sizeof(Int32));
+
+ const char* data = binary_data.data();
+ parse_binary_from_sparse_column(TypeIndex::Array, data, res, field_info);
+ const Array& array = res.get<Array>();
+ EXPECT_EQ(array.size(), 2);
+ EXPECT_EQ(array[0].get<Int32>(), 42);
+ EXPECT_EQ(array[1].get<Int32>(), 43);
+ }
+
+ // Test unsupported types - these should throw exceptions
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::UInt8, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::UInt16, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::UInt32, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::UInt64, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::Date, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::DateTime, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::Decimal32, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::Decimal64, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::Decimal128V2, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::Tuple, nullptr, res, field_info));
+ EXPECT_ANY_THROW(parse_binary_from_sparse_column(TypeIndex::Map, nullptr, res, field_info));
+ EXPECT_ANY_THROW(
+ parse_binary_from_sparse_column(TypeIndex::VARIANT, nullptr, res, field_info));
+ }
+
+ // Test add_sub_column
+ {
+ auto column = VariantUtil::construct_basic_varint_column();
+ PathInData path("test.path");
+
+ // Test normal case
+ column->add_sub_column(path, 10);
+
+ // Test with existing path
+ column->add_sub_column(path, 10);
+
+ // Test with max subcolumns limit
+ for (int i = 0; i < 1000; i++) {
+ PathInData new_path("test.path." + std::to_string(i));
+ column->add_sub_column(new_path, 10);
+ }
+ }
+
+ // Test wrapp_array_nullable
+ {
+ auto column = VariantUtil::construct_advanced_varint_column();
+ EXPECT_TRUE(column->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ PathInData path("v.f");
+ auto* subcolumn = column->get_subcolumn(path);
+ subcolumn->wrapp_array_nullable();
+ EXPECT_TRUE(subcolumn->get_least_common_type()->is_nullable());
+ }
+
+ // Test is_empty_nested
+ {
+ vectorized::ColumnObject container_variant(1, true);
+ // v: {"k": [1,2,3]} ==》 [{"k": 1}, {"k": 2}, {"k": 3}]
+ // {"k": []} => [{}] vs {"k": null} -> [null]
+ // {"k": [4]} => [{"k": 4}]
+ auto col_arr =
+ ColumnArray::create(ColumnInt64::create(), ColumnArray::ColumnOffsets::create());
+ // Array array1 = {1, 2, 3};
+ // Array array2 = {4};
+ // col_arr->insert(array1);
+ // col_arr->insert(array2);
+ Array an;
+ an.push_back(Null());
+ col_arr->insert(an);
+ col_arr->insert(an);
+ col_arr->insert(an);
+ MutableColumnPtr nested_object = ColumnObject::create(
+ container_variant.max_subcolumns_count(), col_arr->get_data().size());
+ MutableColumnPtr offset = col_arr->get_offsets_ptr()->assume_mutable(); // [3, 3, 4]
+ auto* nested_object_ptr = assert_cast<ColumnObject*>(nested_object.get());
+ // flatten nested arrays
+ MutableColumnPtr flattend_column = col_arr->get_data_ptr()->assume_mutable();
+ DataTypePtr flattend_type = DataTypeFactory::instance().create_data_type(
+ FieldType::OLAP_FIELD_TYPE_BIGINT, 0, 0);
+ // add sub path without parent prefix
+ PathInData sub_path("k");
+ nested_object_ptr->add_sub_column(sub_path, std::move(flattend_column),
+ std::move(flattend_type));
+ nested_object = make_nullable(nested_object->get_ptr())->assume_mutable();
+ auto array =
+ make_nullable(ColumnArray::create(std::move(nested_object), std::move(offset)));
+ PathInData path("v.k");
+ container_variant.add_sub_column(path, array->assume_mutable(),
+ container_variant.NESTED_TYPE);
+ container_variant.set_num_rows(3);
+ for (auto subcolumn : container_variant.get_subcolumns()) {
+ if (subcolumn->data.is_root) {
+ // Nothing
+ EXPECT_TRUE(subcolumn->data.is_empty_nested(0));
+ continue;
+ }
+ for (int i = 0; i < 3; ++i) {
+ EXPECT_FALSE(subcolumn->data.is_empty_nested(i));
+ }
+ }
+ }
+
+ // Test is_empty_nested
+ {
+ auto v = ColumnObject::create(1);
+ auto sub_dt = make_nullable(std::make_unique<DataTypeArray>(
+ make_nullable(std::make_unique<DataTypeObject>(1))));
+ auto sub_col = sub_dt->create_column();
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+ Array an;
+ an.push_back(Null());
+ data.emplace_back("v.a", an);
+ // 2. subcolumn path
+ auto vf = VariantUtil::construct_variant_map(data);
+ v->try_insert(vf);
+
+ for (auto subcolumn : v->get_subcolumns()) {
+ for (int i = 0; i < v->size(); ++i) {
+ if (subcolumn->data.is_root) {
+ EXPECT_TRUE(subcolumn->data.is_empty_nested(i));
+ }
+ EXPECT_TRUE(subcolumn->data.is_empty_nested(i));
+ }
+ }
+ Status st = v->finalize(ColumnObject::FinalizeMode::WRITE_MODE);
+ EXPECT_TRUE(st.ok());
+ PathInData path("v.a");
+ for (auto sub : v->get_subcolumns()) {
+ if (sub->data.is_root) {
+ continue;
+ }
+ sub->kind = SubcolumnsTree<ColumnObject::Subcolumn>::Node::NESTED;
+ EXPECT_FALSE(v->try_insert_default_from_nested(sub));
+ }
+ }
+}
+}
diff --git a/be/test/vec/columns/common_column_test.h b/be/test/vec/columns/common_column_test.h
index 9cb455e1de2..720854c7b50 100644
--- a/be/test/vec/columns/common_column_test.h
+++ b/be/test/vec/columns/common_column_test.h
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+#pragma once
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
@@ -22,17 +23,18 @@
#include <filesystem>
#include <fstream>
#include <iostream>
+#include <random>
+#include <string>
-#include "olap/schema.h"
+#include "testutil/test_util.h"
+#include "util/simd/bits.h"
#include "vec/columns/column.h"
#include "vec/columns/column_array.h"
-#include "vec/columns/column_dictionary.h"
#include "vec/columns/column_map.h"
+#include "vec/columns/column_object.h"
#include "vec/columns/columns_number.h"
#include "vec/common/cow.h"
#include "vec/core/field.h"
-#include "vec/core/sort_block.h"
-#include "vec/core/sort_description.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_array.h"
@@ -47,6 +49,20 @@ namespace doris::vectorized {
static bool gen_check_data_in_assert = true;
+template <typename Container>
+std::string join_ints(const Container& nums) {
+ if (nums.empty()) {
+ return "[]";
+ }
+
+ std::ostringstream oss;
+ oss << "[";
+ std::copy(nums.begin(), nums.end() - 1,
+ std::ostream_iterator<typename Container::value_type>(oss, ","));
+ oss << nums.back() << "]";
+ return oss.str();
+}
+
class CommonColumnTest : public ::testing::Test {
protected:
// Helper function to load data from CSV, with index which splited by spliter and load to columns
@@ -243,14 +259,20 @@ public:
assert_callback(columns, check_columns, dataTypes);
}
- static void checkColumn(const IColumn& col1, const IColumn& col2, const IDataType& dataType,
- size_t column_size) {
+ static void ALWAYS_INLINE checkField(const IColumn& col1, const IColumn& col2, size_t idx1,
+ size_t idx2) {
+ Field f1;
+ Field f2;
+ col1.get(idx1, f1);
+ col2.get(idx2, f2);
+ EXPECT_EQ(f1, f2) << "idx1: " << idx1 << " idx2: " << idx2 << " col1: " << col1.get_name()
+ << " col2: " << col2.get_name() << " f1: " << f1.get_type_name()
+ << " f2: " << f2.get_type_name();
+ }
+
+ static void checkColumn(const IColumn& col1, const IColumn& col2, size_t column_size) {
for (size_t i = 0; i < column_size; ++i) {
- Field f1;
- Field f2;
- col1.get(i, f1);
- col2.get(i, f2);
- EXPECT_EQ(f1, f2);
+ checkField(col1, col2, i, i);
}
}
@@ -370,6 +392,47 @@ public:
check_res_file("insert_from", res);
}
+ static void assert_insert_from_with_field_callback(const MutableColumnPtr& source_column) {
+ auto target_column = source_column->clone_empty();
+ for (size_t j = 0; j < source_column->size(); ++j) {
+ target_column->insert_from(*source_column, j);
+ }
+ ASSERT_EQ(target_column->size(), source_column->size());
+ checkColumn(*target_column, *source_column, source_column->size());
+ };
+
+ static void assert_insert_many_from_with_field_callback(const MutableColumnPtr& source_column) {
+ auto src_size = source_column->size();
+ std::vector<size_t> insert_vals_count = {0, 3, 10};
+ std::vector<size_t> src_data_indices = {0, src_size, src_size - 1, (src_size + 1) >> 1};
+
+ auto test_func = [&](size_t clone_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column = source_column->clone_resized(actual_clone_count);
+ for (auto pos = src_data_indices.begin(); pos < src_data_indices.end(); ++pos) {
+ if (*pos >= src_size) {
+ continue;
+ }
+ for (auto n : insert_vals_count) {
+ target_column->resize(actual_clone_count);
+ target_column->insert_many_from(*source_column, *pos, n);
+ EXPECT_EQ(target_column->size(), actual_clone_count + n);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ // field assert
+ checkField(*target_column, *source_column, i, i);
+ }
+ for (size_t j = 0; j < n; ++j, ++i) {
+ // field assert
+ checkField(*target_column, *source_column, i, *pos);
+ }
+ }
+ }
+ };
+ test_func(0);
+ test_func(10);
+ };
+
// insert_from_multi_column (const std::vector< const IColumn * > &srcs, std::vector< size_t > positions)
// speed up for insert_from interface according to avoid virtual call
static void assert_insert_from_multi_column_callback(MutableColumns& load_cols,
@@ -480,6 +543,42 @@ public:
check_res_file("insert_range_from", res);
}
+ static void assert_insert_range_from_with_field_callback(
+ const MutableColumnPtr& source_column) {
+ std::vector<size_t> insert_vals_count = {0, 10, 1000};
+ auto src_size = source_column->size();
+ std::vector<size_t> src_data_indices = {0, src_size - 1, (src_size + 1) >> 1};
+ auto test_func = [&](size_t clone_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column = source_column->clone_resized(actual_clone_count);
+ for (auto pos = src_data_indices.begin(); pos < src_data_indices.end(); ++pos) {
+ if (*pos >= src_size) {
+ continue;
+ }
+ for (auto n : insert_vals_count) {
+ target_column->resize(actual_clone_count);
+ size_t actual_insert_count = std::min(n, src_size - *pos);
+ target_column->insert_range_from_ignore_overflow(*source_column, *pos,
+ actual_insert_count);
+ auto target_size = target_column->size();
+ EXPECT_EQ(target_size, actual_clone_count + actual_insert_count);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ checkField(*target_column, *source_column, i, i);
+ }
+ for (size_t j = *pos; i < target_size; ++i, ++j) {
+ checkField(*target_column, *source_column, i, j);
+ }
+ }
+ }
+ };
+ test_func(0);
+ test_func(10);
+
+ auto target_column = source_column->clone_empty();
+ EXPECT_THROW(target_column->insert_range_from(*source_column, 0, src_size + 1), Exception);
+ }
+
// assert insert_range_from_ignore_overflow which happened in columnStr<UInt32> want to insert from ColumnStr<UInt64> for more column string to be inserted not just limit to the 4G
// Define the custom assert callback function to verify insert_range_from_ignore_overflow behavior
static void assert_insert_range_from_ignore_overflow(MutableColumns& load_cols,
@@ -520,10 +619,8 @@ public:
EXPECT_TRUE(st2.ok()) << "Failed to merge block1: " << st2.to_string();
// step4. to check data in assert_cols
for (size_t i = 0; i < load_cols.size(); ++i) {
- checkColumn(*load_cols[i], *mb.get_column_by_position(i), *types[i],
- load_cols[i]->size());
- checkColumn(*load_cols[i], *assert_mb.get_column_by_position(i), *types[i],
- load_cols[i]->size());
+ checkColumn(*load_cols[i], *mb.get_column_by_position(i), load_cols[i]->size());
+ checkColumn(*load_cols[i], *assert_mb.get_column_by_position(i), load_cols[i]->size());
}
}
@@ -663,6 +760,77 @@ public:
check_res_file("insert_indices_from", res);
}
+ static void assert_insert_indices_from_with_field_callback(
+ const MutableColumnPtr& source_column) {
+ auto src_size = source_column->size();
+ auto target_column = source_column->clone_resized(src_size);
+ {
+ auto tmp_target_column = target_column->clone_empty();
+ std::vector<uint32_t> indices;
+
+ // empty indices array
+ tmp_target_column->insert_indices_from(*source_column, indices.data(), indices.data());
+ EXPECT_EQ(tmp_target_column->size(), 0);
+ }
+ auto test_func2 = [&](size_t clone_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ {
+ auto tmp_target_column = target_column->clone_resized(actual_clone_count);
+ // insert all elements from source column
+ std::vector<uint32_t> indices(src_size);
+ std::iota(indices.begin(), indices.end(), 0);
+ tmp_target_column->insert_indices_from(*source_column, indices.data(),
+ indices.data() + src_size);
+ EXPECT_EQ(tmp_target_column->size(), actual_clone_count + indices.size());
+ size_t j = 0;
+ for (j = 0; j != actual_clone_count; ++j) {
+ checkField(*tmp_target_column, *source_column, j, j);
+ }
+ for (size_t k = 0; j < actual_clone_count + indices.size(); ++j, ++k) {
+ checkField(*tmp_target_column, *source_column, j, indices[k]);
+ }
+ }
+ {
+ // Normal case with random indices
+ auto tmp_target_column = target_column->clone_resized(actual_clone_count);
+ std::vector<uint32_t> indices(src_size);
+ std::iota(indices.begin(), indices.end(), 0);
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(indices.begin(), indices.end(), g);
+ tmp_target_column->insert_indices_from(*source_column, indices.data(),
+ indices.data() + indices.size());
+ EXPECT_EQ(tmp_target_column->size(), actual_clone_count + indices.size());
+ size_t j = 0;
+ for (j = 0; j != actual_clone_count; ++j) {
+ checkField(*tmp_target_column, *source_column, j, j);
+ }
+ for (size_t k = 0; j < actual_clone_count + indices.size(); ++j, ++k) {
+ checkField(*tmp_target_column, *source_column, j, indices[k]);
+ }
+ }
+ {
+ // Normal case with duplicate indices
+ auto tmp_target_column = target_column->clone_resized(actual_clone_count);
+ std::vector<uint32_t> indices = {0, uint32_t(source_column->size() - 1),
+ uint32_t((source_column->size() + 1) >> 1),
+ uint32_t(source_column->size() - 1), 0};
+ tmp_target_column->insert_indices_from(*source_column, indices.data(),
+ indices.data() + indices.size());
+ EXPECT_EQ(tmp_target_column->size(), actual_clone_count + indices.size());
+ size_t j = 0;
+ for (j = 0; j != actual_clone_count; ++j) {
+ checkField(*tmp_target_column, *source_column, j, j);
+ }
+ for (size_t k = 0; j < actual_clone_count + indices.size(); ++j, ++k) {
+ checkField(*tmp_target_column, *source_column, j, indices[k]);
+ }
+ }
+ };
+ test_func2(0);
+ test_func2(10);
+ }
+
static void assert_insert_data_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
// Create an empty column to verify `insert_data` functionality
MutableColumns verify_columns;
@@ -701,6 +869,16 @@ public:
check_res_file("insert_data", res);
}
+ static void assert_insert_data_with_field_callback(const MutableColumnPtr& source_column) {
+ auto target_column = source_column->clone_empty();
+ for (size_t j = 0; j < source_column->size(); ++j) {
+ target_column->insert_data(source_column->get_data_at(j).data,
+ source_column->get_data_at(j).size);
+ }
+ ASSERT_EQ(target_column->size(), source_column->size());
+ checkColumn(*target_column, *source_column, source_column->size());
+ };
+
static void assert_insert_many_raw_data_from_callback(MutableColumns& load_cols,
DataTypeSerDeSPtrs serders) {
// Create an empty column to verify `insert_many_raw_data` functionality
@@ -777,6 +955,58 @@ public:
check_res_file("insert_default", res);
}
+ static void assert_insert_default_with_field_callback(const MutableColumnPtr& source_column) {
+ Field default_field;
+ {
+ auto target_column = source_column->clone_empty();
+ target_column->insert_default();
+ ASSERT_EQ(target_column->size(), 1);
+ target_column->get(0, default_field);
+ std::cout << "default_field: " << default_field.get_type_name() << std::endl;
+ }
+ auto src_size = source_column->size();
+
+ auto test_func = [&](size_t clone_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column = source_column->clone_resized(actual_clone_count);
+ target_column->insert_default();
+ auto target_size = target_column->size();
+ EXPECT_EQ(target_size, actual_clone_count + 1);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ checkField(*target_column, *source_column, i, i);
+ }
+ Field last_field;
+ target_column->get(i, last_field);
+ EXPECT_EQ(last_field, default_field);
+ };
+
+ std::vector<size_t> insert_vals_count = {0, 10, 1000};
+ src_size = source_column->size();
+
+ auto test_func2 = [&](size_t clone_count) {
+ for (auto n : insert_vals_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column = source_column->clone_resized(actual_clone_count);
+ target_column->insert_many_defaults(n);
+ auto target_size = target_column->size();
+ EXPECT_EQ(target_size, actual_clone_count + n);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ checkField(*target_column, *source_column, i, i);
+ }
+ for (; i < target_size; ++i) {
+ Field f;
+ target_column->get(i, f);
+ EXPECT_EQ(f, default_field);
+ }
+ }
+ };
+ test_func(0);
+ test_func(10);
+ test_func2(0);
+ test_func2(10);
+ };
// assert insert_many_default
// Define the custom assert callback function to verify insert_many_defaults behavior
static void assert_insert_many_defaults_callback(MutableColumns& load_cols,
@@ -1139,6 +1369,18 @@ public:
check_res_file("pop_back", res);
}
+ static void assert_pop_back_with_field_callback(const MutableColumnPtr source_column) {
+ auto src_size = source_column->size();
+ std::vector<size_t> pop_back_count = {0, src_size - 1, (src_size + 1) >> 1};
+ for (auto n : pop_back_count) {
+ auto target_column = source_column->clone_resized(src_size);
+ target_column->pop_back(n);
+ EXPECT_EQ(target_column->size(), src_size - n);
+ checkColumn(*target_column, *source_column, target_column->size());
+ }
+ EXPECT_ANY_THROW(source_column->pop_back(src_size + 1)); // pop_back out of range
+ }
+
//virtual MutablePtr
// Creates empty column with the same type.
//clone_empty () const this is clone ,we should also check if the size is 0 after clone and ptr is not the same
@@ -1447,6 +1689,34 @@ public:
}
check_res_file("replicate", res);
}
+
+ static void assert_replicate_with_field(const MutableColumnPtr source_column) {
+ std::vector<size_t> insert_vals_count = {0, 10, 1000};
+ auto src_size = source_column->size();
+ srand((unsigned)time(nullptr));
+ IColumn::Offsets offsets(src_size);
+ IColumn::Offsets counts(src_size);
+ size_t total_size = 0;
+ for (size_t i = 0; i < src_size; ++i) {
+ counts[i] = rand() % 10;
+ total_size += counts[i];
+ offsets[i] = uint32(total_size);
+ }
+ {
+ auto target_column = source_column->clone_empty();
+ IColumn::Offsets empty_offsets;
+ auto tmp_col = target_column->replicate(empty_offsets);
+ EXPECT_EQ(tmp_col->size(), 0);
+ }
+ auto target_column = source_column->replicate(offsets);
+ EXPECT_EQ(target_column->size(), total_size);
+ size_t total_idx = 0;
+ for (size_t i = 0; i < src_size; ++i) {
+ for (size_t j = 0; j < counts[i]; ++j) {
+ checkField(*target_column, *source_column, total_idx++, i);
+ }
+ }
+ }
//virtual void
//for_each_subcolumn (ColumnCallback)
//virtual void
@@ -1664,6 +1934,57 @@ public:
check_res_file("filter", res);
}
+ static void assert_filter_with_field_callback(const MutableColumnPtr source_column) {
+ auto source_size = source_column->size();
+ IColumn::Filter all_filtered(source_size, 0);
+ IColumn::Filter no_filtered(source_size, 1);
+ IColumn::Filter normal_filter(source_size, 1);
+ normal_filter[0] = 0;
+ normal_filter[source_size - 1] = 0;
+ normal_filter[source_size / 2] = 0;
+ std::vector<IColumn::Filter*> filters = {&all_filtered, &no_filtered, &normal_filter};
+ auto test_func = [&](const IColumn::Filter* filter) {
+ const auto* filter_data = (const int8_t*)filter->data();
+ auto expected_size = filter->size() - simd::count_zero_num(filter_data, filter->size());
+ {
+ // empty column
+ auto target_column = source_column->clone_empty();
+ IColumn::Filter tmp_filter;
+ auto ptr = target_column->filter(tmp_filter, expected_size);
+ EXPECT_EQ(ptr->size(), 0);
+ }
+ auto target_column = source_column->clone_resized(source_size);
+ auto ptr = target_column->filter(*filter, expected_size);
+ EXPECT_EQ(ptr->size(), expected_size);
+ // check filter result is right
+ for (size_t i = 0, find_pos = 0; i < expected_size; ++i, ++find_pos) {
+ find_pos = simd::find_byte(filter_data, find_pos, filter->size(), (int8_t)1);
+ EXPECT_TRUE(find_pos < filter->size());
+ checkField(ptr.operator*(), *source_column, i, find_pos);
+ }
+
+ // filter will modify the original column
+ {
+ // empty filter
+ auto target_column_1 = source_column->clone_empty();
+ IColumn::Filter tmp_filter;
+ auto res_size = target_column_1->filter(tmp_filter);
+ EXPECT_EQ(res_size, 0);
+ EXPECT_EQ(target_column_1->size(), 0);
+ }
+ auto result_size = target_column->filter(*filter);
+ EXPECT_EQ(result_size, expected_size);
+ for (size_t i = 0, find_pos = 0; i < expected_size; ++i, ++find_pos) {
+ find_pos = simd::find_byte(filter_data, find_pos, filter->size(), (int8_t)1);
+ EXPECT_TRUE(find_pos < filter->size());
+ checkField(*target_column, *source_column, i, find_pos);
+ }
+ };
+ for (const auto& filter : filters) {
+ test_func(filter);
+ }
+ }
+
// filter with result_hint_size which should return new column ptr
// filter (const Filter &filt, ssize_t result_size_hint) const =0 with a result_size_hint to pass, but we should make sure the result_size_hint is not bigger than the source column size
static void assert_filter_with_result_hint_callback(MutableColumns& load_cols,
@@ -1783,9 +2104,9 @@ public:
}
// this function helps to check sort permutation behavior for column which use column::compare_at
- void stable_get_column_permutation(const IColumn& column, bool ascending, size_t limit,
- int nan_direction_hint,
- IColumn::Permutation& out_permutation) {
+ static void stable_get_column_permutation(const IColumn& column, bool ascending, size_t limit,
+ int nan_direction_hint,
+ IColumn::Permutation& out_permutation) {
(void)(limit);
size_t size = column.size();
@@ -1889,7 +2210,7 @@ public:
LOG(INFO) << "now we are in update_hashes column : " << load_cols[i]->get_name()
<< " for column size : " << source_column->size();
EXPECT_NO_FATAL_FAILURE(source_column->update_crcs_with_value(
- crc_hash_vals.data(), pts[i], source_column->size()));
+ crc_hash_vals.data(), pts[i], uint32_t(source_column->size())));
// check after update_hashes: 1 in selector present the load cols data is selected and data should be default value
auto ser_col = ColumnString::create();
ser_col->reserve(source_column->size());
@@ -1981,7 +2302,7 @@ public:
// check column data equal
for (size_t i = 0; i < columns.size(); ++i) {
EXPECT_EQ(columns[i]->size(), argument_columns[i]->size());
- checkColumn(*columns[i], *argument_columns[i], *data_types[i], columns[0]->size());
+ checkColumn(*columns[i], *argument_columns[i], columns[0]->size());
}
}
}
@@ -2046,7 +2367,7 @@ public:
for (size_t i = 0; i < columns.size(); ++i) {
EXPECT_EQ(columns[i]->size(), check_columns[i]->size());
EXPECT_EQ(columns[i]->size(), check_columns[i]->size());
- checkColumn(*columns[i], *check_columns[i], *dataTypes[i], rows);
+ checkColumn(*columns[i], *check_columns[i], rows);
}
}
}
@@ -2070,7 +2391,7 @@ public:
} else {
EXPECT_EQ(ptr.get(), source_column.get());
// check the column ptr is the same as the source column
- checkColumn(*source_column, *ptr, *typs[i], source_column->size());
+ checkColumn(*source_column, *ptr, source_column->size());
}
}
}
@@ -2131,7 +2452,7 @@ public:
} else {
// just check the column ptr is the same as the source column and res
EXPECT_EQ(ptr.get(), source_column.get());
- checkColumn(*source_column, *ptr, *typs[i], source_column->size());
+ checkColumn(*source_column, *ptr, source_column->size());
}
}
}
@@ -2337,6 +2658,306 @@ public:
ASSERT_EQ(lhs[i], rhs[i]) << "i: " << i << "limit: " << limit;
}
}
+
+ void assert_column_permutations2(vectorized::IColumn& column, DataTypePtr dataType) {
+ IColumn::Permutation actual_permutation;
+ IColumn::Permutation expected_permutation;
+
+ size_t column_size = column.size();
+ std::cout << "column size: " << column_size << std::endl;
+ std::vector<size_t> limits = {
+ 0, 1, 2, 3, 5, 10, column_size - 1, column_size, column_size + 1};
+ for (auto limit : limits) {
+ assert_column_permutation(column, true, limit, -1, actual_permutation,
+ expected_permutation);
+ assert_column_permutation(column, true, limit, 1, actual_permutation,
+ expected_permutation);
+
+ assert_column_permutation(column, false, limit, -1, actual_permutation,
+ expected_permutation);
+ assert_column_permutation(column, false, limit, 1, actual_permutation,
+ expected_permutation);
+ }
+ }
+};
+auto check_permute = [](const IColumn& column, const IColumn::Permutation& permutation,
+ size_t limit, size_t expected_size) {
+ auto res_col = column.permute(permutation, limit);
+ EXPECT_EQ(res_col->size(), expected_size);
+ try {
+ for (size_t j = 0; j < expected_size; ++j) {
+ EXPECT_EQ(res_col->compare_at(j, permutation[j], column, -1), 0);
+ }
+ } catch (doris::Exception& e) {
+ LOG(ERROR) << "Exception: " << e.what();
+ // using field check
+ for (size_t j = 0; j < expected_size; ++j) {
+ Field r;
+ Field l;
+ column.get(permutation[j], r);
+ res_col->get(j, l);
+ EXPECT_EQ(r, l);
+ }
+ }
+};
+auto assert_column_vector_permute = [](MutableColumns& cols, size_t num_rows,
+ bool stable_test = true) {
+ for (const auto& col : cols) {
+ size_t expected_size = num_rows ? std::min(col->size(), num_rows) : col->size();
+ if (stable_test) {
+ IColumn::Permutation permutation;
+ CommonColumnTest::stable_get_column_permutation(*col, true, col->size(), -1,
+ permutation);
+ check_permute(*col, permutation, num_rows, expected_size);
+ }
+ {
+ IColumn::Permutation permutation(col->size());
+ std::iota(permutation.begin(), permutation.end(), IColumn::Permutation::value_type(0));
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(permutation.begin(), permutation.end(), g);
+ check_permute(*col, permutation, num_rows, expected_size);
+ }
+ }
+};
+
+auto assert_column_vector_update_hashes_with_value_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes_with_value` functionality
+ // check update_hashes_with_value with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ size_t rows = source_column->size();
+ NullMap null_map(rows, 0);
+ const uint8_t* null_data = nullptr;
+ if (with_nullmap) {
+ null_data = null_map.data();
+ std::vector<size_t> null_positions {0, rows - 1, rows / 2};
+ for (const auto& pos : null_positions) {
+ null_map[pos] = 1;
+ }
+ }
+
+ std::vector<uint64_t> xx_hash_vals(source_column->size());
+ std::cout << "now we are in update_hashes_with_value column : "
+ << load_cols[i]->get_name() << " for column size : " << source_column->size()
+ << std::endl;
+ auto* __restrict xx_hashes = xx_hash_vals.data();
+ EXPECT_NO_FATAL_FAILURE(source_column->update_hashes_with_value(xx_hashes, null_data));
+ // check after update_hashes_with_value: 1 in selector present the load cols data is selected and data should be default value
+ std::vector<string> data;
+ std::ostringstream oss;
+
+ data.push_back("column: " + source_column->get_name() +
+ " with hashes: " + join_ints(xx_hash_vals));
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "update_hashes_with_value" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
+ test_func(true);
+};
+
+auto assert_column_vector_update_crc_hashes_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ std::vector<PrimitiveType> pts,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ size_t rows = source_column->size();
+ NullMap null_map(rows, 0);
+ const uint8_t* null_data = nullptr;
+ if (with_nullmap) {
+ null_data = null_map.data();
+ std::vector<size_t> null_positions {0, rows - 1, rows / 2};
+ for (const auto& pos : null_positions) {
+ null_map[pos] = 1;
+ }
+ }
+
+ std::vector<uint32_t> crc_hash_vals(source_column->size());
+ std::cout << "now we are in update_hashes column : " << load_cols[i]->get_name()
+ << " for column size : " << source_column->size() << std::endl;
+ EXPECT_NO_FATAL_FAILURE(source_column->update_crcs_with_value(
+ crc_hash_vals.data(), pts[i], uint32_t(source_column->size()), 0, null_data));
+ // check after update_hashes: 1 in selector present the load cols data is selected and data should be default value
+ auto ser_col = ColumnString::create();
+ ser_col->reserve(source_column->size());
+ std::vector<string> data;
+ data.push_back("column: " + source_column->get_name() +
+ " with hashes: " + join_ints(crc_hash_vals));
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "update_crcs_hashes" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
+ test_func(true);
+};
+auto assert_column_vector_update_siphashes_with_value_callback =
+ [](const MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ std::vector<std::vector<string>> res;
+ auto option = DataTypeSerDe::FormatOptions();
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ SipHash hash;
+ LOG(INFO) << "now we are in update_hashes column : " << load_cols[i]->get_name()
+ << " for column size : " << source_column->size();
+ for (size_t j = 0; j < source_column->size(); ++j) {
+ source_column->update_hash_with_value(j, hash);
+ }
+ auto ser_col = ColumnString::create();
+ ser_col->reserve(source_column->size());
+ VectorBufferWriter buffer_writer(*ser_col.get());
+ std::vector<string> data;
+ data.push_back("column: " + source_column->get_name() +
+ " with hashes: " + std::to_string(hash.get64()) +
+ " with ptr: " + std::to_string(source_column->size()));
+ res.push_back(data);
+ }
+ check_or_generate_res_file(
+ res_file_path.empty() ? "update_siphashes_hashes" : res_file_path, res);
+ };
+auto assert_update_xxHash_with_value_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ size_t rows = source_column->size();
+
+ NullMap null_map(rows, 0);
+ const uint8_t* null_data = nullptr;
+ if (with_nullmap) {
+ null_data = null_map.data();
+ std::vector<size_t> null_positions {0, rows - 1, rows / 2};
+ for (const auto& pos : null_positions) {
+ null_map[pos] = 1;
+ }
+ }
+
+ uint64_t hash = 0;
+ source_column->update_xxHash_with_value(0, source_column->size(), hash, null_data);
+ auto ser_col = ColumnString::create();
+ ser_col->reserve(source_column->size());
+ VectorBufferWriter buffer_writer(*ser_col.get());
+ std::vector<string> data;
+ data.push_back("column: " + source_column->get_name() +
+ " with hashes: " + std::to_string(hash) +
+ " with ptr: " + std::to_string(source_column->size()));
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "update_xxHash_with_value" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
+ test_func(true);
+};
+
+auto assert_update_crc_with_value_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+ auto option = DataTypeSerDe::FormatOptions();
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ size_t rows = source_column->size();
+
+ NullMap null_map(rows, 0);
+ const uint8_t* null_data = nullptr;
+ if (with_nullmap) {
+ null_data = null_map.data();
+ std::vector<size_t> null_positions {0, rows - 1, rows / 2};
+ for (const auto& pos : null_positions) {
+ null_map[pos] = 1;
+ }
+ }
+
+ uint32_t hash = 0;
+ source_column->update_crc_with_value(0, source_column->size(), hash, null_data);
+ auto ser_col = ColumnString::create();
+ ser_col->reserve(source_column->size());
+ VectorBufferWriter buffer_writer(*ser_col.get());
+ std::vector<string> data;
+ data.push_back("column: " + source_column->get_name() +
+ " with hashes: " + std::to_string(hash) +
+ " with ptr: " + std::to_string(source_column->size()));
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "update_crc_with_value" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
+ test_func(true);
+};
+
+auto assert_byte_size_with_file_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+ std::vector<string> data;
+ auto actual_str_value = std::to_string(source_column->byte_size());
+ data.push_back("column: " + source_column->get_name() +
+ " with byte_size: " + (actual_str_value));
+ data.push_back(actual_str_value);
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "_byte_size" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
+};
+
+auto assert_allocated_bytes_with_file_callback = [](const MutableColumns& load_cols,
+ DataTypeSerDeSPtrs serders,
+ const std::string& res_file_path) {
+ // Create an empty column to verify `update_hashes` functionality
+ // check update_hashes with different hashes
+ auto test_func = [&](bool with_nullmap) {
+ std::vector<std::vector<string>> res;
+ for (size_t i = 0; i < load_cols.size(); ++i) {
+ const auto& source_column = load_cols[i];
+
+ std::vector<string> data;
+ auto actual_str_value = std::to_string(source_column->allocated_bytes());
+ data.push_back("column: " + source_column->get_name() +
+ " with allocate size: " + (actual_str_value));
+ data.push_back(actual_str_value);
+ res.push_back(data);
+ }
+ std::string file_name = res_file_path.empty() ? "_allocate_size" : res_file_path;
+ file_name += with_nullmap ? "_with_nullmap" : "";
+ check_or_generate_res_file(file_name, res);
+ };
+ test_func(false);
};
} // namespace doris::vectorized
diff --git a/run-be-ut.sh b/run-be-ut.sh
index 96a97bd34f5..0adff685628 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -37,6 +37,7 @@ set +o posix
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+export ROOT
export DORIS_HOME="${ROOT}"
. "${DORIS_HOME}/env.sh"
@@ -49,6 +50,7 @@ Usage: $0 <options>
--clean clean and build ut
--run build and run all ut
--run --filter=xx build and run specified ut
+ --run --gen_out generate expected check data for test
--coverage coverage after run ut
-j build parallel
-h print this help message
@@ -69,7 +71,7 @@ Usage: $0 <options>
exit 1
}
-if ! OPTS="$(getopt -n "$0" -o vhj:f: -l coverage,benchmark,run,clean,filter: -- "$@")"; then
+if ! OPTS="$(getopt -n "$0" -o vhj:f: -l gen_out,coverage,benchmark,run,clean,filter: -- "$@")"; then
usage
fi
@@ -80,6 +82,7 @@ RUN=0
DENABLE_CLANG_COVERAGE='OFF'
BUILD_AZURE='ON'
FILTER=""
+GEN_OUT=""
if [[ "$#" != 1 ]]; then
while true; do
case "$1" in
@@ -95,6 +98,10 @@ if [[ "$#" != 1 ]]; then
DENABLE_CLANG_COVERAGE='ON'
shift
;;
+ --gen_out)
+ GEN_OUT='--gen_out'
+ shift
+ ;;
-f | --filter)
FILTER="--gtest_filter=$2"
shift 2
@@ -457,7 +464,7 @@ profdata=${DORIS_TEST_BINARY_DIR}/doris_be_test.profdata
file_name="${test##*/}"
if [[ -f "${test}" ]]; then
if [[ "_${DENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
- LLVM_PROFILE_FILE="${profraw}" "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
+ LLVM_PROFILE_FILE="${profraw}" "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}" "${GEN_OUT}"
if [[ -d "${DORIS_TEST_BINARY_DIR}"/report ]]; then
rm -rf "${DORIS_TEST_BINARY_DIR}"/report
fi
@@ -471,7 +478,7 @@ if [[ -f "${test}" ]]; then
echo "${cmd2}"
eval "${cmd2}"
else
- "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
+ "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}" "${GEN_OUT}"
fi
echo "=== Finished. Gtest output: ${GTEST_OUTPUT_DIR}"
else
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org