You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/03/09 15:53:39 UTC
[doris] 01/10: [Fix](Lightweight schema Change) query error caused by array default type is unsupported (#17331)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 271ab0ece7364c8e81af6ddacb789d7ced43a4ea
Author: ZhangYu0123 <67...@users.noreply.github.com>
AuthorDate: Tue Mar 7 16:30:41 2023 +0800
[Fix](Lightweight schema Change) query error caused by array default type is unsupported (#17331)
We have supportted array type default [], but when using lightweight schema Change to add column array type, query failed as follows:
Fix "array default type is unsupported" error.
Fix the default value filling assignment digit problem.
---
be/src/olap/rowset/segment_v2/column_reader.cpp | 49 ++++++++++++++++------
be/src/vec/columns/column_nullable.h | 6 +++
.../schema_change_p0/test_alter_table_column.out | 11 +++++
.../test_alter_table_column.groovy | 42 +++++++++++++++++++
4 files changed, 96 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index c8da3c8a7c..ecef8cb4dd 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -993,8 +993,16 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) {
((Slice*)_mem_value.data())->size = _default_value.length();
((Slice*)_mem_value.data())->data = _default_value.data();
} else if (_type_info->type() == OLAP_FIELD_TYPE_ARRAY) {
- // TODO llj for Array default value
- return Status::NotSupported("Array default type is unsupported");
+ if (_default_value != "[]") {
+ return Status::NotSupported("Array default {} is unsupported", _default_value);
+ } else {
+ ((Slice*)_mem_value.data())->size = _default_value.length();
+ ((Slice*)_mem_value.data())->data = _default_value.data();
+ }
+ } else if (_type_info->type() == OLAP_FIELD_TYPE_STRUCT) {
+ return Status::NotSupported("STRUCT default type is unsupported");
+ } else if (_type_info->type() == OLAP_FIELD_TYPE_MAP) {
+ return Status::NotSupported("MAP default type is unsupported");
} else {
s = _type_info->from_string(_mem_value.data(), _default_value, _precision, _scale);
}
@@ -1033,9 +1041,6 @@ Status DefaultValueColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, b
void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, size_t type_size,
void* mem_value,
vectorized::MutableColumnPtr& dst, size_t n) {
- vectorized::Int128 int128;
- char* data_ptr = (char*)&int128;
- size_t data_len = sizeof(int128);
dst = dst->convert_to_predicate_column_if_dictionary();
switch (type_info->type()) {
@@ -1045,18 +1050,26 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info,
break;
}
case OLAP_FIELD_TYPE_DATE: {
+ vectorized::Int64 int64;
+ char* data_ptr = (char*)&int64;
+ size_t data_len = sizeof(int64);
+
assert(type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::CppType)); //uint24_t
std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::to_string(mem_value);
vectorized::VecDateTimeValue value;
value.from_date_str(str.c_str(), str.length());
value.cast_to_date();
- //TODO: here is int128 = int64, here rely on the logic of little endian
- int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
+
+ int64 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
dst->insert_many_data(data_ptr, data_len, n);
break;
}
case OLAP_FIELD_TYPE_DATETIME: {
+ vectorized::Int64 int64;
+ char* data_ptr = (char*)&int64;
+ size_t data_len = sizeof(int64);
+
assert(type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::CppType)); //int64_t
std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::to_string(mem_value);
@@ -1064,11 +1077,15 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info,
value.from_date_str(str.c_str(), str.length());
value.to_datetime();
- int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
+ int64 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
dst->insert_many_data(data_ptr, data_len, n);
break;
}
case OLAP_FIELD_TYPE_DECIMAL: {
+ vectorized::Int128 int128;
+ char* data_ptr = (char*)&int128;
+ size_t data_len = sizeof(int128);
+
assert(type_size ==
sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DECIMAL>::CppType)); //decimal12_t
decimal12_t* d = (decimal12_t*)mem_value;
@@ -1080,14 +1097,22 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info,
case OLAP_FIELD_TYPE_VARCHAR:
case OLAP_FIELD_TYPE_CHAR:
case OLAP_FIELD_TYPE_JSONB: {
- data_ptr = ((Slice*)mem_value)->data;
- data_len = ((Slice*)mem_value)->size;
+ char* data_ptr = ((Slice*)mem_value)->data;
+ size_t data_len = ((Slice*)mem_value)->size;
dst->insert_many_data(data_ptr, data_len, n);
break;
}
+ case OLAP_FIELD_TYPE_ARRAY: {
+ if (dst->is_nullable()) {
+ static_cast<vectorized::ColumnNullable&>(*dst).insert_not_null_elements(n);
+ } else {
+ dst->insert_many_defaults(n);
+ }
+ break;
+ }
default: {
- data_ptr = (char*)mem_value;
- data_len = type_size;
+ char* data_ptr = (char*)mem_value;
+ size_t data_len = type_size;
dst->insert_many_data(data_ptr, data_len, n);
}
}
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index 91c0ac0cf4..2b053ef939 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -151,6 +151,12 @@ public:
_has_null = true;
}
+ void insert_not_null_elements(size_t num) {
+ get_nested_column().insert_many_defaults(num);
+ _get_null_map_column().fill(0, num);
+ _has_null = false;
+ }
+
void insert_null_elements(int num) {
get_nested_column().insert_many_defaults(num);
_get_null_map_column().fill(1, num);
diff --git a/regression-test/data/schema_change_p0/test_alter_table_column.out b/regression-test/data/schema_change_p0/test_alter_table_column.out
index a130d16e16..81a715c2bc 100644
--- a/regression-test/data/schema_change_p0/test_alter_table_column.out
+++ b/regression-test/data/schema_change_p0/test_alter_table_column.out
@@ -18,6 +18,17 @@ value2 INT Yes false \N SUM
-- !sql --
1 1 40 60
+-- !sql --
+k1 INT Yes true \N
+value1 INT Yes false \N NONE
+value2 ARRAY<INT(11)> Yes false [] NONE
+value3 ARRAY<INT(11)> Yes false \N NONE
+value4 ARRAY<INT(11)> No false [] NONE
+
+-- !sql --
+1 2 [] \N []
+3 4 [] \N []
+
-- !select --
\N \N \N
1 1989 1001
diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy
index 8bbf40bef7..097e210384 100644
--- a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy
+++ b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy
@@ -114,6 +114,47 @@ suite("test_alter_table_column") {
qt_sql "select * from ${tbName2};"
sql "DROP TABLE ${tbName2} FORCE;"
+ def tbNameAddArray = "alter_table_add_array_column_dup"
+ sql "DROP TABLE IF EXISTS ${tbNameAddArray}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbNameAddArray} (
+ k1 INT,
+ value1 INT
+ )
+ DUPLICATE KEY (k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(
+ "replication_num" = "1",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "true");
+ """
+
+ sql "insert into ${tbNameAddArray} values(1,2)"
+ sql "insert into ${tbNameAddArray} values(3,4)"
+ sql """
+ ALTER TABLE ${tbNameAddArray}
+ ADD COLUMN value2 ARRAY<INT> DEFAULT '[]' AFTER value1,
+ ADD COLUMN value3 ARRAY<INT> AFTER value2,
+ ADD COLUMN value4 ARRAY<INT> NOT NULL DEFAULT '[]' AFTER value3;
+ """
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbNameAddArray)
+ if (res == "FINISHED") {
+ break
+ } else {
+ Thread.sleep(2000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+
+ Thread.sleep(200)
+ qt_sql "desc ${tbNameAddArray};"
+ qt_sql "select * from ${tbNameAddArray};"
+ sql "DROP TABLE ${tbNameAddArray} FORCE;"
+
// vector search
def check_load_result = {checklabel, testTablex ->
Integer max_try_milli_secs = 10000
@@ -175,4 +216,5 @@ suite("test_alter_table_column") {
def res4 = sql "select k1, k2, k3, null from baseall order by k1"
check2_doris(res3, res4)
sql "DROP TABLE ${tbName3} FORCE;"
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org