You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/19 15:37:36 UTC
(doris) branch master updated: [fix](parquet) return error if schema changed in complex types (#31128)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0484d6b7bff [fix](parquet) return error if schema changed in complex types (#31128)
0484d6b7bff is described below
commit 0484d6b7bfffacab917a8e38ea60b0610164b86c
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Mon Feb 19 23:37:28 2024 +0800
[fix](parquet) return error if schema changed in complex types (#31128)
Check the column type of complex type to prevent core dump in BE. ColumnReader will throw segmentation fault in the following case:
Change complex types in hive:
hive> create table struct_test(
id int,
sf struct<f1: int, f2: map<string, string>>) stored as parquet;
hive> insert into struct_test values
(1, named_struct('f1', 1, 'f2', str_to_map('1:s2,2:s2'))),
(2, named_struct('f1', 2, 'f2', str_to_map('k1:s3,k2:s4'))),
(3, named_struct('f1', 3, 'f2', str_to_map('k1:s5,k2:s6')));
hive> alter table struct_test change sf sf struct<f1:int, f2: string>;
---
be/src/vec/exec/format/parquet/vparquet_column_reader.cpp | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index cfddb5346e8..f813d14b63b 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -418,7 +418,7 @@ Status ScalarColumnReader::_read_nested_column(ColumnPtr& doris_column, DataType
}
RETURN_IF_ERROR(_chunk_reader->decode_values(data_column, type, select_vector, is_dict_filter));
if (ancestor_nulls != 0) {
- static_cast<void>(_chunk_reader->skip_values(ancestor_nulls, false));
+ RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_nulls, false));
}
if (!align_rows) {
@@ -608,6 +608,9 @@ Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, DataTypePtr&
}
data_column = doris_column->assume_mutable();
}
+ if (remove_nullable(type)->get_type_id() != TypeIndex::Array) {
+ return Status::Corruption("Wrong data type for column '{}'", _field_schema->name);
+ }
ColumnPtr& element_column = static_cast<ColumnArray&>(*data_column).get_data_ptr();
DataTypePtr& element_type = const_cast<DataTypePtr&>(
@@ -654,6 +657,9 @@ Status MapColumnReader::read_column_data(ColumnPtr& doris_column, DataTypePtr& t
}
data_column = doris_column->assume_mutable();
}
+ if (remove_nullable(type)->get_type_id() != TypeIndex::Map) {
+ return Status::Corruption("Wrong data type for column '{}'", _field_schema->name);
+ }
auto& map = static_cast<ColumnMap&>(*data_column);
DataTypePtr& key_type = const_cast<DataTypePtr&>(
@@ -717,6 +723,9 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, DataTypePtr
}
data_column = doris_column->assume_mutable();
}
+ if (remove_nullable(type)->get_type_id() != TypeIndex::Struct) {
+ return Status::Corruption("Wrong data type for column '{}'", _field_schema->name);
+ }
auto& doris_struct = static_cast<ColumnStruct&>(*data_column);
if (_child_readers.size() != doris_struct.tuple_size()) {
@@ -731,7 +740,7 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, DataTypePtr
size_t field_rows = 0;
bool field_eof = false;
if (i == 0) {
- static_cast<void>(_child_readers[i]->read_column_data(
+ RETURN_IF_ERROR(_child_readers[i]->read_column_data(
doris_field, doris_type, select_vector, batch_size, &field_rows, &field_eof,
is_dict_filter));
*read_rows = field_rows;
@@ -740,7 +749,7 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, DataTypePtr
while (field_rows < *read_rows && !field_eof) {
size_t loop_rows = 0;
select_vector.reset();
- static_cast<void>(_child_readers[i]->read_column_data(
+ RETURN_IF_ERROR(_child_readers[i]->read_column_data(
doris_field, doris_type, select_vector, *read_rows - field_rows, &loop_rows,
&field_eof, is_dict_filter));
field_rows += loop_rows;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org