You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/07/04 07:52:10 UTC
[doris] branch master updated: [fix](multi-catalog) fix the core dump on hms table (#10573)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 46bff6bba0 [fix](multi-catalog) fix the core dump on hms table (#10573)
46bff6bba0 is described below
commit 46bff6bba01befb84860f5d299f7c8290e6a7e4f
Author: huangzhaowei <hu...@bytedance.com>
AuthorDate: Mon Jul 4 15:52:05 2022 +0800
[fix](multi-catalog) fix the core dump on hms table (#10573)
In the funciton `TextConverter::write_vec_column`, it should execute the statement `nullable_column->get_null_map_data().push_back(0);` for every row.
Otherwise the null map will get error and cause the core dump.
---
be/src/exec/text_converter.h | 2 +-
be/src/exec/text_converter.hpp | 16 +++++++++-------
be/src/vec/exec/file_scanner.cpp | 5 -----
be/src/vec/exec/file_text_scanner.cpp | 10 ----------
.../apache/doris/catalog/HiveMetaStoreClientHelper.java | 1 +
5 files changed, 11 insertions(+), 23 deletions(-)
diff --git a/be/src/exec/text_converter.h b/be/src/exec/text_converter.h
index 18f1e4185f..a3d3fe5ff6 100644
--- a/be/src/exec/text_converter.h
+++ b/be/src/exec/text_converter.h
@@ -53,7 +53,7 @@ public:
bool write_column(const SlotDescriptor* slot_desc, vectorized::MutableColumnPtr* column_ptr,
const char* data, size_t len, bool copy_string, bool need_escape);
- bool write_vec_column(const SlotDescriptor* slot_desc, vectorized::IColumn* column_ptr,
+ bool write_vec_column(const SlotDescriptor* slot_desc, vectorized::IColumn* nullable_col_ptr,
const char* data, size_t len, bool copy_string, bool need_escape);
// Removes escape characters from len characters of the null-terminated string src,
diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp
index b931d4c560..f57c613c04 100644
--- a/be/src/exec/text_converter.hpp
+++ b/be/src/exec/text_converter.hpp
@@ -188,10 +188,17 @@ inline void TextConverter::write_string_column(const SlotDescriptor* slot_desc,
inline bool TextConverter::write_column(const SlotDescriptor* slot_desc,
vectorized::MutableColumnPtr* column_ptr, const char* data,
size_t len, bool copy_string, bool need_escape) {
- vectorized::IColumn* col_ptr = column_ptr->get();
+ vectorized::IColumn* nullable_col_ptr = column_ptr->get();
+ return write_vec_column(slot_desc, nullable_col_ptr, data, len, copy_string, need_escape);
+}
+
+inline bool TextConverter::write_vec_column(const SlotDescriptor* slot_desc,
+ vectorized::IColumn* nullable_col_ptr, const char* data,
+ size_t len, bool copy_string, bool need_escape) {
+ vectorized::IColumn* col_ptr = nullable_col_ptr;
// \N means it's NULL
if (true == slot_desc->is_nullable()) {
- auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(column_ptr->get());
+ auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(nullable_col_ptr);
if ((len == 2 && data[0] == '\\' && data[1] == 'N') || len == SQL_NULL_DATA) {
nullable_column->insert_data(nullptr, 0);
return true;
@@ -200,12 +207,7 @@ inline bool TextConverter::write_column(const SlotDescriptor* slot_desc,
col_ptr = &nullable_column->get_nested_column();
}
}
- return write_vec_column(slot_desc, col_ptr, data, len, copy_string, need_escape);
-}
-inline bool TextConverter::write_vec_column(const SlotDescriptor* slot_desc,
- vectorized::IColumn* col_ptr, const char* data,
- size_t len, bool copy_string, bool need_escape) {
StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
// Parse the raw-text data. Translate the text string to internal format.
diff --git a/be/src/vec/exec/file_scanner.cpp b/be/src/vec/exec/file_scanner.cpp
index 106b94cd73..d9780f8dbe 100644
--- a/be/src/vec/exec/file_scanner.cpp
+++ b/be/src/vec/exec/file_scanner.cpp
@@ -190,11 +190,6 @@ Status FileScanner::_fill_columns_from_path(vectorized::Block* _block) {
auto doris_column = _block->get_by_name(slot_desc->col_name()).column;
IColumn* col_ptr = const_cast<IColumn*>(doris_column.get());
- if (slot_desc->is_nullable()) {
- auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(col_ptr);
- nullable_column->get_null_map_data().push_back(0);
- col_ptr = &nullable_column->get_nested_column();
- }
for (size_t j = 0; j < rows; ++j) {
_text_converter->write_vec_column(slot_desc, col_ptr,
diff --git a/be/src/vec/exec/file_text_scanner.cpp b/be/src/vec/exec/file_text_scanner.cpp
index 03e2dd9275..8f98ad314b 100644
--- a/be/src/vec/exec/file_text_scanner.cpp
+++ b/be/src/vec/exec/file_text_scanner.cpp
@@ -132,16 +132,6 @@ Status FileTextScanner::_fill_file_columns(const Slice& line, vectorized::Block*
auto doris_column = _block->get_by_name(slot_desc->col_name()).column;
IColumn* col_ptr = const_cast<IColumn*>(doris_column.get());
- if (slot_desc->is_nullable()) {
- auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(col_ptr);
- nullable_column->get_null_map_data().push_back(0);
- col_ptr = &nullable_column->get_nested_column();
- }
-
- if (value.size == 2 && value.data[0] == '\\' && value[1] == 'N') {
- col_ptr->insert_default();
- continue;
- }
_text_converter->write_vec_column(slot_desc, col_ptr, value.data, value.size, true, false);
}
_rows++;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 80d311fe54..4979cb1699 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -624,6 +624,7 @@ public class HiveMetaStoreClientHelper {
case CHAR:
return TypeInfoFactory.charTypeInfo;
case VARCHAR:
+ case STRING:
return TypeInfoFactory.varcharTypeInfo;
default:
throw new DdlException("Unsupported column type: " + dorisType);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org