You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2020/08/27 09:53:09 UTC
[incubator-doris] branch master updated: Add
OLAP_ERR_DATE_QUALITY_ERR error status to display schema change failure
(#4388)
This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new ad738fa Add OLAP_ERR_DATE_QUALITY_ERR error status to display schema change failure (#4388)
ad738fa is described below
commit ad738fa1988f627f73a19da0ab40116bf1c50c69
Author: HangyuanLiu <46...@qq.com>
AuthorDate: Thu Aug 27 17:52:53 2020 +0800
Add OLAP_ERR_DATE_QUALITY_ERR error status to display schema change failure (#4388)
In the process of historical data transformation of materialized views, it may occur that the transformation fails due to data quality.
Add an error status code ：OLAP_ERR_DATE_QUALITY_ERR to determine if a data problem is causing the failure
#3344
---
be/src/agent/task_worker_pool.cpp | 3 +
be/src/olap/olap_define.h | 2 +
be/src/olap/schema_change.cpp | 102 ++++++++++-----------
be/src/olap/schema_change.h | 10 +-
be/test/olap/schema_change_test.cpp | 81 +++++++++++++++-
docs/en/administrator-guide/materialized_view.md | 5 +
.../zh-CN/administrator-guide/materialized_view.md | 5 +
.../java/org/apache/doris/master/MasterImpl.java | 1 +
8 files changed, 148 insertions(+), 61 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 2f0ac4d..b557f3e 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -509,6 +509,9 @@ void TaskWorkerPool::_alter_tablet(TaskWorkerPool* worker_pool_this,
&error_msgs, process_name);
OLAPStatus sc_status = worker_pool_this->_env->storage_engine()->execute_task(&engine_task);
if (sc_status != OLAP_SUCCESS) {
+ if (sc_status == OLAP_ERR_DATA_QUALITY_ERR) {
+ error_msgs.push_back("The data quality does not satisfy, please check your data. ");
+ }
status = DORIS_ERROR;
} else {
status = DORIS_SUCCESS;
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index 7d74053..1797fe1 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -304,6 +304,8 @@ enum OLAPStatus {
OLAP_ERR_PREVIOUS_SCHEMA_CHANGE_NOT_FINISHED = -1603,
OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID = -1604,
OLAP_ERR_QUERY_SPLIT_KEY_ERR = -1605,
+ //Error caused by a data quality issue during schema change/materialized view
+ OLAP_ERR_DATA_QUALITY_ERR = -1606,
// Column File
// [-1700, -1800)
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 626c1f8..994c667 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -175,7 +175,7 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)
<< " origin_type=" \
<< ref_block->tablet_schema().column(ref_column).type() \
<< ", alter_type=" << mutable_block->tablet_schema().column(i).type(); \
- return false; \
+ return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID; \
} \
break; \
}
@@ -384,23 +384,23 @@ bool count_field(RowCursor* read_helper, RowCursor* write_helper, const TabletCo
return true;
}
-bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_version,
+OLAPStatus RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_version,
RowBlock* mutable_block, uint64_t* filtered_rows) const {
if (mutable_block == nullptr) {
LOG(FATAL) << "mutable block is uninitialized.";
- return false;
+ return OLAP_ERR_NOT_INITED;
} else if (mutable_block->tablet_schema().num_columns() != _schema_mapping.size()) {
LOG(WARNING) << "mutable block does not match with schema mapping rules. "
<< "block_schema_size=" << mutable_block->tablet_schema().num_columns()
<< ", mapping_schema_size=" << _schema_mapping.size();
- return false;
+ return OLAP_ERR_NOT_INITED;
}
if (mutable_block->capacity() < ref_block->row_block_info().row_num) {
LOG(WARNING) << "mutable block is not large enough for storing the changed block. "
<< "mutable_block_size=" << mutable_block->capacity()
<< ", ref_block_row_num=" << ref_block->row_block_info().row_num;
- return false;
+ return OLAP_ERR_NOT_INITED;
}
mutable_block->clear();
@@ -408,13 +408,13 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
RowCursor write_helper;
if (write_helper.init(mutable_block->tablet_schema()) != OLAP_SUCCESS) {
LOG(WARNING) << "fail to init rowcursor.";
- return false;
+ return OLAP_ERR_NOT_INITED;
}
RowCursor read_helper;
if (read_helper.init(ref_block->tablet_schema()) != OLAP_SUCCESS) {
LOG(WARNING) << "fail to init rowcursor.";
- return false;
+ return OLAP_ERR_NOT_INITED;
}
// a.1 先判断数据是否需要过滤，最终只有标记为1的才是留下需要的
@@ -464,7 +464,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
_do_materialized_transform = count_field;
} else {
LOG(WARNING) << "error materialized view function : " << _schema_mapping[i].materialized_function;
- return false;
+ return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID;
}
VLOG(3) << "_schema_mapping[" << i << "].materialized_function : " << _schema_mapping[i].materialized_function;
for (size_t row_index = 0, new_row_index = 0;
@@ -478,7 +478,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
if (!_do_materialized_transform(&read_helper, &write_helper,
ref_block->tablet_schema().column(ref_column), i, _schema_mapping[i].ref_column, mem_pool)) {
- return false;
+ return OLAP_ERR_DATA_QUALITY_ERR;
}
}
continue;
@@ -577,7 +577,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
<< "the column type which was altered from was unsupported."
<< "status:" << st << ", from_type=" << reftype
<< ", to_type=" << newtype;
- return false;
+ return st;
}
}
}
@@ -606,7 +606,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
LOG(WARNING) << "the column type which was altered from was unsupported."
<< " from_type="
<< ref_block->tablet_schema().column(ref_column).type();
- return false;
+ return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID;
}
if (newtype < reftype) {
@@ -643,7 +643,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v
// （其实在init时就可以重新init成少的，filter留下的new_row_num）
// 在split_table时，可能会出现因为过滤导致没有数据
mutable_block->finalize(new_row_num);
- return true;
+ return OLAP_SUCCESS;
}
#undef CONVERT_FROM_TYPE
@@ -911,7 +911,7 @@ bool RowBlockMerger::_pop_heap() {
return true;
}
-bool LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader,
+OLAPStatus LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader,
RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet,
TabletSharedPtr base_tablet) {
OLAPStatus status = new_rowset_writer->add_rowset_for_linked_schema_change(
@@ -922,10 +922,8 @@ bool LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader,
<< ", base_tablet=" << base_tablet->full_name()
<< ", version=" << new_rowset_writer->version().first << "-"
<< new_rowset_writer->version().second;
- return false;
}
-
- return true;
+ return status;
}
SchemaChangeDirectly::SchemaChangeDirectly(const RowBlockChanger& row_block_changer)
@@ -949,13 +947,13 @@ bool SchemaChangeDirectly::_write_row_block(RowsetWriter* rowset_writer, RowBloc
return true;
}
-bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
+OLAPStatus SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) {
if (_row_block_allocator == nullptr) {
_row_block_allocator = new RowBlockAllocator(new_tablet->tablet_schema(), 0);
if (_row_block_allocator == nullptr) {
LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator);
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
}
@@ -963,12 +961,12 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
_cursor = new (nothrow) RowCursor();
if (nullptr == _cursor) {
LOG(WARNING) << "fail to allocate row cursor.";
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
if (OLAP_SUCCESS != _cursor->init(new_tablet->tablet_schema())) {
LOG(WARNING) << "fail to init row cursor.";
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
}
@@ -990,14 +988,13 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
LOG(WARNING) << "create empty version for schema change failed."
<< "version=" << rowset_writer->version().first << "-"
<< rowset_writer->version().second;
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
- return true;
+ return OLAP_SUCCESS;
}
VLOG(3) << "init writer. new_tablet=" << new_tablet->full_name()
<< ", block_row_number=" << new_tablet->num_rows_per_row_block();
- bool result = true;
RowBlock* new_row_block = nullptr;
// Reset filtered_rows and merged_rows statistic
@@ -1014,12 +1011,10 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
_row_block_allocator->release(new_row_block);
new_row_block = nullptr;
}
-
- if (OLAP_SUCCESS !=
- _row_block_allocator->allocate(&new_row_block,
- ref_row_block->row_block_info().row_num, true)) {
+ res = _row_block_allocator->allocate(&new_row_block,
+ ref_row_block->row_block_info().row_num, true);
+ if (OLAP_SUCCESS != res) {
LOG(WARNING) << "failed to allocate RowBlock.";
- result = false;
goto DIRECTLY_PROCESS_ERR;
}
} else {
@@ -1028,10 +1023,9 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
// 将ref改为new。这一步按道理来说确实需要等大的块，但理论上和writer无关。
uint64_t filtered_rows = 0;
- if (!_row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second,
- new_row_block, &filtered_rows)) {
+ if ((res = _row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second,
+ new_row_block, &filtered_rows)) != OLAP_SUCCESS) {
LOG(WARNING) << "failed to change data in row block.";
- result = false;
goto DIRECTLY_PROCESS_ERR;
}
// rows filtered by delete handler one by one
@@ -1039,7 +1033,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
if (!_write_row_block(rowset_writer, new_row_block)) {
LOG(WARNING) << "failed to write row block.";
- result = false;
+ res = OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID;
goto DIRECTLY_PROCESS_ERR;
}
@@ -1048,7 +1042,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
}
if (OLAP_SUCCESS != rowset_writer->flush()) {
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
goto DIRECTLY_PROCESS_ERR;
}
@@ -1064,7 +1058,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr
<< ", merged_rows=" << merged_rows()
<< ", filtered_rows=" << filtered_rows()
<< ", new_index_rows=" << rowset_writer->num_rows();
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
}
LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows()
<< ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows()
@@ -1080,7 +1074,7 @@ DIRECTLY_PROCESS_ERR:
_row_block_allocator->release(new_row_block);
new_row_block = nullptr;
}
- return result;
+ return res;
}
SchemaChangeWithSorting::SchemaChangeWithSorting(const RowBlockChanger& row_block_changer,
@@ -1102,7 +1096,7 @@ SchemaChangeWithSorting::~SchemaChangeWithSorting() {
SAFE_DELETE(_row_block_allocator);
}
-bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
+OLAPStatus SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet,
TabletSharedPtr base_tablet) {
if (_row_block_allocator == nullptr) {
@@ -1110,7 +1104,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
new (nothrow) RowBlockAllocator(new_tablet->tablet_schema(), _memory_limitation);
if (_row_block_allocator == nullptr) {
LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator);
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
}
@@ -1127,12 +1121,11 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
LOG(WARNING) << "create empty version for schema change failed."
<< " version=" << new_rowset_writer->version().first << "-"
<< new_rowset_writer->version().second;
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
- return true;
+ return OLAP_SUCCESS;
}
- bool result = true;
RowBlockSorter row_block_sorter(_row_block_allocator);
// for internal sorting
@@ -1162,7 +1155,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
ref_row_block->row_block_info().row_num,
true)) {
LOG(WARNING) << "failed to allocate RowBlock.";
- result = false;
+ res = OLAP_ERR_INPUT_PARAMETER_ERROR;
goto SORTING_PROCESS_ERR;
}
@@ -1170,7 +1163,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
if (row_block_arr.size() < 1) {
LOG(WARNING) << "Memory limitation is too small for Schema Change."
<< "memory_limitation=" << _memory_limitation;
- return false;
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
// enter here while memory limitation is reached.
@@ -1185,7 +1178,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
rowset_reader->version_hash(), new_tablet, new_rowset_type,
segments_overlap, &rowset)) {
LOG(WARNING) << "failed to sorting internally.";
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
goto SORTING_PROCESS_ERR;
}
@@ -1207,7 +1200,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
if (!_row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second,
new_row_block, &filtered_rows)) {
LOG(WARNING) << "failed to change data in row block.";
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
goto SORTING_PROCESS_ERR;
}
add_filtered_rows(filtered_rows);
@@ -1215,7 +1208,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
if (new_row_block->row_block_info().row_num > 0) {
if (!row_block_sorter.sort(&new_row_block)) {
LOG(WARNING) << "failed to sort row block.";
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
OLAP_GOTO(SORTING_PROCESS_ERR);
}
@@ -1243,7 +1236,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
rowset_reader->version_hash(), new_tablet, new_rowset_type,
segments_overlap, &rowset)) {
LOG(WARNING) << "failed to sorting internally.";
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
goto SORTING_PROCESS_ERR;
}
@@ -1266,11 +1259,11 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
LOG(WARNING) << "create empty version for schema change failed."
<< " version=" << new_rowset_writer->version().first << "-"
<< new_rowset_writer->version().second;
- return false;
+ return OLAP_ERR_ALTER_STATUS_ERR;
}
} else if (!_external_sorting(src_rowsets, new_rowset_writer, new_tablet)) {
LOG(WARNING) << "failed to sorting externally.";
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
goto SORTING_PROCESS_ERR;
}
@@ -1285,7 +1278,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
<< ", merged_rows=" << merged_rows()
<< ", filtered_rows=" << filtered_rows()
<< ", new_index_rows=" << new_rowset_writer->num_rows();
- result = false;
+ res = OLAP_ERR_ALTER_STATUS_ERR;
}
LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows()
<< ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows()
@@ -1309,7 +1302,7 @@ SORTING_PROCESS_ERR:
}
row_block_arr.clear();
- return result;
+ return res;
}
bool SchemaChangeWithSorting::_internal_sorting(const vector<RowBlock*>& row_block_arr,
@@ -1721,7 +1714,8 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl
std::unique_ptr<RowsetWriter> rowset_writer;
RowsetFactory::create_rowset_writer(writer_context, &rowset_writer);
- if (!sc_procedure->process(rowset_reader, rowset_writer.get(), new_tablet, base_tablet)) {
+ if ((res = sc_procedure->process(rowset_reader, rowset_writer.get(), new_tablet, base_tablet)) != OLAP_SUCCESS) {
+
if ((*base_rowset)->is_pending()) {
LOG(WARNING) << "failed to process the transaction when schema change. "
<< "tablet=" << new_tablet->full_name() << "'"
@@ -1731,7 +1725,6 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl
<< "version=" << (*base_rowset)->version().first << "-"
<< (*base_rowset)->version().second;
}
- res = OLAP_ERR_INPUT_PARAMETER_ERROR;
new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX +
rowset_writer->rowset_id().to_string());
goto SCHEMA_VERSION_CONVERT_ERR;
@@ -1929,12 +1922,11 @@ OLAPStatus SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangePa
goto PROCESS_ALTER_EXIT;
}
- if (!sc_procedure->process(rs_reader, rowset_writer.get(), sc_params.new_tablet,
- sc_params.base_tablet)) {
+ if ((res = sc_procedure->process(rs_reader, rowset_writer.get(), sc_params.new_tablet,
+ sc_params.base_tablet)) != OLAP_SUCCESS) {
LOG(WARNING) << "failed to process the version."
<< " version=" << rs_reader->version().first << "-"
<< rs_reader->version().second;
- res = OLAP_ERR_INPUT_PARAMETER_ERROR;
new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX +
rowset_writer->rowset_id().to_string());
goto PROCESS_ALTER_EXIT;
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 7771ff7..6ce83dd 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -59,7 +59,7 @@ public:
SchemaMapping get_schema_mapping() const { return _schema_mapping; }
- bool change_row_block(const RowBlock* ref_block, int32_t data_version, RowBlock* mutable_block,
+ OLAPStatus change_row_block(const RowBlock* ref_block, int32_t data_version, RowBlock* mutable_block,
uint64_t* filtered_rows) const;
private:
@@ -92,7 +92,7 @@ public:
SchemaChange() : _filtered_rows(0), _merged_rows(0) {}
virtual ~SchemaChange() {}
- virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder,
+ virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder,
TabletSharedPtr tablet, TabletSharedPtr base_tablet) = 0;
void add_filtered_rows(uint64_t filtered_rows) { _filtered_rows += filtered_rows; }
@@ -118,7 +118,7 @@ public:
: _row_block_changer(row_block_changer) {}
~LinkedSchemaChange() {}
- bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer,
+ virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer,
TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override;
private:
@@ -134,7 +134,7 @@ public:
explicit SchemaChangeDirectly(const RowBlockChanger& row_block_changer);
virtual ~SchemaChangeDirectly();
- virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer,
+ virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer,
TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override;
private:
@@ -154,7 +154,7 @@ public:
size_t memory_limitation);
virtual ~SchemaChangeWithSorting();
- virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder,
+ virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder,
TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override;
private:
diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp
index 8367fa8..e7ede79 100644
--- a/be/test/olap/schema_change_test.cpp
+++ b/be/test/olap/schema_change_test.cpp
@@ -781,7 +781,6 @@ TEST_F(TestColumn, ConvertIntToBitmap) {
column_mapping->ref_column = 2;
column_mapping->materialized_function = "to_bitmap";
-
RowBlock mutable_block(&mv_tablet_schema);
mutable_block.init(block_info);
uint64_t filtered_rows = 0;
@@ -878,6 +877,86 @@ TEST_F(TestColumn, ConvertCharToHLL) {
HyperLogLog hll(*dst_slice);
ASSERT_EQ(hll.estimate_cardinality(), 1);
}
+
+TEST_F(TestColumn, ConvertCharToCount) {
+ //Base Tablet
+ TabletSchema tablet_schema;
+ CreateTabletSchema(tablet_schema);
+
+ //Base row block
+ CreateColumnWriter(tablet_schema);
+
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+ write_row.allocate_memory_for_string_type(tablet_schema);
+ std::vector<std::string> val_string_array;
+ val_string_array.emplace_back("1");
+ val_string_array.emplace_back("1");
+ val_string_array.emplace_back("2");
+ val_string_array.emplace_back("3");
+ OlapTuple tuple(val_string_array);
+ write_row.from_tuple(tuple);
+ block.set_row(0, write_row);
+
+ block.finalize(1);
+ ColumnDataHeaderMessage header;
+ ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
+
+ //Materialized View tablet schema
+ TabletSchemaPB mv_tablet_schema_pb;
+ mv_tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS);
+ mv_tablet_schema_pb.set_num_short_key_columns(2);
+ mv_tablet_schema_pb.set_num_rows_per_row_block(1024);
+ mv_tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
+ mv_tablet_schema_pb.set_next_column_unique_id(3);
+
+ ColumnPB* mv_column_1 = mv_tablet_schema_pb.add_column();
+ mv_column_1->set_unique_id(1);
+ mv_column_1->set_name("k1");
+ mv_column_1->set_type("INT");
+ mv_column_1->set_is_key(true);
+ mv_column_1->set_length(4);
+ mv_column_1->set_index_length(4);
+ mv_column_1->set_is_nullable(false);
+ mv_column_1->set_is_bf_column(false);
+
+ ColumnPB* mv_column_2 = mv_tablet_schema_pb.add_column();
+ mv_column_2->set_unique_id(2);
+ mv_column_2->set_name("v1");
+ mv_column_2->set_type("BIGINT");
+ mv_column_2->set_length(4);
+ mv_column_2->set_is_key(false);
+ mv_column_2->set_is_nullable(false);
+ mv_column_2->set_is_bf_column(false);
+ mv_column_2->set_aggregation("SUM");
+
+ TabletSchema mv_tablet_schema;
+ mv_tablet_schema.init_from_pb(mv_tablet_schema_pb);
+
+ RowBlockChanger row_block_changer(mv_tablet_schema);
+ ColumnMapping* column_mapping = row_block_changer.get_mutable_column_mapping(0);
+ column_mapping->ref_column = 0;
+ column_mapping = row_block_changer.get_mutable_column_mapping(1);
+ column_mapping->ref_column = 1;
+ column_mapping->materialized_function = "count_field";
+
+ RowBlock mutable_block(&mv_tablet_schema);
+ mutable_block.init(block_info);
+ uint64_t filtered_rows = 0;
+ row_block_changer.change_row_block(&block, 0, &mutable_block, &filtered_rows);
+
+ RowCursor mv_row_cursor;
+ mv_row_cursor.init(mv_tablet_schema);
+ mutable_block.get_row(0, &mv_row_cursor);
+
+ auto dst = mv_row_cursor.cell_ptr(1);
+ ASSERT_EQ(*(int64_t *)dst, 1);
+}
}
int main(int argc, char** argv) {
diff --git a/docs/en/administrator-guide/materialized_view.md b/docs/en/administrator-guide/materialized_view.md
index 52e35b1..a98fc9f 100644
--- a/docs/en/administrator-guide/materialized_view.md
+++ b/docs/en/administrator-guide/materialized_view.md
@@ -486,3 +486,8 @@ This problem can be solved by creating a materialized view with k3 as the first
2. If the conditional column of the delete statement does not exist in the materialized view, the delete operation cannot be performed. If you must delete data, you need to delete the materialized view before deleting the data.
3. Too many materialized views on a single table will affect the efficiency of importing: When importing data, the materialized view and base table data are updated synchronously. If a table has more than 10 materialized view tables, it may cause the import speed to be very high. slow. This is the same as a single import needs to import 10 tables at the same time.
4. The same column with different aggregate functions cannot appear in a materialized view at the same time. For example, select sum(a), min(a) from table are not supported.
+
+## Error
+1. DATA_QUALITY_ERR: "The data quality does not satisfy, please check your data"
+Materialized view creation failed due to data quality issues.
+Note: The bitmap type only supports positive integers. If there are negative Numbers in the original data, the materialized view will fail to be created
\ No newline at end of file
diff --git a/docs/zh-CN/administrator-guide/materialized_view.md b/docs/zh-CN/administrator-guide/materialized_view.md
index d91f717..173b5ac 100644
--- a/docs/zh-CN/administrator-guide/materialized_view.md
+++ b/docs/zh-CN/administrator-guide/materialized_view.md
@@ -488,3 +488,8 @@ MySQL [test]> desc advertiser_view_record;
2. 如果删除语句的条件列，在物化视图中不存在，则不能进行删除操作。如果一定要删除数据，则需要先将物化视图删除，然后方可删除数据。
3. 单表上过多的物化视图会影响导入的效率：导入数据时，物化视图和 base 表数据是同步更新的，如果一张表的物化视图表超过10张，则有可能导致导入速度很慢。这就像单次导入需要同时导入10张表数据是一样的。
4. 相同列，不同聚合函数，不能同时出现在一张物化视图中，比如：select sum(a), min(a) from table 不支持。
+
+## 异常错误
+1. DATA_QUALITY_ERR: "The data quality does not satisfy, please check your data"
+ 由于数据质量问题导致物化视图创建失败。
+ 注意：bitmap类型仅支持正整型, 如果原始数据中存在负数，会导致物化视图创建失败
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index 45c5033..e01232f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -135,6 +135,7 @@ public class MasterImpl {
if (taskStatus.getStatus_code() != TStatusCode.OK) {
task.failed();
String errMsg = "task type: " + taskType + ", status_code: " + taskStatus.getStatus_code().toString() +
+ (taskStatus.isSetError_msgs() ? (", status_message: " + taskStatus.getError_msgs()) : "") +
", backendId: " + backend + ", signature: " + signature;
task.setErrorMsg(errMsg);
// We start to let FE perceive the task's error msg
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org