You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/06/11 14:33:01 UTC
[GitHub] [incubator-doris] morningman commented on a change in pull request #3739: Support materialized view schema change
morningman commented on a change in pull request #3739:
URL: https://github.com/apache/incubator-doris/pull/3739#discussion_r438823024
##########
File path: be/src/olap/schema_change.cpp
##########
@@ -194,6 +188,105 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)
break; \
}
+
+bool to_bitmap(RowCursor* read_helper, RowCursor* write_helper, const TabletColumn& ref_column,
+ int field_idx, int ref_field_idx, MemPool* mem_pool) {
+ write_helper->set_not_null(ref_field_idx);
+ BitmapValue bitmap;
+ if (!read_helper->is_null(ref_field_idx)) {
+ uint64_t origin_value;
+ char *src = read_helper->cell_ptr(ref_field_idx);
+ switch (ref_column.type()) {
+ case OLAP_FIELD_TYPE_TINYINT:
+ if (*(int8_t *) src < 0) {
+ LOG(WARNING) << "The input: " << *(int8_t *) src
+ << " is not valid, to_bitmap only support bigint value from 0 to 18446744073709551615 currently";
+ return false;
+ }
+ origin_value = *(int8_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_UNSIGNED_TINYINT:
+ origin_value = *(uint8_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_SMALLINT:
+ if (*(int16_t *) src < 0) {
+ LOG(WARNING) << "The input: " << *(int16_t *) src
+ << " is not valid, to_bitmap only support bigint value from 0 to 18446744073709551615 currently";
+ return false;
+ }
+ origin_value = *(int16_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_UNSIGNED_SMALLINT:
+ origin_value = *(uint16_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_INT:
+ if (*(int32_t *) src < 0) {
+ LOG(WARNING) << "The input: " << *(int32_t *) src
+ << " is not valid, to_bitmap only support bigint value from 0 to 18446744073709551615 currently";
+ return false;
+ }
+ origin_value = *(int32_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_UNSIGNED_INT:
+ origin_value = *(uint32_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_BIGINT:
+ if (*(int64_t *) src < 0) {
+ LOG(WARNING) << "The input: " << *(int64_t *) src
+ << " is not valid, to_bitmap only support bigint value from 0 to 18446744073709551615 currently";
+ return false;
+ }
+ origin_value = *(int64_t *) src;
+ break;
+ case OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
+ origin_value = *(uint64_t *) src;
+ break;
+ default:
+ LOG(WARNING) << "the column type which was altered from was unsupported."
+ << " from_type="
+ << ref_column.type();
+ return false;
+ }
+ bitmap.add(origin_value);
+ }
+ char *buf = reinterpret_cast<char *>(mem_pool->allocate(bitmap.getSizeInBytes()));
+ Slice dst(buf, bitmap.getSizeInBytes());
+ bitmap.write(dst.data);
+ write_helper->set_field_content(field_idx, reinterpret_cast<char *>(&dst), mem_pool);
+ return true;
+}
+
+bool hll_hash(RowCursor* read_helper, RowCursor* write_helper, const TabletColumn& ref_column,
+ int field_idx, int ref_field_idx, MemPool* mem_pool) {
+ write_helper->set_not_null(field_idx);
+ HyperLogLog hll;
+ if (!read_helper->is_null(ref_field_idx)) {
+ Slice src;
+ if (ref_column.type() != OLAP_FIELD_TYPE_VARCHAR) {
+ src.data = read_helper->cell_ptr(ref_field_idx);
+ src.size = ref_column.length();
+ } else {
+ src = *reinterpret_cast<Slice *>(read_helper->cell_ptr(ref_field_idx));
+ }
+ uint64_t hash_value = HashUtil::murmur_hash64A(src.data, src.size, HashUtil::MURMUR_SEED);
+ hll.update(hash_value);
+ }
+ std::string buf;
+ buf.resize(hll.max_serialized_size());
+ buf.resize(hll.serialize((uint8_t *) buf.c_str()));
+ Slice dst(buf);
+ write_helper->set_field_content(field_idx, reinterpret_cast<char *>(&dst), mem_pool);
+ return true;
+}
+
+bool count(RowCursor* read_helper, RowCursor* write_helper, const TabletColumn& ref_column,
+ int field_idx, int ref_field_idx, MemPool* mem_pool) {
+ write_helper->set_not_null(field_idx);
+ int64_t count = read_helper->is_null(field_idx) ? 0 : 1;
Review comment:
```suggestion
int8_t count = read_helper->is_null(field_idx) ? 0 : 1;
```
##########
File path: gensrc/thrift/AgentService.thrift
##########
@@ -104,6 +104,13 @@ struct TAlterTabletReqV2 {
// version of data which this alter task should transform
5: optional Types.TVersion alter_version
6: optional Types.TVersionHash alter_version_hash // Deprecated
+ 7: optional list<TAlterMaterializedViewParam> materialized_view_params
+}
+
+struct TAlterMaterializedViewParam {
+ 1: required string column_name
+ 2: required string origin_column_name
+ 3: optional string mv_expr
Review comment:
Why not using TExpr?
##########
File path: be/src/olap/schema_change.cpp
##########
@@ -194,6 +188,105 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)
break; \
}
+
+bool to_bitmap(RowCursor* read_helper, RowCursor* write_helper, const TabletColumn& ref_column,
+ int field_idx, int ref_field_idx, MemPool* mem_pool) {
+ write_helper->set_not_null(ref_field_idx);
+ BitmapValue bitmap;
+ if (!read_helper->is_null(ref_field_idx)) {
+ uint64_t origin_value;
+ char *src = read_helper->cell_ptr(ref_field_idx);
+ switch (ref_column.type()) {
+ case OLAP_FIELD_TYPE_TINYINT:
+ if (*(int8_t *) src < 0) {
+ LOG(WARNING) << "The input: " << *(int8_t *) src
Review comment:
How about return Status with this error message?
And do not print log here, it may lead to lots of log, and this error info should return to user.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org