You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by "github-actions[bot] (via GitHub)" <gi...@apache.org> on 2023/03/07 04:40:48 UTC
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #17444: [Feature-WIP](parquet-reader) Implements dict filter functionality in parquet reader.
github-actions[bot] commented on code in PR #17444:
URL: https://github.com/apache/doris/pull/17444#discussion_r1127333462
##########
be/src/vec/exec/format/parquet/vparquet_group_reader.cpp:
##########
@@ -608,4 +1055,195 @@ ParquetColumnReader::Statistics RowGroupReader::statistics() {
return st;
}
+void RowGroupReader::_set_column_id(VExpr* root) {
+ for (VExpr* expr : root->children()) {
+ if (expr->is_slot_ref()) {
+ VSlotRef* slotRef = (VSlotRef*)expr;
+ slotRef->set_temp_column_id(0);
+ }
+ _set_column_id(expr);
+ }
+}
+
+void RowGroupReader::_restore_column_id(VExpr* root) {
+ for (VExpr* expr : root->children()) {
+ if (expr->is_slot_ref()) {
+ VSlotRef* slotRef = (VSlotRef*)expr;
+ slotRef->restore_orig_column_id();
+ }
+ _restore_column_id(expr);
+ }
+}
+
+Status RowGroupReader::_execute_conjuncts(const std::vector<VExprContext*>& ctxs, Block* block,
+ IColumn::Filter *result_filter, bool *can_filter_all) {
+ auto* __restrict result_filter_data = result_filter->data();
+ int j = 0;
+ for (auto* ctx : ctxs) {
+ ++j;
+ int result_column_id = -1;
+ RETURN_IF_ERROR(ctx->execute(block, &result_column_id));
+ ColumnPtr& filter_column = block->get_by_position(result_column_id).column;
+ if (auto* nullable_column = check_and_get_column<ColumnNullable>(*filter_column)) {
+ size_t column_size = nullable_column->size();
+ if (column_size == 0) {
+ *can_filter_all = true;
+ break;
+ } else {
+ const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr();
+
+ MutableColumnPtr mutable_holder =
+ nested_column->use_count() == 1
+ ? nested_column->assume_mutable()
+ : nested_column->clone_resized(nested_column->size());
+
+ ColumnUInt8* concrete_column = typeid_cast<ColumnUInt8*>(mutable_holder.get());
+ if (!concrete_column) {
+ return Status::InvalidArgument(
+ "Illegal type {} of column for filter. Must be UInt8 or Nullable(UInt8).",
+ filter_column->get_name());
+ }
+ auto* __restrict null_map_data = nullable_column->get_null_map_data().data();
+ IColumn::Filter& filter = concrete_column->get_data();
+ auto* __restrict filter_data = filter.data();
+ const size_t size = filter.size();
+
+ if (_position_delete_ctx.has_filter) {
+ auto* __restrict pos_delete_filter_data = _pos_delete_filter_ptr->data();
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &=
+ (!null_map_data[i]) & filter_data[i] & pos_delete_filter_data[i];
+ }
+ } else {
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &= (!null_map_data[i]) & filter_data[i];
+ }
+ }
+ }
+ } else if (auto* const_column = check_and_get_column<ColumnConst>(*filter_column)) {
+ // filter all
+ *can_filter_all = !const_column->get_bool(0);
+ break;
+ } else {
+ MutableColumnPtr mutable_holder =
+ filter_column->use_count() == 1
+ ? filter_column->assume_mutable()
+ : filter_column->clone_resized(filter_column->size());
+ ColumnUInt8* mutable_filter_column = typeid_cast<ColumnUInt8*>(mutable_holder.get());
+ if (!mutable_filter_column) {
+ return Status::InvalidArgument(
+ "Illegal type {} of column for filter. Must be UInt8 or Nullable(UInt8).",
+ filter_column->get_name());
+ }
+
+ IColumn::Filter& filter = mutable_filter_column->get_data();
+ auto* __restrict filter_data = filter.data();
+
+ if (_position_delete_ctx.has_filter) {
+ auto* __restrict pos_delete_filter_data = _pos_delete_filter_ptr->data();
+ const size_t size = filter.size();
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &= filter_data[i] & pos_delete_filter_data[i];
+ }
+ } else {
+ const size_t size = filter.size();
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &= filter_data[i];
+ }
+ }
+ }
+
+// size_t filter_size = result_filter->size();
+// size_t count = filter_size - simd::count_zero_num((int8_t*)result_filter->data(), filter_size);
+// fprintf(stderr, "filter[%d] count %ld\n", j, count);
+ }
+ return Status::OK();
+}
+
+Status RowGroupReader::_execute_conjuncts_and_filter_block(const std::vector<VExprContext*>& ctxs, Block* block,
+ std::vector<uint32_t>& columns_to_filter,
+ int column_to_keep) {
+ IColumn::Filter result_filter(block->rows(), 1);
+ auto* __restrict result_filter_data = result_filter.data();
+ int j = 0;
+ for (auto* ctx : ctxs) {
+ ++j;
+ int result_column_id = -1;
+ RETURN_IF_ERROR(ctx->execute(block, &result_column_id));
+ ColumnPtr& filter_column = block->get_by_position(result_column_id).column;
+ if (auto* nullable_column = check_and_get_column<ColumnNullable>(*filter_column)) {
+ const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr();
+
+ MutableColumnPtr mutable_holder =
+ nested_column->use_count() == 1
+ ? nested_column->assume_mutable()
+ : nested_column->clone_resized(nested_column->size());
+
+ ColumnUInt8* concrete_column = typeid_cast<ColumnUInt8*>(mutable_holder.get());
+ if (!concrete_column) {
+ return Status::InvalidArgument(
+ "Illegal type {} of column for filter. Must be UInt8 or Nullable(UInt8).",
+ filter_column->get_name());
+ }
+ auto* __restrict null_map_data = nullable_column->get_null_map_data().data();
+ IColumn::Filter& filter = concrete_column->get_data();
+ auto* __restrict filter_data = filter.data();
+ const size_t size = filter.size();
+
+ if (_position_delete_ctx.has_filter) {
+ auto* __restrict pos_delete_filter_data = _pos_delete_filter_ptr->data();
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &=
+ (!null_map_data[i]) & filter_data[i] & pos_delete_filter_data[i];
+ }
+ } else {
+ for (size_t i = 0; i < size; ++i) {
+ result_filter_data[i] &= (!null_map_data[i]) & filter_data[i];
+ }
+ }
+ } else if (auto* const_column = check_and_get_column<ColumnConst>(*filter_column)) {
+ bool ret = const_column->get_bool(0);
+ if (!ret) {
+ for (auto& col : columns_to_filter) {
+ std::move(*block->get_by_position(col).column).assume_mutable()->clear();
Review Comment:
warning: std::move of the const expression has no effect; remove std::move() [performance-move-const-arg]
```suggestion
*block->get_by_position(col).column.assume_mutable()->clear();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org