You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/09/12 02:07:40 UTC
[incubator-doris] branch master updated: [Sprak-Doris-Connector]
support boolean data type (#6601)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new b3ae607 [Sprak-Doris-Connector] support boolean data type (#6601)
b3ae607 is described below
commit b3ae607fe99100cebfe82c3a0b07c28dd36c8811
Author: Yunfeng,Wu <wu...@baidu.com>
AuthorDate: Sun Sep 12 10:07:23 2021 +0800
[Sprak-Doris-Connector] support boolean data type (#6601)
1. Support boolean data type for spark-doris-connector because Doris has previously supported the boolean data type
2. Bug-Fix for the Doris BE core when spark request data from be
---
be/src/runtime/fragment_mgr.cpp | 1 +
be/src/util/arrow/row_batch.cpp | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 281df63..e607266 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -751,6 +751,7 @@ Status FragmentMgr::exec_external_plan_fragment(const TScanOpenParams& params,
// assign the param used to execute PlanFragment
TExecPlanFragmentParams exec_fragment_params;
exec_fragment_params.protocol_version = (PaloInternalServiceVersion::type)0;
+ exec_fragment_params.__set_is_simplified_param(false);
exec_fragment_params.__set_fragment(t_query_plan_info.plan_fragment);
exec_fragment_params.__set_desc_tbl(t_query_plan_info.desc_tbl);
diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp
index b616a54..0667595 100644
--- a/be/src/util/arrow/row_batch.cpp
+++ b/be/src/util/arrow/row_batch.cpp
@@ -82,6 +82,9 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
case TYPE_DECIMALV2:
*result = std::make_shared<arrow::Decimal128Type>(27, 9);
break;
+ case TYPE_BOOLEAN:
+ *result = arrow::boolean();
+ break;
default:
return Status::InvalidArgument(
strings::Substitute("Unknown primitive type($0)", type.type));
@@ -130,6 +133,9 @@ Status convert_to_doris_type(const arrow::DataType& type, TSlotDescriptorBuilder
case arrow::Type::DOUBLE:
builder->type(TYPE_DOUBLE);
break;
+ case arrow::Type::BOOL:
+ builder->type(TYPE_BOOLEAN);
+ break;
default:
return Status::InvalidArgument(strings::Substitute("Unknown arrow type id($0)", type.id()));
}
@@ -268,6 +274,22 @@ public:
}
return builder.Finish(&_arrays[_cur_field_idx]);
}
+ // process boolean
+ arrow::Status Visit(const arrow::BooleanType& type) {
+ arrow::BooleanBuilder builder(_pool);
+ size_t num_rows = _batch.num_rows();
+ builder.Reserve(num_rows);
+ for (size_t i = 0; i < num_rows; ++i) {
+ bool is_null = _cur_slot_ref->is_null_bit_set(_batch.get_row(i));
+ if (is_null) {
+ ARROW_RETURN_NOT_OK(builder.AppendNull());
+ continue;
+ }
+ auto cell_ptr = _cur_slot_ref->get_slot(_batch.get_row(i));
+ ARROW_RETURN_NOT_OK(builder.Append(*(bool*)cell_ptr));
+ }
+ return builder.Finish(&_arrays[_cur_field_idx]);
+ }
Status convert(std::shared_ptr<arrow::RecordBatch>* out);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org