You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/05/29 11:13:05 UTC
[GitHub] [incubator-doris] morningman commented on a diff in pull request #9803: [fix](brpc) Embed serialized request into the attachment and transmit it through http brpc
morningman commented on code in PR #9803:
URL: https://github.com/apache/incubator-doris/pull/9803#discussion_r884250864
##########
be/src/util/proto_util.h:
##########
@@ -74,4 +84,88 @@ inline void attachment_transfer_request_block(const Params* brpc_request, brpc::
}
}
+// Embed tuple_data and brpc request serialization string in controller attachment.
+template <typename Params, typename Closure>
+inline void request_embed_attachment_contain_tuple(Params* brpc_request,
+ const std::string& tuple_data,
+ Closure* closure) {
+ auto row_batch = brpc_request->mutable_row_batch();
+ row_batch->set_tuple_data("");
+ request_embed_attachment(brpc_request, tuple_data, closure);
+}
+
+// Embed column_values and brpc request serialization string in controller attachment.
+template <typename Params, typename Closure>
+inline void request_embed_attachment_contain_block(Params* brpc_request,
+ const std::string& column_values,
+ Closure* closure) {
+ auto block = brpc_request->mutable_block();
+ block->set_column_values("");
+ request_embed_attachment(brpc_request, column_values, closure);
+}
+
+template <typename Params, typename Closure>
+inline void request_embed_attachment(Params* brpc_request, const std::string& data,
+ Closure* closure) {
+ LOG(WARNING) << "http_brpc, request_embed_attachment start, data.size: " << data.size()
Review Comment:
VLOG
##########
be/src/common/config.h:
##########
@@ -553,9 +553,10 @@ CONF_String(default_rowset_type, "BETA");
CONF_Int64(brpc_max_body_size, "3147483648");
// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
CONF_Int64(brpc_socket_max_unwritten_bytes, "1073741824");
-// Whether to transfer RowBatch in ProtoBuf Request to Controller Attachment and send it
-// through brpc, this will be faster and avoid the error of Request length overflow.
-CONF_mBool(transfer_data_by_brpc_attachment, "false");
+// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
+// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
+// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
+CONF_mBool(brpc_request_embed_attachment_send_by_http, "true");
Review Comment:
The default value is true. It is compatible that an new version BE send request to a old version BE?
##########
be/src/vec/sink/vdata_stream_sender.cpp:
##########
@@ -138,13 +144,27 @@ Status VDataStreamSender::Channel::send_block(PBlock* block, bool eos) {
_closure->ref();
_closure->cntl.set_timeout_ms(_brpc_timeout_ms);
- if (_brpc_request.has_block()) {
- request_block_transfer_attachment<PTransmitDataParams,
- RefCountClosure<PTransmitDataResult>>(
+ if (_parent->_column_values_buffer_ptr != nullptr && _brpc_request.has_block() &&
+ !_brpc_request.block().has_column_values()) {
+ DCHECK(_parent->_column_values_buffer.size() != 0);
+ request_embed_attachment_contain_block<PTransmitDataParams,
+ RefCountClosure<PTransmitDataResult>>(
&_brpc_request, _parent->_column_values_buffer, _closure);
+ std::string brpc_url;
+ brpc_url =
Review Comment:
this `brpc_url` and `_brpc_http_stub` can be initialized when creating data_stream_sender.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org