You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/01/24 07:03:29 UTC
[9/9] impala git commit: IMPALA-6395: Add a flag for data stream
sender's buffer size
IMPALA-6395: Add a flag for data stream sender's buffer size
This change adds a flag to control the maximum size in bytes
a row batch in a data stream sender's channel can accumulate
before the row batch is sent over the wire. Increasing this
value will better amortize the cost of compression and RPC
per row batch. The default value is 16KB per channel.
Change-Id: I385f4b7a0671bb2d7872bee60d476c375680b5c2
Reviewed-on: http://gerrit.cloudera.org:8080/9026
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/92fd7e56
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/92fd7e56
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/92fd7e56
Branch: refs/heads/master
Commit: 92fd7e56eb83b67f7f4ec836ebf8e495c75cbdb7
Parents: 11d1784
Author: Michael Ho <kw...@cloudera.com>
Authored: Fri Jan 12 23:19:35 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Jan 24 06:56:56 2018 +0000
----------------------------------------------------------------------
be/src/exec/data-sink.cc | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/92fd7e56/be/src/exec/data-sink.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/data-sink.cc b/be/src/exec/data-sink.cc
index d0e7f4a..f8f068e 100644
--- a/be/src/exec/data-sink.cc
+++ b/be/src/exec/data-sink.cc
@@ -39,6 +39,9 @@
#include "common/names.h"
DECLARE_bool(use_krpc);
+DEFINE_int64(data_stream_sender_buffer_size, 16 * 1024,
+ "(Advanced) Max size in bytes which a row batch in a data stream sender's channel "
+ "can accumulate before the row batch is sent over the wire.");
using strings::Substitute;
@@ -65,12 +68,13 @@ Status DataSink::Create(const TPlanFragmentCtx& fragment_ctx,
if (FLAGS_use_krpc) {
*sink = pool->Add(new KrpcDataStreamSender(fragment_instance_ctx.sender_id,
- row_desc, thrift_sink.stream_sink, fragment_ctx.destinations, 16 * 1024,
- state));
+ row_desc, thrift_sink.stream_sink, fragment_ctx.destinations,
+ FLAGS_data_stream_sender_buffer_size, state));
} else {
// TODO: figure out good buffer size based on size of output row
*sink = pool->Add(new DataStreamSender(fragment_instance_ctx.sender_id, row_desc,
- thrift_sink.stream_sink, fragment_ctx.destinations, 16 * 1024, state));
+ thrift_sink.stream_sink, fragment_ctx.destinations,
+ FLAGS_data_stream_sender_buffer_size, state));
}
break;
case TDataSinkType::TABLE_SINK: