You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/01/25 21:46:25 UTC
[07/11] impala git commit: IMPALA-6395: Add a flag for data stream
sender's buffer size
IMPALA-6395: Add a flag for data stream sender's buffer size
This change adds a flag to control the maximum size in bytes
a row batch in a data stream sender's channel can accumulate
before the row batch is sent over the wire. Increasing this
value will better amortize the cost of compression and RPC
per row batch. The default value is 16KB per channel.
Change-Id: I385f4b7a0671bb2d7872bee60d476c375680b5c2
Reviewed-on: http://gerrit.cloudera.org:8080/9026
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/42ce1f95
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/42ce1f95
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/42ce1f95
Branch: refs/heads/2.x
Commit: 42ce1f959cb1bf53ae1b76bf2779628892038679
Parents: f4b6818
Author: Michael Ho <kw...@cloudera.com>
Authored: Fri Jan 12 23:19:35 2018 -0800
Committer: Philip Zeyliger <ph...@cloudera.com>
Committed: Wed Jan 24 10:17:57 2018 -0800
----------------------------------------------------------------------
be/src/exec/data-sink.cc | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/42ce1f95/be/src/exec/data-sink.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/data-sink.cc b/be/src/exec/data-sink.cc
index d0e7f4a..f8f068e 100644
--- a/be/src/exec/data-sink.cc
+++ b/be/src/exec/data-sink.cc
@@ -39,6 +39,9 @@
#include "common/names.h"
DECLARE_bool(use_krpc);
+DEFINE_int64(data_stream_sender_buffer_size, 16 * 1024,
+ "(Advanced) Max size in bytes which a row batch in a data stream sender's channel "
+ "can accumulate before the row batch is sent over the wire.");
using strings::Substitute;
@@ -65,12 +68,13 @@ Status DataSink::Create(const TPlanFragmentCtx& fragment_ctx,
if (FLAGS_use_krpc) {
*sink = pool->Add(new KrpcDataStreamSender(fragment_instance_ctx.sender_id,
- row_desc, thrift_sink.stream_sink, fragment_ctx.destinations, 16 * 1024,
- state));
+ row_desc, thrift_sink.stream_sink, fragment_ctx.destinations,
+ FLAGS_data_stream_sender_buffer_size, state));
} else {
// TODO: figure out good buffer size based on size of output row
*sink = pool->Add(new DataStreamSender(fragment_instance_ctx.sender_id, row_desc,
- thrift_sink.stream_sink, fragment_ctx.destinations, 16 * 1024, state));
+ thrift_sink.stream_sink, fragment_ctx.destinations,
+ FLAGS_data_stream_sender_buffer_size, state));
}
break;
case TDataSinkType::TABLE_SINK: