You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by wz...@apache.org on 2022/02/16 17:18:34 UTC
[impala] 02/03: IMPALA-10931 (part 1): Rebase copied Kudu source code
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e36f2b55716c7d6246988ee125668542778becd3
Author: wzhou-code <wz...@cloudera.com>
AuthorDate: Mon Feb 14 14:08:34 2022 -0800
IMPALA-10931 (part 1): Rebase copied Kudu source code
Removed be/src/kudu/{util|security|rpc} and copied back over from Kudu
commit: 136a8c21549c9d81d96d8ef3cfa4d200b9325f32
Change-Id: Ie7bc10e9436db48169f67765159a3838df5f473a
Reviewed-on: http://gerrit.cloudera.org:8080/18235
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Wenzhe Zhou <wz...@cloudera.com>
---
be/src/kudu/rpc/CMakeLists.txt | 29 +-
be/src/kudu/rpc/acceptor_pool.cc | 34 +-
be/src/kudu/rpc/client_negotiation.cc | 84 +-
be/src/kudu/rpc/client_negotiation.h | 14 +-
be/src/kudu/rpc/connection.cc | 103 +-
be/src/kudu/rpc/connection.h | 44 +-
be/src/kudu/rpc/connection_id.cc | 4 +-
be/src/kudu/rpc/connection_id.h | 4 +
be/src/kudu/rpc/exactly_once_rpc-test.cc | 79 +-
be/src/kudu/rpc/inbound_call.cc | 31 +-
be/src/kudu/rpc/inbound_call.h | 14 +-
be/src/kudu/rpc/messenger.cc | 191 +-
be/src/kudu/rpc/messenger.h | 216 +-
be/src/kudu/rpc/mt-rpc-test.cc | 157 +-
be/src/kudu/rpc/negotiation-test.cc | 173 +-
be/src/kudu/rpc/negotiation.cc | 13 +-
be/src/kudu/rpc/negotiation.h | 1 +
be/src/kudu/rpc/outbound_call.cc | 101 +-
be/src/kudu/rpc/outbound_call.h | 103 +-
be/src/kudu/rpc/periodic-test.cc | 16 +-
be/src/kudu/rpc/protoc-gen-krpc.cc | 499 +--
be/src/kudu/rpc/proxy-test.cc | 269 ++
be/src/kudu/rpc/proxy.cc | 198 +-
be/src/kudu/rpc/proxy.h | 69 +-
be/src/kudu/rpc/reactor-test.cc | 18 +-
be/src/kudu/rpc/reactor.cc | 94 +-
be/src/kudu/rpc/reactor.h | 47 +-
be/src/kudu/rpc/remote_method.h | 4 +-
be/src/kudu/rpc/response_callback.h | 4 +-
be/src/kudu/rpc/result_tracker.cc | 6 +-
be/src/kudu/rpc/result_tracker.h | 15 +-
be/src/kudu/rpc/retriable_rpc.h | 11 +-
be/src/kudu/rpc/rpc-bench.cc | 4 +-
be/src/kudu/rpc/rpc-test-base.h | 213 +-
be/src/kudu/rpc/rpc-test.cc | 663 ++--
be/src/kudu/rpc/rpc.cc | 5 +-
be/src/kudu/rpc/rpc.h | 4 +-
be/src/kudu/rpc/rpc_context.cc | 12 +-
be/src/kudu/rpc/rpc_context.h | 19 +-
be/src/kudu/rpc/rpc_controller.cc | 22 +-
be/src/kudu/rpc/rpc_controller.h | 23 +
be/src/kudu/rpc/rpc_introspection.proto | 34 +-
be/src/kudu/rpc/rpc_service.h | 8 +-
be/src/kudu/rpc/rpc_sidecar.cc | 49 +-
be/src/kudu/rpc/rpc_sidecar.h | 34 +-
be/src/kudu/rpc/rpc_stub-test.cc | 24 +-
be/src/kudu/rpc/rpc_verification_util.cc | 24 +-
be/src/kudu/rpc/rpc_verification_util.h | 9 +-
be/src/kudu/rpc/rpcz_store.cc | 16 +-
be/src/kudu/rpc/rtest.proto | 17 +-
be/src/kudu/rpc/sasl_common.cc | 31 +-
be/src/kudu/rpc/sasl_common.h | 8 +-
be/src/kudu/rpc/serialization.cc | 3 +-
be/src/kudu/rpc/server_negotiation.cc | 90 +-
be/src/kudu/rpc/server_negotiation.h | 12 +-
be/src/kudu/rpc/service_if.cc | 24 +-
be/src/kudu/rpc/service_if.h | 21 +-
be/src/kudu/rpc/service_pool.cc | 13 +-
be/src/kudu/rpc/service_pool.h | 6 +-
be/src/kudu/rpc/transfer.cc | 168 +-
be/src/kudu/rpc/transfer.h | 59 +-
be/src/kudu/security/CMakeLists.txt | 16 -
be/src/kudu/security/ca/cert_management-test.cc | 3 +-
be/src/kudu/security/ca/cert_management.cc | 17 +-
be/src/kudu/security/ca/cert_management.h | 7 +-
be/src/kudu/security/cert-test.cc | 7 +-
be/src/kudu/security/cert.cc | 16 +-
be/src/kudu/security/cert.h | 7 +-
be/src/kudu/security/crypto-test.cc | 7 +-
be/src/kudu/security/crypto.cc | 27 +-
be/src/kudu/security/crypto.h | 8 +-
be/src/kudu/security/init.cc | 135 +-
be/src/kudu/security/init.h | 17 +
be/src/kudu/security/kinit_context.h | 13 +
be/src/kudu/security/security-test-util.cc | 19 +-
be/src/kudu/security/security_flags.cc | 24 +-
be/src/kudu/security/security_flags.h | 6 +-
be/src/kudu/security/simple_acl.cc | 5 +-
be/src/kudu/security/simple_acl.h | 2 +-
be/src/kudu/security/test/mini_kdc-test.cc | 4 +-
be/src/kudu/security/test/mini_kdc.cc | 31 +-
be/src/kudu/security/test/test_certs.cc | 385 +--
be/src/kudu/security/test/test_pass.h | 5 +-
be/src/kudu/security/tls_context.cc | 217 +-
be/src/kudu/security/tls_context.h | 47 +-
be/src/kudu/security/tls_handshake-test.cc | 247 +-
be/src/kudu/security/tls_handshake.cc | 144 +-
be/src/kudu/security/tls_handshake.h | 53 +-
be/src/kudu/security/tls_socket-test.cc | 19 +-
be/src/kudu/security/tls_socket.cc | 93 +-
be/src/kudu/security/tls_socket.h | 18 +-
be/src/kudu/security/token-test.cc | 44 +-
be/src/kudu/security/token_signing_key.cc | 2 +-
be/src/kudu/security/token_verifier.cc | 37 +-
be/src/kudu/security/token_verifier.h | 10 +-
be/src/kudu/security/x509_check_host.cc | 6 +-
be/src/kudu/util/CMakeLists.txt | 88 +-
be/src/kudu/util/async_util-test.cc | 17 +-
be/src/kudu/util/async_util.h | 9 +-
be/src/kudu/util/atomic-test.cc | 2 +-
be/src/kudu/util/auto_release_pool.h | 99 -
be/src/kudu/util/bit-stream-utils.h | 16 +-
be/src/kudu/util/bit-stream-utils.inline.h | 19 +-
be/src/kudu/util/bit-util-test.cc | 17 +-
be/src/kudu/util/bit-util.h | 15 +-
be/src/kudu/util/bitmap-test.cc | 81 +-
be/src/kudu/util/bitmap.cc | 1 +
be/src/kudu/util/bitmap.h | 134 +-
be/src/kudu/util/bitset-test.cc | 2 +
be/src/kudu/util/block_bloom_filter-test.cc | 38 +-
be/src/kudu/util/block_bloom_filter.cc | 115 +-
be/src/kudu/util/block_cache_metrics.cc | 4 +-
be/src/kudu/util/blocking_queue-test.cc | 280 +-
be/src/kudu/util/blocking_queue.h | 176 +-
be/src/kudu/util/bloom_filter.h | 9 +-
be/src/kudu/util/cache-bench.cc | 2 +-
be/src/kudu/util/cache-test.cc | 9 +-
be/src/kudu/util/cache.cc | 16 +-
be/src/kudu/util/cache.h | 18 +-
be/src/kudu/util/callback_bind-test.cc | 119 -
be/src/kudu/util/char_util.cc | 4 +
be/src/kudu/util/cloud/instance_detector-test.cc | 65 +-
be/src/kudu/util/cloud/instance_detector.cc | 31 +-
be/src/kudu/util/cloud/instance_detector.h | 35 +-
be/src/kudu/util/cloud/instance_metadata.cc | 107 +-
be/src/kudu/util/cloud/instance_metadata.h | 73 +-
be/src/kudu/util/coding-inl.h | 14 +
be/src/kudu/util/coding.cc | 4 +-
be/src/kudu/util/compression/compression-test.cc | 12 +-
be/src/kudu/util/compression/compression_codec.cc | 1 -
be/src/kudu/util/countdown_latch-test.cc | 23 +-
be/src/kudu/util/countdown_latch.h | 15 +-
be/src/kudu/util/cow_object.h | 47 +
be/src/kudu/util/crc-test.cc | 25 +-
be/src/kudu/util/crc.h | 4 +-
be/src/kudu/util/curl_util.cc | 2 +-
be/src/kudu/util/debug-util-test.cc | 99 +-
be/src/kudu/util/debug-util.cc | 8 +-
be/src/kudu/util/debug-util.h | 4 +
be/src/kudu/util/debug/trace_event.h | 12 +-
be/src/kudu/util/debug/trace_event_impl.cc | 105 +-
be/src/kudu/util/debug/trace_event_impl.h | 48 +-
.../kudu/util/debug/trace_event_synthetic_delay.h | 10 +-
be/src/kudu/util/decimal_util.h | 2 +-
be/src/kudu/util/env-test.cc | 217 +-
be/src/kudu/util/env.cc | 34 +-
be/src/kudu/util/env.h | 103 +-
be/src/kudu/util/env_posix.cc | 684 +++-
be/src/kudu/util/env_util-test.cc | 8 +-
be/src/kudu/util/env_util.cc | 27 +-
be/src/kudu/util/env_util.h | 9 +-
be/src/kudu/util/faststring-test.cc | 52 +
be/src/kudu/util/faststring.h | 40 +
be/src/kudu/util/fault_injection.cc | 1 -
be/src/kudu/util/file_cache-stress-test.cc | 17 +-
be/src/kudu/util/file_cache-test.cc | 76 +-
be/src/kudu/util/file_cache.cc | 22 +-
be/src/kudu/util/flags.cc | 75 +-
be/src/kudu/util/flags.h | 9 +-
be/src/kudu/util/group_varint-test.cc | 18 +-
be/src/kudu/util/group_varint.cc | 4 +-
be/src/kudu/util/hash_util-test.cc | 27 +-
be/src/kudu/util/hash_util.h | 18 +-
be/src/kudu/util/hdr_histogram.cc | 7 +-
be/src/kudu/util/hdr_histogram.h | 4 +-
be/src/kudu/util/init.cc | 10 +
be/src/kudu/util/inline_slice-test.cc | 6 +-
be/src/kudu/util/jsonwriter-test.cc | 2 +-
be/src/kudu/util/kernel_stack_watchdog.cc | 6 +-
be/src/kudu/util/kernel_stack_watchdog.h | 7 +-
be/src/kudu/util/knapsack_solver-test.cc | 4 +-
be/src/kudu/util/kudu_export.h | 62 -
be/src/kudu/util/locks.h | 7 +-
be/src/kudu/util/logging-test.cc | 4 +-
be/src/kudu/util/logging.cc | 28 +-
be/src/kudu/util/logging.h | 2 +-
be/src/kudu/util/logging_callback.h | 20 +-
be/src/kudu/util/maintenance_manager-test.cc | 521 +++-
be/src/kudu/util/maintenance_manager.cc | 403 ++-
be/src/kudu/util/maintenance_manager.h | 119 +-
be/src/kudu/util/maintenance_manager.proto | 1 +
be/src/kudu/util/maintenance_manager_metrics.cc | 58 +
.../maintenance_manager_metrics.h} | 44 +-
be/src/kudu/util/make_shared.h | 2 +-
be/src/kudu/util/malloc.h | 2 +-
be/src/kudu/util/map-util-test.cc | 27 +
be/src/kudu/util/mem_tracker-test.cc | 3 +-
be/src/kudu/util/memcmpable_varint-test.cc | 12 +-
be/src/kudu/util/memory/memory.cc | 10 +-
be/src/kudu/util/memory/memory.h | 12 +-
be/src/kudu/util/metrics-test.cc | 51 +-
be/src/kudu/util/metrics.cc | 39 +-
be/src/kudu/util/metrics.h | 83 +-
be/src/kudu/util/minidump-test.cc | 13 +-
be/src/kudu/util/minidump.cc | 13 +-
be/src/kudu/util/monotime-test.cc | 100 +-
be/src/kudu/util/monotime.cc | 36 +-
be/src/kudu/util/monotime.h | 99 +-
be/src/kudu/util/mt-hdr_histogram-test.cc | 43 +-
be/src/kudu/util/mt-metrics-test.cc | 33 +-
be/src/kudu/util/mt-threadlocal-test.cc | 85 +-
be/src/kudu/util/mutex.h | 4 +-
be/src/kudu/util/net/dns_resolver-test.cc | 94 +-
be/src/kudu/util/net/dns_resolver.cc | 43 +-
be/src/kudu/util/net/dns_resolver.h | 11 +-
be/src/kudu/util/net/net_util-test.cc | 138 +-
be/src/kudu/util/net/net_util.cc | 213 +-
be/src/kudu/util/net/net_util.h | 25 +-
be/src/kudu/util/net/sockaddr.cc | 216 +-
be/src/kudu/util/net/sockaddr.h | 151 +-
be/src/kudu/util/net/socket-test.cc | 158 +-
be/src/kudu/util/net/socket.cc | 63 +-
be/src/kudu/util/net/socket.h | 4 +-
be/src/kudu/util/notification.h | 141 +
be/src/kudu/util/nvm_cache.cc | 60 +-
be/src/kudu/util/nvm_cache.h | 2 +
be/src/kudu/util/object_pool-test.cc | 1 -
be/src/kudu/util/object_pool.h | 12 +-
be/src/kudu/util/once-test.cc | 23 +-
be/src/kudu/{security => util}/openssl_util.cc | 24 +-
be/src/kudu/{security => util}/openssl_util.h | 32 +-
be/src/kudu/{security => util}/openssl_util_bio.h | 5 +-
be/src/kudu/util/path_util.cc | 60 +-
be/src/kudu/util/path_util.h | 4 +-
be/src/kudu/util/pb_util-internal.cc | 1 -
be/src/kudu/util/pb_util-test.cc | 115 +-
be/src/kudu/util/pb_util.cc | 72 +-
be/src/kudu/util/pb_util.h | 22 +-
be/src/kudu/util/process_memory.cc | 6 +-
be/src/kudu/util/protobuf_util.h | 8 +-
be/src/kudu/util/protoc-gen-insertions.cc | 9 +-
be/src/kudu/util/pstack_watcher.cc | 5 +-
be/src/kudu/util/random.h | 56 +
be/src/kudu/util/random_util-test.cc | 84 +
be/src/kudu/util/random_util.h | 61 +-
be/src/kudu/util/rle-encoding.h | 27 +-
be/src/kudu/util/rle-test.cc | 29 +-
be/src/kudu/util/rolling_log-test.cc | 1 +
be/src/kudu/util/rolling_log.cc | 5 +-
be/src/kudu/util/rw_mutex-test.cc | 6 +-
be/src/kudu/util/safe_math-test.cc | 2 +-
be/src/kudu/util/sanitizer_options.cc | 19 +-
be/src/kudu/util/slice.h | 62 +-
be/src/kudu/util/spinlock_profiling.cc | 7 +-
be/src/kudu/util/sse2neon.h | 3286 +++++++++++++++++++-
be/src/kudu/util/status.h | 11 +-
be/src/kudu/util/status_callback.h | 16 +-
be/src/kudu/util/string_case-test.cc | 21 +
be/src/kudu/util/string_case.cc | 29 +
be/src/kudu/util/string_case.h | 3 +
be/src/kudu/util/striped64-test.cc | 35 +-
be/src/kudu/util/striped64.cc | 8 +-
be/src/kudu/util/subprocess-test.cc | 1 +
be/src/kudu/util/subprocess.cc | 26 +-
be/src/kudu/util/subprocess.h | 6 +
be/src/kudu/util/test_graph.cc | 25 +-
be/src/kudu/util/test_graph.h | 27 +-
be/src/kudu/util/test_macros.h | 4 +-
be/src/kudu/util/test_main.cc | 5 +-
be/src/kudu/util/test_util.cc | 123 +-
be/src/kudu/util/test_util.h | 28 +-
be/src/kudu/util/thread-test.cc | 25 +-
be/src/kudu/util/thread.cc | 84 +-
be/src/kudu/util/thread.h | 102 +-
be/src/kudu/util/thread_restrictions.cc | 4 +-
be/src/kudu/util/threadlocal.cc | 5 +-
be/src/kudu/util/threadpool-test.cc | 570 +++-
be/src/kudu/util/threadpool.cc | 293 +-
be/src/kudu/util/threadpool.h | 198 +-
be/src/kudu/util/timer.h | 67 +
be/src/kudu/util/trace-test.cc | 6 +-
be/src/kudu/util/trace.h | 3 +-
be/src/kudu/util/ttl_cache-test.cc | 1 +
be/src/kudu/util/ttl_cache.h | 8 +-
be/src/kudu/util/url-coding-test.cc | 8 +-
be/src/kudu/util/url-coding.cc | 11 +-
be/src/kudu/util/user.cc | 12 +-
be/src/kudu/util/version_util.cc | 1 +
be/src/kudu/util/web_callback_registry.h | 15 +-
be/src/kudu/util/yamlreader-test.cc | 5 +-
be/src/kudu/util/zlib.cc | 31 +-
281 files changed, 13981 insertions(+), 4616 deletions(-)
diff --git a/be/src/kudu/rpc/CMakeLists.txt b/be/src/kudu/rpc/CMakeLists.txt
index 1c1118e..c8d831e 100644
--- a/be/src/kudu/rpc/CMakeLists.txt
+++ b/be/src/kudu/rpc/CMakeLists.txt
@@ -15,9 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-# Target including all protobuf-generated code.
-add_custom_target(kudu-rpc-proto-deps)
-
#### Global header protobufs
PROTOBUF_GENERATE_CPP(
RPC_HEADER_PROTO_SRCS RPC_HEADER_PROTO_HDRS RPC_HEADER_PROTO_TGTS
@@ -29,8 +26,6 @@ ADD_EXPORTABLE_LIBRARY(rpc_header_proto
DEPS protobuf pb_util_proto token_proto
NONLINK_DEPS ${RPC_HEADER_PROTO_TGTS})
-add_dependencies(kudu-rpc-proto-deps ${RPC_HEADER_PROTO_TGTS})
-
PROTOBUF_GENERATE_CPP(
RPC_INTROSPECTION_PROTO_SRCS RPC_INTROSPECTION_PROTO_HDRS RPC_INTROSPECTION_PROTO_TGTS
SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..
@@ -44,10 +39,6 @@ ADD_EXPORTABLE_LIBRARY(rpc_introspection_proto
DEPS ${RPC_INTROSPECTION_PROTO_LIBS}
NONLINK_DEPS ${RPC_INTROSPECTION_PROTO_TGTS})
-add_dependencies(kudu-rpc-proto-deps ${RPC_INTROSPECTION_PROTO_TGTS})
-
-add_definitions(-DKUDU_HEADERS_USE_SHORT_STATUS_MACROS)
-
### RPC library
set(KRPC_SRCS
acceptor_pool.cc
@@ -99,16 +90,13 @@ ADD_EXPORTABLE_LIBRARY(krpc
DEPS ${KRPC_LIBS})
### RPC generator tool
-add_executable(protoc-gen-krpc protoc-gen-krpc.cc
- # Impala - add stub for kudu::VersionInfo
- ${CMAKE_CURRENT_SOURCE_DIR}/../../common/kudu_version.cc
- # Impala - add definition for any flag names shared between Impala / Kudu.
- # TODO: Consider either removing code that depends on these flags, or namespacing them
- # somehow.
- ${CMAKE_CURRENT_SOURCE_DIR}/../../common/global-flags.cc)
-# IMPALA-8642: kudu_version.cc depends on gen-cpp/Status_types.h in target thrift-deps
-add_dependencies(protoc-gen-krpc thrift-deps)
-target_link_libraries(protoc-gen-krpc gutil glog gflags protoc protobuf rpc_header_proto ${KUDU_BASE_LIBS})
+add_executable(protoc-gen-krpc protoc-gen-krpc.cc)
+target_link_libraries(protoc-gen-krpc
+ ${KUDU_BASE_LIBS}
+ rpc_header_proto
+ protoc
+ protobuf
+ gutil)
#### RPC test
PROTOBUF_GENERATE_CPP(
@@ -141,9 +129,10 @@ ADD_KUDU_TEST(exactly_once_rpc-test PROCESSORS 10)
ADD_KUDU_TEST(mt-rpc-test RUN_SERIAL true)
ADD_KUDU_TEST(negotiation-test)
ADD_KUDU_TEST(periodic-test)
+ADD_KUDU_TEST(proxy-test)
ADD_KUDU_TEST(reactor-test)
ADD_KUDU_TEST(request_tracker-test)
ADD_KUDU_TEST(rpc-bench RUN_SERIAL true)
-ADD_KUDU_TEST(rpc-test)
+ADD_KUDU_TEST(rpc-test NUM_SHARDS 8)
ADD_KUDU_TEST(rpc_stub-test)
ADD_KUDU_TEST(service_queue-test RUN_SERIAL true)
diff --git a/be/src/kudu/rpc/acceptor_pool.cc b/be/src/kudu/rpc/acceptor_pool.cc
index 88d48da..84fbd18 100644
--- a/be/src/kudu/rpc/acceptor_pool.cc
+++ b/be/src/kudu/rpc/acceptor_pool.cc
@@ -17,8 +17,9 @@
#include "kudu/rpc/acceptor_pool.h"
-#include <string>
+#include <functional>
#include <ostream>
+#include <string>
#include <vector>
#include <gflags/gflags.h>
@@ -53,6 +54,12 @@ METRIC_DEFINE_counter(server, rpc_connections_accepted,
"Number of incoming TCP connections made to the RPC server",
kudu::MetricLevel::kInfo);
+METRIC_DEFINE_counter(server, rpc_connections_accepted_unix_domain_socket,
+ "RPC Connections Accepted via UNIX Domain Socket",
+ kudu::MetricUnit::kConnections,
+ "Number of incoming UNIX Domain Socket connections made to the RPC server",
+ kudu::MetricLevel::kInfo);
+
DEFINE_int32(rpc_acceptor_listen_backlog, 128,
"Socket backlog parameter used when listening for RPC connections. "
"This defines the maximum length to which the queue of pending "
@@ -70,9 +77,12 @@ AcceptorPool::AcceptorPool(Messenger* messenger, Socket* socket,
: messenger_(messenger),
socket_(socket->Release()),
bind_address_(bind_address),
- rpc_connections_accepted_(METRIC_rpc_connections_accepted.Instantiate(
- messenger->metric_entity())),
- closing_(false) {}
+ closing_(false) {
+ auto& accept_metric = bind_address.is_ip() ?
+ METRIC_rpc_connections_accepted :
+ METRIC_rpc_connections_accepted_unix_domain_socket;
+ rpc_connections_accepted_ = accept_metric.Instantiate(messenger->metric_entity());
+}
AcceptorPool::~AcceptorPool() {
Shutdown();
@@ -84,7 +94,7 @@ Status AcceptorPool::Start(int num_threads) {
for (int i = 0; i < num_threads; i++) {
scoped_refptr<kudu::Thread> new_thread;
Status s = kudu::Thread::Create("acceptor pool", "acceptor",
- &AcceptorPool::RunThread, this, &new_thread);
+ [this]() { this->RunThread(); }, &new_thread);
if (!s.ok()) {
Shutdown();
return s;
@@ -159,12 +169,14 @@ void AcceptorPool::RunThread() {
<< THROTTLE_MSG;
continue;
}
- s = new_sock.SetNoDelay(true);
- if (!s.ok()) {
- KLOG_EVERY_N_SECS(WARNING, 1) << "Acceptor with remote = " << remote.ToString()
- << " failed to set TCP_NODELAY on a newly accepted socket: "
- << s.ToString() << THROTTLE_MSG;
- continue;
+ if (remote.is_ip()) {
+ s = new_sock.SetNoDelay(true);
+ if (!s.ok()) {
+ KLOG_EVERY_N_SECS(WARNING, 1) << "Acceptor with remote = " << remote.ToString()
+ << " failed to set TCP_NODELAY on a newly accepted socket: "
+ << s.ToString() << THROTTLE_MSG;
+ continue;
+ }
}
rpc_connections_accepted_->Increment();
messenger_->RegisterInboundSocket(&new_sock, remote);
diff --git a/be/src/kudu/rpc/client_negotiation.cc b/be/src/kudu/rpc/client_negotiation.cc
index aeb13e4..59f3886 100644
--- a/be/src/kudu/rpc/client_negotiation.cc
+++ b/be/src/kudu/rpc/client_negotiation.cc
@@ -17,9 +17,13 @@
#include "kudu/rpc/client_negotiation.h"
+#include <gssapi/gssapi.h>
+#include <gssapi/gssapi_krb5.h>
+#include <sasl/sasl.h>
+
#include <cstdint>
#include <cstring>
-#include <map>
+#include <functional>
#include <memory>
#include <ostream>
#include <set>
@@ -27,9 +31,6 @@
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
-#include <gssapi/gssapi.h>
-#include <gssapi/gssapi_krb5.h>
-#include <sasl/sasl.h>
#include "kudu/gutil/basictypes.h"
#include "kudu/gutil/map-util.h"
@@ -38,7 +39,6 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/blocking_ops.h"
#include "kudu/rpc/constants.h"
-#include "kudu/rpc/messenger.h"
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/sasl_common.h"
#include "kudu/rpc/sasl_helper.h"
@@ -47,6 +47,7 @@
#include "kudu/security/gssapi.h"
#include "kudu/security/tls_context.h"
#include "kudu/security/tls_handshake.h"
+#include "kudu/security/token.pb.h"
#include "kudu/util/debug/leakcheck_disabler.h"
#include "kudu/util/faststring.h"
#include "kudu/util/net/sockaddr.h"
@@ -61,15 +62,14 @@
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif // #if defined(__APPLE__)
-using std::map;
+DECLARE_bool(rpc_encrypt_loopback_connections);
+
+using kudu::security::RpcEncryption;
using std::set;
using std::string;
using std::unique_ptr;
-
using strings::Substitute;
-DECLARE_bool(rpc_encrypt_loopback_connections);
-
namespace kudu {
namespace rpc {
@@ -119,12 +119,15 @@ ClientNegotiation::ClientNegotiation(unique_ptr<Socket> socket,
const security::TlsContext* tls_context,
boost::optional<security::SignedTokenPB> authn_token,
RpcEncryption encryption,
+ bool encrypt_loopback,
std::string sasl_proto_name)
: socket_(std::move(socket)),
helper_(SaslHelper::CLIENT),
tls_context_(tls_context),
+ tls_handshake_(security::TlsHandshakeType::CLIENT),
encryption_(encryption),
tls_negotiated_(false),
+ encrypt_loopback_(encrypt_loopback),
authn_token_(std::move(authn_token)),
psecret_(nullptr, std::free),
negotiated_authn_(AuthenticationType::INVALID),
@@ -188,8 +191,7 @@ Status ClientNegotiation::Negotiate(unique_ptr<ErrorStatusPB>* rpc_error) {
// TODO(KUDU-1921): allow the client to require TLS.
if (encryption_ != RpcEncryption::DISABLED &&
ContainsKey(server_features_, TLS)) {
- RETURN_NOT_OK(tls_context_->InitiateHandshake(security::TlsHandshakeType::CLIENT,
- &tls_handshake_));
+ RETURN_NOT_OK(tls_context_->InitiateHandshake(&tls_handshake_));
if (negotiated_authn_ == AuthenticationType::SASL) {
// When using SASL authentication, verifying the server's certificate is
@@ -244,7 +246,7 @@ Status ClientNegotiation::SendNegotiatePB(const NegotiatePB& msg) {
DCHECK(msg.has_step()) << "message must have a step";
TRACE("Sending $0 NegotiatePB request", NegotiatePB::NegotiateStep_Name(msg.step()));
- return SendFramedMessageBlocking(socket(), header, msg, deadline_);
+ return SendFramedMessageBlocking(socket_.get(), header, msg, deadline_);
}
Status ClientNegotiation::RecvNegotiatePB(NegotiatePB* msg,
@@ -252,7 +254,8 @@ Status ClientNegotiation::RecvNegotiatePB(NegotiatePB* msg,
unique_ptr<ErrorStatusPB>* rpc_error) {
ResponseHeader header;
Slice param_buf;
- RETURN_NOT_OK(ReceiveFramedMessageBlocking(socket(), buffer, &header, ¶m_buf, deadline_));
+ RETURN_NOT_OK(ReceiveFramedMessageBlocking(
+ socket_.get(), buffer, &header, ¶m_buf, deadline_));
RETURN_NOT_OK(helper_.CheckNegotiateCallId(header.call_id()));
if (header.is_error()) {
@@ -260,7 +263,8 @@ Status ClientNegotiation::RecvNegotiatePB(NegotiatePB* msg,
}
RETURN_NOT_OK(helper_.ParseNegotiatePB(param_buf, msg));
- TRACE("Received $0 NegotiatePB response", NegotiatePB::NegotiateStep_Name(msg->step()));
+ TRACE("Received $0 NegotiatePB response",
+ NegotiatePB::NegotiateStep_Name(msg->step()));
return Status::OK();
}
@@ -285,24 +289,28 @@ Status ClientNegotiation::SendConnectionHeader() {
uint8_t buf[buflen];
serialization::SerializeConnHeader(buf);
size_t nsent;
- return socket()->BlockingWrite(buf, buflen, &nsent, deadline_);
+ return socket_->BlockingWrite(buf, buflen, &nsent, deadline_);
}
Status ClientNegotiation::InitSaslClient() {
// TODO(KUDU-1922): consider setting SASL_SUCCESS_DATA
unsigned flags = 0;
+ const auto desc = Substitute("creating new SASL $0 client", sasl_proto_name_);
sasl_conn_t* sasl_conn = nullptr;
- RETURN_NOT_OK_PREPEND(WrapSaslCall(nullptr /* no conn */, [&]() {
- return sasl_client_new(
- sasl_proto_name_.c_str(), // Registered name of the service using SASL. Required.
- helper_.server_fqdn(), // The fully qualified domain name of the remote server.
- nullptr, // Local and remote IP address strings. (we don't use
- nullptr, // any mechanisms which require this info.)
- &callbacks_[0], // Connection-specific callbacks.
- flags,
- &sasl_conn);
- }), Substitute("unable to create new SASL $0 client", sasl_proto_name_));
+ RETURN_NOT_OK_PREPEND(WrapSaslCall(
+ nullptr /* no conn */,
+ [&]() {
+ return sasl_client_new(
+ sasl_proto_name_.c_str(), // Registered name of the service using SASL. Required.
+ helper_.server_fqdn(), // The fully qualified domain name of the remote server.
+ nullptr, // Local and remote IP address strings. (we don't use
+ nullptr, // any mechanisms which require this info.)
+ &callbacks_[0], // Connection-specific callbacks.
+ flags,
+ &sasl_conn);
+ },
+ desc.c_str()), desc + " failed");
sasl_conn_.reset(sasl_conn);
return Status::OK();
}
@@ -318,7 +326,7 @@ Status ClientNegotiation::SendNegotiate() {
client_features_.insert(TLS);
// If the remote peer is local, then we allow using TLS for authentication
// without encryption or integrity.
- if (socket_->IsLoopbackConnection() && !FLAGS_rpc_encrypt_loopback_connections) {
+ if (socket_->IsLoopbackConnection() && !encrypt_loopback_) {
client_features_.insert(TLS_AUTHENTICATION_ONLY);
}
}
@@ -345,8 +353,7 @@ Status ClientNegotiation::SendNegotiate() {
return Status::NotAuthorized("client is not configured with an authentication type");
}
- RETURN_NOT_OK(SendNegotiatePB(msg));
- return Status::OK();
+ return SendNegotiatePB(msg);
}
Status ClientNegotiation::HandleNegotiate(const NegotiatePB& response) {
@@ -427,7 +434,6 @@ Status ClientNegotiation::HandleNegotiate(const NegotiatePB& response) {
// TODO(KUDU-1921): allow the client to require authentication.
if (ContainsKey(client_mechs, SaslMechanism::GSSAPI) &&
ContainsKey(server_mechs, SaslMechanism::GSSAPI)) {
-
// Check that the client has local Kerberos credentials, and if not fall
// back to an alternate mechanism.
Status s = CheckGSSAPI();
@@ -492,8 +498,7 @@ Status ClientNegotiation::HandleTlsHandshake(const NegotiatePB& response) {
string token;
Status s = tls_handshake_.Continue(response.tls_handshake(), &token);
- if (s.IsIncomplete()) {
- // Another roundtrip is required to complete the handshake.
+ if (tls_handshake_.NeedsExtraStep(s, token)) {
RETURN_NOT_OK(SendTlsHandshake(std::move(token)));
}
@@ -592,7 +597,8 @@ Status ClientNegotiation::SendSaslInitiate() {
* SASL_NOMECH -- no mechanism meets requested properties
* SASL_INTERACT -- user interaction needed to fill in prompt_need list
*/
- TRACE("Calling sasl_client_start()");
+ static constexpr const char* const kDesc = "calling sasl_client_start()";
+ TRACE(kDesc);
const Status s = WrapSaslCall(sasl_conn_.get(), [&]() {
return sasl_client_start(
sasl_conn_.get(), // The SASL connection context created by init()
@@ -601,9 +607,9 @@ Status ClientNegotiation::SendSaslInitiate() {
&init_msg, // Filled in on success.
&init_msg_len, // Filled in on success.
&negotiated_mech); // Filled in on success.
- });
+ }, kDesc);
- if (PREDICT_FALSE(!s.IsIncomplete() && !s.ok())) {
+ if (PREDICT_FALSE(!s.ok() && !s.IsIncomplete())) {
return s;
}
@@ -695,11 +701,13 @@ Status ClientNegotiation::HandleSaslSuccess(const NegotiatePB& response) {
}
Status ClientNegotiation::DoSaslStep(const string& in, const char** out, unsigned* out_len) {
- TRACE("Calling sasl_client_step()");
+ static constexpr const char* const kDesc = "calling sasl_client_step()";
+ TRACE(kDesc);
return WrapSaslCall(sasl_conn_.get(), [&]() {
- return sasl_client_step(sasl_conn_.get(), in.c_str(), in.length(), nullptr, out, out_len);
- });
+ return sasl_client_step(
+ sasl_conn_.get(), in.c_str(), in.length(), nullptr, out, out_len);
+ }, kDesc);
}
Status ClientNegotiation::SendConnectionContext() {
@@ -720,7 +728,7 @@ Status ClientNegotiation::SendConnectionContext() {
*conn_context.mutable_encoded_nonce() = ciphertext.ToString();
}
- return SendFramedMessageBlocking(socket(), header, conn_context, deadline_);
+ return SendFramedMessageBlocking(socket_.get(), header, conn_context, deadline_);
}
int ClientNegotiation::GetOptionCb(const char* plugin_name, const char* option,
diff --git a/be/src/kudu/rpc/client_negotiation.h b/be/src/kudu/rpc/client_negotiation.h
index 06fb2b8..4d3e6d5 100644
--- a/be/src/kudu/rpc/client_negotiation.h
+++ b/be/src/kudu/rpc/client_negotiation.h
@@ -17,6 +17,8 @@
#pragma once
+#include <sasl/sasl.h>
+
#include <cstdlib>
#include <memory>
#include <set>
@@ -26,19 +28,16 @@
#include <boost/optional/optional.hpp>
#include <glog/logging.h>
-#include <sasl/sasl.h>
-#include "kudu/rpc/messenger.h"
+#include "kudu/gutil/port.h"
#include "kudu/rpc/negotiation.h"
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/sasl_common.h"
#include "kudu/rpc/sasl_helper.h"
#include "kudu/security/security_flags.h"
#include "kudu/security/tls_handshake.h"
-#include "kudu/security/token.pb.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/socket.h"
-#include "kudu/gutil/port.h"
#include "kudu/util/status.h"
namespace kudu {
@@ -47,6 +46,7 @@ class Slice;
class faststring;
namespace security {
+class SignedTokenPB;
class TlsContext;
}
@@ -65,7 +65,8 @@ class ClientNegotiation {
ClientNegotiation(std::unique_ptr<Socket> socket,
const security::TlsContext* tls_context,
boost::optional<security::SignedTokenPB> authn_token,
- RpcEncryption encryption,
+ security::RpcEncryption encryption,
+ bool encrypt_loopback,
std::string sasl_proto_name);
// Enable PLAIN authentication.
@@ -227,8 +228,9 @@ class ClientNegotiation {
// TLS state.
const security::TlsContext* tls_context_;
security::TlsHandshake tls_handshake_;
- const RpcEncryption encryption_;
+ const security::RpcEncryption encryption_;
bool tls_negotiated_;
+ bool encrypt_loopback_;
// TSK state.
boost::optional<security::SignedTokenPB> authn_token_;
diff --git a/be/src/kudu/rpc/connection.cc b/be/src/kudu/rpc/connection.cc
index 0b78d46..11298d7 100644
--- a/be/src/kudu/rpc/connection.cc
+++ b/be/src/kudu/rpc/connection.cc
@@ -18,6 +18,7 @@
#include "kudu/rpc/connection.h"
#include <netinet/in.h>
+#include <netinet/tcp.h>
#include <string.h>
#include <algorithm>
@@ -32,8 +33,8 @@
#include <ev.h>
#include <glog/logging.h>
-#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/human_readable.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/inbound_call.h"
@@ -44,17 +45,20 @@
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/rpc_introspection.pb.h"
#include "kudu/rpc/transfer.h"
+#include "kudu/security/tls_socket.h"
+#include "kudu/util/errno.h"
+#include "kudu/util/faststring.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/net/socket.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+#include <sys/socket.h>
#ifdef __linux__
#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <linux/tcp.h>
#endif
+using kudu::security::TlsSocket;
using std::includes;
using std::set;
using std::shared_ptr;
@@ -206,7 +210,7 @@ Connection::Connection(ReactorThread *reactor_thread,
direction_(direction),
last_activity_time_(MonoTime::Now()),
is_epoll_registered_(false),
- next_call_id_(1),
+ call_id_(std::numeric_limits<int32_t>::max()),
credentials_policy_(policy),
negotiation_complete_(false),
is_confidential_(false),
@@ -344,10 +348,11 @@ void Connection::QueueOutbound(unique_ptr<OutboundTransfer> transfer) {
outbound_transfers_.push_back(*transfer.release());
if (negotiation_complete_ && !write_io_.is_active()) {
- // If we weren't currently in the middle of sending anything,
- // then our write_io_ interest is stopped. Need to re-start it.
- // Only do this after connection negotiation is done doing its work.
- write_io_.start();
+ // Optimistically assume that the socket is writable if we didn't already
+ // have something queued.
+ if (ProcessOutboundTransfers() == kMoreToSend) {
+ write_io_.start();
+ }
}
}
@@ -478,7 +483,7 @@ void Connection::QueueOutboundCall(shared_ptr<OutboundCall> call) {
// Serialize the actual bytes to be put on the wire.
TransferPayload tmp_slices;
- size_t n_slices = call->SerializeTo(&tmp_slices);
+ call->SerializeTo(&tmp_slices);
call->SetQueued();
@@ -536,7 +541,7 @@ void Connection::QueueOutboundCall(shared_ptr<OutboundCall> call) {
TransferCallbacks *cb = new CallTransferCallbacks(std::move(call), this);
awaiting_response_[call_id] = car.release();
QueueOutbound(unique_ptr<OutboundTransfer>(
- OutboundTransfer::CreateForCallRequest(call_id, tmp_slices, n_slices, cb)));
+ OutboundTransfer::CreateForCallRequest(call_id, tmp_slices, cb)));
}
// Callbacks for sending an RPC call response from the server.
@@ -608,14 +613,14 @@ void Connection::QueueResponseForCall(unique_ptr<InboundCall> call) {
// ResponseTransferCallbacks::NotifyTransferAborted.
TransferPayload tmp_slices;
- size_t n_slices = call->SerializeResponseTo(&tmp_slices);
+ call->SerializeResponseTo(&tmp_slices);
TransferCallbacks *cb = new ResponseTransferCallbacks(std::move(call), this);
// After the response is sent, can delete the InboundCall object.
// We set a dummy call ID and required feature set, since these are not needed
// when sending responses.
unique_ptr<OutboundTransfer> t(
- OutboundTransfer::CreateForCallResponse(tmp_slices, n_slices, cb));
+ OutboundTransfer::CreateForCallResponse(tmp_slices, cb));
QueueTransferTask *task = new QueueTransferTask(std::move(t), this);
reactor_thread_->reactor()->ScheduleReactorTask(task);
@@ -646,11 +651,12 @@ void Connection::ReadHandler(ev::io &watcher, int revents) {
}
last_activity_time_ = reactor_thread_->cur_time();
+ faststring extra_buf;
while (true) {
if (!inbound_) {
inbound_.reset(new InboundTransfer());
}
- Status status = inbound_->ReceiveBuffer(*socket_);
+ Status status = inbound_->ReceiveBuffer(socket_.get(), &extra_buf);
if (PREDICT_FALSE(!status.ok())) {
if (status.posix_code() == ESHUTDOWN) {
VLOG(1) << ToString() << " shut down by remote end.";
@@ -674,14 +680,11 @@ void Connection::ReadHandler(ev::io &watcher, int revents) {
LOG(FATAL) << "Invalid direction: " << direction_;
}
- // TODO: it would seem that it would be good to loop around and see if
- // there is more data on the socket by trying another recv(), but it turns
- // out that it really hurts throughput to do so. A better approach
- // might be for each InboundTransfer to actually try to read an extra byte,
- // and if it succeeds, then we'd copy that byte into a new InboundTransfer
- // and loop around, since it's likely the next call also arrived at the
- // same time.
- break;
+ if (extra_buf.size() > 0) {
+ inbound_.reset(new InboundTransfer(std::move(extra_buf)));
+ } else {
+ break;
+ }
}
}
@@ -748,19 +751,22 @@ void Connection::WriteHandler(ev::io &watcher, int revents) {
}
DVLOG(3) << ToString() << ": writeHandler: revents = " << revents;
- OutboundTransfer *transfer;
if (outbound_transfers_.empty()) {
LOG(WARNING) << ToString() << " got a ready-to-write callback, but there is "
"nothing to write.";
write_io_.stop();
return;
}
+ if (ProcessOutboundTransfers() == kNoMoreToSend) {
+ write_io_.stop();
+ }
+}
+Connection::ProcessOutboundTransfersResult Connection::ProcessOutboundTransfers() {
while (!outbound_transfers_.empty()) {
- transfer = &(outbound_transfers_.front());
+ OutboundTransfer* transfer = &(outbound_transfers_.front());
if (!transfer->TransferStarted()) {
-
if (transfer->is_for_outbound_call()) {
CallAwaitingResponse* car = FindOrDie(awaiting_response_, transfer->call_id());
if (!car->call) {
@@ -799,25 +805,23 @@ void Connection::WriteHandler(ev::io &watcher, int revents) {
}
last_activity_time_ = reactor_thread_->cur_time();
- Status status = transfer->SendBuffer(*socket_);
+ Status status = transfer->SendBuffer(socket_.get());
if (PREDICT_FALSE(!status.ok())) {
LOG(WARNING) << ToString() << " send error: " << status.ToString();
reactor_thread_->DestroyConnection(this, status);
- return;
+ return kConnectionDestroyed;
}
if (!transfer->TransferFinished()) {
DVLOG(3) << ToString() << ": writeHandler: xfer not finished.";
- return;
+ return kMoreToSend;
}
outbound_transfers_.pop_front();
delete transfer;
}
- // If we were able to write all of our outbound transfers,
- // we don't have any more to write.
- write_io_.stop();
+ return kNoMoreToSend;
}
std::string Connection::ToString() const {
@@ -886,7 +890,7 @@ Status Connection::DumpPB(const DumpConnectionsRequestPB& req,
if (direction_ == CLIENT) {
for (const car_map_t::value_type& entry : awaiting_response_) {
- CallAwaitingResponse *c = entry.second;
+ CallAwaitingResponse* c = entry.second;
if (c->call) {
c->call->DumpPB(req, resp->add_calls_in_flight());
}
@@ -907,7 +911,7 @@ Status Connection::DumpPB(const DumpConnectionsRequestPB& req,
LOG(FATAL);
}
#ifdef __linux__
- if (negotiation_complete_) {
+ if (negotiation_complete_ && remote_.is_ip()) {
// TODO(todd): it's a little strange to not set socket level stats during
// negotiation, but we don't have access to the socket here until negotiation
// is complete.
@@ -915,6 +919,11 @@ Status Connection::DumpPB(const DumpConnectionsRequestPB& req,
"could not fill in TCP info for RPC connection");
}
#endif // __linux__
+
+ if (negotiation_complete_ && remote_.is_ip()) {
+ WARN_NOT_OK(GetTransportDetailsPB(resp->mutable_transport_details()),
+ "could not fill in transport info for RPC connection");
+ }
return Status::OK();
}
@@ -996,5 +1005,35 @@ Status Connection::GetSocketStatsPB(SocketStatsPB* pb) const {
}
#endif // __linux__
+Status Connection::GetTransportDetailsPB(TransportDetailsPB* pb) const {
+ DCHECK(reactor_thread_->IsCurrentThread());
+ DCHECK(pb);
+
+ // As for the dynamic_cast below: this is not very elegant or performant code,
+ // but introducing a generic virtual method with vague semantics into the base
+ // Socket class doesn't look like a good choice either. Also, the
+ // GetTransportDetailsPB() method isn't supposed to be a part of any hot path.
+ const TlsSocket* tls_socket = dynamic_cast<TlsSocket*>(socket_.get());
+ if (tls_socket) {
+ auto* tls = pb->mutable_tls();
+ tls->set_protocol(tls_socket->GetProtocolName());
+ tls->set_cipher_suite(tls_socket->GetCipherDescription());
+ }
+
+ int fd = socket_->GetFd();
+ CHECK_GE(fd, 0);
+ int32_t max_seg_size = 0;
+ socklen_t optlen = sizeof(max_seg_size);
+ int ret = ::getsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &max_seg_size, &optlen);
+ if (ret) {
+ int err = errno;
+ return Status::NetworkError(
+ "getsockopt(TCP_MAXSEG) failed", ErrnoToString(err), err);
+ }
+ pb->mutable_tcp()->set_max_segment_size(max_seg_size);
+
+ return Status::OK();
+}
+
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/connection.h b/be/src/kudu/rpc/connection.h
index ab3b029..5b3638e 100644
--- a/be/src/kudu/rpc/connection.h
+++ b/be/src/kudu/rpc/connection.h
@@ -22,6 +22,7 @@
#include <memory>
#include <set>
#include <string>
+#include <type_traits>
#include <unordered_map>
#include <utility>
@@ -30,12 +31,11 @@
#include <ev++.h>
#include <glog/logging.h>
-#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/connection_id.h"
+#include "kudu/rpc/remote_user.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/rpc_header.pb.h"
-#include "kudu/rpc/remote_user.h"
#include "kudu/rpc/transfer.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/sockaddr.h"
@@ -50,10 +50,12 @@ namespace rpc {
class DumpConnectionsRequestPB;
class InboundCall;
class OutboundCall;
-class RpcConnectionPB;
class ReactorThread;
+class RpcConnectionPB;
class RpczStore;
class SocketStatsPB;
+class TransportDetailsPB;
+
enum class CredentialsPolicy;
//
@@ -184,6 +186,26 @@ class Connection : public RefCountedThreadSafe<Connection> {
// libev callback when we may write to the socket.
void WriteHandler(ev::io &watcher, int revents);
+ enum ProcessOutboundTransfersResult {
+ // All of the transfers in the queue have been sent successfully.
+ // The queue is now empty.
+ kNoMoreToSend,
+ // Not all transfers were able to be sent. The caller should
+ // ensure that write_io_ is enabled in order to continue attempting
+ // to send transfers.
+ kMoreToSend,
+ // An error occurred trying to write to the connection, and the
+ // connection was destroyed. NOTE: 'this' may be deleted if this
+ // value is returned.
+ kConnectionDestroyed
+ };
+
+ // Process any pending outbound transfers in outbound_transfers_.
+ // Result indicates the state of the connection following the attempt.
+ //
+ // NOTE: This may invoke DestroyConnection() on 'this'.
+ ProcessOutboundTransfersResult ProcessOutboundTransfers();
+
// Safe to be called from other threads.
std::string ToString() const;
@@ -272,13 +294,8 @@ class Connection : public RefCountedThreadSafe<Connection> {
// and ensuring we roll over from INT32_MAX to 0.
// Negative numbers are reserved for special purposes.
int32_t GetNextCallId() {
- int32_t call_id = next_call_id_;
- if (PREDICT_FALSE(next_call_id_ == std::numeric_limits<int32_t>::max())) {
- next_call_id_ = 0;
- } else {
- next_call_id_++;
- }
- return call_id;
+ static constexpr uint32_t kCallIdMask = std::numeric_limits<int32_t>::max(); // 0x7fffffff
+ return static_cast<int32_t>(++call_id_ & kCallIdMask);
}
// An incoming packet has completed transferring on the server side.
@@ -305,6 +322,8 @@ class Connection : public RefCountedThreadSafe<Connection> {
Status GetSocketStatsPB(SocketStatsPB* pb) const;
+ Status GetTransportDetailsPB(TransportDetailsPB* pb) const;
+
// The reactor thread that created this connection.
ReactorThread* const reactor_thread_;
@@ -351,8 +370,9 @@ class Connection : public RefCountedThreadSafe<Connection> {
// being handled.
inbound_call_map_t calls_being_handled_;
- // the next call ID to use
- int32_t next_call_id_;
+ // The internal counter to generate next call ID to use. The call ID is
+ // a signed integer: 31 LSBs of the internal counter are used to represent it.
+ uint32_t call_id_;
// Starts as Status::OK, gets set to a shutdown status upon Shutdown().
Status shutdown_status_;
diff --git a/be/src/kudu/rpc/connection_id.cc b/be/src/kudu/rpc/connection_id.cc
index ab98715..8a14d83 100644
--- a/be/src/kudu/rpc/connection_id.cc
+++ b/be/src/kudu/rpc/connection_id.cc
@@ -20,7 +20,7 @@
#include <cstddef>
#include <utility>
-#include <boost/functional/hash/hash.hpp>
+#include <boost/container_hash/extensions.hpp>
#include <glog/logging.h>
#include "kudu/gutil/strings/substitute.h"
@@ -52,7 +52,7 @@ void ConnectionId::set_network_plane(string network_plane) {
string ConnectionId::ToString() const {
string remote;
- if (hostname_ != remote_.host()) {
+ if (remote_.is_initialized() && remote_.is_ip() && hostname_ != remote_.host()) {
remote = strings::Substitute("$0 ($1)", remote_.ToString(), hostname_);
} else {
remote = remote_.ToString();
diff --git a/be/src/kudu/rpc/connection_id.h b/be/src/kudu/rpc/connection_id.h
index 6ec98f7..0aed953 100644
--- a/be/src/kudu/rpc/connection_id.h
+++ b/be/src/kudu/rpc/connection_id.h
@@ -45,6 +45,10 @@ class ConnectionId {
const std::string& hostname() const { return hostname_; }
+ void set_remote(const Sockaddr& remote) {
+ remote_ = remote;
+ }
+
// The credentials of the user associated with this connection, if any.
void set_user_credentials(UserCredentials user_credentials);
diff --git a/be/src/kudu/rpc/exactly_once_rpc-test.cc b/be/src/kudu/rpc/exactly_once_rpc-test.cc
index 638072b..b2841af 100644
--- a/be/src/kudu/rpc/exactly_once_rpc-test.cc
+++ b/be/src/kudu/rpc/exactly_once_rpc-test.cc
@@ -20,9 +20,11 @@
#include <atomic>
#include <cstdint>
#include <cstdlib>
+#include <functional>
#include <memory>
#include <ostream>
#include <string>
+#include <thread>
#include <utility>
#include <vector>
@@ -30,7 +32,6 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "kudu/gutil/callback.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/request_tracker.h"
#include "kudu/rpc/response_callback.h"
@@ -50,7 +51,6 @@
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
-#include "kudu/util/thread.h"
DECLARE_int64(remember_clients_ttl_ms);
DECLARE_int64(remember_responses_ttl_ms);
@@ -60,6 +60,7 @@ using kudu::pb_util::SecureDebugString;
using kudu::pb_util::SecureShortDebugString;
using std::atomic_int;
using std::shared_ptr;
+using std::thread;
using std::unique_ptr;
using std::vector;
@@ -88,13 +89,13 @@ class TestServerPicker : public ServerPicker<CalculatorServiceProxy> {
public:
explicit TestServerPicker(CalculatorServiceProxy* proxy) : proxy_(proxy) {}
- void PickLeader(const ServerPickedCallback& callback, const MonoTime& deadline) override {
- callback.Run(Status::OK(), proxy_);
+ void PickLeader(const ServerPickedCallback& callback, const MonoTime& /*deadline*/) override {
+ callback(Status::OK(), proxy_);
}
- void MarkServerFailed(CalculatorServiceProxy*, const Status&) override {}
- void MarkReplicaNotLeader(CalculatorServiceProxy*) override {}
- void MarkResourceNotFound(CalculatorServiceProxy*) override {}
+ void MarkServerFailed(CalculatorServiceProxy* /*server*/, const Status& /*status*/) override {}
+ void MarkReplicaNotLeader(CalculatorServiceProxy* /*replica*/) override {}
+ void MarkResourceNotFound(CalculatorServiceProxy* /*replica*/) override {}
private:
CalculatorServiceProxy* proxy_;
@@ -218,33 +219,30 @@ class ExactlyOnceRpcTest : public RpcTestBase {
const scoped_refptr<RequestTracker>& request_tracker,
shared_ptr<Messenger> messenger,
int value,
- int server_sleep = 0) : latch_(1) {
+ int server_sleep = 0) : latch(1) {
MonoTime now = MonoTime::Now();
now.AddDelta(MonoDelta::FromMilliseconds(10000));
- rpc_ = new CalculatorServiceRpc(server_picker,
- request_tracker,
- now,
- std::move(messenger),
- value,
- &latch_,
- server_sleep);
+ rpc = new CalculatorServiceRpc(server_picker,
+ request_tracker,
+ now,
+ std::move(messenger),
+ value,
+ &latch,
+ server_sleep);
}
void Start() {
- CHECK_OK(kudu::Thread::Create(
- "test",
- "test",
- &RetriableRpcExactlyOnceAdder::SleepAndSend, this, &thread));
+ thr = thread([this]() { this->SleepAndSend(); });
}
void SleepAndSend() {
- rpc_->SendRpc();
- latch_.Wait();
+ rpc->SendRpc();
+ latch.Wait();
}
- CountDownLatch latch_;
- scoped_refptr<kudu::Thread> thread;
- CalculatorServiceRpc* rpc_;
+ CountDownLatch latch;
+ thread thr;
+ CalculatorServiceRpc* rpc;
};
// An exactly once adder that sends multiple, simultaneous calls, to the server
@@ -264,10 +262,7 @@ class ExactlyOnceRpcTest : public RpcTestBase {
}
void Start() {
- CHECK_OK(kudu::Thread::Create(
- "test",
- "test",
- &SimultaneousExactlyOnceAdder::SleepAndSend, this, &thread));
+ thr = thread([this]() { this->SleepAndSend(); });
}
// Sleeps the preset number of msecs before sending the call.
@@ -282,7 +277,7 @@ class ExactlyOnceRpcTest : public RpcTestBase {
RpcController controller;
ExactlyOnceRequestPB req;
ExactlyOnceResponsePB resp;
- scoped_refptr<kudu::Thread> thread;
+ thread thr;
};
@@ -314,7 +309,7 @@ class ExactlyOnceRpcTest : public RpcTestBase {
// This thread is used in the stress test where we're constantly running GC.
// So, once we get a "success" response, it's likely that the result will be
// GCed on the server side, and thus it's not safe to spuriously retry.
- adder->rpc_->sometimes_retry_successful_ = false;
+ adder->rpc->sometimes_retry_successful_ = false;
adder->SleepAndSend();
SleepFor(MonoDelta::FromMilliseconds(rand() % 10));
counter++;
@@ -390,7 +385,7 @@ class ExactlyOnceRpcTest : public RpcTestBase {
TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsAfterRpcCompleted) {
ASSERT_OK(StartServer());
ExactlyOnceResponsePB original_resp;
- size_t mem_consumption = mem_tracker_->consumption();
+ int mem_consumption = mem_tracker_->consumption();
{
RpcController controller;
ExactlyOnceRequestPB req;
@@ -471,7 +466,7 @@ TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsWithReplicatedRpc) {
count += j;
}
for (int j = 0; j < kNumRpcs; j++) {
- CHECK_OK(ThreadJoiner(adders[j]->thread.get()).Join());
+ adders[j]->thr.join();
}
CheckValueMatches(count);
}
@@ -513,7 +508,7 @@ TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsWithConcurrentUpdaters) {
}
uint64_t time_micros = 0;
for (int j = 0; j < kNumThreads; j++) {
- CHECK_OK(ThreadJoiner(adders[j]->thread.get()).Join());
+ adders[j]->thr.join();
ASSERT_EQ(adders[j]->resp.current_val(), i + 1);
if (time_micros == 0) {
time_micros = adders[j]->resp.current_time_micros();
@@ -602,18 +597,16 @@ TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsGarbageCollectionStressTest)
CHECK_OK(request_tracker_->NewSeqNo(&stubborn_req_seq_num));
ASSERT_EQ(stubborn_req_seq_num, 0);
- scoped_refptr<kudu::Thread> stubborn_thread;
- CHECK_OK(kudu::Thread::Create(
- "stubborn", "stubborn", &ExactlyOnceRpcTest::StubbornlyWriteTheSameRequestThread,
- this, stubborn_req_seq_num, stubborn_run_for, &stubborn_thread));
+ thread stubborn_thread([this, stubborn_req_seq_num, stubborn_run_for]() {
+ this->StubbornlyWriteTheSameRequestThread(stubborn_req_seq_num, stubborn_run_for);
+ });
- scoped_refptr<kudu::Thread> write_thread;
- CHECK_OK(kudu::Thread::Create(
- "write", "write", &ExactlyOnceRpcTest::DoLongWritesThread,
- this, writes_run_for, &write_thread));
+ thread write_thread([this, writes_run_for]() {
+ this->DoLongWritesThread(writes_run_for);
+ });
- write_thread->Join();
- stubborn_thread->Join();
+ write_thread.join();
+ stubborn_thread.join();
// Within a few seconds, the consumption should be back to zero.
// Really, this should be within 100ms, but we'll give it a bit of
diff --git a/be/src/kudu/rpc/inbound_call.cc b/be/src/kudu/rpc/inbound_call.cc
index edb1d3c..2e0e222 100644
--- a/be/src/kudu/rpc/inbound_call.cc
+++ b/be/src/kudu/rpc/inbound_call.cc
@@ -22,6 +22,7 @@
#include <memory>
#include <ostream>
+#include <boost/container/vector.hpp>
#include <glog/logging.h>
#include <google/protobuf/message.h>
#include <google/protobuf/message_lite.h>
@@ -46,8 +47,8 @@ class FieldDescriptor;
}
}
+using google::protobuf::ArenaOptions;
using google::protobuf::FieldDescriptor;
-using google::protobuf::Message;
using google::protobuf::MessageLite;
using std::string;
using std::unique_ptr;
@@ -57,11 +58,18 @@ using strings::Substitute;
namespace kudu {
namespace rpc {
+static ArenaOptions MakeArenaOptions() {
+ ArenaOptions opts;
+ opts.start_block_size = 4096;
+ return opts;
+}
+
InboundCall::InboundCall(Connection* conn)
: conn_(conn),
trace_(new Trace),
method_info_(nullptr),
- deadline_(MonoTime::Max()) {
+ deadline_(MonoTime::Max()),
+ arena_(MakeArenaOptions()) {
RecordCallReceived();
}
@@ -94,7 +102,7 @@ Status InboundCall::ParseFrom(unique_ptr<InboundTransfer> transfer) {
}
RETURN_NOT_OK(RpcSidecar::ParseSidecars(
- header_.sidecar_offsets(), serialized_request_, inbound_sidecar_slices_));
+ header_.sidecar_offsets(), serialized_request_, &inbound_sidecar_slices_));
if (header_.sidecar_offsets_size() > 0) {
// Trim the request to just the message
serialized_request_ = Slice(serialized_request_.data(), header_.sidecar_offsets(0));
@@ -187,7 +195,7 @@ void InboundCall::SerializeResponseBuffer(const MessageLite& response,
int32_t sidecar_byte_size = 0;
for (const unique_ptr<RpcSidecar>& car : outbound_sidecars_) {
resp_hdr.add_sidecar_offsets(sidecar_byte_size + protobuf_msg_size);
- int32_t sidecar_bytes = car->AsSlice().size();
+ size_t sidecar_bytes = car->TotalSize();
DCHECK_LE(sidecar_byte_size, TransferLimits::kMaxTotalSidecarBytes - sidecar_bytes);
sidecar_byte_size += sidecar_bytes;
}
@@ -199,20 +207,15 @@ void InboundCall::SerializeResponseBuffer(const MessageLite& response,
&response_hdr_buf_);
}
-size_t InboundCall::SerializeResponseTo(TransferPayload* slices) const {
+void InboundCall::SerializeResponseTo(TransferPayload* slices) const {
TRACE_EVENT0("rpc", "InboundCall::SerializeResponseTo");
DCHECK_GT(response_hdr_buf_.size(), 0);
DCHECK_GT(response_msg_buf_.size(), 0);
- size_t n_slices = 2 + outbound_sidecars_.size();
- DCHECK_LE(n_slices, slices->size());
- auto slice_iter = slices->begin();
- *slice_iter++ = Slice(response_hdr_buf_);
- *slice_iter++ = Slice(response_msg_buf_);
+ slices->push_back(Slice(response_hdr_buf_));
+ slices->push_back(Slice(response_msg_buf_));
for (auto& sidecar : outbound_sidecars_) {
- *slice_iter++ = sidecar->AsSlice();
+ sidecar->AppendSlices(slices);
}
- DCHECK_EQ(slice_iter - slices->begin(), n_slices);
- return n_slices;
}
Status InboundCall::AddOutboundSidecar(unique_ptr<RpcSidecar> car, int* idx) {
@@ -222,7 +225,7 @@ Status InboundCall::AddOutboundSidecar(unique_ptr<RpcSidecar> car, int* idx) {
if (outbound_sidecars_.size() > TransferLimits::kMaxSidecars) {
return Status::ServiceUnavailable("All available sidecars already used");
}
- int64_t sidecar_bytes = car->AsSlice().size();
+ size_t sidecar_bytes = car->TotalSize();
if (outbound_sidecars_total_bytes_ >
TransferLimits::kMaxTotalSidecarBytes - sidecar_bytes) {
return Status::RuntimeError(Substitute("Total size of sidecars $0 would exceed limit $1",
diff --git a/be/src/kudu/rpc/inbound_call.h b/be/src/kudu/rpc/inbound_call.h
index 892b893..1c83c7b 100644
--- a/be/src/kudu/rpc/inbound_call.h
+++ b/be/src/kudu/rpc/inbound_call.h
@@ -25,11 +25,13 @@
#include <vector>
#include <glog/logging.h>
+#include <google/protobuf/arena.h>
#include "kudu/gutil/macros.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/remote_method.h"
#include "kudu/rpc/rpc_header.pb.h"
+#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/rpc/service_if.h"
#include "kudu/rpc/transfer.h"
#include "kudu/util/faststring.h"
@@ -55,7 +57,6 @@ class Connection;
class DumpConnectionsRequestPB;
class RemoteUser;
class RpcCallInProgressPB;
-class RpcSidecar;
struct InboundCallTiming {
MonoTime time_received; // Time the call was first accepted.
@@ -133,8 +134,7 @@ class InboundCall {
// Serialize the response packet for the finished call into 'slices'.
// The resulting slices refer to memory in this object.
- // Returns the number of slices in the serialized response.
- size_t SerializeResponseTo(TransferPayload* slices) const;
+ void SerializeResponseTo(TransferPayload* slices) const;
// See RpcContext::AddRpcSidecar()
Status AddOutboundSidecar(std::unique_ptr<RpcSidecar> car, int* idx);
@@ -151,6 +151,10 @@ class InboundCall {
Trace* trace();
+ google::protobuf::Arena* pb_arena() {
+ return &arena_;
+ }
+
const InboundCallTiming& timing() const {
return timing_;
}
@@ -265,7 +269,7 @@ class InboundCall {
// Inbound sidecars from the request. The slices are views onto transfer_. There are as
// many slices as header_.sidecar_offsets_size().
- Slice inbound_sidecar_slices_[TransferLimits::kMaxSidecars];
+ SidecarSliceVector inbound_sidecar_slices_;
// The trace buffer.
scoped_refptr<Trace> trace_;
@@ -286,6 +290,8 @@ class InboundCall {
// client did not pass a timeout.
MonoTime deadline_;
+ google::protobuf::Arena arena_;
+
DISALLOW_COPY_AND_ASSIGN(InboundCall);
};
diff --git a/be/src/kudu/rpc/messenger.cc b/be/src/kudu/rpc/messenger.cc
index 48af449..af8d724 100644
--- a/be/src/kudu/rpc/messenger.cc
+++ b/be/src/kudu/rpc/messenger.cc
@@ -42,146 +42,48 @@
#include "kudu/rpc/sasl_common.h"
#include "kudu/rpc/server_negotiation.h"
#include "kudu/rpc/service_if.h"
-#include "kudu/security/openssl_util.h"
#include "kudu/security/tls_context.h"
#include "kudu/security/token_verifier.h"
#include "kudu/util/flags.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/socket.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/status.h"
#include "kudu/util/thread_restrictions.h"
#include "kudu/util/threadpool.h"
+using kudu::security::RpcAuthentication;
+using kudu::security::RpcEncryption;
using std::string;
using std::shared_ptr;
-using std::make_shared;
using std::unique_ptr;
using strings::Substitute;
-namespace boost {
-template <typename Signature> class function;
-}
-
namespace kudu {
namespace rpc {
-MessengerBuilder::MessengerBuilder(std::string name)
+const int64_t MessengerBuilder::kRpcNegotiationTimeoutMs = 3000;
+
+MessengerBuilder::MessengerBuilder(string name)
: name_(std::move(name)),
connection_keepalive_time_(MonoDelta::FromMilliseconds(65000)),
num_reactors_(4),
min_negotiation_threads_(0),
max_negotiation_threads_(4),
coarse_timer_granularity_(MonoDelta::FromMilliseconds(100)),
- rpc_negotiation_timeout_ms_(3000),
+ rpc_negotiation_timeout_ms_(kRpcNegotiationTimeoutMs),
sasl_proto_name_("kudu"),
rpc_authentication_("optional"),
rpc_encryption_("optional"),
+ rpc_loopback_encryption_(false),
rpc_tls_ciphers_(kudu::security::SecurityDefaults::kDefaultTlsCiphers),
+ rpc_tls_ciphersuites_(kudu::security::SecurityDefaults::kDefaultTlsCipherSuites),
rpc_tls_min_protocol_(kudu::security::SecurityDefaults::kDefaultTlsMinVersion),
enable_inbound_tls_(false),
reuseport_(false) {
}
-MessengerBuilder& MessengerBuilder::set_connection_keepalive_time(const MonoDelta &keepalive) {
- connection_keepalive_time_ = keepalive;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_num_reactors(int num_reactors) {
- num_reactors_ = num_reactors;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_min_negotiation_threads(int min_negotiation_threads) {
- min_negotiation_threads_ = min_negotiation_threads;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_max_negotiation_threads(int max_negotiation_threads) {
- max_negotiation_threads_ = max_negotiation_threads;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_coarse_timer_granularity(const MonoDelta &granularity) {
- coarse_timer_granularity_ = granularity;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_metric_entity(
- const scoped_refptr<MetricEntity>& metric_entity) {
- metric_entity_ = metric_entity;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_connection_keep_alive_time(int32_t time_in_ms) {
- connection_keepalive_time_ = MonoDelta::FromMilliseconds(time_in_ms);
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_rpc_negotiation_timeout_ms(int64_t time_in_ms) {
- rpc_negotiation_timeout_ms_ = time_in_ms;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_sasl_proto_name(const std::string& sasl_proto_name) {
- sasl_proto_name_ = sasl_proto_name;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_rpc_authentication(const std::string& rpc_authentication) {
- rpc_authentication_ = rpc_authentication;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_rpc_encryption(const std::string& rpc_encryption) {
- rpc_encryption_ = rpc_encryption;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_rpc_tls_ciphers(const std::string& rpc_tls_ciphers) {
- rpc_tls_ciphers_ = rpc_tls_ciphers;
- return *this;
-}
-
-MessengerBuilder &MessengerBuilder::set_rpc_tls_min_protocol(
- const std::string& rpc_tls_min_protocol) {
- rpc_tls_min_protocol_ = rpc_tls_min_protocol;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_epki_cert_key_files(
- const std::string& cert, const std::string& private_key) {
- rpc_certificate_file_ = cert;
- rpc_private_key_file_ = private_key;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_epki_certificate_authority_file(const std::string& ca) {
- rpc_ca_certificate_file_ = ca;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_epki_private_password_key_cmd(const std::string& cmd) {
- rpc_private_key_password_cmd_ = cmd;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_keytab_file(const std::string& keytab_file) {
- keytab_file_ = keytab_file;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::enable_inbound_tls() {
- enable_inbound_tls_ = true;
- return *this;
-}
-
-MessengerBuilder& MessengerBuilder::set_reuseport() {
- reuseport_ = true;
- return *this;
-}
-
Status MessengerBuilder::Build(shared_ptr<Messenger>* msgr) {
// Initialize SASL library before we start making requests
RETURN_NOT_OK(SaslInit(!keytab_file_.empty()));
@@ -191,15 +93,13 @@ Status MessengerBuilder::Build(shared_ptr<Messenger>* msgr) {
// Note: can't use make_shared() as it doesn't support custom deleters.
shared_ptr<Messenger> new_msgr(new Messenger(*this),
std::mem_fn(&Messenger::AllExternalReferencesDropped));
-
RETURN_NOT_OK(ParseTriState("--rpc_authentication",
rpc_authentication_,
&new_msgr->authentication_));
-
RETURN_NOT_OK(ParseTriState("--rpc_encryption",
rpc_encryption_,
&new_msgr->encryption_));
-
+ new_msgr->loopback_encryption_ = rpc_loopback_encryption_;
RETURN_NOT_OK(new_msgr->Init());
if (new_msgr->encryption_ != RpcEncryption::DISABLED && enable_inbound_tls_) {
auto* tls_context = new_msgr->mutable_tls_context();
@@ -273,11 +173,11 @@ void Messenger::ShutdownInternal(ShutdownMode mode) {
RpcServicesMap services_to_release;
{
std::lock_guard<percpu_rwlock> guard(lock_);
- if (closing_) {
+ if (state_ == kClosing) {
return;
}
VLOG(1) << "shutting down messenger " << name_;
- closing_ = true;
+ state_ = kClosing;
services_to_release = std::move(rpc_services_);
pools_to_shutdown = std::move(acceptor_pools_);
@@ -300,7 +200,7 @@ void Messenger::ShutdownInternal(ShutdownMode mode) {
}
}
-Status Messenger::AddAcceptorPool(const Sockaddr &accept_addr,
+Status Messenger::AddAcceptorPool(const Sockaddr& accept_addr,
shared_ptr<AcceptorPool>* pool) {
// Before listening, if we expect to require Kerberos, we want to verify
// that everything is set up correctly. This way we'll generate errors on
@@ -311,7 +211,7 @@ Status Messenger::AddAcceptorPool(const Sockaddr &accept_addr,
}
Socket sock;
- RETURN_NOT_OK(sock.Init(0));
+ RETURN_NOT_OK(sock.Init(accept_addr.family(), 0));
RETURN_NOT_OK(sock.SetReuseAddr(true));
if (reuseport_) {
RETURN_NOT_OK(sock.SetReusePort(true));
@@ -319,7 +219,7 @@ Status Messenger::AddAcceptorPool(const Sockaddr &accept_addr,
RETURN_NOT_OK(sock.Bind(accept_addr));
Sockaddr remote;
RETURN_NOT_OK(sock.GetSocketAddress(&remote));
- auto acceptor_pool(make_shared<AcceptorPool>(this, &sock, remote));
+ auto acceptor_pool(std::make_shared<AcceptorPool>(this, &sock, remote));
std::lock_guard<percpu_rwlock> guard(lock_);
acceptor_pools_.push_back(acceptor_pool);
@@ -332,11 +232,12 @@ Status Messenger::RegisterService(const string& service_name,
const scoped_refptr<RpcService>& service) {
DCHECK(service);
std::lock_guard<percpu_rwlock> guard(lock_);
+ DCHECK_NE(kServicesUnregistered, state_);
+ DCHECK_NE(kClosing, state_);
if (InsertIfNotPresent(&rpc_services_, service_name, service)) {
return Status::OK();
- } else {
- return Status::AlreadyPresent("This service is already present");
}
+ return Status::AlreadyPresent("This service is already present");
}
void Messenger::UnregisterAllServices() {
@@ -344,6 +245,7 @@ void Messenger::UnregisterAllServices() {
{
std::lock_guard<percpu_rwlock> guard(lock_);
to_release = std::move(rpc_services_);
+ state_ = kServicesUnregistered;
}
// Release the map outside of the lock.
}
@@ -378,10 +280,16 @@ void Messenger::QueueInboundCall(unique_ptr<InboundCall> call) {
scoped_refptr<RpcService>* service = FindOrNull(rpc_services_,
call->remote_method().service_name());
if (PREDICT_FALSE(!service)) {
- Status s = Status::ServiceUnavailable(Substitute("service $0 not registered on $1",
- call->remote_method().service_name(), name_));
- LOG(INFO) << s.ToString();
- call.release()->RespondFailure(ErrorStatusPB::ERROR_NO_SUCH_SERVICE, s);
+ const auto msg = Substitute("service $0 not registered on $1",
+ call->remote_method().service_name(), name_);
+ if (state_ == kServicesRegistered) {
+ // NOTE: this message is only actually interesting if it's not transient.
+ LOG(INFO) << msg;
+ call.release()->RespondFailure(ErrorStatusPB::ERROR_NO_SUCH_SERVICE, Status::NotFound(msg));
+ } else {
+ call.release()->RespondFailure(
+ ErrorStatusPB::ERROR_UNAVAILABLE, Status::ServiceUnavailable(msg));
+ }
return;
}
@@ -402,19 +310,22 @@ void Messenger::RegisterInboundSocket(Socket *new_socket, const Sockaddr &remote
}
Messenger::Messenger(const MessengerBuilder &bld)
- : name_(bld.name_),
- closing_(false),
- authentication_(RpcAuthentication::REQUIRED),
- encryption_(RpcEncryption::REQUIRED),
- tls_context_(new security::TlsContext(bld.rpc_tls_ciphers_, bld.rpc_tls_min_protocol_)),
- token_verifier_(new security::TokenVerifier()),
- rpcz_store_(new RpczStore()),
- metric_entity_(bld.metric_entity_),
- rpc_negotiation_timeout_ms_(bld.rpc_negotiation_timeout_ms_),
- sasl_proto_name_(bld.sasl_proto_name_),
- keytab_file_(bld.keytab_file_),
- reuseport_(bld.reuseport_),
- retain_self_(this) {
+ : name_(bld.name_),
+ state_(kStarted),
+ authentication_(RpcAuthentication::REQUIRED),
+ encryption_(RpcEncryption::REQUIRED),
+ tls_context_(new security::TlsContext(bld.rpc_tls_ciphers_,
+ bld.rpc_tls_ciphersuites_,
+ bld.rpc_tls_min_protocol_,
+ bld.rpc_tls_excluded_protocols_)),
+ token_verifier_(new security::TokenVerifier),
+ rpcz_store_(new RpczStore),
+ metric_entity_(bld.metric_entity_),
+ rpc_negotiation_timeout_ms_(bld.rpc_negotiation_timeout_ms_),
+ sasl_proto_name_(bld.sasl_proto_name_),
+ keytab_file_(bld.keytab_file_),
+ reuseport_(bld.reuseport_),
+ retain_self_(this) {
for (int i = 0; i < bld.num_reactors_; i++) {
reactors_.push_back(new Reactor(retain_self_, i, bld));
}
@@ -429,16 +340,14 @@ Messenger::Messenger(const MessengerBuilder &bld)
}
Messenger::~Messenger() {
- CHECK(closing_) << "Should have already shut down";
+ CHECK_EQ(state_, kClosing) << "Should have already shut down";
STLDeleteElements(&reactors_);
}
-Reactor* Messenger::RemoteToReactor(const Sockaddr &remote) {
- uint32_t hashCode = remote.HashCode();
- int reactor_idx = hashCode % reactors_.size();
+Reactor* Messenger::RemoteToReactor(const Sockaddr& remote) {
// This is just a static partitioning; we could get a lot
// fancier with assigning Sockaddrs to Reactors.
- return reactors_[reactor_idx];
+ return reactors_[remote.HashCode() % reactors_.size()];
}
Status Messenger::Init() {
@@ -458,7 +367,7 @@ Status Messenger::DumpConnections(const DumpConnectionsRequestPB& req,
return Status::OK();
}
-void Messenger::ScheduleOnReactor(const boost::function<void(const Status&)>& func,
+void Messenger::ScheduleOnReactor(std::function<void(const Status&)> func,
MonoDelta when) {
DCHECK(!reactors_.empty());
@@ -474,7 +383,7 @@ void Messenger::ScheduleOnReactor(const boost::function<void(const Status&)>& fu
chosen = reactors_[rand() % reactors_.size()];
}
- DelayedTask* task = new DelayedTask(func, when);
+ DelayedTask* task = new DelayedTask(std::move(func), when);
chosen->ScheduleReactorTask(task);
}
diff --git a/be/src/kudu/rpc/messenger.h b/be/src/kudu/rpc/messenger.h
index 6d95615..4c99f6f 100644
--- a/be/src/kudu/rpc/messenger.h
+++ b/be/src/kudu/rpc/messenger.h
@@ -17,6 +17,7 @@
#pragma once
#include <cstdint>
+#include <functional>
#include <memory>
#include <mutex>
#include <string>
@@ -39,11 +40,6 @@
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/status.h"
-namespace boost {
-template <typename Signature>
-class function;
-}
-
namespace kudu {
class Socket;
@@ -52,13 +48,10 @@ class ThreadPool;
namespace security {
class TlsContext;
class TokenVerifier;
-}
+} // namespace security
namespace rpc {
-using security::RpcAuthentication;
-using security::RpcEncryption;
-
class AcceptorPool;
class DumpConnectionsRequestPB;
class DumpConnectionsResponsePB;
@@ -87,83 +80,177 @@ class MessengerBuilder {
friend class Messenger;
friend class ReactorThread;
+ static const int64_t kRpcNegotiationTimeoutMs;
+
explicit MessengerBuilder(std::string name);
// Set the length of time we will keep a TCP connection will alive with no traffic.
- MessengerBuilder &set_connection_keepalive_time(const MonoDelta &keepalive);
+ MessengerBuilder& set_connection_keepalive_time(const MonoDelta& keepalive) {
+ connection_keepalive_time_ = keepalive;
+ return *this;
+ }
// Set the number of reactor threads that will be used for sending and
// receiving.
- MessengerBuilder &set_num_reactors(int num_reactors);
+ MessengerBuilder& set_num_reactors(int num_reactors) {
+ num_reactors_ = num_reactors;
+ return *this;
+ }
// Set the minimum number of connection-negotiation threads that will be used
// to handle the blocking connection-negotiation step.
- MessengerBuilder &set_min_negotiation_threads(int min_negotiation_threads);
+ MessengerBuilder& set_min_negotiation_threads(int min_negotiation_threads) {
+ min_negotiation_threads_ = min_negotiation_threads;
+ return *this;
+ }
// Set the maximum number of connection-negotiation threads that will be used
// to handle the blocking connection-negotiation step.
- MessengerBuilder &set_max_negotiation_threads(int max_negotiation_threads);
+ MessengerBuilder& set_max_negotiation_threads(int max_negotiation_threads) {
+ max_negotiation_threads_ = max_negotiation_threads;
+ return *this;
+ }
// Set the granularity with which connections are checked for keepalive.
- MessengerBuilder &set_coarse_timer_granularity(const MonoDelta &granularity);
+ MessengerBuilder& set_coarse_timer_granularity(const MonoDelta& granularity) {
+ coarse_timer_granularity_ = granularity;
+ return *this;
+ }
// Set metric entity for use by RPC systems.
- MessengerBuilder &set_metric_entity(const scoped_refptr<MetricEntity>& metric_entity);
+ MessengerBuilder& set_metric_entity(
+ const scoped_refptr<MetricEntity>& metric_entity) {
+ metric_entity_ = metric_entity;
+ return *this;
+ }
// Set the time in milliseconds after which an idle connection from a client will be
// disconnected by the server.
- MessengerBuilder &set_connection_keep_alive_time(int32_t time_in_ms);
+ MessengerBuilder& set_connection_keep_alive_time(int32_t time_in_ms) {
+ connection_keepalive_time_ = MonoDelta::FromMilliseconds(time_in_ms);
+ return *this;
+ }
// Set the timeout for negotiating an RPC connection.
- MessengerBuilder &set_rpc_negotiation_timeout_ms(int64_t time_in_ms);
+ MessengerBuilder& set_rpc_negotiation_timeout_ms(int64_t time_in_ms) {
+ rpc_negotiation_timeout_ms_ = time_in_ms;
+ return *this;
+ }
// Set the SASL protocol name that is used for the SASL negotiation.
- MessengerBuilder &set_sasl_proto_name(const std::string& sasl_proto_name);
+ MessengerBuilder& set_sasl_proto_name(const std::string& sasl_proto_name) {
+ sasl_proto_name_ = sasl_proto_name;
+ return *this;
+ }
// Set the state of authentication required. If 'optional', authentication will be used when
// the remote end supports it. If 'required', connections which are not able to authenticate
// (because the remote end lacks support) are rejected.
- MessengerBuilder &set_rpc_authentication(const std::string& rpc_authentication);
+ MessengerBuilder& set_rpc_authentication(const std::string& rpc_authentication) {
+ rpc_authentication_ = rpc_authentication;
+ return *this;
+ }
// Set the state of encryption required. If 'optional', encryption will be used when the
// remote end supports it. If 'required', connections which are not able to use encryption
// (because the remote end lacks support) are rejected. If 'disabled', encryption will not
// be used, and RPC authentication (--rpc_authentication) must also be disabled as well.
- MessengerBuilder &set_rpc_encryption(const std::string& rpc_encryption);
+ MessengerBuilder& set_rpc_encryption(const std::string& rpc_encryption) {
+ rpc_encryption_ = rpc_encryption;
+ return *this;
+ }
+
+ MessengerBuilder& set_rpc_loopback_encryption(bool rpc_loopback_encryption) {
+ rpc_loopback_encryption_ = rpc_loopback_encryption;
+ return *this;
+ }
+
+ // Set TLSv1.2 and earlier cipher suite preferences to use for TLS-secured RPC
+ // connections. Uses the OpenSSL cipher preference list format. Under the
+ // hood, SSL_CTX_set_cipher_list() is eventually being called with
+ // 'rpc_tls_ciphers'. See 'man (1) ciphers' for more information on the syntax
+ // of the cipher suite preference list and
+ // https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_ciphersuites.html
+ // for SSL_CTX_set_cipher_list() API details.
+ MessengerBuilder& set_rpc_tls_ciphers(const std::string& rpc_tls_ciphers) {
+ rpc_tls_ciphers_ = rpc_tls_ciphers;
+ return *this;
+ }
- // Set the cipher suite preferences to use for TLS-secured RPC connections. Uses the OpenSSL
- // cipher preference list format. See man (1) ciphers for more information.
- MessengerBuilder &set_rpc_tls_ciphers(const std::string& rpc_tls_ciphers);
+ // Set TLSv1.3-specific cipher suite preferences to use for TLS-secured RPC
+ // connections. Uses the OpenSSL ciphersuite preference list format for
+ // TLSv1.3. Under the hood, SSL_CTX_set_ciphersuites() is eventually being
+ // called with 'rpc_tls_ciphersuites'. See 'man (1) ciphers' for more
+ // information on the TLSv1.3-specific syntax for the cipher suite preference
+ // list and
+ // https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_ciphersuites.html
+ // for SSL_CTX_set_ciphersuites() API details.
+ MessengerBuilder &set_rpc_tls_ciphersuites(
+ const std::string& rpc_tls_ciphersuites) {
+ rpc_tls_ciphersuites_ = rpc_tls_ciphersuites;
+ return *this;
+ }
- // Set the minimum protocol version to allow when for securing RPC connections with TLS. May be
- // one of 'TLSv1', 'TLSv1.1', or 'TLSv1.2'.
- MessengerBuilder &set_rpc_tls_min_protocol(const std::string& rpc_tls_min_protocol);
+ // Set the minimum protocol version to allow when for securing RPC connections
+ // with TLS. May be one of 'TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3'.
+ MessengerBuilder &set_rpc_tls_min_protocol(
+ const std::string& rpc_tls_min_protocol) {
+ rpc_tls_min_protocol_ = rpc_tls_min_protocol;
+ return *this;
+ }
+
+ // Set the list of TLS protocols to avoid when securing RPC connections. The
+ // elements might be from the list of 'TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3'.
+ MessengerBuilder& set_rpc_tls_excluded_protocols(
+ std::vector<std::string> rpc_tls_excluded_protocols) {
+ rpc_tls_excluded_protocols_ = std::move(rpc_tls_excluded_protocols);
+ return *this;
+ }
// Set the TLS server certificate and private key files paths. If this is set in conjunction
// with enable_inbound_tls(), internal PKI will not be used for encrypted communication and
// external PKI will be used instead.
- MessengerBuilder &set_epki_cert_key_files(
- const std::string& cert, const std::string& private_key);
+ MessengerBuilder& set_epki_cert_key_files(
+ const std::string& cert, const std::string& private_key) {
+ rpc_certificate_file_ = cert;
+ rpc_private_key_file_ = private_key;
+ return *this;
+ }
// Set the TLS Certificate Authority file path. Must always be set with set_epki_cert_key_files().
// If this is set in conjunction with enable_inbound_tls(), internal PKI will not be used for
// encrypted communication and external PKI will be used instead.
- MessengerBuilder &set_epki_certificate_authority_file(const std::string& ca);
+ MessengerBuilder& set_epki_certificate_authority_file(const std::string& ca) {
+ rpc_ca_certificate_file_ = ca;
+ return *this;
+ }
// Set a Unix command whose output returns the password used to decrypt the RPC server's private
// key file specified via set_epki_cert_key_files(). If the .PEM key file is not
// password-protected, this flag does not need to be set. Trailing whitespace will be trimmed
// before it is used to decrypt the private key.
- MessengerBuilder &set_epki_private_password_key_cmd(const std::string& cmd);
+ MessengerBuilder& set_epki_private_password_key_cmd(const std::string& cmd) {
+ rpc_private_key_password_cmd_ = cmd;
+ return *this;
+ }
// Set the path to the Kerberos Keytab file for this server.
- MessengerBuilder &set_keytab_file(const std::string& keytab_file);
+ MessengerBuilder& set_keytab_file(const std::string& keytab_file) {
+ keytab_file_ = keytab_file;
+ return *this;
+ }
// Configure the messenger to enable TLS encryption on inbound connections.
- MessengerBuilder& enable_inbound_tls();
+ MessengerBuilder& enable_inbound_tls() {
+ enable_inbound_tls_ = true;
+ return *this;
+ }
// Configure the messenger to set the SO_REUSEPORT socket option.
- MessengerBuilder& set_reuseport();
+ MessengerBuilder& set_reuseport() {
+ reuseport_ = true;
+ return *this;
+ }
Status Build(std::shared_ptr<Messenger>* msgr);
@@ -179,8 +266,11 @@ class MessengerBuilder {
std::string sasl_proto_name_;
std::string rpc_authentication_;
std::string rpc_encryption_;
- std::string rpc_tls_ciphers_;
+ bool rpc_loopback_encryption_;
+ std::string rpc_tls_ciphers_; // pre-TLSv1.3 cipher suites
+ std::string rpc_tls_ciphersuites_; // TLSv1.3-related cipher suites
std::string rpc_tls_min_protocol_;
+ std::vector<std::string> rpc_tls_excluded_protocols_;
std::string rpc_certificate_file_;
std::string rpc_private_key_file_;
std::string rpc_ca_certificate_file_;
@@ -234,7 +324,7 @@ class Messenger {
// If Kerberos is enabled, this also runs a pre-flight check that makes
// sure the environment is appropriately configured to authenticate
// clients via Kerberos. If not, this returns a RuntimeError.
- Status AddAcceptorPool(const Sockaddr &accept_addr,
+ Status AddAcceptorPool(const Sockaddr& accept_addr,
std::shared_ptr<AcceptorPool>* pool);
// Register a new RpcService to handle inbound requests.
@@ -248,21 +338,23 @@ class Messenger {
// Returns an error if no service with this name can be found.
Status UnregisterService(const std::string& service_name);
- // Unregisters all RPC services.
+ // Unregisters all RPC services. Once called, no new services can be
+ // registered, and attempts to access missing services will result in a
+ // retriable error code.
void UnregisterAllServices();
// Queue a call for transmission. This will pick the appropriate reactor,
// and enqueue a task on that reactor to assign and send the call.
- void QueueOutboundCall(const std::shared_ptr<OutboundCall> &call);
+ void QueueOutboundCall(const std::shared_ptr<OutboundCall>& call);
// Enqueue a call for processing on the server.
void QueueInboundCall(std::unique_ptr<InboundCall> call);
// Queue a cancellation for the given outbound call.
- void QueueCancellation(const std::shared_ptr<OutboundCall> &call);
+ void QueueCancellation(const std::shared_ptr<OutboundCall>& call);
// Take ownership of the socket via Socket::Release
- void RegisterInboundSocket(Socket *new_socket, const Sockaddr &remote);
+ void RegisterInboundSocket(Socket* new_socket, const Sockaddr& remote);
// Dump info on related TCP connections into the given protobuf.
Status DumpConnections(const DumpConnectionsRequestPB& req,
@@ -272,7 +364,7 @@ class Messenger {
//
// The status argument conveys whether 'func' was run correctly (i.e.
// after the elapsed time) or not.
- void ScheduleOnReactor(const boost::function<void(const Status&)>& func,
+ void ScheduleOnReactor(std::function<void(const Status&)> func,
MonoDelta when);
const security::TlsContext& tls_context() const { return *tls_context_; }
@@ -293,8 +385,9 @@ class Messenger {
authn_token_ = token;
}
- RpcAuthentication authentication() const { return authentication_; }
- RpcEncryption encryption() const { return encryption_; }
+ security::RpcAuthentication authentication() const { return authentication_; }
+ security::RpcEncryption encryption() const { return encryption_; }
+ bool loopback_encryption() const { return loopback_encryption_; }
ThreadPool* negotiation_pool(Connection::Direction dir);
@@ -306,14 +399,21 @@ class Messenger {
return name_;
}
+ void SetServicesRegistered() {
+ std::lock_guard<percpu_rwlock> guard(lock_);
+ state_ = kServicesRegistered;
+ }
+
bool closing() const {
shared_lock<rw_spinlock> l(lock_.get_lock());
- return closing_;
+ return state_ == kClosing;
}
scoped_refptr<MetricEntity> metric_entity() const { return metric_entity_; }
- const int64_t rpc_negotiation_timeout_ms() const { return rpc_negotiation_timeout_ms_; }
+ int64_t rpc_negotiation_timeout_ms() const {
+ return rpc_negotiation_timeout_ms_;
+ }
const std::string& sasl_proto_name() const {
return sasl_proto_name_;
@@ -331,9 +431,9 @@ class Messenger {
FRIEND_TEST(TestRpc, TestConnectionNetworkPlane);
FRIEND_TEST(TestRpc, TestReopenOutboundConnections);
- explicit Messenger(const MessengerBuilder &bld);
+ explicit Messenger(const MessengerBuilder& bld);
- Reactor* RemoteToReactor(const Sockaddr &remote);
+ Reactor* RemoteToReactor(const Sockaddr& remote);
Status Init();
void RunTimeoutThread();
void UpdateCurTime();
@@ -356,14 +456,31 @@ class Messenger {
// Protects closing_, acceptor_pools_, rpc_services_.
mutable percpu_rwlock lock_;
- bool closing_;
+ enum State {
+ // The Messenger has been started; not all services may be registered yet.
+ kStarted,
+
+ // The Messenger is fully up running. All services have been registered and
+ // are accepting requests.
+ // NOTE: Messengers that do not register services never enter this state.
+ kServicesRegistered,
+
+ // All services have been unregistered. No further requests will succeed.
+ // NOTE: Messengers that do not register services never enter this state.
+ kServicesUnregistered,
+
+ // The Messenger is being closed. Its resources may be freed.
+ kClosing,
+ };
+ State state_;
// Whether to require authentication and encryption on the connections managed
// by this messenger.
// TODO(KUDU-1928): scope these to individual proxies, so that messengers can be
// reused by different clients.
- RpcAuthentication authentication_;
- RpcEncryption encryption_;
+ security::RpcAuthentication authentication_;
+ security::RpcEncryption encryption_;
+ bool loopback_encryption_;
// Pools which are listening on behalf of this messenger.
// Note that the user may have called Shutdown() on one of these
@@ -456,4 +573,3 @@ class Messenger {
} // namespace rpc
} // namespace kudu
-
diff --git a/be/src/kudu/rpc/mt-rpc-test.cc b/be/src/kudu/rpc/mt-rpc-test.cc
index 0245ce6..773eb39 100644
--- a/be/src/kudu/rpc/mt-rpc-test.cc
+++ b/be/src/kudu/rpc/mt-rpc-test.cc
@@ -19,6 +19,7 @@
#include <memory>
#include <ostream>
#include <string>
+#include <thread>
#include <utility>
#include <vector>
@@ -27,14 +28,16 @@
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
-#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/acceptor_pool.h"
#include "kudu/rpc/messenger.h"
#include "kudu/rpc/proxy.h"
#include "kudu/rpc/rpc-test-base.h"
+#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/rpc_service.h"
+#include "kudu/rpc/rtest.pb.h"
#include "kudu/rpc/service_if.h"
#include "kudu/rpc/service_pool.h"
+#include "kudu/util/barrier.h"
#include "kudu/util/countdown_latch.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
@@ -43,17 +46,17 @@
#include "kudu/util/status.h"
#include "kudu/util/stopwatch.h"
#include "kudu/util/test_macros.h"
-#include "kudu/util/thread.h"
-
+METRIC_DECLARE_counter(queue_overflow_rejections_kudu_rpc_test_CalculatorService_Add);
+METRIC_DECLARE_counter(queue_overflow_rejections_kudu_rpc_test_CalculatorService_Sleep);
METRIC_DECLARE_counter(rpc_connections_accepted);
METRIC_DECLARE_counter(rpcs_queue_overflow);
using std::string;
using std::shared_ptr;
+using std::thread;
using std::unique_ptr;
using std::vector;
-using strings::Substitute;
namespace kudu {
namespace rpc {
@@ -61,19 +64,19 @@ namespace rpc {
class MultiThreadedRpcTest : public RpcTestBase {
public:
// Make a single RPC call.
- void SingleCall(Sockaddr server_addr, const char* method_name,
+ void SingleCall(Sockaddr server_addr, const string& method_name,
Status* result, CountDownLatch* latch) {
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
CHECK_OK(CreateMessenger("ClientSC", &client_messenger));
Proxy p(client_messenger, server_addr, server_addr.host(),
GenericCalculatorService::static_service_name());
- *result = DoTestSyncCall(p, method_name);
+ *result = DoTestSyncCall(&p, method_name);
latch->CountDown();
}
// Make RPC calls until we see a failure.
- void HammerServer(Sockaddr server_addr, const char* method_name,
+ void HammerServer(Sockaddr server_addr, const string& method_name,
Status* last_result) {
shared_ptr<Messenger> client_messenger;
CHECK_OK(CreateMessenger("ClientHS", &client_messenger));
@@ -81,7 +84,7 @@ class MultiThreadedRpcTest : public RpcTestBase {
}
void HammerServerWithMessenger(
- Sockaddr server_addr, const char* method_name, Status* last_result,
+ Sockaddr server_addr, const string& method_name, Status* last_result,
const shared_ptr<Messenger>& messenger) {
LOG(INFO) << "Connecting to " << server_addr.ToString();
Proxy p(messenger, server_addr, server_addr.host(),
@@ -90,7 +93,7 @@ class MultiThreadedRpcTest : public RpcTestBase {
int i = 0;
while (true) {
i++;
- Status s = DoTestSyncCall(p, method_name);
+ Status s = DoTestSyncCall(&p, method_name);
if (!s.ok()) {
// Return on first failure.
LOG(INFO) << "Call failed. Shutting down client thread. Ran " << i << " calls: "
@@ -102,8 +105,8 @@ class MultiThreadedRpcTest : public RpcTestBase {
}
};
-static void AssertShutdown(kudu::Thread* thread, const Status* status) {
- ASSERT_OK(ThreadJoiner(thread).warn_every_ms(500).Join());
+static void AssertShutdown(thread* thread, const Status* status) {
+ thread->join();
string msg = status->ToString();
ASSERT_TRUE(msg.find("Service unavailable") != string::npos ||
msg.find("Network error") != string::npos)
@@ -117,13 +120,14 @@ TEST_F(MultiThreadedRpcTest, TestShutdownDuringService) {
Sockaddr server_addr;
ASSERT_OK(StartTestServer(&server_addr));
- const int kNumThreads = 4;
- scoped_refptr<kudu::Thread> threads[kNumThreads];
+ constexpr int kNumThreads = 4;
+ thread threads[kNumThreads];
Status statuses[kNumThreads];
for (int i = 0; i < kNumThreads; i++) {
- ASSERT_OK(kudu::Thread::Create("test", strings::Substitute("t$0", i),
- &MultiThreadedRpcTest::HammerServer, this, server_addr,
- GenericCalculatorService::kAddMethodName, &statuses[i], &threads[i]));
+ auto* my_status = &statuses[i];
+ threads[i] = thread([this, server_addr, my_status]() {
+ this->HammerServer(server_addr, GenericCalculatorService::kAddMethodName, my_status);
+ });
}
SleepFor(MonoDelta::FromMilliseconds(50));
@@ -134,7 +138,7 @@ TEST_F(MultiThreadedRpcTest, TestShutdownDuringService) {
server_messenger_->Shutdown();
for (int i = 0; i < kNumThreads; i++) {
- AssertShutdown(threads[i].get(), &statuses[i]);
+ AssertShutdown(&threads[i], &statuses[i]);
}
}
@@ -148,11 +152,11 @@ TEST_F(MultiThreadedRpcTest, TestShutdownClientWhileCallsPending) {
shared_ptr<Messenger> client_messenger;
ASSERT_OK(CreateMessenger("Client", &client_messenger));
- scoped_refptr<kudu::Thread> thread;
Status status;
- ASSERT_OK(kudu::Thread::Create("test", "test",
- &MultiThreadedRpcTest::HammerServerWithMessenger, this, server_addr,
- GenericCalculatorService::kAddMethodName, &status, client_messenger, &thread));
+ thread thread([this, server_addr, &status, client_messenger]() {
+ this->HammerServerWithMessenger(server_addr, GenericCalculatorService::kAddMethodName,
+ &status, client_messenger);
+ });
// Shut down the messenger after a very brief sleep. This often will race so that the
// call gets submitted to the messenger before shutdown, but the negotiation won't have
@@ -162,7 +166,7 @@ TEST_F(MultiThreadedRpcTest, TestShutdownClientWhileCallsPending) {
client_messenger->Shutdown();
client_messenger.reset();
- ASSERT_OK(ThreadJoiner(thread.get()).warn_every_ms(500).Join());
+ thread.join();
ASSERT_TRUE(status.IsAborted() ||
status.IsServiceUnavailable());
string msg = status.ToString();
@@ -210,7 +214,7 @@ TEST_F(MultiThreadedRpcTest, TestBlowOutServiceQueue) {
CHECK_OK(bld.Build(&server_messenger_));
shared_ptr<AcceptorPool> pool;
- ASSERT_OK(server_messenger_->AddAcceptorPool(Sockaddr(), &pool));
+ ASSERT_OK(server_messenger_->AddAcceptorPool(Sockaddr::Wildcard(), &pool));
ASSERT_OK(pool->Start(kMaxConcurrency));
Sockaddr server_addr = pool->bind_address();
@@ -222,13 +226,16 @@ TEST_F(MultiThreadedRpcTest, TestBlowOutServiceQueue) {
ASSERT_OK(service_pool_->Init(n_worker_threads_));
server_messenger_->RegisterService(service_name_, service_pool_);
- scoped_refptr<kudu::Thread> threads[3];
- Status status[3];
+ constexpr int kNumThreads = 3;
+ thread threads[kNumThreads];
+ Status status[kNumThreads];
CountDownLatch latch(1);
- for (int i = 0; i < 3; i++) {
- ASSERT_OK(kudu::Thread::Create("test", strings::Substitute("t$0", i),
- &MultiThreadedRpcTest::SingleCall, this, server_addr,
- GenericCalculatorService::kAddMethodName, &status[i], &latch, &threads[i]));
+ for (int i = 0; i < kNumThreads; i++) {
+ auto* my_status = &status[i];
+ threads[i] = thread([this, server_addr, my_status, &latch]() {
+ this->SingleCall(server_addr, GenericCalculatorService::kAddMethodName,
+ my_status, &latch);
+ });
}
// One should immediately fail due to backpressure. The latch is only initialized
@@ -240,8 +247,8 @@ TEST_F(MultiThreadedRpcTest, TestBlowOutServiceQueue) {
service_pool_->Shutdown();
server_messenger_->Shutdown();
- for (const auto& thread : threads) {
- ASSERT_OK(ThreadJoiner(thread.get()).warn_every_ms(500).Join());
+ for (auto& t : threads) {
+ t.join();
}
// Verify that one error was due to backpressure.
@@ -261,10 +268,81 @@ TEST_F(MultiThreadedRpcTest, TestBlowOutServiceQueue) {
ASSERT_EQ(1, rpcs_queue_overflow->value());
}
+TEST_F(MultiThreadedRpcTest, PerMethodQueueOverflowRejectionCounter) {
+ n_acceptor_pool_threads_ = 1;
+ n_server_reactor_threads_ = 1;
+ n_worker_threads_ = 1;
+ service_queue_length_ = 1;
+
+ Sockaddr server_addr;
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, false /*enable_ssl*/));
+
+ auto* rpcs_queue_overflow = METRIC_rpcs_queue_overflow.Instantiate(
+ server_messenger_->metric_entity()).get();
+ ASSERT_EQ(0, rpcs_queue_overflow->value());
+
+ auto* queue_overflow_rejections_add =
+ METRIC_queue_overflow_rejections_kudu_rpc_test_CalculatorService_Add.Instantiate(
+ server_messenger_->metric_entity()).get();
+ ASSERT_EQ(0, queue_overflow_rejections_add->value());
+
+ auto* queue_overflow_rejections_sleep =
+ METRIC_queue_overflow_rejections_kudu_rpc_test_CalculatorService_Sleep.Instantiate(
+ server_messenger_->metric_entity()).get();
+ ASSERT_EQ(0, queue_overflow_rejections_sleep->value());
+
+ // It seems that even in case of scheduling anomalies, 16 concurrent
+ // requests should be enough to overflow the RPC service queue of size 1,
+ // where the queue is served by a single worker thread and it takes at least
+ // 100ms for a request to complete.
+ constexpr size_t kNumThreads = 16;
+ vector<thread> threads;
+ threads.reserve(kNumThreads);
+ vector<Status> status(kNumThreads);
+ Barrier barrier(kNumThreads);
+ for (size_t i = 0; i < kNumThreads; ++i) {
+ const size_t idx = i;
+ threads.emplace_back([&, idx]() {
+ shared_ptr<Messenger> client_messenger;
+ CHECK_OK(CreateMessenger("ClientMessenger", &client_messenger));
+ Proxy p(client_messenger, server_addr, server_addr.host(),
+ CalculatorService::static_service_name());
+ SleepRequestPB req;
+ req.set_sleep_micros(100 * 1000); // 100ms
+ SleepResponsePB resp;
+ RpcController controller;
+ controller.set_timeout(MonoDelta::FromMilliseconds(10000));
+ barrier.Wait();
+ status[idx] = p.SyncRequest(
+ GenericCalculatorService::kSleepMethodName, req, &resp, &controller);
+ });
+ }
+
+ for (auto& t : threads) {
+ t.join();
+ }
+
+ size_t num_errors = 0;
+ for (const auto& s : status) {
+ if (!s.ok()) {
+ ++num_errors;
+ }
+ }
+
+ // Check that there were some RPC queue overflows.
+ const auto queue_overflow_num = rpcs_queue_overflow->value();
+ ASSERT_GT(num_errors, 0);
+ ASSERT_EQ(num_errors, queue_overflow_num);
+
+ // Check corresponding per-method rejection counters.
+ ASSERT_EQ(0, queue_overflow_rejections_add->value());
+ ASSERT_EQ(queue_overflow_num, queue_overflow_rejections_sleep->value());
+}
+
static void HammerServerWithTCPConns(const Sockaddr& addr) {
while (true) {
Socket socket;
- CHECK_OK(socket.Init(0));
+ CHECK_OK(socket.Init(addr.family(), 0));
Status s;
LOG_SLOW_EXECUTION(INFO, 100, "Connect took long") {
s = socket.Connect(addr);
@@ -285,12 +363,11 @@ TEST_F(MultiThreadedRpcTest, TestShutdownWithIncomingConnections) {
ASSERT_OK(StartTestServer(&server_addr));
// Start a number of threads which just hammer the server with TCP connections.
- vector<scoped_refptr<kudu::Thread> > threads;
- for (int i = 0; i < 8; i++) {
- scoped_refptr<kudu::Thread> new_thread;
- CHECK_OK(kudu::Thread::Create("test", strings::Substitute("t$0", i),
- &HammerServerWithTCPConns, server_addr, &new_thread));
- threads.push_back(new_thread);
+ constexpr int kNumThreads = 8;
+ vector<thread> threads;
+ threads.reserve(kNumThreads);
+ for (int i = 0; i < kNumThreads; i++) {
+ threads.emplace_back([server_addr]() { HammerServerWithTCPConns(server_addr); });
}
// Sleep until the server has started to actually accept some connections from the
@@ -306,8 +383,8 @@ TEST_F(MultiThreadedRpcTest, TestShutdownWithIncomingConnections) {
service_pool_->Shutdown();
server_messenger_->Shutdown();
- for (scoped_refptr<kudu::Thread>& t : threads) {
- ASSERT_OK(ThreadJoiner(t.get()).warn_every_ms(500).Join());
+ for (auto& t : threads) {
+ t.join();
}
}
diff --git a/be/src/kudu/rpc/negotiation-test.cc b/be/src/kudu/rpc/negotiation-test.cc
index bebc5e9..e0a8321 100644
--- a/be/src/kudu/rpc/negotiation-test.cc
+++ b/be/src/kudu/rpc/negotiation-test.cc
@@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-#include "kudu/rpc/rpc-test-base.h"
+#include "kudu/rpc/negotiation.h"
+#include <krb5/krb5.h> // IWYU pragma: keep
+#include <sasl/sasl.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -32,19 +34,16 @@
#include <boost/optional/optional.hpp>
#include <gflags/gflags.h>
-#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include <sasl/sasl.h>
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/walltime.h"
#include "kudu/rpc/client_negotiation.h"
-#include "kudu/rpc/messenger.h"
-#include "kudu/rpc/negotiation.h"
#include "kudu/rpc/remote_user.h"
+#include "kudu/rpc/rpc-test-base.h"
#include "kudu/rpc/sasl_common.h"
#include "kudu/rpc/server_negotiation.h"
#include "kudu/security/cert.h"
@@ -93,19 +92,19 @@ DEFINE_bool(is_test_child, false,
DECLARE_bool(rpc_encrypt_loopback_connections);
DECLARE_bool(rpc_trace_negotiation);
-using std::string;
-using std::thread;
-using std::unique_ptr;
-using std::vector;
-
using kudu::security::Cert;
using kudu::security::PkiConfig;
using kudu::security::PrivateKey;
+using kudu::security::RpcEncryption;
using kudu::security::SignedTokenPB;
using kudu::security::TlsContext;
using kudu::security::TokenSigner;
using kudu::security::TokenSigningPrivateKey;
using kudu::security::TokenVerifier;
+using std::string;
+using std::thread;
+using std::unique_ptr;
+using std::vector;
namespace kudu {
namespace rpc {
@@ -232,13 +231,13 @@ TEST_P(TestNegotiation, TestNegotiation) {
// Create the listening socket, client socket, and server socket.
Socket listening_socket;
- ASSERT_OK(listening_socket.Init(0));
- ASSERT_OK(listening_socket.BindAndListen(Sockaddr(), 1));
- Sockaddr server_addr;
+ Sockaddr server_addr = Sockaddr::Wildcard();
+ ASSERT_OK(listening_socket.Init(server_addr.family(), 0));
+ ASSERT_OK(listening_socket.BindAndListen(server_addr, 1));
ASSERT_OK(listening_socket.GetSocketAddress(&server_addr));
unique_ptr<Socket> client_socket(new Socket());
- ASSERT_OK(client_socket->Init(0));
+ ASSERT_OK(client_socket->Init(server_addr.family(), 0));
client_socket->Connect(server_addr);
unique_ptr<Socket> server_socket(desc.use_test_socket ?
@@ -253,11 +252,13 @@ TEST_P(TestNegotiation, TestNegotiation) {
&client_tls_context,
authn_token,
desc.client.encryption,
+ desc.rpc_encrypt_loopback,
"kudu");
ServerNegotiation server_negotiation(std::move(server_socket),
&server_tls_context,
&token_verifier,
desc.server.encryption,
+ desc.rpc_encrypt_loopback,
"kudu");
// Set client and server SASL mechanisms.
@@ -397,9 +398,9 @@ TEST_P(TestNegotiation, TestNegotiation) {
}
}
-INSTANTIATE_TEST_CASE_P(NegotiationCombinations,
- TestNegotiation,
- ::testing::Values(
+INSTANTIATE_TEST_SUITE_P(NegotiationCombinations,
+ TestNegotiation,
+ ::testing::Values(
// client: no authn/mechs
// server: no authn/mechs
@@ -1004,14 +1005,14 @@ static void RunAcceptingDelegator(Socket* acceptor,
static void RunNegotiationTest(const SocketCallable& server_runner,
const SocketCallable& client_runner) {
Socket server_sock;
- CHECK_OK(server_sock.Init(0));
- ASSERT_OK(server_sock.BindAndListen(Sockaddr(), 1));
- Sockaddr server_bind_addr;
+ Sockaddr server_bind_addr = Sockaddr::Wildcard();
+ CHECK_OK(server_sock.Init(server_bind_addr.family(), 0));
+ ASSERT_OK(server_sock.BindAndListen(server_bind_addr, 1));
ASSERT_OK(server_sock.GetSocketAddress(&server_bind_addr));
thread server(RunAcceptingDelegator, &server_sock, server_runner);
unique_ptr<Socket> client_sock(new Socket());
- CHECK_OK(client_sock->Init(0));
+ CHECK_OK(client_sock->Init(server_bind_addr.family(), 0));
ASSERT_OK(client_sock->Connect(server_bind_addr));
thread client(client_runner, std::move(client_sock));
@@ -1026,34 +1027,34 @@ static void RunNegotiationTest(const SocketCallable& server_runner,
////////////////////////////////////////////////////////////////////////////////
#ifndef __APPLE__
-template<class T>
-using CheckerFunction = std::function<void(const Status&, T&)>;
// Run GSSAPI negotiation from the server side. Runs
// 'post_check' after negotiation to verify the result.
static void RunGSSAPINegotiationServer(unique_ptr<Socket> socket,
- const CheckerFunction<ServerNegotiation>& post_check) {
+ const std::function<void(const Status&)>& post_check) {
TlsContext tls_context;
CHECK_OK(tls_context.Init());
TokenVerifier token_verifier;
ServerNegotiation server_negotiation(std::move(socket), &tls_context,
- &token_verifier, RpcEncryption::OPTIONAL, "kudu");
+ &token_verifier, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
server_negotiation.set_server_fqdn("127.0.0.1");
CHECK_OK(server_negotiation.EnableGSSAPI());
- post_check(server_negotiation.Negotiate(), server_negotiation);
+ post_check(server_negotiation.Negotiate());
}
// Run GSSAPI negotiation from the client side. Runs
// 'post_check' after negotiation to verify the result.
static void RunGSSAPINegotiationClient(unique_ptr<Socket> conn,
- const CheckerFunction<ClientNegotiation>& post_check) {
+ const std::function<void(const Status&)>& post_check) {
TlsContext tls_context;
CHECK_OK(tls_context.Init());
ClientNegotiation client_negotiation(std::move(conn), &tls_context,
- boost::none, RpcEncryption::OPTIONAL, "kudu");
+ boost::none, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
client_negotiation.set_server_fqdn("127.0.0.1");
CHECK_OK(client_negotiation.EnableGSSAPI());
- post_check(client_negotiation.Negotiate(), client_negotiation);
+ post_check(client_negotiation.Negotiate());
}
// Test invalid SASL negotiations using the GSSAPI (kerberos) mechanism over a socket.
@@ -1067,23 +1068,29 @@ TEST_F(TestNegotiation, TestGSSAPIInvalidNegotiation) {
// Try to negotiate with no krb5 credentials on either side. It should fail on both
// sides.
RunNegotiationTest(
- std::bind(RunGSSAPINegotiationServer, std::placeholders::_1,
- [](const Status& s, ServerNegotiation& server) {
- // The client notices there are no credentials and
- // doesn't send any failure message to the server.
- // Instead, it just disconnects.
- //
- // TODO(todd): it might be preferable to have the server
- // fail to start if it has no valid keytab.
- CHECK(s.IsNetworkError());
- }),
- std::bind(RunGSSAPINegotiationClient, std::placeholders::_1,
- [](const Status& s, ClientNegotiation& client) {
- CHECK(s.IsNotAuthorized());
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationServer(
+ std::move(socket),
+ [](const Status& s) {
+ // The client notices there are no credentials and
+ // doesn't send any failure message to the server.
+ // Instead, it just disconnects.
+ //
+ // TODO(todd): it might be preferable to have the server
+ // fail to start if it has no valid keytab.
+ CHECK(s.IsNetworkError());
+ });
+ },
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationClient(
+ std::move(socket),
+ [](const Status& s) {
+ CHECK(s.IsNotAuthorized());
#ifndef KRB5_VERSION_LE_1_10
- CHECK_GT(s.ToString().find("No Kerberos credentials available"), 0);
+ CHECK_GT(s.ToString().find("No Kerberos credentials available"), 0);
#endif
- }));
+ });
+ });
// Create the server principal and keytab.
string kt_path;
@@ -1093,20 +1100,26 @@ TEST_F(TestNegotiation, TestGSSAPIInvalidNegotiation) {
// Try to negotiate with no krb5 credentials on the client. It should fail on both
// sides.
RunNegotiationTest(
- std::bind(RunGSSAPINegotiationServer, std::placeholders::_1,
- [](const Status& s, ServerNegotiation& server) {
- // The client notices there are no credentials and
- // doesn't send any failure message to the server.
- // Instead, it just disconnects.
- CHECK(s.IsNetworkError());
- }),
- std::bind(RunGSSAPINegotiationClient, std::placeholders::_1,
- [](const Status& s, ClientNegotiation& client) {
- CHECK(s.IsNotAuthorized());
- ASSERT_STR_MATCHES(s.ToString(),
- "Not authorized: server requires authentication, "
- "but client does not have Kerberos credentials available");
- }));
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationServer(
+ std::move(socket),
+ [](const Status& s) {
+ // The client notices there are no credentials and
+ // doesn't send any failure message to the server.
+ // Instead, it just disconnects.
+ CHECK(s.IsNetworkError());
+ });
+ },
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationClient(
+ std::move(socket),
+ [](const Status& s) {
+ CHECK(s.IsNotAuthorized());
+ ASSERT_STR_MATCHES(s.ToString(),
+ "Not authorized: server requires authentication, "
+ "but client does not have Kerberos credentials available");
+ });
+ });
// Create and kinit as a client user.
ASSERT_OK(kdc.CreateUserPrincipal("testuser"));
@@ -1120,22 +1133,28 @@ TEST_F(TestNegotiation, TestGSSAPIInvalidNegotiation) {
CHECK_ERR(setenv("KRB5_KTNAME", kt_path.c_str(), 1 /*replace*/));
RunNegotiationTest(
- std::bind(RunGSSAPINegotiationServer, std::placeholders::_1,
- [](const Status& s, ServerNegotiation& server) {
- CHECK(s.IsNotAuthorized());
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationServer(
+ std::move(socket),
+ [](const Status& s) {
+ CHECK(s.IsNotAuthorized());
#ifndef KRB5_VERSION_LE_1_10
- ASSERT_STR_CONTAINS(s.ToString(),
- "No key table entry found matching kudu/127.0.0.1");
+ ASSERT_STR_CONTAINS(s.ToString(),
+ "No key table entry found matching kudu/127.0.0.1");
#endif
- }),
- std::bind(RunGSSAPINegotiationClient, std::placeholders::_1,
- [](const Status& s, ClientNegotiation& client) {
- CHECK(s.IsNotAuthorized());
+ });
+ },
+ [](unique_ptr<Socket> socket) {
+ RunGSSAPINegotiationClient(
+ std::move(socket),
+ [](const Status& s) {
+ CHECK(s.IsNotAuthorized());
#ifndef KRB5_VERSION_LE_1_10
- ASSERT_STR_CONTAINS(s.ToString(),
- "No key table entry found matching kudu/127.0.0.1");
+ ASSERT_STR_CONTAINS(s.ToString(),
+ "No key table entry found matching kudu/127.0.0.1");
#endif
- }));
+ });
+ });
}
#endif
@@ -1195,7 +1214,8 @@ static void RunTimeoutExpectingServer(unique_ptr<Socket> socket) {
CHECK_OK(tls_context.Init());
TokenVerifier token_verifier;
ServerNegotiation server_negotiation(std::move(socket), &tls_context,
- &token_verifier, RpcEncryption::OPTIONAL, "kudu");
+ &token_verifier, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
CHECK_OK(server_negotiation.EnablePlain());
Status s = server_negotiation.Negotiate();
ASSERT_TRUE(s.IsNetworkError()) << "Expected client to time out and close the connection. Got: "
@@ -1206,7 +1226,8 @@ static void RunTimeoutNegotiationClient(unique_ptr<Socket> sock) {
TlsContext tls_context;
CHECK_OK(tls_context.Init());
ClientNegotiation client_negotiation(std::move(sock), &tls_context,
- boost::none, RpcEncryption::OPTIONAL, "kudu");
+ boost::none, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
CHECK_OK(client_negotiation.EnablePlain("test", "test"));
MonoTime deadline = MonoTime::Now() - MonoDelta::FromMilliseconds(100L);
client_negotiation.set_deadline(deadline);
@@ -1227,7 +1248,8 @@ static void RunTimeoutNegotiationServer(unique_ptr<Socket> socket) {
CHECK_OK(tls_context.Init());
TokenVerifier token_verifier;
ServerNegotiation server_negotiation(std::move(socket), &tls_context,
- &token_verifier, RpcEncryption::OPTIONAL, "kudu");
+ &token_verifier, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
CHECK_OK(server_negotiation.EnablePlain());
MonoTime deadline = MonoTime::Now() - MonoDelta::FromMilliseconds(100L);
server_negotiation.set_deadline(deadline);
@@ -1240,7 +1262,8 @@ static void RunTimeoutExpectingClient(unique_ptr<Socket> socket) {
TlsContext tls_context;
CHECK_OK(tls_context.Init());
ClientNegotiation client_negotiation(std::move(socket), &tls_context,
- boost::none, RpcEncryption::OPTIONAL, "kudu");
+ boost::none, RpcEncryption::OPTIONAL,
+ /* encrypt_loopback */ false, "kudu");
CHECK_OK(client_negotiation.EnablePlain("test", "test"));
Status s = client_negotiation.Negotiate();
ASSERT_TRUE(s.IsNetworkError()) << "Expected server to time out and close the connection. Got: "
@@ -1275,7 +1298,7 @@ class TestDisableInit : public KuduTest {
// Invoke the currently-running test case in a new subprocess.
string filter_flag = strings::Substitute("--gtest_filter=$0.$1",
- CURRENT_TEST_CASE_NAME(), CURRENT_TEST_NAME());
+ CURRENT_TEST_SUITE_NAME(), CURRENT_TEST_NAME());
string executable_path;
CHECK_OK(env_->GetExecutablePath(&executable_path));
string stdout;
diff --git a/be/src/kudu/rpc/negotiation.cc b/be/src/kudu/rpc/negotiation.cc
index 31e0b33..e429b4c 100644
--- a/be/src/kudu/rpc/negotiation.cc
+++ b/be/src/kudu/rpc/negotiation.cc
@@ -43,7 +43,6 @@
#include "kudu/rpc/server_negotiation.h"
#include "kudu/rpc/user_credentials.h"
#include "kudu/security/tls_context.h"
-#include "kudu/security/token.pb.h"
#include "kudu/util/errno.h"
#include "kudu/util/flag_tags.h"
#include "kudu/util/logging.h"
@@ -72,6 +71,8 @@ DEFINE_bool(rpc_encrypt_loopback_connections, false,
"an attacker.");
TAG_FLAG(rpc_encrypt_loopback_connections, advanced);
+using kudu::security::RpcAuthentication;
+using kudu::security::RpcEncryption;
using std::string;
using std::unique_ptr;
using strings::Substitute;
@@ -165,6 +166,7 @@ static Status DisableSocketTimeouts(Socket* socket) {
static Status DoClientNegotiation(Connection* conn,
RpcAuthentication authentication,
RpcEncryption encryption,
+ bool encrypt_loopback,
MonoTime deadline,
unique_ptr<ErrorStatusPB>* rpc_error) {
const auto* messenger = conn->reactor_thread()->reactor()->messenger();
@@ -175,6 +177,7 @@ static Status DoClientNegotiation(Connection* conn,
&messenger->tls_context(),
authn_token,
encryption,
+ encrypt_loopback,
messenger->sasl_proto_name());
client_negotiation.set_server_fqdn(conn->outbound_connection_id().hostname());
@@ -235,6 +238,7 @@ static Status DoClientNegotiation(Connection* conn,
static Status DoServerNegotiation(Connection* conn,
RpcAuthentication authentication,
RpcEncryption encryption,
+ bool encrypt_loopback,
const MonoTime& deadline) {
const auto* messenger = conn->reactor_thread()->reactor()->messenger();
if (authentication == RpcAuthentication::REQUIRED &&
@@ -256,6 +260,7 @@ static Status DoServerNegotiation(Connection* conn,
&messenger->tls_context(),
&messenger->token_verifier(),
encryption,
+ encrypt_loopback,
messenger->sasl_proto_name());
if (authentication != RpcAuthentication::DISABLED && !messenger->keytab_file().empty()) {
@@ -285,13 +290,15 @@ static Status DoServerNegotiation(Connection* conn,
void Negotiation::RunNegotiation(const scoped_refptr<Connection>& conn,
RpcAuthentication authentication,
RpcEncryption encryption,
+ bool loopback_encryption,
MonoTime deadline) {
Status s;
unique_ptr<ErrorStatusPB> rpc_error;
+ bool encrypt_loopback = FLAGS_rpc_encrypt_loopback_connections || loopback_encryption;
if (conn->direction() == Connection::SERVER) {
- s = DoServerNegotiation(conn.get(), authentication, encryption, deadline);
+ s = DoServerNegotiation(conn.get(), authentication, encryption, encrypt_loopback, deadline);
} else {
- s = DoClientNegotiation(conn.get(), authentication, encryption, deadline,
+ s = DoClientNegotiation(conn.get(), authentication, encryption, encrypt_loopback, deadline,
&rpc_error);
}
diff --git a/be/src/kudu/rpc/negotiation.h b/be/src/kudu/rpc/negotiation.h
index 9f06c64..8f48e71 100644
--- a/be/src/kudu/rpc/negotiation.h
+++ b/be/src/kudu/rpc/negotiation.h
@@ -47,6 +47,7 @@ class Negotiation {
static void RunNegotiation(const scoped_refptr<Connection>& conn,
security::RpcAuthentication authentication,
security::RpcEncryption encryption,
+ bool loopback_encryption,
MonoTime deadline);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Negotiation);
diff --git a/be/src/kudu/rpc/outbound_call.cc b/be/src/kudu/rpc/outbound_call.cc
index 11ca221..151195f 100644
--- a/be/src/kudu/rpc/outbound_call.cc
+++ b/be/src/kudu/rpc/outbound_call.cc
@@ -17,6 +17,7 @@
#include "kudu/rpc/outbound_call.h"
+#include <cstddef>
#include <cstdint>
#include <functional>
#include <limits>
@@ -27,6 +28,7 @@
#include <utility>
#include <vector>
+#include <boost/container/vector.hpp>
#include <gflags/gflags.h>
#include <google/protobuf/message.h>
@@ -79,21 +81,24 @@ static const double kMicrosPerSecond = 1000000.0;
OutboundCall::OutboundCall(const ConnectionId& conn_id,
const RemoteMethod& remote_method,
+ unique_ptr<RequestPayload> payload,
+ CallbackBehavior cb_behavior,
google::protobuf::Message* response_storage,
RpcController* controller,
ResponseCallback callback)
- : state_(READY),
+ : cb_behavior_(cb_behavior),
+ state_(READY),
remote_method_(remote_method),
conn_id_(conn_id),
callback_(std::move(callback)),
controller_(DCHECK_NOTNULL(controller)),
response_(DCHECK_NOTNULL(response_storage)),
+ payload_(std::move(payload)),
cancellation_requested_(false) {
- DVLOG(4) << "OutboundCall " << this << " constructed with state_: " << StateName(state_)
- << " and RPC timeout: "
- << (controller->timeout().Initialized() ? controller->timeout().ToString() : "none");
- header_.set_call_id(kInvalidCallId);
- remote_method.ToPB(header_.mutable_remote_method());
+ DVLOG(4) << Substitute("OutboundCall $0 constructed with the state_: $1 and RPC timeout: $2",
+ this, StateName(state_),
+ (controller->timeout().Initialized() ? controller->timeout().ToString() : "none"));
+ payload_->header_.set_call_id(kInvalidCallId);
start_time_ = MonoTime::Now();
if (!controller_->required_server_features().empty()) {
@@ -101,45 +106,66 @@ OutboundCall::OutboundCall(const ConnectionId& conn_id,
}
if (controller_->request_id_) {
- header_.set_allocated_request_id(controller_->request_id_.release());
+ payload_->header_.set_allocated_request_id(controller_->request_id_.release());
}
}
+
+OutboundCall::OutboundCall(const ConnectionId& conn_id,
+ const RemoteMethod& remote_method,
+ google::protobuf::Message* response_storage,
+ RpcController* controller,
+ ResponseCallback callback)
+ : OutboundCall(conn_id, remote_method, std::make_unique<RequestPayload>(remote_method),
+ CallbackBehavior::kFreeSidecars, response_storage, controller,
+ std::move(callback)) {
+}
+
OutboundCall::~OutboundCall() {
DCHECK(IsFinished());
DVLOG(4) << "OutboundCall " << this << " destroyed with state_: " << StateName(state_);
}
-size_t OutboundCall::SerializeTo(TransferPayload* slices) {
- DCHECK_LT(0, request_buf_.size())
+void OutboundCall::SerializeTo(TransferPayload* slices) {
+ DCHECK_LT(0, payload_->request_buf_.size())
<< "Must call SetRequestPayload() before SerializeTo()";
- const MonoDelta &timeout = controller_->timeout();
- if (timeout.Initialized()) {
- header_.set_timeout_millis(timeout.ToMilliseconds());
+ if (controller_->timeout().Initialized()) {
+ payload_->header_.set_timeout_millis(controller_->timeout().ToMilliseconds());
}
for (uint32_t feature : controller_->required_server_features()) {
- header_.add_required_feature_flags(feature);
+ payload_->header_.add_required_feature_flags(feature);
}
- DCHECK_LE(0, sidecar_byte_size_);
+ DCHECK_LE(0, payload_->sidecar_byte_size_);
serialization::SerializeHeader(
- header_, sidecar_byte_size_ + request_buf_.size(), &header_buf_);
-
- size_t n_slices = 2 + sidecars_.size();
- DCHECK_LE(n_slices, slices->size());
- auto slice_iter = slices->begin();
- *slice_iter++ = Slice(header_buf_);
- *slice_iter++ = Slice(request_buf_);
- for (auto& sidecar : sidecars_) {
- *slice_iter++ = sidecar->AsSlice();
+ payload_->header_, payload_->sidecar_byte_size_ + payload_->request_buf_.size(),
+ &payload_->header_buf_);
+
+ slices->clear();
+ slices->push_back(payload_->header_buf_);
+ slices->push_back(payload_->request_buf_);
+ for (auto& sidecar : payload_->sidecars_) {
+ sidecar->AppendSlices(slices);
}
- DCHECK_EQ(slice_iter - slices->begin(), n_slices);
- return n_slices;
}
-void OutboundCall::SetRequestPayload(const Message& req,
+RequestPayload::RequestPayload(const RemoteMethod& remote_method) {
+ header_.set_call_id(kInvalidCallId);
+ remote_method.ToPB(header_.mutable_remote_method());
+}
+
+unique_ptr<RequestPayload> RequestPayload::CreateRequestPayload(
+ const RemoteMethod& remote_method,
+ const Message& req,
+ vector<unique_ptr<RpcSidecar>>&& sidecars) {
+ auto payload = std::make_unique<RequestPayload>(remote_method);
+ payload->PopulateRequestPayload(req, std::move(sidecars));
+ return payload;
+}
+
+void RequestPayload::PopulateRequestPayload(const Message& req,
vector<unique_ptr<RpcSidecar>>&& sidecars) {
DCHECK_EQ(-1, sidecar_byte_size_);
@@ -153,7 +179,7 @@ void OutboundCall::SetRequestPayload(const Message& req,
sidecar_byte_size_ = 0;
for (const unique_ptr<RpcSidecar>& car: sidecars_) {
header_.add_sidecar_offsets(sidecar_byte_size_ + message_size);
- int32_t sidecar_bytes = car->AsSlice().size();
+ size_t sidecar_bytes = car->TotalSize();
DCHECK_LE(sidecar_byte_size_, TransferLimits::kMaxTotalSidecarBytes - sidecar_bytes);
sidecar_byte_size_ += sidecar_bytes;
}
@@ -161,6 +187,11 @@ void OutboundCall::SetRequestPayload(const Message& req,
serialization::SerializeMessage(req, &request_buf_, sidecar_byte_size_, true);
}
+void OutboundCall::SetRequestPayload(const Message& req,
+ vector<unique_ptr<RpcSidecar>>&& sidecars) {
+ DCHECK_NOTNULL(payload_)->PopulateRequestPayload(req, std::move(sidecars));
+}
+
Status OutboundCall::status() const {
std::lock_guard<simple_spinlock> l(lock_);
return status_;
@@ -268,7 +299,9 @@ void OutboundCall::Cancel() {
void OutboundCall::CallCallback() {
// Clear references to outbound sidecars before invoking callback.
- sidecars_.clear();
+ if (cb_behavior_ == CallbackBehavior::kFreeSidecars) {
+ FreeSidecars();
+ }
int64_t start_cycles = CycleClock::Now();
{
@@ -319,6 +352,10 @@ void OutboundCall::SetResponse(unique_ptr<CallResponse> resp) {
}
}
+unique_ptr<RequestPayload> OutboundCall::ReleaseRequestPayload() {
+ return std::move(payload_);
+}
+
void OutboundCall::SetQueued() {
set_state(ON_OUTBOUND_QUEUE);
}
@@ -335,9 +372,9 @@ void OutboundCall::SetSent() {
// behavior is a lot more efficient if memory is freed from the same thread
// which allocated it -- this lets it keep to thread-local operations instead
// of taking a mutex to put memory back on the global freelist.
- delete [] header_buf_.release();
+ delete [] payload_->header_buf_.release();
- // request_buf_ is also done being used here, but since it was allocated by
+ // payload_ is also done being used here, but since it was allocated by
// the caller thread, we would rather let that thread free it whenever it
// deletes the RpcController.
@@ -454,7 +491,7 @@ string OutboundCall::ToString() const {
void OutboundCall::DumpPB(const DumpConnectionsRequestPB& req,
RpcCallInProgressPB* resp) {
std::lock_guard<simple_spinlock> l(lock_);
- resp->mutable_header()->CopyFrom(header_);
+ resp->mutable_header()->CopyFrom(payload_->header_);
resp->set_micros_elapsed((MonoTime::Now() - start_time_).ToMicroseconds());
switch (state_) {
@@ -517,7 +554,7 @@ Status CallResponse::ParseFrom(unique_ptr<InboundTransfer> transfer) {
// Use information from header to extract the payload slices.
RETURN_NOT_OK(RpcSidecar::ParseSidecars(header_.sidecar_offsets(),
- serialized_response_, sidecar_slices_));
+ serialized_response_, &sidecar_slices_));
if (header_.sidecar_offsets_size() > 0) {
serialized_response_ =
diff --git a/be/src/kudu/rpc/outbound_call.h b/be/src/kudu/rpc/outbound_call.h
index 7cd5f01..620a28b 100644
--- a/be/src/kudu/rpc/outbound_call.h
+++ b/be/src/kudu/rpc/outbound_call.h
@@ -16,7 +16,6 @@
// under the License.
#pragma once
-#include <cstddef>
#include <cstdint>
#include <memory>
#include <ostream>
@@ -34,6 +33,7 @@
#include "kudu/rpc/remote_method.h"
#include "kudu/rpc/response_callback.h"
#include "kudu/rpc/rpc_header.pb.h"
+#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/rpc/transfer.h"
#include "kudu/util/faststring.h"
#include "kudu/util/locks.h"
@@ -56,7 +56,43 @@ class CallResponse;
class DumpConnectionsRequestPB;
class RpcCallInProgressPB;
class RpcController;
-class RpcSidecar;
+
+// Encapsulates the request payload being sent by a call.
+class RequestPayload {
+ public:
+ // Creates a payload for the given remote method, serializing the given
+ // request, taking ownership of the sidecars, and populating the header as
+ // necessary.
+ static std::unique_ptr<RequestPayload> CreateRequestPayload(
+ const RemoteMethod& remote_method,
+ const google::protobuf::Message& req,
+ std::vector<std::unique_ptr<RpcSidecar>>&& sidecars);
+
+ // Creates an "empty" payload for the given remote method. Callers should
+ // also call PopulateRequestPayload() to form a usable payload.
+ explicit RequestPayload(const RemoteMethod& remote_method);
+
+ // Serializes the given 'req' and takes ownership of 'sidecars', populating
+ // the header as necessary.
+ void PopulateRequestPayload(const google::protobuf::Message& req,
+ std::vector<std::unique_ptr<RpcSidecar>>&& sidecars);
+ private:
+ friend class OutboundCall;
+
+ // The RPC header.
+ // Parts of this (eg the call ID) are only assigned once this request has
+ // been passed to the reactor thread and assigned a connection. Calls should
+ // re-assign the call ID if this payload is used in multiple calls (e.g.
+ // retries after re-resolving the address).
+ RequestHeader header_;
+ faststring header_buf_;
+ faststring request_buf_;
+ std::vector<std::unique_ptr<RpcSidecar>> sidecars_;
+
+ // Total size in bytes of all sidecars in 'sidecars_'. Set in SetRequestPayload().
+ // This cannot exceed TransferLimits::kMaxTotalSidecarBytes.
+ int32_t sidecar_byte_size_ = -1;
+};
// Tracks the status of a call on the client side.
//
@@ -81,6 +117,20 @@ class OutboundCall {
REMOTE_CALL,
};
+ // Behavior when running the callback with regards to freeing resources. Some
+ // callers may expect the call itself free sidecars upon completion, while
+ // others may attempt to reuse the sidecars in another call attempt upon
+ // failure.
+ enum class CallbackBehavior {
+ kFreeSidecars,
+ kDontFreeSidecars,
+ };
+
+ OutboundCall(const ConnectionId& conn_id, const RemoteMethod& remote_method,
+ std::unique_ptr<RequestPayload> req_payload, CallbackBehavior cb_behavior,
+ google::protobuf::Message* response_storage,
+ RpcController* controller, ResponseCallback callback);
+
OutboundCall(const ConnectionId& conn_id, const RemoteMethod& remote_method,
google::protobuf::Message* response_storage,
RpcController* controller, ResponseCallback callback);
@@ -98,14 +148,13 @@ class OutboundCall {
// Assign the call ID for this call. This is called from the reactor
// thread once a connection has been assigned. Must only be called once.
void set_call_id(int32_t call_id) {
- DCHECK_EQ(header_.call_id(), kInvalidCallId) << "Already has a call ID";
- header_.set_call_id(call_id);
+ DCHECK_EQ(payload_->header_.call_id(), kInvalidCallId) << "Already has a call ID";
+ payload_->header_.set_call_id(call_id);
}
// Serialize the call for the wire. Requires that SetRequestPayload()
// is called first. This is called from the Reactor thread.
- // Returns the number of slices in the serialized call.
- size_t SerializeTo(TransferPayload* slices);
+ void SerializeTo(TransferPayload* slices);
// Mark in the call that cancellation has been requested. If the call hasn't yet
// started sending or has finished sending the RPC request but is waiting for a
@@ -150,6 +199,12 @@ class OutboundCall {
return required_rpc_features_;
}
+ std::unique_ptr<RequestPayload> ReleaseRequestPayload();
+
+ void FreeSidecars() {
+ DCHECK_NOTNULL(payload_)->sidecars_.clear();
+ }
+
std::string ToString() const;
void DumpPB(const DumpConnectionsRequestPB& req, RpcCallInProgressPB* resp);
@@ -166,12 +221,12 @@ class OutboundCall {
// Return true if a call ID has been assigned to this call.
bool call_id_assigned() const {
- return header_.call_id() != kInvalidCallId;
+ return payload_->header_.call_id() != kInvalidCallId;
}
int32_t call_id() const {
DCHECK(call_id_assigned());
- return header_.call_id();
+ return payload_->header_.call_id();
}
// Returns true if cancellation has been requested. Must be called from
@@ -229,6 +284,16 @@ class OutboundCall {
// This will only be non-NULL if status().IsRemoteError().
const ErrorStatusPB* error_pb() const;
+ // Call the user-provided callback. Note that entries in 'sidecars_' are cleared
+ // prior to invoking the callback so the client can assume that the call doesn't
+ // hold references to outbound sidecars.
+ void CallCallback();
+
+ // The behavior defining whether to free sidecars upon calling the callback.
+ // Certain callbacks may perfer freeing the sidecars manually from within the
+ // callback.
+ const CallbackBehavior cb_behavior_;
+
// Lock for state_ status_, error_pb_ fields, since they
// may be mutated by the reactor thread while the client thread
// reads them.
@@ -237,16 +302,6 @@ class OutboundCall {
Status status_;
std::unique_ptr<ErrorStatusPB> error_pb_;
- // Call the user-provided callback. Note that entries in 'sidecars_' are cleared
- // prior to invoking the callback so the client can assume that the call doesn't
- // hold references to outbound sidecars.
- void CallCallback();
-
- // The RPC header.
- // Parts of this (eg the call ID) are only assigned once this call has been
- // passed to the reactor thread and assigned a connection.
- RequestHeader header_;
-
// The remote method being called.
RemoteMethod remote_method_;
@@ -261,20 +316,12 @@ class OutboundCall {
google::protobuf::Message* response_;
// Buffers for storing segments of the wire-format request.
- faststring header_buf_;
- faststring request_buf_;
+ std::unique_ptr<RequestPayload> payload_;
// Once a response has been received for this call, contains that response.
// Otherwise NULL.
std::unique_ptr<CallResponse> call_response_;
- // All sidecars to be sent with this call.
- std::vector<std::unique_ptr<RpcSidecar>> sidecars_;
-
- // Total size in bytes of all sidecars in 'sidecars_'. Set in SetRequestPayload().
- // This cannot exceed TransferLimits::kMaxTotalSidecarBytes.
- int32_t sidecar_byte_size_ = -1;
-
// True if cancellation was requested on this call.
bool cancellation_requested_;
@@ -331,7 +378,7 @@ class CallResponse {
Slice serialized_response_;
// Slices of data for rpc sidecars. They point into memory owned by transfer_.
- Slice sidecar_slices_[TransferLimits::kMaxSidecars];
+ SidecarSliceVector sidecar_slices_;
// The incoming transfer data - retained because serialized_response_
// and sidecar_slices_ refer into its data.
diff --git a/be/src/kudu/rpc/periodic-test.cc b/be/src/kudu/rpc/periodic-test.cc
index cd793bd..a6082ae 100644
--- a/be/src/kudu/rpc/periodic-test.cc
+++ b/be/src/kudu/rpc/periodic-test.cc
@@ -15,8 +15,11 @@
// specific language governing permissions and limitations
// under the License.
+#include "kudu/rpc/periodic.h"
+
#include <atomic>
#include <cstdint>
+#include <functional>
#include <memory>
#include <ostream>
#include <string>
@@ -27,7 +30,6 @@
#include <gtest/gtest.h>
#include "kudu/rpc/messenger.h"
-#include "kudu/rpc/periodic.h"
#include "kudu/util/monotime.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/stopwatch.h"
@@ -97,9 +99,9 @@ class JitteredPeriodicTimerTest : public PeriodicTimerTest,
shared_ptr<PeriodicTimer> timer_;
};
-INSTANTIATE_TEST_CASE_P(AllJitterModes,
- JitteredPeriodicTimerTest,
- ::testing::Values(0.0, 0.25));
+INSTANTIATE_TEST_SUITE_P(AllJitterModes,
+ JitteredPeriodicTimerTest,
+ ::testing::Values(0.0, 0.25));
TEST_P(JitteredPeriodicTimerTest, TestStartStop) {
// Before the timer starts, the counter's value should not change.
@@ -208,9 +210,9 @@ class JitteredOneShotPeriodicTimerTest : public JitteredPeriodicTimerTest {
}
};
-INSTANTIATE_TEST_CASE_P(AllJitterModes,
- JitteredOneShotPeriodicTimerTest,
- ::testing::Values(0.0, 0.25));
+INSTANTIATE_TEST_SUITE_P(AllJitterModes,
+ JitteredOneShotPeriodicTimerTest,
+ ::testing::Values(0.0, 0.25));
TEST_P(JitteredOneShotPeriodicTimerTest, TestBasics) {
// Kick off the one-shot timer a few times.
diff --git a/be/src/kudu/rpc/protoc-gen-krpc.cc b/be/src/kudu/rpc/protoc-gen-krpc.cc
index 840e549..bdf11f7 100644
--- a/be/src/kudu/rpc/protoc-gen-krpc.cc
+++ b/be/src/kudu/rpc/protoc-gen-krpc.cc
@@ -155,26 +155,25 @@ class FileSubstitutions : public Substituter {
class MethodSubstitutions : public Substituter {
public:
explicit MethodSubstitutions(const MethodDescriptor* method)
- : method_(method) {
+ : method_(method) {
}
void InitSubstitutionMap(map<string, string>* map) const override {
- (*map)["rpc_name"] = method_->name();
- (*map)["rpc_full_name"] = method_->full_name();
- (*map)["rpc_full_name_plainchars"] =
+ auto& m = *map;
+ m["rpc_name"] = method_->name();
+ m["rpc_full_name"] = method_->full_name();
+ m["rpc_full_name_plainchars"] =
StringReplace(method_->full_name(), ".", "_", true);
- (*map)["request"] =
- ReplaceNamespaceDelimiters(
- StripNamespaceIfPossible(method_->service()->full_name(),
- method_->input_type()->full_name()));
- (*map)["response"] =
- ReplaceNamespaceDelimiters(
- StripNamespaceIfPossible(method_->service()->full_name(),
- method_->output_type()->full_name()));
- (*map)["metric_enum_key"] = Substitute("kMetricIndex$0", method_->name());
+ m["request"] = ReplaceNamespaceDelimiters(
+ StripNamespaceIfPossible(method_->service()->full_name(),
+ method_->input_type()->full_name()));
+ m["response"] = ReplaceNamespaceDelimiters(
+ StripNamespaceIfPossible(method_->service()->full_name(),
+ method_->output_type()->full_name()));
+ m["metric_enum_key"] = Substitute("kMetricIndex$0", method_->name());
bool track_result = static_cast<bool>(method_->options().GetExtension(track_rpc_result));
- (*map)["track_result"] = track_result ? " true" : "false";
- (*map)["authz_method"] = GetAuthzMethod(*method_).get_value_or("AuthorizeAllowAll");
+ m["track_result"] = track_result ? " true" : "false";
+ m["authz_method"] = GetAuthzMethod(*method_).get_value_or("AuthorizeAllowAll");
}
// Strips the package from method arguments if they are in the same package as
@@ -209,15 +208,16 @@ class MethodSubstitutions : public Substituter {
class ServiceSubstitutions : public Substituter {
public:
explicit ServiceSubstitutions(const ServiceDescriptor* service)
- : service_(service)
- {}
+ : service_(service) {
+ }
void InitSubstitutionMap(map<string, string>* map) const override {
- (*map)["service_name"] = service_->name();
- (*map)["full_service_name"] = service_->full_name();
- (*map)["service_method_count"] = SimpleItoa(service_->method_count());
+ auto& m = *map;
+ m["service_name"] = service_->name();
+ m["full_service_name"] = service_->full_name();
+ m["service_method_count"] = SimpleItoa(service_->method_count());
- // TODO: upgrade to protobuf 2.5.x and attach service comments
+ // TODO(todd): upgrade to protobuf 2.5.x and attach service comments
// to the generated service classes using the SourceLocation API.
}
@@ -225,7 +225,6 @@ class ServiceSubstitutions : public Substituter {
const ServiceDescriptor* service_;
};
-
class SubstitutionContext {
public:
// Takes ownership of the substituter.
@@ -256,8 +255,6 @@ class SubstitutionContext {
vector<unique_ptr<const Substituter>> subs_;
};
-
-
class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
public:
CodeGenerator() { }
@@ -268,7 +265,7 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
const string&/* parameter */,
google::protobuf::compiler::GeneratorContext* gen_context,
string* error) const override {
- unique_ptr<FileSubstitutions> name_info(new FileSubstitutions());
+ unique_ptr<FileSubstitutions> name_info(new FileSubstitutions);
bool ret = name_info->Init(file, error);
if (!ret) {
return false;
@@ -316,31 +313,30 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
SubstitutionContext* subs,
const FileDescriptor* file) {
Print(printer, *subs,
- "// THIS FILE IS AUTOGENERATED FROM $path$\n"
- "\n"
- "#pragma once\n"
- "\n"
- "#include <string>\n"
- "\n"
- "#include \"kudu/gutil/ref_counted.h\"\n"
- "#include \"kudu/rpc/service_if.h\"\n"
- "\n"
- "namespace google {\n"
- "namespace protobuf {\n"
- "class Message;\n"
- "} // namespace protobuf\n"
- "} // namespace google\n"
- "\n"
- "namespace kudu {\n"
- "class MetricEntity;\n"
- "namespace rpc {\n"
- "class ResultTracker;\n"
- "class RpcContext;\n"
- "} // namespace rpc\n"
- "} // namespace kudu\n"
- "\n"
- "$open_namespace$"
- "\n");
+ "// THIS FILE IS AUTOGENERATED FROM $path$\n"
+ "\n"
+ "#pragma once\n"
+ "\n"
+ "#include <string>\n"
+ "\n"
+ "#include \"kudu/gutil/ref_counted.h\"\n"
+ "#include \"kudu/rpc/service_if.h\"\n"
+ "\n"
+ "namespace google {\n"
+ "namespace protobuf {\n"
+ "class Message;\n"
+ "} // namespace protobuf\n"
+ "} // namespace google\n"
+ "\n"
+ "namespace kudu {\n"
+ "class MetricEntity;\n"
+ "namespace rpc {\n"
+ "class ResultTracker;\n"
+ "class RpcContext;\n"
+ "} // namespace rpc\n"
+ "} // namespace kudu\n"
+ "\n"
+ "$open_namespace$");
for (int service_idx = 0; service_idx < file->service_count();
++service_idx) {
@@ -348,26 +344,30 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
subs->PushService(service);
Print(printer, *subs,
- "class $service_name$If : public ::kudu::rpc::GeneratedServiceIf {\n"
- " public:\n"
- " explicit $service_name$If(const scoped_refptr<::kudu::MetricEntity>& entity,"
- " const scoped_refptr<::kudu::rpc::ResultTracker>& result_tracker);\n"
- " virtual ~$service_name$If();\n"
- " std::string service_name() const override;\n"
- " static std::string static_service_name();\n"
- "\n"
- );
+ "\n"
+ "class $service_name$If : public ::kudu::rpc::GeneratedServiceIf {\n"
+ " public:\n"
+ " static const std::string& static_service_name();\n"
+ "\n"
+ " $service_name$If(\n"
+ " const scoped_refptr<::kudu::MetricEntity>& entity,\n"
+ " const scoped_refptr<::kudu::rpc::ResultTracker>& result_tracker);\n"
+ " virtual ~$service_name$If();\n"
+ "\n"
+ " const std::string& service_name() const override;\n");
set<string> authz_methods;
for (int method_idx = 0; method_idx < service->method_count();
++method_idx) {
- const MethodDescriptor *method = service->method(method_idx);
+ const MethodDescriptor* method = service->method(method_idx);
subs->PushMethod(method);
Print(printer, *subs,
- " virtual void $rpc_name$(const class $request$ *req,\n"
- " class $response$ *resp, ::kudu::rpc::RpcContext *context) = 0;\n"
- );
+ "\n"
+ " virtual void $rpc_name$(\n"
+ " const class $request$* req,\n"
+ " class $response$* resp,\n"
+ " ::kudu::rpc::RpcContext* context) = 0;\n");
subs->Pop(); // method
if (auto m = GetAuthzMethod(*method)) {
authz_methods.insert(m.get());
@@ -376,50 +376,54 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
if (!authz_methods.empty()) {
printer->Print(
- "\n\n"
- " // Authorization methods\n"
- " // ---------------------\n\n");
+ "\n"
+ " // Authorization methods\n"
+ " // ---------------------\n");
}
for (const string& m : authz_methods) {
printer->Print({ {"m", m} },
- " virtual bool $m$(const google::protobuf::Message* req,\n"
- " google::protobuf::Message* resp, ::kudu::rpc::RpcContext *context) = 0;\n");
+ "\n"
+ " virtual bool $m$(\n"
+ " const google::protobuf::Message* req,\n"
+ " google::protobuf::Message* resp,\n"
+ " ::kudu::rpc::RpcContext* context) = 0;\n");
}
Print(printer, *subs,
- "\n"
- "};\n"
- );
+ "\n"
+ " private:\n"
+ " static const std::string kServiceName_;\n"
+ "};\n");
subs->Pop(); // Service
}
Print(printer, *subs,
- "\n"
- "$close_namespace$"
- "\n");
+ "\n"
+ "$close_namespace$");
}
static void GenerateServiceIf(Printer* printer,
SubstitutionContext* subs,
const FileDescriptor* file) {
Print(printer, *subs,
- "// THIS FILE IS AUTOGENERATED FROM $path$\n"
- "\n"
- "#include <functional>\n"
- "#include <memory>\n"
- "#include <unordered_map>\n"
- "#include <utility>\n"
- "\n"
- "#include <google/protobuf/message.h>\n"
- "\n"
- "#include \"$path_no_extension$.pb.h\"\n"
- "#include \"$path_no_extension$.service.h\"\n"
- "\n"
- "#include \"kudu/rpc/result_tracker.h\"\n"
- "#include \"kudu/rpc/service_if.h\"\n"
- "#include \"kudu/util/metrics.h\"\n"
- "\n");
+ "// THIS FILE IS AUTOGENERATED FROM $path$\n"
+ "\n"
+ "#include <functional>\n"
+ "#include <memory>\n"
+ "#include <unordered_map>\n"
+ "#include <utility>\n"
+ "\n"
+ "#include <google/protobuf/message.h>\n"
+ "\n"
+ "#include \"$path_no_extension$.pb.h\"\n"
+ "#include \"$path_no_extension$.service.h\"\n"
+ "\n"
+ "#include \"kudu/rpc/result_tracker.h\"\n"
+ "#include \"kudu/rpc/service_if.h\"\n"
+ "#include \"kudu/util/metrics.h\"\n"
+ "#include \"kudu/util/net/dns_resolver.h\"\n"
+ "\n");
// Define metric prototypes for each method in the service.
for (int service_idx = 0; service_idx < file->service_count();
@@ -432,13 +436,22 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
const MethodDescriptor* method = service->method(method_idx);
subs->PushMethod(method);
Print(printer, *subs,
- "METRIC_DEFINE_histogram(server, handler_latency_$rpc_full_name_plainchars$,\n"
- " \"$rpc_full_name$ RPC Time\",\n"
- " kudu::MetricUnit::kMicroseconds,\n"
- " \"Microseconds spent handling $rpc_full_name$() RPC requests\",\n"
- " kudu::MetricLevel::kInfo,\n"
- " 60000000LU, 2);\n"
- "\n");
+ "METRIC_DEFINE_histogram(server,\n"
+ " handler_latency_$rpc_full_name_plainchars$,\n"
+ " \"$rpc_full_name$ RPC Time\",\n"
+ " kudu::MetricUnit::kMicroseconds,\n"
+ " \"Microseconds spent handling $rpc_full_name$ RPC requests\",\n"
+ " kudu::MetricLevel::kInfo,\n"
+ " 60000000LU, 2);\n"
+ "\n");
+ Print(printer, *subs,
+ "METRIC_DEFINE_counter(server,\n"
+ " queue_overflow_rejections_$rpc_full_name_plainchars$,\n"
+ " \"$rpc_full_name$ RPC Rejections\",\n"
+ " kudu::MetricUnit::kRequests,\n"
+ " \"Number of rejected $rpc_full_name$ requests due to RPC queue overflow\",\n"
+ " kudu::MetricLevel::kInfo);\n"
+ "\n");
subs->Pop(); // method
}
@@ -446,15 +459,14 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
}
Print(printer, *subs,
- "using google::protobuf::Message;\n"
- "using kudu::MetricEntity;\n"
- "using kudu::rpc::ResultTracker;\n"
- "using kudu::rpc::RpcContext;\n"
- "using kudu::rpc::RpcMethodInfo;\n"
- "using std::unique_ptr;\n"
- "\n"
- "$open_namespace$"
- "\n");
+ "using google::protobuf::Message;\n"
+ "using kudu::MetricEntity;\n"
+ "using kudu::rpc::ResultTracker;\n"
+ "using kudu::rpc::RpcContext;\n"
+ "using kudu::rpc::RpcMethodInfo;\n"
+ "using std::string;\n"
+ "\n"
+ "$open_namespace$");
for (int service_idx = 0; service_idx < file->service_count();
++service_idx) {
@@ -462,9 +474,17 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
subs->PushService(service);
Print(printer, *subs,
- "$service_name$If::$service_name$If(const scoped_refptr<MetricEntity>& entity,"
- " const scoped_refptr<ResultTracker>& result_tracker) {\n"
- "result_tracker_ = result_tracker;\n"
+ "\n"
+ "const string $service_name$If::kServiceName_ = \"$full_service_name$\";\n"
+ "\n"
+ "const string& $service_name$If::static_service_name() {\n"
+ " return kServiceName_;\n"
+ "}\n"
+ "\n"
+ "$service_name$If::$service_name$If(\n"
+ " const scoped_refptr<MetricEntity>& entity,\n"
+ " const scoped_refptr<ResultTracker>& result_tracker)\n"
+ " : GeneratedServiceIf(result_tracker) {\n"
);
for (int method_idx = 0; method_idx < service->method_count();
++method_idx) {
@@ -472,79 +492,76 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
subs->PushMethod(method);
Print(printer, *subs,
- " {\n"
- " scoped_refptr<RpcMethodInfo> mi(new RpcMethodInfo());\n"
- " mi->req_prototype.reset(new $request$());\n"
- " mi->resp_prototype.reset(new $response$());\n"
- " mi->authz_method = [this](const Message* req, Message* resp,\n"
- " RpcContext* ctx) {\n"
- " return this->$authz_method$(static_cast<const $request$*>(req),\n"
- " static_cast<$response$*>(resp),\n"
- " ctx);\n"
- " };\n"
- " mi->track_result = $track_result$;\n"
- " mi->handler_latency_histogram =\n"
- " METRIC_handler_latency_$rpc_full_name_plainchars$.Instantiate(entity);\n"
- " mi->func = [this](const Message* req, Message* resp, RpcContext* ctx) {\n"
- " this->$rpc_name$(static_cast<const $request$*>(req),\n"
- " static_cast<$response$*>(resp),\n"
- " ctx);\n"
- " };\n"
- " methods_by_name_[\"$rpc_name$\"] = std::move(mi);\n"
- " }\n");
+ " {\n"
+ " scoped_refptr<RpcMethodInfo> mi(new RpcMethodInfo);\n"
+ " mi->req_prototype.reset(new $request$);\n"
+ " mi->resp_prototype.reset(new $response$);\n"
+ " mi->authz_method = [this](const Message* req, Message* resp,\n"
+ " RpcContext* ctx) {\n"
+ " return this->$authz_method$(\n"
+ " static_cast<const $request$*>(req),\n"
+ " static_cast<$response$*>(resp),\n"
+ " ctx);\n"
+ " };\n"
+ " mi->track_result = $track_result$;\n"
+ " mi->handler_latency_histogram =\n"
+ " METRIC_handler_latency_$rpc_full_name_plainchars$.Instantiate(entity);\n"
+ " mi->queue_overflow_rejections =\n"
+ " METRIC_queue_overflow_rejections_$rpc_full_name_plainchars$.Instantiate("
+ "entity);\n"
+ " mi->func = [this](const Message* req, Message* resp, RpcContext* ctx) {\n"
+ " this->$rpc_name$(\n"
+ " static_cast<const $request$*>(req),\n"
+ " static_cast<$response$*>(resp),\n"
+ " ctx);\n"
+ " };\n"
+ " methods_by_name_[\"$rpc_name$\"] = std::move(mi);\n"
+ " }\n");
subs->Pop(); // method
}
Print(printer, *subs,
- "}\n"
- "\n"
- "$service_name$If::~$service_name$If() {\n"
- "}\n"
- "\n"
- "std::string $service_name$If::service_name() const {\n"
- " return \"$full_service_name$\";\n"
- "}\n"
- "std::string $service_name$If::static_service_name() {\n"
- " return \"$full_service_name$\";\n"
- "}\n"
- "\n"
- );
+ "}\n"
+ "\n"
+ "$service_name$If::~$service_name$If() {\n"
+ "}\n"
+ "\n"
+ "const string& $service_name$If::service_name() const {\n"
+ " return kServiceName_;\n"
+ "}\n");
subs->Pop(); // service
}
Print(printer, *subs,
- "\n"
- "$close_namespace$"
- "\n"
- );
+ "\n"
+ "$close_namespace$");
}
static void GenerateProxyHeader(Printer* printer,
SubstitutionContext* subs,
const FileDescriptor* file) {
Print(printer, *subs,
- "// THIS FILE IS AUTOGENERATED FROM $path$\n"
- "\n"
- "#pragma once\n"
- "\n"
- "#include <memory>\n"
- "#include <string>\n"
- "\n"
- "#include \"kudu/rpc/proxy.h\"\n"
- "#include \"kudu/rpc/response_callback.h\"\n"
- "#include \"kudu/util/status.h\"\n"
- "\n"
- "namespace kudu { class Sockaddr; }\n"
- "namespace kudu {\n"
- "namespace rpc {\n"
- "class Messenger;\n"
- "class RpcController;\n"
- "} // namespace rpc\n"
- "} // namespace kudu\n"
- "\n"
- "$open_namespace$"
- "\n");
+ "// THIS FILE IS AUTOGENERATED FROM $path$\n"
+ "\n"
+ "#pragma once\n"
+ "\n"
+ "#include <memory>\n"
+ "#include <string>\n"
+ "\n"
+ "#include \"kudu/rpc/proxy.h\"\n"
+ "#include \"kudu/rpc/response_callback.h\"\n"
+ "#include \"kudu/util/status.h\"\n"
+ "\n"
+ "namespace kudu { class Sockaddr; }\n"
+ "namespace kudu {\n"
+ "namespace rpc {\n"
+ "class Messenger;\n"
+ "class RpcController;\n"
+ "} // namespace rpc\n"
+ "} // namespace kudu\n"
+ "\n"
+ "$open_namespace$");
for (int service_idx = 0; service_idx < file->service_count();
++service_idx) {
@@ -552,14 +569,18 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
subs->PushService(service);
Print(printer, *subs,
- "class $service_name$Proxy : public ::kudu::rpc::Proxy {\n"
- " public:\n"
- " $service_name$Proxy(std::shared_ptr<::kudu::rpc::Messenger>\n"
- " messenger, const ::kudu::Sockaddr &sockaddr,"
- " std::string hostname);\n"
- " ~$service_name$Proxy();\n"
- "\n"
- );
+ "\n"
+ "class $service_name$Proxy : public ::kudu::rpc::Proxy {\n"
+ " public:\n"
+ " $service_name$Proxy(\n"
+ " std::shared_ptr<::kudu::rpc::Messenger> messenger,\n"
+ " const ::kudu::Sockaddr& sockaddr,\n"
+ " std::string hostname);\n"
+ " $service_name$Proxy(\n"
+ " std::shared_ptr<::kudu::rpc::Messenger> messenger,\n"
+ " const ::kudu::HostPort& hp,\n"
+ " ::kudu::DnsResolver* dns_resolver);\n"
+ " ~$service_name$Proxy();\n");
for (int method_idx = 0; method_idx < service->method_count();
++method_idx) {
@@ -567,92 +588,108 @@ class CodeGenerator : public ::google::protobuf::compiler::CodeGenerator {
subs->PushMethod(method);
Print(printer, *subs,
- "\n"
- " ::kudu::Status $rpc_name$(const class $request$ &req,\n"
- " class $response$ *resp,\n"
- " ::kudu::rpc::RpcController *controller);\n"
- " void $rpc_name$Async(const class $request$ &req,\n"
- " class $response$ *response,\n"
- " ::kudu::rpc::RpcController *controller,\n"
- " const ::kudu::rpc::ResponseCallback &callback);\n"
- );
+ "\n"
+ " ::kudu::Status $rpc_name$(\n"
+ " const class $request$& req,\n"
+ " class $response$* resp,\n"
+ " ::kudu::rpc::RpcController* controller);\n"
+ " void $rpc_name$Async(\n"
+ " const class $request$& req,\n"
+ " class $response$* response,\n"
+ " ::kudu::rpc::RpcController* controller,\n"
+ " const ::kudu::rpc::ResponseCallback& callback);\n");
subs->Pop(); // method
}
- Print(printer, *subs,
- "};\n");
+ Print(printer, *subs, "};\n");
subs->Pop(); // service
}
Print(printer, *subs,
- "\n"
- "$close_namespace$"
- "\n"
- );
+ "\n"
+ "$close_namespace$");
}
static void GenerateProxy(Printer* printer,
SubstitutionContext* subs,
const FileDescriptor* file) {
Print(printer, *subs,
- "// THIS FILE IS AUTOGENERATED FROM $path$\n"
- "\n"
- "#include <string>\n"
- "#include <utility>\n"
- "\n"
- "#include \"$path_no_extension$.pb.h\"\n"
- "#include \"$path_no_extension$.proxy.h\"\n"
- "\n"
- "namespace kudu {\n"
- "namespace rpc {\n"
- "class Messenger;\n"
- "class RpcController;\n"
- "} // namespace rpc\n"
- "} // namespace kudu\n"
- "\n"
- "$open_namespace$"
- "\n");
+ "// THIS FILE IS AUTOGENERATED FROM $path$\n"
+ "\n"
+ "#include <string>\n"
+ "#include <utility>\n"
+ "\n"
+ "#include \"$path_no_extension$.pb.h\"\n"
+ "#include \"$path_no_extension$.proxy.h\"\n"
+ "\n"
+ "namespace kudu {\n"
+ "class DnsResolver;\n"
+ "namespace rpc {\n"
+ "class Messenger;\n"
+ "class RpcController;\n"
+ "} // namespace rpc\n"
+ "} // namespace kudu\n"
+ "\n"
+ "$open_namespace$");
for (int service_idx = 0; service_idx < file->service_count();
++service_idx) {
const ServiceDescriptor* service = file->service(service_idx);
subs->PushService(service);
Print(printer, *subs,
- "$service_name$Proxy::$service_name$Proxy(\n"
- " std::shared_ptr< ::kudu::rpc::Messenger> messenger,\n"
- " const ::kudu::Sockaddr &remote, std::string hostname)\n"
- " : Proxy(std::move(messenger), remote, std::move(hostname), \"$full_service_name$\") {\n"
- "}\n"
- "\n"
- "$service_name$Proxy::~$service_name$Proxy() {\n"
- "}\n"
- "\n"
- "\n");
+ "\n"
+ "$service_name$Proxy::$service_name$Proxy(\n"
+ " std::shared_ptr<::kudu::rpc::Messenger> messenger,\n"
+ " const ::kudu::Sockaddr& remote,\n"
+ " std::string hostname)\n"
+ " : Proxy(std::move(messenger),\n"
+ " remote,\n"
+ " std::move(hostname),\n"
+ " \"$full_service_name$\") {\n"
+ "}\n"
+ "$service_name$Proxy::$service_name$Proxy(\n"
+ " std::shared_ptr<::kudu::rpc::Messenger> messenger,\n"
+ " const ::kudu::HostPort& hp,\n"
+ " ::kudu::DnsResolver* dns_resolver)\n"
+ " : Proxy(std::move(messenger),\n"
+ " hp,\n"
+ " dns_resolver,\n"
+ " \"$full_service_name$\") {\n"
+ "}\n"
+ "\n"
+ "$service_name$Proxy::~$service_name$Proxy() {\n"
+ "}\n");
for (int method_idx = 0; method_idx < service->method_count();
++method_idx) {
const MethodDescriptor* method = service->method(method_idx);
subs->PushMethod(method);
Print(printer, *subs,
- "::kudu::Status $service_name$Proxy::$rpc_name$(const $request$ &req, $response$ *resp,\n"
- " ::kudu::rpc::RpcController *controller) {\n"
- " return SyncRequest(\"$rpc_name$\", req, resp, controller);\n"
- "}\n"
- "\n"
- "void $service_name$Proxy::$rpc_name$Async(const $request$ &req,\n"
- " $response$ *resp, ::kudu::rpc::RpcController *controller,\n"
- " const ::kudu::rpc::ResponseCallback &callback) {\n"
- " AsyncRequest(\"$rpc_name$\", req, resp, controller, callback);\n"
- "}\n"
- "\n");
+ "\n"
+ "::kudu::Status $service_name$Proxy::$rpc_name$(\n"
+ " const $request$& req,\n"
+ " $response$* resp,\n"
+ " ::kudu::rpc::RpcController* controller) {\n"
+ " static const std::string kRpcName = \"$rpc_name$\";\n"
+ " return SyncRequest(kRpcName, req, resp, controller);\n"
+ "}\n"
+ "\n"
+ "void $service_name$Proxy::$rpc_name$Async(\n"
+ " const $request$& req,\n"
+ " $response$* resp,\n"
+ " ::kudu::rpc::RpcController* controller,\n"
+ " const ::kudu::rpc::ResponseCallback& callback) {\n"
+ " static const std::string kRpcName = \"$rpc_name$\";\n"
+ " AsyncRequest(kRpcName, req, resp, controller, callback);\n"
+ "}\n");
subs->Pop(); // method
}
subs->Pop(); // service
}
Print(printer, *subs,
- "\n"
- "$close_namespace$"
- "\n");
+ "\n"
+ "$close_namespace$");
}
};
+
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/proxy-test.cc b/be/src/kudu/rpc/proxy-test.cc
new file mode 100644
index 0000000..c5ad52b
--- /dev/null
+++ b/be/src/kudu/rpc/proxy-test.cc
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/rpc/proxy.h"
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <gflags/gflags_declare.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/rpc/response_callback.h"
+#include "kudu/rpc/rpc-test-base.h"
+#include "kudu/rpc/rpc_controller.h"
+#include "kudu/rpc/rtest.pb.h"
+#include "kudu/rpc/service_pool.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/net/dns_resolver.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/net/sockaddr.h"
+#include "kudu/util/notification.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+DECLARE_string(dns_addr_resolution_override);
+
+using std::shared_ptr;
+using std::string;
+using std::thread;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+namespace rpc {
+
+class Messenger;
+
+namespace {
+
+constexpr uint16_t kPort = 1111;
+constexpr const char* kFakeHost = "fakehost";
+const HostPort kFakeHostPort(kFakeHost, kPort);
+
+void SendRequestAsync(const ResponseCallback& cb,
+ Proxy* p, RpcController* controller, SleepResponsePB* resp) {
+ SleepRequestPB req;
+ req.set_sleep_micros(100 * 1000); // 100ms
+ p->AsyncRequest(GenericCalculatorService::kSleepMethodName, req, resp, controller, cb);
+}
+
+Status SendRequest(Proxy* p) {
+ SleepRequestPB req;
+ req.set_sleep_micros(100 * 1000); // 100ms
+ SleepResponsePB resp;
+ RpcController controller;
+ controller.set_timeout(MonoDelta::FromMilliseconds(10000));
+ return p->SyncRequest(GenericCalculatorService::kSleepMethodName, req, &resp, &controller);
+}
+
+} // anonymous namespace
+
+class RpcProxyTest : public RpcTestBase {
+};
+
+TEST_F(RpcProxyTest, TestProxyRetriesWhenRequestLeavesScope) {
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+
+ // We're providing a fake hostport to encourage retries of DNS resolution,
+ // which will attempt to send the request payload after the request protobuf
+ // has already left scope.
+ Proxy p(client_messenger, kFakeHostPort, &dns_resolver,
+ CalculatorService::static_service_name());
+ p.Init();
+
+ SleepResponsePB resp;
+ {
+ Notification note;
+ RpcController controller;
+ controller.set_timeout(MonoDelta::FromMilliseconds(10000));
+ SendRequestAsync([¬e] () { note.Notify(); }, &p, &controller, &resp);
+ note.WaitForNotification();
+ Status s = controller.status();
+ ASSERT_TRUE(s.IsNetworkError()) << s.ToString();
+ }
+
+ // Now try a successful call.
+ string ip = GetBindIpForDaemon(/*index*/2, kDefaultBindMode);
+ Sockaddr addr;
+ ASSERT_OK(addr.ParseString(ip, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, addr.ToString());
+
+ Notification note;
+ RpcController controller;
+ controller.set_timeout(MonoDelta::FromMilliseconds(10000));
+ SendRequestAsync([¬e] () { note.Notify(); }, &p, &controller, &resp);
+ note.WaitForNotification();
+ ASSERT_OK(controller.status());
+}
+
+// Test that proxies initialized with a DnsResolver return errors when
+// receiving a non-transient error.
+TEST_F(RpcProxyTest, TestProxyReturnsOnNonTransientError) {
+ SKIP_IF_SLOW_NOT_ALLOWED(); // This test waits for a timeout.
+
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ Proxy p(client_messenger, kFakeHostPort, &dns_resolver,
+ CalculatorService::static_service_name());
+ p.Init();
+ Status s = SendRequest(&p);
+ ASSERT_TRUE(s.IsNetworkError()) << s.ToString();
+
+ // If we do resolve to an address that turns out to be bogus, we should
+ // time out when negotiating.
+ FLAGS_dns_addr_resolution_override = Substitute("$0=1.1.1.1", kFakeHost);
+ s = SendRequest(&p);
+ ASSERT_TRUE(s.IsTimedOut()) << s.ToString();
+}
+
+// Test that ensures a proxy initialized with an address will use that address.
+TEST_F(RpcProxyTest, TestProxyUsesInitialAddr) {
+ string ip1 = GetBindIpForDaemon(/*index*/1, kDefaultBindMode);
+ Sockaddr server_addr;
+ ASSERT_OK(server_addr.ParseString(ip1, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr));
+
+ // Despite our proxy being configured with a fake host, our request should
+ // still go through since we call Init() with a valid address.
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ Proxy p(client_messenger, kFakeHostPort, &dns_resolver,
+ CalculatorService::static_service_name());
+ p.Init(server_addr);
+ ASSERT_OK(SendRequest(&p));
+
+ server_messenger_.reset();
+ service_pool_.reset();
+
+ // With our server down, the request should fail.
+ Status s = SendRequest(&p);
+ ASSERT_TRUE(s.IsNetworkError()) << s.ToString();
+
+ // Once we bring up a new server and allow our proxy to resolve it, the
+ // request should succeed.
+ string ip2 = GetBindIpForDaemon(/*index*/2, kDefaultBindMode);
+ Sockaddr second_addr;
+ ASSERT_OK(second_addr.ParseString(ip2, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&second_addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, second_addr.ToString());
+ ASSERT_OK(SendRequest(&p));
+}
+
+TEST_F(RpcProxyTest, TestNonResolvingProxyIgnoresInit) {
+ string ip = GetBindIpForDaemon(/*index*/1, kDefaultBindMode);
+ Sockaddr server_addr;
+ ASSERT_OK(server_addr.ParseString(ip, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr));
+
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ HostPort hp(ip, kPort);
+ Proxy p(client_messenger, hp, &dns_resolver, CalculatorService::static_service_name());
+
+ // Call Init() with a fake address. Because this proxy isn't configured for
+ // address re-resolution, the new address is ignored.
+ Sockaddr fake_addr;
+ ASSERT_OK(fake_addr.ParseString("1.1.1.1", kPort));
+ p.Init(fake_addr);
+
+ // We should thus have no trouble sending a request.
+ ASSERT_OK(SendRequest(&p));
+}
+
+// Start a proxy with a DNS resolver that maps a hostname to the address bound
+// by the server. Then restart the server but bind to a different address, and
+// update the DNS resolver to map the same hostname to the different address.
+// The proxy should eventually be usable.
+TEST_F(RpcProxyTest, TestProxyReresolvesAddress) {
+ string ip1 = GetBindIpForDaemon(/*index*/1, kDefaultBindMode);
+ Sockaddr server_addr;
+ ASSERT_OK(server_addr.ParseString(ip1, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, server_addr.ToString());
+
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ Proxy p(client_messenger, kFakeHostPort, &dns_resolver,
+ CalculatorService::static_service_name());
+ p.Init();
+ ASSERT_OK(SendRequest(&p));
+
+ string ip2 = GetBindIpForDaemon(/*index*/2, kDefaultBindMode);
+ Sockaddr second_addr;
+ ASSERT_OK(second_addr.ParseString(ip2, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&second_addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, second_addr.ToString());
+ ASSERT_OK(SendRequest(&p));
+}
+
+TEST_F(RpcProxyTest, TestProxyReresolvesAddressFromThreads) {
+ constexpr const int kNumThreads = 4;
+
+ string ip1 = GetBindIpForDaemon(/*index*/1, kDefaultBindMode);
+ Sockaddr server_addr;
+ ASSERT_OK(server_addr.ParseString(ip1, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, server_addr.ToString());
+
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("client_messenger", &client_messenger));
+ DnsResolver dns_resolver(1, 1024 * 1024);
+ Proxy p(client_messenger, kFakeHostPort, &dns_resolver,
+ CalculatorService::static_service_name());
+ p.Init();
+ ASSERT_OK(SendRequest(&p));
+
+ string ip2 = GetBindIpForDaemon(/*index*/2, kDefaultBindMode);
+ Sockaddr second_addr;
+ ASSERT_OK(second_addr.ParseString(ip2, kPort));
+ ASSERT_OK(StartTestServerWithGeneratedCode(&second_addr));
+ FLAGS_dns_addr_resolution_override = Substitute("$0=$1", kFakeHost, second_addr.ToString());
+
+ vector<Status> errors(kNumThreads);
+ vector<thread> threads;
+ threads.reserve(kNumThreads);
+ for (int i = 0; i < kNumThreads; i++) {
+ threads.emplace_back([&, i] {
+ errors[i] = SendRequest(&p);
+ });
+ }
+ for (auto& t : threads) {
+ t.join();
+ }
+ for (const auto& e : errors) {
+ EXPECT_OK(e);
+ }
+}
+
+} // namespace rpc
+} // namespace kudu
diff --git a/be/src/kudu/rpc/proxy.cc b/be/src/kudu/rpc/proxy.cc
index 24668ab..9501a38 100644
--- a/be/src/kudu/rpc/proxy.cc
+++ b/be/src/kudu/rpc/proxy.cc
@@ -17,29 +17,37 @@
#include "kudu/rpc/proxy.h"
+#include <functional>
#include <iostream>
#include <memory>
#include <utility>
+#include <vector>
-#include <boost/bind.hpp> // IWYU pragma: keep
-#include <boost/core/ref.hpp>
#include <glog/logging.h>
+#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/rpc/outbound_call.h"
#include "kudu/rpc/messenger.h"
+#include "kudu/rpc/outbound_call.h"
#include "kudu/rpc/remote_method.h"
#include "kudu/rpc/response_callback.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/user_credentials.h"
+#include "kudu/util/kernel_stack_watchdog.h"
+#include "kudu/util/logging.h"
+#include "kudu/util/net/dns_resolver.h"
+#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
-#include "kudu/util/countdown_latch.h"
+#include "kudu/util/notification.h"
#include "kudu/util/status.h"
#include "kudu/util/user.h"
using google::protobuf::Message;
using std::string;
using std::shared_ptr;
+using std::unique_ptr;
+using std::vector;
+using strings::Substitute;
namespace kudu {
namespace rpc {
@@ -49,11 +57,12 @@ Proxy::Proxy(std::shared_ptr<Messenger> messenger,
string hostname,
string service_name)
: service_name_(std::move(service_name)),
+ dns_resolver_(nullptr),
messenger_(std::move(messenger)),
is_started_(false) {
CHECK(messenger_ != nullptr);
DCHECK(!service_name_.empty()) << "Proxy service name must not be blank";
-
+ DCHECK(remote.is_initialized());
// By default, we set the real user to the currently logged-in user.
// Effective user and password remain blank.
string real_user;
@@ -68,37 +77,186 @@ Proxy::Proxy(std::shared_ptr<Messenger> messenger,
conn_id_ = ConnectionId(remote, std::move(hostname), std::move(creds));
}
+Proxy::Proxy(std::shared_ptr<Messenger> messenger,
+ HostPort hp,
+ DnsResolver* dns_resolver,
+ string service_name)
+ : service_name_(std::move(service_name)),
+ hp_(std::move(hp)),
+ dns_resolver_(dns_resolver),
+ messenger_(std::move(messenger)),
+ is_started_(false) {
+ CHECK(messenger_ != nullptr);
+ DCHECK(!service_name_.empty()) << "Proxy service name must not be blank";
+ DCHECK(hp_.Initialized());
+}
+
+Sockaddr* Proxy::GetSingleSockaddr(std::vector<Sockaddr>* addrs) const {
+ DCHECK(!addrs->empty());
+ if (PREDICT_FALSE(addrs->size() > 1)) {
+ LOG(WARNING) << Substitute(
+ "$0 proxy host/port $1 resolves to $2 different addresses. Using $3",
+ service_name_, hp_.ToString(), addrs->size(), (*addrs)[0].ToString());
+ }
+ return &(*addrs)[0];
+}
+
+void Proxy::Init(Sockaddr addr) {
+ if (!dns_resolver_) {
+ return;
+ }
+ // By default, we set the real user to the currently logged-in user.
+ // Effective user and password remain blank.
+ string real_user;
+ Status s = GetLoggedInUser(&real_user);
+ if (!s.ok()) {
+ LOG(WARNING) << "Proxy for " << service_name_ << ": Unable to get logged-in user name: "
+ << s.ToString() << " before connecting to host/port: " << hp_.ToString();
+ }
+ vector<Sockaddr> addrs;
+ if (!addr.is_initialized()) {
+ s = dns_resolver_->ResolveAddresses(hp_, &addrs);
+ if (PREDICT_TRUE(s.ok() && !addrs.empty())) {
+ addr = *GetSingleSockaddr(&addrs);
+ DCHECK(addr.is_initialized());
+ addr.set_port(hp_.port());
+ // NOTE: it's ok to proceed on failure -- the address will remain
+ // uninitialized and be re-resolved when sending the next request.
+ }
+ }
+
+ UserCredentials creds;
+ creds.set_real_user(std::move(real_user));
+ conn_id_ = ConnectionId(addr, hp_.host(), std::move(creds));
+}
+
Proxy::~Proxy() {
}
+void Proxy::EnqueueRequest(const string& method,
+ unique_ptr<RequestPayload> req_payload,
+ google::protobuf::Message* response,
+ RpcController* controller,
+ const ResponseCallback& callback,
+ OutboundCall::CallbackBehavior cb_behavior) const {
+ ConnectionId connection = conn_id();
+ DCHECK(connection.remote().is_initialized());
+ controller->call_.reset(
+ new OutboundCall(connection, {service_name_, method}, std::move(req_payload),
+ cb_behavior, response, controller, callback));
+ controller->SetMessenger(messenger_.get());
+
+ // If this fails to queue, the callback will get called immediately
+ // and the controller will be in an ERROR state.
+ messenger_->QueueOutboundCall(controller->call_);
+}
+
+void Proxy::RefreshDnsAndEnqueueRequest(const std::string& method,
+ unique_ptr<RequestPayload> req_payload,
+ google::protobuf::Message* response,
+ RpcController* controller,
+ const ResponseCallback& callback) {
+ DCHECK(!controller->call_);
+ vector<Sockaddr>* addrs = new vector<Sockaddr>();
+ DCHECK_NOTNULL(dns_resolver_)->RefreshAddressesAsync(hp_, addrs,
+ [this, req_raw = req_payload.release(),
+ &method, callback, response, controller, addrs] (const Status& s) mutable {
+ unique_ptr<RequestPayload> req_payload(req_raw);
+ unique_ptr<vector<Sockaddr>> unique_addrs(addrs);
+ // If we fail to resolve the address, treat the call as failed.
+ if (!s.ok() || addrs->empty()) {
+ DCHECK(!controller->call_);
+ // NOTE: we need to keep a reference here because the callback may end up
+ // destructing the controller and the outbound call, _while_ the callback
+ // is running from within the call!
+ auto shared_call = std::make_shared<OutboundCall>(
+ conn_id(), RemoteMethod{service_name_, method}, response, controller, callback);
+ controller->call_ = shared_call;
+ controller->call_->SetFailed(s.CloneAndPrepend("failed to refresh physical address"));
+ return;
+ }
+ auto* addr = GetSingleSockaddr(addrs);
+ DCHECK(addr->is_initialized());
+ addr->set_port(hp_.port());
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ conn_id_.set_remote(*addr);
+ }
+ // NOTE: we don't expect the user-provided callback to free sidecars, so
+ // make sure the outbound call frees it for us.
+ EnqueueRequest(method, std::move(req_payload), response, controller, callback,
+ OutboundCall::CallbackBehavior::kFreeSidecars);
+ });
+}
+
void Proxy::AsyncRequest(const string& method,
const google::protobuf::Message& req,
google::protobuf::Message* response,
RpcController* controller,
- const ResponseCallback& callback) const {
+ const ResponseCallback& callback) {
CHECK(!controller->call_) << "Controller should be reset";
base::subtle::NoBarrier_Store(&is_started_, true);
- RemoteMethod remote_method(service_name_, method);
- controller->call_.reset(
- new OutboundCall(conn_id_, remote_method, response, controller, callback));
- controller->SetRequestParam(req);
- controller->SetMessenger(messenger_.get());
+ // TODO(awong): it would be great if we didn't have to heap allocate the
+ // payload.
+ auto req_payload = RequestPayload::CreateRequestPayload(
+ RemoteMethod{service_name_, method},
+ req, controller->ReleaseOutboundSidecars());
+ if (!dns_resolver_) {
+ // NOTE: we don't expect the user-provided callback to free sidecars, so
+ // make sure the outbound call frees it for us.
+ EnqueueRequest(method, std::move(req_payload), response, controller, callback,
+ OutboundCall::CallbackBehavior::kFreeSidecars);
+ return;
+ }
- // If this fails to queue, the callback will get called immediately
- // and the controller will be in an ERROR state.
- messenger_->QueueOutboundCall(controller->call_);
+ // If we haven't successfully initialized the remote, e.g. because the DNS
+ // lookup failed, refresh the DNS entry and enqueue the request.
+ bool remote_initialized;
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ remote_initialized = conn_id_.remote().is_initialized();
+ }
+ if (!remote_initialized) {
+ RefreshDnsAndEnqueueRequest(method, std::move(req_payload), response, controller, callback);
+ return;
+ }
+
+ // Otherwise, just enqueue the request, but retry if there's an error, since
+ // it's possible the physical address of the host was changed. We only retry
+ // once more before calling the callback.
+ auto refresh_dns_and_cb = [this, &method, callback, response, controller] () {
+ // TODO(awong): we should be more specific here -- consider having the RPC
+ // layer set a flag in the controller that warrants a retry.
+ if (PREDICT_FALSE(!controller->status().ok())) {
+ KLOG_EVERY_N_SECS(WARNING, 5)
+ << Substitute("Call had error, refreshing address and retrying: $0",
+ controller->status().ToString());
+ auto req_payload = controller->ReleaseRequestPayload();
+ controller->Reset();
+ RefreshDnsAndEnqueueRequest(method, std::move(req_payload), response, controller, callback);
+ return;
+ }
+ // For any other status, OK or otherwise, just run the callback.
+ controller->FreeOutboundSidecars();
+ SCOPED_WATCH_STACK(100);
+ callback();
+ };
+ // Since we may end up using the request payload in the event of a retry,
+ // ensure the outbound call doesn't free the sidecars, and instead free
+ // manually from within our callback.
+ EnqueueRequest(method, std::move(req_payload), response, controller, refresh_dns_and_cb,
+ OutboundCall::CallbackBehavior::kDontFreeSidecars);
}
Status Proxy::SyncRequest(const string& method,
const google::protobuf::Message& req,
google::protobuf::Message* resp,
- RpcController* controller) const {
- CountDownLatch latch(1);
+ RpcController* controller) {
+ Notification note;
AsyncRequest(method, req, DCHECK_NOTNULL(resp), controller,
- boost::bind(&CountDownLatch::CountDown, boost::ref(latch)));
-
- latch.Wait();
+ [¬e]() { note.Notify(); });
+ note.WaitForNotification();
return controller->status();
}
@@ -115,7 +273,7 @@ void Proxy::set_network_plane(string network_plane) {
}
std::string Proxy::ToString() const {
- return strings::Substitute("$0@$1", service_name_, conn_id_.ToString());
+ return Substitute("$0@$1", service_name_, conn_id_.ToString());
}
} // namespace rpc
diff --git a/be/src/kudu/rpc/proxy.h b/be/src/kudu/rpc/proxy.h
index ccf5f18..cb82205 100644
--- a/be/src/kudu/rpc/proxy.h
+++ b/be/src/kudu/rpc/proxy.h
@@ -17,12 +17,17 @@
#pragma once
#include <memory>
+#include <mutex>
#include <string>
+#include <vector>
#include "kudu/gutil/atomicops.h"
#include "kudu/gutil/macros.h"
#include "kudu/rpc/connection_id.h"
+#include "kudu/rpc/outbound_call.h"
#include "kudu/rpc/response_callback.h"
+#include "kudu/util/locks.h"
+#include "kudu/util/net/net_util.h"
#include "kudu/util/status.h"
namespace google {
@@ -33,6 +38,7 @@ class Message;
namespace kudu {
+class DnsResolver;
class Sockaddr;
namespace rpc {
@@ -65,8 +71,27 @@ class Proxy {
std::string hostname,
std::string service_name);
+ // TODO(awong): consider a separate auto-resolving proxy class?
+ Proxy(std::shared_ptr<Messenger> messenger,
+ HostPort hp,
+ DnsResolver* dns_resolver,
+ std::string service_name);
+
~Proxy();
+ // If the proxy is configured for address re-resolution (by supplying a
+ // DnsResolver and HostPort in the constructor), performs an initial
+ // resolution of the address using the HostPort. If 'addr' is supplied, it is
+ // used instead of performing resolution (this is useful to initialize
+ // several proxies with a single external DNS resolution).
+ //
+ // Otherwise, this is a no-op.
+ //
+ // NOTE: it is always OK to skip calling this method -- if this proxy is
+ // configured for address re-resolution and this is skipped, the resolution
+ // will happen upon sending the first request.
+ void Init(Sockaddr addr = {});
+
// Call a remote method asynchronously.
//
// Typically, users will not call this directly, but rather through
@@ -97,14 +122,14 @@ class Proxy {
const google::protobuf::Message& req,
google::protobuf::Message* resp,
RpcController* controller,
- const ResponseCallback& callback) const;
+ const ResponseCallback& callback);
// The same as AsyncRequest(), except that the call blocks until the call
// finishes. If the call fails, returns a non-OK result.
Status SyncRequest(const std::string& method,
const google::protobuf::Message& req,
google::protobuf::Message* resp,
- RpcController* controller) const;
+ RpcController* controller);
// Set the user credentials which should be used to log in.
void set_user_credentials(UserCredentials user_credentials);
@@ -121,9 +146,49 @@ class Proxy {
std::string ToString() const;
private:
+ // Asynchronously refreshes the DNS, enqueueing the given request upon
+ // success, or failing the call and calling the callback upon failure.
+ void RefreshDnsAndEnqueueRequest(const std::string& method,
+ std::unique_ptr<RequestPayload> req_payload,
+ google::protobuf::Message* response,
+ RpcController* controller,
+ const ResponseCallback& callback);
+
+ // Queues the given request as an outbound call using the given messenger,
+ // controller, and response.
+ void EnqueueRequest(const std::string& method,
+ std::unique_ptr<RequestPayload> req_payload,
+ google::protobuf::Message* response,
+ RpcController* controller,
+ const ResponseCallback& callback,
+ OutboundCall::CallbackBehavior cb_behavior) const;
+
+ // Returns a single Sockaddr from the 'addrs', logging a warning if there is
+ // more than one to choose from.
+ Sockaddr* GetSingleSockaddr(std::vector<Sockaddr>* addrs) const;
+
+ ConnectionId conn_id() const {
+ std::lock_guard<simple_spinlock> l(lock_);
+ return conn_id_;
+ }
+
const std::string service_name_;
+ HostPort hp_;
+ DnsResolver* dns_resolver_;
std::shared_ptr<Messenger> messenger_;
+
+ // TODO(awong): consider implementing some lock-free list of ConnectionIds
+ // instead of taking a lock every time we want to get the "current"
+ // ConnectionId.
+ //
+ // Connection ID used by this proxy. Once the proxy has started sending
+ // requests, the connection ID may be updated in response to calls (e.g. if
+ // we re-resolved the physical address in response to an invalid DNS entry).
+ // As such, 'conn_id_' is protected by 'lock_', and should be copied and
+ // passed around, rather than used directly
+ mutable simple_spinlock lock_;
ConnectionId conn_id_;
+
mutable Atomic32 is_started_;
DISALLOW_COPY_AND_ASSIGN(Proxy);
diff --git a/be/src/kudu/rpc/reactor-test.cc b/be/src/kudu/rpc/reactor-test.cc
index 2de5f58..0168f0e 100644
--- a/be/src/kudu/rpc/reactor-test.cc
+++ b/be/src/kudu/rpc/reactor-test.cc
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
+#include <functional>
#include <memory>
-#include <boost/bind.hpp> // IWYU pragma: keep
-#include <boost/function.hpp>
#include <glog/logging.h>
#include <gtest/gtest.h>
@@ -57,10 +56,10 @@ class ReactorTest : public RpcTestBase {
latch_.CountDown();
}
- void ScheduledTaskScheduleAgain(const Status& status) {
+ void ScheduledTaskScheduleAgain(const Status& /*status*/) {
+ auto* current = Thread::current_thread();
messenger_->ScheduleOnReactor(
- boost::bind(&ReactorTest::ScheduledTaskCheckThread, this, _1,
- Thread::current_thread()),
+ [=](const Status& s) { this->ScheduledTaskCheckThread(s, current); },
MonoDelta::FromMilliseconds(0));
latch_.CountDown();
}
@@ -72,7 +71,7 @@ class ReactorTest : public RpcTestBase {
TEST_F(ReactorTest, TestFunctionIsCalled) {
messenger_->ScheduleOnReactor(
- boost::bind(&ReactorTest::ScheduledTask, this, _1, Status::OK()),
+ [=](const Status& s) { this->ScheduledTask(s, Status::OK()); },
MonoDelta::FromSeconds(0));
latch_.Wait();
}
@@ -80,7 +79,7 @@ TEST_F(ReactorTest, TestFunctionIsCalled) {
TEST_F(ReactorTest, TestFunctionIsCalledAtTheRightTime) {
MonoTime before = MonoTime::Now();
messenger_->ScheduleOnReactor(
- boost::bind(&ReactorTest::ScheduledTask, this, _1, Status::OK()),
+ [=](const Status& s) { this->ScheduledTask(s, Status::OK()); },
MonoDelta::FromMilliseconds(100));
latch_.Wait();
MonoTime after = MonoTime::Now();
@@ -90,8 +89,7 @@ TEST_F(ReactorTest, TestFunctionIsCalledAtTheRightTime) {
TEST_F(ReactorTest, TestFunctionIsCalledIfReactorShutdown) {
messenger_->ScheduleOnReactor(
- boost::bind(&ReactorTest::ScheduledTask, this, _1,
- Status::Aborted("doesn't matter")),
+ [=](const Status& s) { this->ScheduledTask(s, Status::Aborted("doesn't matter")); },
MonoDelta::FromSeconds(60));
messenger_->Shutdown();
latch_.Wait();
@@ -102,7 +100,7 @@ TEST_F(ReactorTest, TestReschedulesOnSameReactorThread) {
latch_.Reset(2);
messenger_->ScheduleOnReactor(
- boost::bind(&ReactorTest::ScheduledTaskScheduleAgain, this, _1),
+ [=](const Status& s) { this->ScheduledTaskScheduleAgain(s); },
MonoDelta::FromSeconds(0));
latch_.Wait();
latch_.Wait();
diff --git a/be/src/kudu/rpc/reactor.cc b/be/src/kudu/rpc/reactor.cc
index 4ed2f5b..4f276ea 100644
--- a/be/src/kudu/rpc/reactor.cc
+++ b/be/src/kudu/rpc/reactor.cc
@@ -17,31 +17,36 @@
#include "kudu/rpc/reactor.h"
+#include <openssl/crypto.h>
+#include <openssl/err.h> // IWYU pragma: keep
+#include <sys/socket.h>
+
#include <cerrno>
+#include <functional>
#include <memory>
#include <mutex>
#include <ostream>
#include <string>
#include <utility>
-#include <boost/bind.hpp> // IWYU pragma: keep
#include <boost/intrusive/list.hpp>
-#include <boost/optional.hpp>
#include <ev++.h>
+#include <ev.h>
#include <gflags/gflags.h>
#include <glog/logging.h>
-#include "kudu/gutil/bind.h"
+#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/stringprintf.h"
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/rpc/client_negotiation.h"
+#include "kudu/gutil/sysinfo.h"
+#include "kudu/gutil/walltime.h"
#include "kudu/rpc/connection.h"
#include "kudu/rpc/messenger.h"
#include "kudu/rpc/negotiation.h"
#include "kudu/rpc/outbound_call.h"
+#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/rpc_introspection.pb.h"
-#include "kudu/rpc/server_negotiation.h"
#include "kudu/util/countdown_latch.h"
#include "kudu/util/debug/sanitizer_scopes.h"
#include "kudu/util/flag_tags.h"
@@ -144,7 +149,7 @@ void DoInitLibEv() {
} // anonymous namespace
-ReactorThread::ReactorThread(Reactor *reactor, const MessengerBuilder& bld)
+ReactorThread::ReactorThread(Reactor* reactor, const MessengerBuilder& bld)
: loop_(kDefaultLibEvFlags),
cur_time_(MonoTime::Now()),
last_unused_tcp_scan_(cur_time_),
@@ -185,7 +190,8 @@ Status ReactorThread::Init() {
ev_set_invoke_pending_cb(loop_, &ReactorThread::InvokePendingCb);
// Create Reactor thread.
- return kudu::Thread::Create("reactor", "rpc reactor", &ReactorThread::RunThread, this, &thread_);
+ return kudu::Thread::Create("reactor", "rpc reactor",
+ [this]() { this->RunThread(); }, &thread_);
}
void ReactorThread::InvokePendingCb(struct ev_loop* loop) {
@@ -421,7 +427,7 @@ void ReactorThread::TimerHandler(ev::timer& /*watcher*/, int revents) {
ScanIdleConnections();
}
-void ReactorThread::RegisterTimeout(ev::timer *watcher) {
+void ReactorThread::RegisterTimeout(ev::timer* watcher) {
watcher->set(loop_);
}
@@ -474,7 +480,7 @@ void ReactorThread::ScanIdleConnections() {
VLOG_IF(1, shutdown > 0) << name() << ": shutdown " << shutdown << " TCP connections.";
}
-const std::string& ReactorThread::name() const {
+const string& ReactorThread::name() const {
return reactor_->name();
}
@@ -482,7 +488,7 @@ MonoTime ReactorThread::cur_time() const {
return cur_time_;
}
-Reactor *ReactorThread::reactor() {
+Reactor* ReactorThread::reactor() {
return reactor_;
}
@@ -564,7 +570,7 @@ Status ReactorThread::FindOrStartConnection(const ConnectionId& conn_id,
// Create a new socket and start connecting to the remote.
Socket sock;
- RETURN_NOT_OK(CreateClientSocket(&sock));
+ RETURN_NOT_OK(CreateClientSocket(conn_id.remote().family(), &sock));
RETURN_NOT_OK(StartConnect(&sock, conn_id.remote()));
unique_ptr<Socket> new_socket(new Socket(sock.Release()));
@@ -603,10 +609,14 @@ Status ReactorThread::StartConnectionNegotiation(const scoped_refptr<Connection>
TRACE("Submitting negotiation task for $0", conn->ToString());
auto authentication = reactor()->messenger()->authentication();
auto encryption = reactor()->messenger()->encryption();
+ auto loopback_encryption = reactor()->messenger()->loopback_encryption();
ThreadPool* negotiation_pool =
reactor()->messenger()->negotiation_pool(conn->direction());
- RETURN_NOT_OK(negotiation_pool->SubmitClosure(
- Bind(&Negotiation::RunNegotiation, conn, authentication, encryption, deadline)));
+ RETURN_NOT_OK(negotiation_pool->Submit([conn, authentication, encryption, loopback_encryption,
+ deadline]() {
+ Negotiation::RunNegotiation(conn, authentication, encryption, loopback_encryption,
+ deadline);
+ }));
return Status::OK();
}
@@ -628,7 +638,8 @@ void ReactorThread::CompleteConnectionNegotiation(
return;
}
- if (FLAGS_tcp_keepalive_probe_period_s > 0) {
+ if (conn->remote().is_ip() &&
+ FLAGS_tcp_keepalive_probe_period_s > 0) {
// Try spreading out the idle poll period to avoid thundering herd in case connections
// are all created at the same time (e.g. after a cluster is restarted).
Status keepalive_status = conn->SetTcpKeepAlive(
@@ -646,9 +657,9 @@ void ReactorThread::CompleteConnectionNegotiation(
conn->EpollRegister(loop_);
}
-Status ReactorThread::CreateClientSocket(Socket *sock) {
- Status ret = sock->Init(Socket::FLAG_NONBLOCKING);
- if (ret.ok()) {
+Status ReactorThread::CreateClientSocket(int family, Socket* sock) {
+ Status ret = sock->Init(family, Socket::FLAG_NONBLOCKING);
+ if (ret.ok() && family == AF_INET) {
ret = sock->SetNoDelay(true);
}
LOG_IF(WARNING, !ret.ok())
@@ -657,7 +668,7 @@ Status ReactorThread::CreateClientSocket(Socket *sock) {
return ret;
}
-Status ReactorThread::StartConnect(Socket *sock, const Sockaddr& remote) {
+Status ReactorThread::StartConnect(Socket* sock, const Sockaddr& remote) {
const Status ret = sock->Connect(remote);
if (ret.ok()) {
VLOG(3) << "StartConnect: connect finished immediately for " << remote.ToString();
@@ -675,7 +686,7 @@ Status ReactorThread::StartConnect(Socket *sock, const Sockaddr& remote) {
return ret;
}
-void ReactorThread::DestroyConnection(Connection *conn,
+void ReactorThread::DestroyConnection(Connection* conn,
const Status& conn_status,
unique_ptr<ErrorStatusPB> rpc_error) {
DCHECK(IsCurrentThread());
@@ -706,7 +717,7 @@ void ReactorThread::DestroyConnection(Connection *conn,
}
}
-DelayedTask::DelayedTask(boost::function<void(const Status&)> func,
+DelayedTask::DelayedTask(std::function<void(const Status&)> func,
MonoDelta when)
: func_(std::move(func)),
when_(when),
@@ -787,7 +798,7 @@ Reactor::~Reactor() {
Shutdown(Messenger::ShutdownMode::ASYNC);
}
-const std::string& Reactor::name() const {
+const string& Reactor::name() const {
return name_;
}
@@ -799,7 +810,7 @@ bool Reactor::closing() const {
// Task to call an arbitrary function within the reactor thread.
class RunFunctionTask : public ReactorTask {
public:
- explicit RunFunctionTask(boost::function<Status()> f)
+ explicit RunFunctionTask(std::function<Status()> f)
: function_(std::move(f)), latch_(1) {}
void Run(ReactorThread* /*reactor*/) override {
@@ -819,25 +830,24 @@ class RunFunctionTask : public ReactorTask {
}
private:
- boost::function<Status()> function_;
+ const std::function<Status()> function_;
Status status_;
CountDownLatch latch_;
};
-Status Reactor::GetMetrics(ReactorMetrics *metrics) {
- return RunOnReactorThread(boost::bind(&ReactorThread::GetMetrics, &thread_, metrics));
+Status Reactor::GetMetrics(ReactorMetrics* metrics) {
+ return RunOnReactorThread([&]() { return this->thread_.GetMetrics(metrics); });
}
-Status Reactor::RunOnReactorThread(const boost::function<Status()>& f) {
- RunFunctionTask task(f);
+Status Reactor::RunOnReactorThread(std::function<Status()> f) {
+ RunFunctionTask task(std::move(f));
ScheduleReactorTask(&task);
return task.Wait();
}
Status Reactor::DumpConnections(const DumpConnectionsRequestPB& req,
DumpConnectionsResponsePB* resp) {
- return RunOnReactorThread(boost::bind(&ReactorThread::DumpConnections,
- &thread_, boost::ref(req), resp));
+ return RunOnReactorThread([&]() { return this->thread_.DumpConnections(req, resp); });
}
class RegisterConnectionTask : public ReactorTask {
@@ -859,10 +869,10 @@ class RegisterConnectionTask : public ReactorTask {
}
private:
- scoped_refptr<Connection> conn_;
+ const scoped_refptr<Connection> conn_;
};
-void Reactor::RegisterInboundSocket(Socket *socket, const Sockaddr& remote) {
+void Reactor::RegisterInboundSocket(Socket* socket, const Sockaddr& remote) {
VLOG(3) << name_ << ": new inbound connection to " << remote.ToString();
unique_ptr<Socket> new_socket(new Socket(socket->Release()));
auto task = new RegisterConnectionTask(
@@ -890,17 +900,17 @@ class AssignOutboundCallTask : public ReactorTask {
}
private:
- shared_ptr<OutboundCall> call_;
+ const shared_ptr<OutboundCall> call_;
};
-void Reactor::QueueOutboundCall(const shared_ptr<OutboundCall>& call) {
+void Reactor::QueueOutboundCall(shared_ptr<OutboundCall> call) {
DVLOG(3) << name_ << ": queueing outbound call "
<< call->ToString() << " to remote " << call->conn_id().remote().ToString();
// Test cancellation when 'call_' is in 'READY' state.
if (PREDICT_FALSE(call->ShouldInjectCancellation())) {
QueueCancellation(call);
}
- ScheduleReactorTask(new AssignOutboundCallTask(call));
+ ScheduleReactorTask(new AssignOutboundCallTask(std::move(call)));
}
class CancellationTask : public ReactorTask {
@@ -918,14 +928,15 @@ class CancellationTask : public ReactorTask {
}
private:
- shared_ptr<OutboundCall> call_;
+ const shared_ptr<OutboundCall> call_;
};
-void Reactor::QueueCancellation(const shared_ptr<OutboundCall>& call) {
- ScheduleReactorTask(new CancellationTask(call));
+void Reactor::QueueCancellation(shared_ptr<OutboundCall> call) {
+ ScheduleReactorTask(new CancellationTask(std::move(call)));
}
-void Reactor::ScheduleReactorTask(ReactorTask *task) {
+void Reactor::ScheduleReactorTask(ReactorTask* task) {
+ bool was_empty;
{
std::unique_lock<LockType> l(lock_);
if (closing_) {
@@ -934,12 +945,15 @@ void Reactor::ScheduleReactorTask(ReactorTask *task) {
task->Abort(ShutdownError(false));
return;
}
+ was_empty = pending_tasks_.empty();
pending_tasks_.push_back(*task);
}
- thread_.WakeThread();
+ if (was_empty) {
+ thread_.WakeThread();
+ }
}
-bool Reactor::DrainTaskQueue(boost::intrusive::list<ReactorTask> *tasks) { // NOLINT(*)
+bool Reactor::DrainTaskQueue(boost::intrusive::list<ReactorTask>* tasks) { // NOLINT(*)
std::lock_guard<LockType> l(lock_);
if (closing_) {
return false;
diff --git a/be/src/kudu/rpc/reactor.h b/be/src/kudu/rpc/reactor.h
index 1767357..8a966c0 100644
--- a/be/src/kudu/rpc/reactor.h
+++ b/be/src/kudu/rpc/reactor.h
@@ -17,12 +17,12 @@
#pragma once
#include <cstdint>
+#include <functional>
#include <list>
#include <memory>
#include <string>
#include <unordered_map>
-#include <boost/function.hpp> // IWYU pragma: keep
#include <boost/intrusive/list.hpp>
#include <boost/intrusive/list_hook.hpp>
#include <ev++.h>
@@ -54,6 +54,7 @@ class DumpConnectionsResponsePB;
class OutboundCall;
class Reactor;
class ReactorThread;
+
enum class CredentialsPolicy;
// Simple metrics information from within a reactor.
@@ -76,7 +77,7 @@ class ReactorTask : public boost::intrusive::list_base_hook<> {
ReactorTask();
// Run the task. 'reactor' is guaranteed to be the current thread.
- virtual void Run(ReactorThread *reactor) = 0;
+ virtual void Run(ReactorThread* reactor) = 0;
// Abort the task, in the case that the reactor shut down before the
// task could be processed. This may or may not run on the reactor thread
@@ -84,7 +85,7 @@ class ReactorTask : public boost::intrusive::list_base_hook<> {
//
// The Reactor guarantees that the Reactor lock is free when this
// method is called.
- virtual void Abort(const Status &abort_status) {}
+ virtual void Abort(const Status& abort_status) {}
virtual ~ReactorTask();
@@ -101,7 +102,7 @@ class ReactorTask : public boost::intrusive::list_base_hook<> {
// receives a Status as its first argument.
class DelayedTask : public ReactorTask {
public:
- DelayedTask(boost::function<void(const Status &)> func, MonoDelta when);
+ DelayedTask(std::function<void(const Status&)> func, MonoDelta when);
// Schedules the task for running later but doesn't actually run it yet.
void Run(ReactorThread* thread) override;
@@ -114,7 +115,7 @@ class DelayedTask : public ReactorTask {
void TimerHandler(ev::timer& watcher, int revents);
// User function to invoke when timer fires or when task is aborted.
- const boost::function<void(const Status&)> func_;
+ const std::function<void(const Status&)> func_;
// Delay to apply to this task.
const MonoDelta when_;
@@ -143,7 +144,7 @@ class ReactorThread {
ConnectionIdHash, ConnectionIdEqual>
conn_multimap_t;
- ReactorThread(Reactor *reactor, const MessengerBuilder &bld);
+ ReactorThread(Reactor* reactor, const MessengerBuilder& bld);
// This may be called from another thread.
Status Init();
@@ -162,14 +163,14 @@ class ReactorThread {
void WakeThread();
// libev callback for handling async notifications in our epoll thread.
- void AsyncHandler(ev::async &watcher, int revents);
+ void AsyncHandler(ev::async& watcher, int revents);
// libev callback for handling timer events in our epoll thread.
- void TimerHandler(ev::timer &watcher, int revents);
+ void TimerHandler(ev::timer& watcher, int revents);
// Register an epoll timer watcher with our event loop.
// Does not set a timeout or start it.
- void RegisterTimeout(ev::timer *watcher);
+ void RegisterTimeout(ev::timer* watcher);
// This may be called from another thread.
const std::string &name() const;
@@ -240,7 +241,7 @@ class ReactorThread {
// The connection is not explicitly deleted -- shared_ptr reference counting
// may hold on to the object after this, but callers should assume that it
// _may_ be deleted by this call.
- void DestroyConnection(Connection *conn, const Status &conn_status,
+ void DestroyConnection(Connection* conn, const Status& conn_status,
std::unique_ptr<ErrorStatusPB> rpc_error = {});
// Scan any open connections for idle ones that have been idle longer than
@@ -249,10 +250,10 @@ class ReactorThread {
void ScanIdleConnections();
// Create a new client socket (non-blocking, NODELAY)
- static Status CreateClientSocket(Socket *sock);
+ static Status CreateClientSocket(int family, Socket* sock);
// Initiate a new connection on the given socket.
- static Status StartConnect(Socket *sock, const Sockaddr &remote);
+ static Status StartConnect(Socket* sock, const Sockaddr& remote);
// Assign a new outbound call to the appropriate connection object.
// If this fails, the call is marked failed and completed.
@@ -263,7 +264,7 @@ class ReactorThread {
// Also mark the call as slated for cancellation so the callback
// may be invoked early if the RPC hasn't yet been sent or if it's
// waiting for a response from the remote.
- void CancelOutboundCall(const std::shared_ptr<OutboundCall> &call);
+ void CancelOutboundCall(const std::shared_ptr<OutboundCall>& call);
// Register a new connection.
void RegisterConnection(scoped_refptr<Connection> conn);
@@ -344,7 +345,7 @@ class Reactor {
public:
Reactor(std::shared_ptr<Messenger> messenger,
int index,
- const MessengerBuilder &bld);
+ const MessengerBuilder& bld);
Status Init();
// Shuts down the reactor and its corresponding thread, optionally waiting
@@ -353,10 +354,10 @@ class Reactor {
~Reactor();
- const std::string &name() const;
+ const std::string& name() const;
// Collect metrics about the reactor.
- Status GetMetrics(ReactorMetrics *metrics);
+ Status GetMetrics(ReactorMetrics* metrics);
// Add any connections on this reactor thread into the given status dump.
Status DumpConnections(const DumpConnectionsRequestPB& req,
@@ -365,14 +366,14 @@ class Reactor {
// Queue a new incoming connection. Takes ownership of the underlying fd from
// 'socket', but not the Socket object itself.
// If the reactor is already shut down, takes care of closing the socket.
- void RegisterInboundSocket(Socket *socket, const Sockaddr &remote);
+ void RegisterInboundSocket(Socket* socket, const Sockaddr& remote);
// Queue a new call to be sent. If the reactor is already shut down, marks
// the call as failed.
- void QueueOutboundCall(const std::shared_ptr<OutboundCall> &call);
+ void QueueOutboundCall(std::shared_ptr<OutboundCall> call);
// Queue a new reactor task to cancel an outbound call.
- void QueueCancellation(const std::shared_ptr<OutboundCall> &call);
+ void QueueCancellation(std::shared_ptr<OutboundCall> call);
// Schedule the given task's Run() method to be called on the
// reactor thread.
@@ -380,15 +381,15 @@ class Reactor {
// called.
// Does _not_ take ownership of 'task' -- the task should take care of
// deleting itself after running if it is allocated on the heap.
- void ScheduleReactorTask(ReactorTask *task);
+ void ScheduleReactorTask(ReactorTask* task);
- Status RunOnReactorThread(const boost::function<Status()>& f);
+ Status RunOnReactorThread(std::function<Status()> f);
// If the Reactor is closing, returns false.
// Otherwise, drains the pending_tasks_ queue into the provided list.
- bool DrainTaskQueue(boost::intrusive::list<ReactorTask> *tasks);
+ bool DrainTaskQueue(boost::intrusive::list<ReactorTask>* tasks);
- Messenger *messenger() const {
+ Messenger* messenger() const {
return messenger_.get();
}
diff --git a/be/src/kudu/rpc/remote_method.h b/be/src/kudu/rpc/remote_method.h
index b5e42ae..9c8ae72 100644
--- a/be/src/kudu/rpc/remote_method.h
+++ b/be/src/kudu/rpc/remote_method.h
@@ -30,8 +30,8 @@ class RemoteMethod {
public:
RemoteMethod() {}
RemoteMethod(std::string service_name, std::string method_name);
- std::string service_name() const { return service_name_; }
- std::string method_name() const { return method_name_; }
+ const std::string& service_name() const { return service_name_; }
+ const std::string& method_name() const { return method_name_; }
// Encode/decode to/from 'pb'.
void FromPB(const RemoteMethodPB& pb);
diff --git a/be/src/kudu/rpc/response_callback.h b/be/src/kudu/rpc/response_callback.h
index 5a70a79..98065b5 100644
--- a/be/src/kudu/rpc/response_callback.h
+++ b/be/src/kudu/rpc/response_callback.h
@@ -16,12 +16,12 @@
// under the License.
#pragma once
-#include <boost/function.hpp>
+#include <functional>
namespace kudu {
namespace rpc {
-typedef boost::function<void()> ResponseCallback;
+typedef std::function<void()> ResponseCallback;
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/result_tracker.cc b/be/src/kudu/rpc/result_tracker.cc
index f8f5c46..d5a376c 100644
--- a/be/src/kudu/rpc/result_tracker.cc
+++ b/be/src/kudu/rpc/result_tracker.cc
@@ -375,7 +375,7 @@ void ResultTracker::FailAndRespondInternal(const RequestIdPB& request_id,
// It is possible for this method to be called for an RPC that was never actually
// tracked (though RecordCompletionAndRespond() can't). One such case is when a
- // follower transaction fails on the TransactionManager, for some reason, before it
+ // follower op fails on the OpDriver, for some reason, before it
// was tracked. The CompletionCallback still calls this method. In this case, do
// nothing.
if (completion_record == nullptr) {
@@ -459,8 +459,8 @@ void ResultTracker::FailAndRespond(const RequestIdPB& request_id,
void ResultTracker::StartGCThread() {
CHECK(!gc_thread_);
- CHECK_OK(Thread::Create("server", "result-tracker", &ResultTracker::RunGCThread,
- this, &gc_thread_));
+ CHECK_OK(Thread::Create("server", "result-tracker",
+ [this]() { this->RunGCThread(); }, &gc_thread_));
}
void ResultTracker::RunGCThread() {
diff --git a/be/src/kudu/rpc/result_tracker.h b/be/src/kudu/rpc/result_tracker.h
index f095728..79d43c1 100644
--- a/be/src/kudu/rpc/result_tracker.h
+++ b/be/src/kudu/rpc/result_tracker.h
@@ -129,18 +129,18 @@ class RpcContext;
// In order to make sure there is only one driver, there must be an _external_ serialization
// point, before the final response is produced, after which only one of the handlers will
// be marked as the driver. For instance, for writes, this serialization point is in
-// TransactionDriver, in a synchronized block where a logic such as this one happens (here
+// OpDriver, in a synchronized block where a logic such as this one happens (here
// in pseudo-ish code):
//
// {
// lock_guard<simple_spinlock> l(lock_);
-// if (follower_transaction) {
+// if (follower_op) {
// result_tracker_->TrackRpcOrChangeDriver(request_id);
-// continue_with_transaction();
-// } else if (client_transaction) {
+// continue_with_op();
+// } else if (client_op) {
// bool is_still_driver = result_tracker_->IsCurrentDriver(request_id);
-// if (is_still_driver) continue_with_transaction();
-// else abort_transaction();
+// if (is_still_driver) continue_with_op();
+// else abort_op();
// }
// }
//
@@ -282,8 +282,7 @@ class ResultTracker : public RefCountedThreadSafe<ResultTracker> {
// Calculates the memory footprint of this struct.
int64_t memory_footprint() const {
return kudu_malloc_usable_size(this)
- + (ongoing_rpcs.capacity() > 0 ? kudu_malloc_usable_size(ongoing_rpcs.data()) :
- 0)
+ + (ongoing_rpcs.capacity() > 0 ? kudu_malloc_usable_size(ongoing_rpcs.data()) : 0)
+ (response ? response->SpaceUsedLong() : 0);
}
};
diff --git a/be/src/kudu/rpc/retriable_rpc.h b/be/src/kudu/rpc/retriable_rpc.h
index 9a0b8f5..3668b55 100644
--- a/be/src/kudu/rpc/retriable_rpc.h
+++ b/be/src/kudu/rpc/retriable_rpc.h
@@ -16,6 +16,7 @@
// under the License.
#pragma once
+#include <functional>
#include <memory>
#include <string>
@@ -159,9 +160,11 @@ void RetriableRpc<Server, RequestPB, ResponsePB>::SendRpc() {
if (sequence_number_ == RequestTracker::kNoSeqNo) {
CHECK_OK(request_tracker_->NewSeqNo(&sequence_number_));
}
- server_picker_->PickLeader(Bind(&RetriableRpc::ReplicaFoundCb,
- Unretained(this)),
- retrier().deadline());
+ server_picker_->PickLeader(
+ [this](const Status& status, Server* server) {
+ this->ReplicaFoundCb(status, server);
+ },
+ retrier().deadline());
}
template <class Server, class RequestPB, class ResponsePB>
@@ -301,7 +304,7 @@ void RetriableRpc<Server, RequestPB, ResponsePB>::ReplicaFoundCb(const Status& s
DCHECK_EQ(result.result, RetriableRpcStatus::OK);
current_ = server;
- Try(server, boost::bind(&RetriableRpc::SendRpcCb, this, Status::OK()));
+ Try(server, [this]() { this->SendRpcCb(Status::OK()); });
}
template <class Server, class RequestPB, class ResponsePB>
diff --git a/be/src/kudu/rpc/rpc-bench.cc b/be/src/kudu/rpc/rpc-bench.cc
index a825900..dca665e 100644
--- a/be/src/kudu/rpc/rpc-bench.cc
+++ b/be/src/kudu/rpc/rpc-bench.cc
@@ -25,7 +25,6 @@
#include <vector>
#include <gflags/gflags.h>
-#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
@@ -46,7 +45,6 @@
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
-using std::bind;
using std::shared_ptr;
using std::string;
using std::thread;
@@ -232,7 +230,7 @@ class ClientAsyncWorkload {
proxy_->AddAsync(req_,
&resp_,
&controller_,
- bind(&ClientAsyncWorkload::CallOneRpc, this));
+ [this]() { this->CallOneRpc(); });
}
void Start() {
diff --git a/be/src/kudu/rpc/rpc-test-base.h b/be/src/kudu/rpc/rpc-test-base.h
index fcc7452..0aa46ce 100644
--- a/be/src/kudu/rpc/rpc-test-base.h
+++ b/be/src/kudu/rpc/rpc-test-base.h
@@ -20,6 +20,8 @@
#include <atomic>
#include <memory>
#include <string>
+#include <thread>
+#include <vector>
#include "kudu/gutil/walltime.h"
#include "kudu/rpc/acceptor_pool.h"
@@ -37,6 +39,7 @@
#include "kudu/rpc/service_if.h"
#include "kudu/rpc/service_pool.h"
#include "kudu/security/security-test-util.h"
+#include "kudu/util/crc.h"
#include "kudu/util/env.h"
#include "kudu/util/faststring.h"
#include "kudu/util/mem_tracker.h"
@@ -53,9 +56,6 @@
DECLARE_bool(rpc_encrypt_loopback_connections);
-namespace kudu {
-namespace rpc {
-
using kudu::rpc_test::AddRequestPB;
using kudu::rpc_test::AddResponsePB;
using kudu::rpc_test::CalculatorError;
@@ -68,8 +68,8 @@ using kudu::rpc_test::ExactlyOnceResponsePB;
using kudu::rpc_test::FeatureFlags;
using kudu::rpc_test::PanicRequestPB;
using kudu::rpc_test::PanicResponsePB;
-using kudu::rpc_test::PushTwoStringsRequestPB;
-using kudu::rpc_test::PushTwoStringsResponsePB;
+using kudu::rpc_test::PushStringsRequestPB;
+using kudu::rpc_test::PushStringsResponsePB;
using kudu::rpc_test::SendTwoStringsRequestPB;
using kudu::rpc_test::SendTwoStringsResponsePB;
using kudu::rpc_test::SleepRequestPB;
@@ -83,17 +83,20 @@ using kudu::rpc_test::WhoAmIResponsePB;
using kudu::rpc_test_diff_package::ReqDiffPackagePB;
using kudu::rpc_test_diff_package::RespDiffPackagePB;
+namespace kudu {
+namespace rpc {
+
// Implementation of CalculatorService which just implements the generic
// RPC handler (no generated code).
class GenericCalculatorService : public ServiceIf {
public:
- static const char *kFullServiceName;
- static const char *kAddMethodName;
- static const char *kSleepMethodName;
- static const char *kSleepWithSidecarMethodName;
- static const char *kPushTwoStringsMethodName;
- static const char *kSendTwoStringsMethodName;
- static const char *kAddExactlyOnce;
+ static const std::string kFullServiceName;
+ static const std::string kAddMethodName;
+ static const std::string kSleepMethodName;
+ static const std::string kSleepWithSidecarMethodName;
+ static const std::string kPushStringsMethodName;
+ static const std::string kSendTwoStringsMethodName;
+ static const std::string kAddExactlyOnce;
static const char* kFirstString;
static const char* kSecondString;
@@ -116,16 +119,16 @@ class GenericCalculatorService : public ServiceIf {
DoSleepWithSidecar(incoming);
} else if (incoming->remote_method().method_name() == kSendTwoStringsMethodName) {
DoSendTwoStrings(incoming);
- } else if (incoming->remote_method().method_name() == kPushTwoStringsMethodName) {
- DoPushTwoStrings(incoming);
+ } else if (incoming->remote_method().method_name() == kPushStringsMethodName) {
+ DoPushStrings(incoming);
} else {
incoming->RespondFailure(ErrorStatusPB::ERROR_NO_SUCH_METHOD,
Status::InvalidArgument("bad method"));
}
}
- std::string service_name() const override { return kFullServiceName; }
- static std::string static_service_name() { return kFullServiceName; }
+ const std::string& service_name() const override { return kFullServiceName; }
+ static const std::string& static_service_name() { return kFullServiceName; }
private:
void DoAdd(InboundCall *incoming) {
@@ -147,50 +150,55 @@ class GenericCalculatorService : public ServiceIf {
LOG(FATAL) << "couldn't parse: " << param.ToDebugString();
}
- std::unique_ptr<faststring> first(new faststring);
- std::unique_ptr<faststring> second(new faststring);
+ faststring first;
Random r(req.random_seed());
- first->resize(req.size1());
- RandomString(first->data(), req.size1(), &r);
+ first.resize(req.size1());
+ RandomString(first.data(), req.size1(), &r);
- second->resize(req.size2());
- RandomString(second->data(), req.size2(), &r);
+ // The second string gets sent in two separate buffers, which get
+ // concatenated on the client side.
+ faststring second_data;
+ second_data.resize(req.size2());
+ RandomString(second_data.data(), second_data.size(), &r);
+
+ std::vector<faststring> second(2);
+ second[0].append(second_data.data(), second_data.size() / 3);
+ second[1].append(second_data.data() + second[0].size(),
+ second_data.size() - second[0].size());
SendTwoStringsResponsePB resp;
int idx1, idx2;
CHECK_OK(incoming->AddOutboundSidecar(
RpcSidecar::FromFaststring(std::move(first)), &idx1));
+
CHECK_OK(incoming->AddOutboundSidecar(
- RpcSidecar::FromFaststring(std::move(second)), &idx2));
+ RpcSidecar::FromFaststrings(std::move(second)), &idx2));
resp.set_sidecar1(idx1);
resp.set_sidecar2(idx2);
incoming->RespondSuccess(resp);
}
- void DoPushTwoStrings(InboundCall* incoming) {
+ static void DoPushStrings(InboundCall* incoming) {
Slice param(incoming->serialized_request());
- PushTwoStringsRequestPB req;
+ PushStringsRequestPB req;
if (!req.ParseFromArray(param.data(), param.size())) {
LOG(FATAL) << "couldn't parse: " << param.ToDebugString();
}
- Slice sidecar1;
- CHECK_OK(incoming->GetInboundSidecar(req.sidecar1_idx(), &sidecar1));
-
- Slice sidecar2;
- CHECK_OK(incoming->GetInboundSidecar(req.sidecar2_idx(), &sidecar2));
-
// Check that reading non-existant sidecars doesn't work.
Slice tmp;
- CHECK(!incoming->GetInboundSidecar(req.sidecar2_idx() + 2, &tmp).ok());
+ CHECK(!incoming->GetInboundSidecar(req.sidecar_indexes_size() + 2, &tmp).ok());
- PushTwoStringsResponsePB resp;
- resp.set_size1(sidecar1.size());
- resp.set_data1(reinterpret_cast<const char*>(sidecar1.data()), sidecar1.size());
- resp.set_size2(sidecar2.size());
- resp.set_data2(reinterpret_cast<const char*>(sidecar2.data()), sidecar2.size());
+ PushStringsResponsePB resp;
+ for (const auto& sidecar_idx : req.sidecar_indexes()) {
+ Slice sidecar;
+ CHECK_OK(incoming->GetInboundSidecar(sidecar_idx, &sidecar));
+
+ resp.add_sizes(sidecar.size());
+ resp.add_crcs(crc::Crc32c(sidecar.data(), sidecar.size()));
+ }
// Drop the sidecars etc, just to confirm that it's safe to do so.
CHECK_GT(incoming->GetTransferSize(), 0);
@@ -253,6 +261,7 @@ class CalculatorService : public CalculatorServiceIf {
void Add(const AddRequestPB *req, AddResponsePB *resp, RpcContext *context) override {
CHECK_GT(context->GetTransferSize(), 0);
+ CHECK_GE(context->call_id(), 0);
resp->set_result(req->x() + req->y());
context->RespondSuccess();
}
@@ -280,11 +289,8 @@ class CalculatorService : public CalculatorServiceIf {
}
if (req->deferred()) {
- // Spawn a new thread which does the sleep and responds later.
- scoped_refptr<Thread> thread;
- CHECK_OK(Thread::Create("rpc-test", "deferred",
- &CalculatorService::DoSleep, this, req, context,
- &thread));
+ std::thread t([this, req, context]() { this->DoSleep(req, context); });
+ t.detach();
return;
}
DoSleep(req, context);
@@ -394,13 +400,14 @@ class CalculatorService : public CalculatorServiceIf {
};
-const char *GenericCalculatorService::kFullServiceName = "kudu.rpc.GenericCalculatorService";
-const char *GenericCalculatorService::kAddMethodName = "Add";
-const char *GenericCalculatorService::kSleepMethodName = "Sleep";
-const char *GenericCalculatorService::kSleepWithSidecarMethodName = "SleepWithSidecar";
-const char *GenericCalculatorService::kPushTwoStringsMethodName = "PushTwoStrings";
-const char *GenericCalculatorService::kSendTwoStringsMethodName = "SendTwoStrings";
-const char *GenericCalculatorService::kAddExactlyOnce = "AddExactlyOnce";
+const std::string GenericCalculatorService::kFullServiceName =
+ "kudu.rpc.GenericCalculatorService";
+const std::string GenericCalculatorService::kAddMethodName = "Add";
+const std::string GenericCalculatorService::kSleepMethodName = "Sleep";
+const std::string GenericCalculatorService::kSleepWithSidecarMethodName = "SleepWithSidecar";
+const std::string GenericCalculatorService::kPushStringsMethodName = "PushStrings";
+const std::string GenericCalculatorService::kSendTwoStringsMethodName = "SendTwoStrings";
+const std::string GenericCalculatorService::kAddExactlyOnce = "AddExactlyOnce";
const char *GenericCalculatorService::kFirstString =
"1111111111111111111111111111111111111111111111111111111111";
@@ -410,11 +417,12 @@ const char *GenericCalculatorService::kSecondString =
class RpcTestBase : public KuduTest {
public:
RpcTestBase()
- : n_worker_threads_(3),
- service_queue_length_(100),
- n_server_reactor_threads_(3),
- keepalive_time_ms_(1000),
- metric_entity_(METRIC_ENTITY_server.Instantiate(&metric_registry_, "test.rpc_test")) {
+ : n_acceptor_pool_threads_(2),
+ n_server_reactor_threads_(3),
+ n_worker_threads_(3),
+ keepalive_time_ms_(1000),
+ service_queue_length_(200),
+ metric_entity_(METRIC_ENTITY_server.Instantiate(&metric_registry_, "test.rpc_test")) {
}
void TearDown() override {
@@ -461,8 +469,8 @@ class RpcTestBase : public KuduTest {
return bld.Build(messenger);
}
- Status DoTestSyncCall(const Proxy &p, const char *method,
- CredentialsPolicy policy = CredentialsPolicy::ANY_CREDENTIALS) {
+ static Status DoTestSyncCall(Proxy* p, const std::string& method,
+ CredentialsPolicy policy = CredentialsPolicy::ANY_CREDENTIALS) {
AddRequestPB req;
req.set_x(rand());
req.set_y(rand());
@@ -470,13 +478,13 @@ class RpcTestBase : public KuduTest {
RpcController controller;
controller.set_timeout(MonoDelta::FromMilliseconds(10000));
controller.set_credentials_policy(policy);
- RETURN_NOT_OK(p.SyncRequest(method, req, &resp, &controller));
+ RETURN_NOT_OK(p->SyncRequest(method, req, &resp, &controller));
CHECK_EQ(req.x() + req.y(), resp.result());
return Status::OK();
}
- void DoTestSidecar(const Proxy &p, int size1, int size2) {
+static void DoTestSidecar(Proxy* p, int size1, int size2) {
const uint32_t kSeed = 12345;
SendTwoStringsRequestPB req;
@@ -487,8 +495,8 @@ class RpcTestBase : public KuduTest {
SendTwoStringsResponsePB resp;
RpcController controller;
controller.set_timeout(MonoDelta::FromMilliseconds(10000));
- CHECK_OK(p.SyncRequest(GenericCalculatorService::kSendTwoStringsMethodName,
- req, &resp, &controller));
+ CHECK_OK(p->SyncRequest(GenericCalculatorService::kSendTwoStringsMethodName,
+ req, &resp, &controller));
Slice first = GetSidecarPointer(controller, resp.sidecar1(), size1);
Slice second = GetSidecarPointer(controller, resp.sidecar2(), size2);
@@ -497,43 +505,44 @@ class RpcTestBase : public KuduTest {
expected.resize(size1);
RandomString(expected.data(), size1, &rng);
- CHECK_EQ(0, first.compare(Slice(expected)));
+ CHECK_EQ(Slice(expected), first);
expected.resize(size2);
RandomString(expected.data(), size2, &rng);
- CHECK_EQ(0, second.compare(Slice(expected)));
+ CHECK_EQ(Slice(expected), second);
}
- Status DoTestOutgoingSidecar(const Proxy &p, int size1, int size2) {
- PushTwoStringsRequestPB request;
- RpcController controller;
+ static Status DoTestOutgoingSidecar(Proxy* p, int size1, int size2) {
+ return DoTestOutgoingSidecar(p, {std::string(size1, 'a'), std::string(size2, 'b')});
+ }
- int idx1;
- std::string s1(size1, 'a');
- CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(Slice(s1)), &idx1));
+ static Status DoTestOutgoingSidecar(Proxy* p, const std::vector<std::string>& strings) {
+ PushStringsRequestPB request;
+ RpcController controller;
- int idx2;
- std::string s2(size2, 'b');
- CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(Slice(s2)), &idx2));
+ for (const auto& s : strings) {
+ int idx;
+ CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(Slice(s)), &idx));
+ request.add_sidecar_indexes(idx);
+ }
- request.set_sidecar1_idx(idx1);
- request.set_sidecar2_idx(idx2);
+ PushStringsResponsePB resp;
+ KUDU_RETURN_NOT_OK(p->SyncRequest(GenericCalculatorService::kPushStringsMethodName,
+ request, &resp, &controller));
+ for (int i = 0; i < strings.size(); i++) {
+ CHECK_EQ(strings[i].size(), resp.sizes(i));
+ CHECK_EQ(crc::Crc32c(strings[i].data(), strings[i].size()),
+ resp.crcs(i));
+ }
- PushTwoStringsResponsePB resp;
- KUDU_RETURN_NOT_OK(p.SyncRequest(GenericCalculatorService::kPushTwoStringsMethodName,
- request, &resp, &controller));
- CHECK_EQ(size1, resp.size1());
- CHECK_EQ(resp.data1(), s1);
- CHECK_EQ(size2, resp.size2());
- CHECK_EQ(resp.data2(), s2);
return Status::OK();
}
- void DoTestOutgoingSidecarExpectOK(const Proxy &p, int size1, int size2) {
+ static void DoTestOutgoingSidecarExpectOK(Proxy* p, int size1, int size2) {
CHECK_OK(DoTestOutgoingSidecar(p, size1, size2));
}
- static void DoTestExpectTimeout(const Proxy& p,
+ static void DoTestExpectTimeout(Proxy* p,
const MonoDelta& timeout,
bool will_be_cancelled = false,
bool* is_negotiaton_error = nullptr) {
@@ -547,7 +556,7 @@ class RpcTestBase : public KuduTest {
c.set_timeout(timeout);
Stopwatch sw;
sw.start();
- Status s = p.SyncRequest(GenericCalculatorService::kSleepMethodName, req, &resp, &c);
+ Status s = p->SyncRequest(GenericCalculatorService::kSleepMethodName, req, &resp, &c);
sw.stop();
ASSERT_FALSE(s.ok());
if (is_negotiaton_error != nullptr) {
@@ -565,9 +574,11 @@ class RpcTestBase : public KuduTest {
// And we also shouldn't take the full time that we asked for
EXPECT_LT(elapsed_millis * 1000, sleep_micros);
if (will_be_cancelled) {
- EXPECT_TRUE(s.IsAborted());
+ // Cancellation is best effort, so even if we cancel the rpc it may still end up
+ // with a timed out status.
+ EXPECT_TRUE(s.IsAborted() || s.IsTimedOut()) << s.ToString();
} else {
- EXPECT_TRUE(s.IsTimedOut());
+ EXPECT_TRUE(s.IsTimedOut()) << s.ToString();
}
LOG(INFO) << "status: " << s.ToString() << ", seconds elapsed: " << sw.elapsed().wall_seconds();
}
@@ -595,10 +606,10 @@ class RpcTestBase : public KuduTest {
// Start a simple socket listening on a local port, returning the address.
// This isn't an RPC server -- just a plain socket which can be helpful for testing.
- Status StartFakeServer(Socket *listen_sock, Sockaddr *listen_addr) {
- Sockaddr bind_addr;
+ static Status StartFakeServer(Socket *listen_sock, Sockaddr *listen_addr) {
+ Sockaddr bind_addr = Sockaddr::Wildcard();
bind_addr.set_port(0);
- RETURN_NOT_OK(listen_sock->Init(0));
+ RETURN_NOT_OK(listen_sock->Init(bind_addr.family(), 0));
RETURN_NOT_OK(listen_sock->BindAndListen(bind_addr, 1));
RETURN_NOT_OK(listen_sock->GetSocketAddress(listen_addr));
LOG(INFO) << "Bound to: " << listen_addr->ToString();
@@ -616,13 +627,17 @@ class RpcTestBase : public KuduTest {
}
template<class ServiceClass>
- Status DoStartTestServer(Sockaddr *server_addr,
+ Status DoStartTestServer(Sockaddr* server_addr,
bool enable_ssl = false,
const std::string& rpc_certificate_file = "",
const std::string& rpc_private_key_file = "",
const std::string& rpc_ca_certificate_file = "",
const std::string& rpc_private_key_password_cmd = "",
const std::shared_ptr<Messenger>& messenger = nullptr) {
+ // Default to binding on wildcard unless specified as an in-out parameter.
+ if (!server_addr->is_initialized()) {
+ *server_addr = Sockaddr::Wildcard();
+ }
if (!messenger) {
RETURN_NOT_OK(CreateMessenger(
"TestServer", &server_messenger_, n_server_reactor_threads_, enable_ssl,
@@ -632,8 +647,8 @@ class RpcTestBase : public KuduTest {
server_messenger_ = messenger;
}
std::shared_ptr<AcceptorPool> pool;
- RETURN_NOT_OK(server_messenger_->AddAcceptorPool(Sockaddr(), &pool));
- RETURN_NOT_OK(pool->Start(2));
+ RETURN_NOT_OK(server_messenger_->AddAcceptorPool(*server_addr, &pool));
+ RETURN_NOT_OK(pool->Start(n_acceptor_pool_threads_));
*server_addr = pool->bind_address();
mem_tracker_ = MemTracker::CreateTracker(-1, "result_tracker");
result_tracker_.reset(new ResultTracker(mem_tracker_));
@@ -642,22 +657,22 @@ class RpcTestBase : public KuduTest {
service_name_ = service->service_name();
scoped_refptr<MetricEntity> metric_entity = server_messenger_->metric_entity();
service_pool_ = new ServicePool(std::move(service), metric_entity, service_queue_length_);
- server_messenger_->RegisterService(service_name_, service_pool_);
- RETURN_NOT_OK(service_pool_->Init(n_worker_threads_));
-
- return Status::OK();
+ RETURN_NOT_OK(server_messenger_->RegisterService(service_name_, service_pool_));
+ return service_pool_->Init(n_worker_threads_);
}
protected:
+ int n_acceptor_pool_threads_;
+ int n_server_reactor_threads_;
+ int n_worker_threads_;
+ int keepalive_time_ms_;
+ int service_queue_length_;
+
std::string service_name_;
std::shared_ptr<Messenger> server_messenger_;
scoped_refptr<ServicePool> service_pool_;
std::shared_ptr<kudu::MemTracker> mem_tracker_;
scoped_refptr<ResultTracker> result_tracker_;
- int n_worker_threads_;
- int service_queue_length_;
- int n_server_reactor_threads_;
- int keepalive_time_ms_;
MetricRegistry metric_registry_;
scoped_refptr<MetricEntity> metric_entity_;
diff --git a/be/src/kudu/rpc/rpc-test.cc b/be/src/kudu/rpc/rpc-test.cc
index 4258de2..4a3aedd 100644
--- a/be/src/kudu/rpc/rpc-test.cc
+++ b/be/src/kudu/rpc/rpc-test.cc
@@ -15,26 +15,29 @@
// specific language governing permissions and limitations
// under the License.
+#include <unistd.h>
+
#include <cerrno>
#include <cstdint>
#include <cstdlib>
#include <cstring>
+#include <functional>
#include <limits>
#include <memory>
#include <ostream>
#include <set>
#include <string>
+#include <thread>
+#include <tuple>
#include <unordered_map>
+#include <utility>
#include <vector>
-#include <boost/bind.hpp>
-#include <boost/core/ref.hpp>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "kudu/gutil/casts.h"
-#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/stl_util.h"
@@ -45,8 +48,10 @@
#include "kudu/rpc/outbound_call.h"
#include "kudu/rpc/proxy.h"
#include "kudu/rpc/reactor.h"
+#include "kudu/rpc/result_tracker.h"
#include "kudu/rpc/rpc-test-base.h"
#include "kudu/rpc/rpc_controller.h"
+#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/rpc_introspection.pb.h"
#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/rpc/rtest.pb.h"
@@ -60,13 +65,15 @@
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/net/socket.h"
#include "kudu/util/random.h"
+#include "kudu/util/random_util.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+#include "kudu/util/stopwatch.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
-#include "kudu/util/thread.h"
+METRIC_DECLARE_counter(queue_overflow_rejections_kudu_rpc_test_CalculatorService_Sleep);
METRIC_DECLARE_histogram(handler_latency_kudu_rpc_test_CalculatorService_Sleep);
METRIC_DECLARE_histogram(rpc_incoming_queue_time);
@@ -76,39 +83,76 @@ DECLARE_int32(tcp_keepalive_probe_period_s);
DECLARE_int32(tcp_keepalive_retry_period_s);
DECLARE_int32(tcp_keepalive_retry_count);
+using std::tuple;
using std::shared_ptr;
using std::string;
+using std::thread;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
namespace kudu {
+
+using strings::Substitute;
+
namespace rpc {
-class TestRpc : public RpcTestBase, public ::testing::WithParamInterface<bool> {
+// RPC proxies require a hostname to be passed. In this test we're just connecting to
+// the wildcard, so we'll hard-code this hostname instead.
+static const char* const kRemoteHostName = "localhost";
+
+class TestRpc : public RpcTestBase, public ::testing::WithParamInterface<tuple<bool,bool>> {
+ protected:
+ TestRpc() {
+ }
+
+ bool enable_ssl() const {
+ return std::get<0>(GetParam());
+ }
+ bool use_unix_socket() const {
+ return std::get<1>(GetParam());
+ }
+ Sockaddr bind_addr() const {
+ if (use_unix_socket()) {
+ // Ensure multiple calls to bind_addr work by unlinking the socket file.
+ // The only way to reuse a socket file is to remove it with unlink().
+ unlink(socket_path_.c_str());
+ Sockaddr addr;
+ CHECK_OK(addr.ParseUnixDomainPath(socket_path_));
+ return addr;
+ }
+ return Sockaddr::Wildcard();
+ }
+ static string expected_remote_str(const Sockaddr& bound_addr) {
+ if (bound_addr.is_ip()) {
+ return Substitute("$0 ($1)", bound_addr.ToString(), kRemoteHostName);
+ }
+ return bound_addr.ToString();
+ }
+ void TearDown() override {
+ RpcTestBase::TearDown();
+ // Ensure we cleanup the socket file on teardown.
+ unlink(socket_path_.c_str());
+ }
+
+ std::string socket_path_ = GetTestSocketPath("rpc-test");
};
-// This is used to run all parameterized tests with and without SSL.
-INSTANTIATE_TEST_CASE_P(OptionalSSL, TestRpc, testing::Values(false, true));
-
-TEST_F(TestRpc, TestSockaddr) {
- Sockaddr addr1, addr2;
- addr1.set_port(1000);
- addr2.set_port(2000);
- // port is ignored when comparing Sockaddr objects
- ASSERT_FALSE(addr1 < addr2);
- ASSERT_FALSE(addr2 < addr1);
- ASSERT_EQ(1000, addr1.port());
- ASSERT_EQ(2000, addr2.port());
- ASSERT_EQ(string("0.0.0.0:1000"), addr1.ToString());
- ASSERT_EQ(string("0.0.0.0:2000"), addr2.ToString());
- Sockaddr addr3(addr1);
- ASSERT_EQ(string("0.0.0.0:1000"), addr3.ToString());
-}
+// This is used to run all parameterized tests with and without SSL, on Unix sockets
+// and TCP.
+INSTANTIATE_TEST_SUITE_P(Parameters, TestRpc,
+ testing::Combine(testing::Values(false, true),
+ testing::Values(false, true)),
+ [](const testing::TestParamInfo<tuple<bool, bool>>& info) {
+ return Substitute("$0_$1",
+ std::get<0>(info.param) ? "SSL" : "NoSSL",
+ std::get<1>(info.param) ? "UnixSocket" : "TCP");
+ });
+
TEST_P(TestRpc, TestMessengerCreateDestroy) {
shared_ptr<Messenger> messenger;
- ASSERT_OK(CreateMessenger("TestCreateDestroy", &messenger, 1, GetParam()));
+ ASSERT_OK(CreateMessenger("TestCreateDestroy", &messenger, 1, enable_ssl()));
LOG(INFO) << "started messenger " << messenger->name();
messenger->Shutdown();
}
@@ -121,12 +165,15 @@ TEST_P(TestRpc, TestAcceptorPoolStartStop) {
int n_iters = AllowSlowTests() ? 100 : 5;
for (int i = 0; i < n_iters; i++) {
shared_ptr<Messenger> messenger;
- ASSERT_OK(CreateMessenger("TestAcceptorPoolStartStop", &messenger, 1, GetParam()));
+ ASSERT_OK(CreateMessenger("TestAcceptorPoolStartStop", &messenger, 1, enable_ssl()));
shared_ptr<AcceptorPool> pool;
- ASSERT_OK(messenger->AddAcceptorPool(Sockaddr(), &pool));
+ ASSERT_OK(messenger->AddAcceptorPool(bind_addr(), &pool));
Sockaddr bound_addr;
ASSERT_OK(pool->GetBoundAddress(&bound_addr));
- ASSERT_NE(0, bound_addr.port());
+ ASSERT_TRUE(bound_addr.is_initialized());
+ if (!use_unix_socket()) {
+ ASSERT_NE(0, bound_addr.port());
+ }
ASSERT_OK(pool->Start(2));
messenger->Shutdown();
}
@@ -142,7 +189,6 @@ TEST_F(TestRpc, TestConnHeaderValidation) {
// Regression test for KUDU-2041
TEST_P(TestRpc, TestNegotiationDeadlock) {
- bool enable_ssl = GetParam();
// The deadlock would manifest in cases where the number of concurrent connection
// requests >= the number of threads. 1 thread and 1 cnxn to ourself is just the easiest
@@ -152,46 +198,44 @@ TEST_P(TestRpc, TestNegotiationDeadlock) {
mb.set_min_negotiation_threads(1)
.set_max_negotiation_threads(1)
.set_metric_entity(metric_entity_);
- if (enable_ssl) mb.enable_inbound_tls();
+ if (enable_ssl()) mb.enable_inbound_tls();
shared_ptr<Messenger> messenger;
CHECK_OK(mb.Build(&messenger));
- Sockaddr server_addr;
- ASSERT_OK(StartTestServerWithCustomMessenger(&server_addr, messenger, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServerWithCustomMessenger(&server_addr, messenger, enable_ssl()));
- Proxy p(messenger, server_addr, server_addr.host(),
+ Proxy p(messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
// Test making successful RPC calls.
TEST_P(TestRpc, TestCall) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- ASSERT_STR_CONTAINS(p.ToString(), strings::Substitute("kudu.rpc.GenericCalculatorService@"
- "{remote=$0, user_credentials=",
- server_addr.ToString()));
+ ASSERT_STR_CONTAINS(p.ToString(), Substitute("kudu.rpc.GenericCalculatorService@"
+ "{remote=$0, user_credentials=",
+ expected_remote_str(server_addr)));
for (int i = 0; i < 10; i++) {
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
}
// Test for KUDU-2091 and KUDU-2220.
TEST_P(TestRpc, TestCallWithChainCertAndChainCA) {
- bool enable_ssl = GetParam();
// We're only interested in running this test with TLS enabled.
- if (!enable_ssl) return;
+ if (!enable_ssl()) return;
string rpc_certificate_file;
string rpc_private_key_file;
@@ -201,29 +245,28 @@ TEST_P(TestRpc, TestCallWithChainCertAndChainCA) {
&rpc_private_key_file,
&rpc_ca_certificate_file));
// Set up server.
- Sockaddr server_addr;
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
SCOPED_TRACE(strings::Substitute("Connecting to $0", server_addr.ToString()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl,
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl(),
rpc_certificate_file, rpc_private_key_file, rpc_ca_certificate_file));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
ASSERT_STR_CONTAINS(p.ToString(), strings::Substitute("kudu.rpc.GenericCalculatorService@"
"{remote=$0, user_credentials=",
- server_addr.ToString()));
+ expected_remote_str(server_addr)));
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
// Test for KUDU-2041.
TEST_P(TestRpc, TestCallWithChainCertAndRootCA) {
- bool enable_ssl = GetParam();
// We're only interested in running this test with TLS enabled.
- if (!enable_ssl) return;
+ if (!enable_ssl()) return;
string rpc_certificate_file;
string rpc_private_key_file;
@@ -233,30 +276,29 @@ TEST_P(TestRpc, TestCallWithChainCertAndRootCA) {
&rpc_private_key_file,
&rpc_ca_certificate_file));
// Set up server.
- Sockaddr server_addr;
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
SCOPED_TRACE(strings::Substitute("Connecting to $0", server_addr.ToString()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl,
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl(),
rpc_certificate_file, rpc_private_key_file, rpc_ca_certificate_file));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
ASSERT_STR_CONTAINS(p.ToString(), strings::Substitute("kudu.rpc.GenericCalculatorService@"
"{remote=$0, user_credentials=",
- server_addr.ToString()));
+ expected_remote_str(server_addr)));
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
// Test making successful RPC calls while using a TLS certificate with a password protected
// private key.
TEST_P(TestRpc, TestCallWithPasswordProtectedKey) {
- bool enable_ssl = GetParam();
// We're only interested in running this test with TLS enabled.
- if (!enable_ssl) return;
+ if (!enable_ssl()) return;
string rpc_certificate_file;
string rpc_private_key_file;
@@ -270,30 +312,29 @@ TEST_P(TestRpc, TestCallWithPasswordProtectedKey) {
rpc_ca_certificate_file = rpc_certificate_file;
rpc_private_key_password_cmd = strings::Substitute("echo $0", passwd);
// Set up server.
- Sockaddr server_addr;
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
SCOPED_TRACE(strings::Substitute("Connecting to $0", server_addr.ToString()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl,
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl(),
rpc_certificate_file, rpc_private_key_file, rpc_ca_certificate_file,
rpc_private_key_password_cmd));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
ASSERT_STR_CONTAINS(p.ToString(), strings::Substitute("kudu.rpc.GenericCalculatorService@"
"{remote=$0, user_credentials=",
- server_addr.ToString()));
+ expected_remote_str(server_addr)));
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
// Test that using a TLS certificate with a password protected private key and providing
// the wrong password for that private key, causes a server startup failure.
TEST_P(TestRpc, TestCallWithBadPasswordProtectedKey) {
- bool enable_ssl = GetParam();
// We're only interested in running this test with TLS enabled.
- if (!enable_ssl) return;
+ if (!enable_ssl()) return;
string rpc_certificate_file;
string rpc_private_key_file;
@@ -309,8 +350,8 @@ TEST_P(TestRpc, TestCallWithBadPasswordProtectedKey) {
rpc_ca_certificate_file = rpc_certificate_file;
rpc_private_key_password_cmd = strings::Substitute("echo $0", passwd);
// Verify that the server fails to start up.
- Sockaddr server_addr;
- Status s = StartTestServer(&server_addr, enable_ssl, rpc_certificate_file, rpc_private_key_file,
+ Sockaddr server_addr = bind_addr();
+ Status s = StartTestServer(&server_addr, enable_ssl(), rpc_certificate_file, rpc_private_key_file,
rpc_ca_certificate_file, rpc_private_key_password_cmd);
ASSERT_TRUE(s.IsRuntimeError());
ASSERT_STR_CONTAINS(s.ToString(), "failed to load private key file");
@@ -319,8 +360,8 @@ TEST_P(TestRpc, TestCallWithBadPasswordProtectedKey) {
// Test that connecting to an invalid server properly throws an error.
TEST_P(TestRpc, TestCallToBadServer) {
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, GetParam()));
- Sockaddr addr;
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Sockaddr addr = Sockaddr::Wildcard();
addr.set_port(0);
Proxy p(client_messenger, addr, addr.host(),
GenericCalculatorService::static_service_name());
@@ -328,7 +369,7 @@ TEST_P(TestRpc, TestCallToBadServer) {
// Loop a few calls to make sure that we properly set up and tear down
// the connections.
for (int i = 0; i < 5; i++) {
- Status s = DoTestSyncCall(p, GenericCalculatorService::kAddMethodName);
+ Status s = DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName);
LOG(INFO) << "Status: " << s.ToString();
ASSERT_TRUE(s.IsNetworkError()) << "unexpected status: " << s.ToString();
}
@@ -337,19 +378,18 @@ TEST_P(TestRpc, TestCallToBadServer) {
// Test that RPC calls can be failed with an error status on the server.
TEST_P(TestRpc, TestInvalidMethodCall) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Call the method which fails.
- Status s = DoTestSyncCall(p, "ThisMethodDoesNotExist");
+ Status s = DoTestSyncCall(&p, "ThisMethodDoesNotExist");
ASSERT_TRUE(s.IsRemoteError()) << "unexpected status: " << s.ToString();
ASSERT_STR_CONTAINS(s.ToString(), "bad method");
}
@@ -358,21 +398,29 @@ TEST_P(TestRpc, TestInvalidMethodCall) {
// is reasonable.
TEST_P(TestRpc, TestWrongService) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client with the wrong service name.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
Proxy p(client_messenger, server_addr, "localhost", "WrongServiceName");
// Call the method which fails.
- Status s = DoTestSyncCall(p, "ThisMethodDoesNotExist");
+ Status s = DoTestSyncCall(&p, "ThisMethodDoesNotExist");
ASSERT_TRUE(s.IsRemoteError()) << "unexpected status: " << s.ToString();
ASSERT_STR_CONTAINS(s.ToString(),
"Service unavailable: service WrongServiceName "
"not registered on TestServer");
+
+ // If the server has been marked as having registered all services, we should
+ // expect a "not found" error instead.
+ server_messenger_->SetServicesRegistered();
+ s = DoTestSyncCall(&p, "ThisMethodDoesNotExist");
+ ASSERT_TRUE(s.IsRemoteError()) << "unexpected status: " << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(),
+ "Not found: service WrongServiceName "
+ "not registered on TestServer");
}
// Test that we can still make RPC connections even if many fds are in use.
@@ -396,14 +444,13 @@ TEST_P(TestRpc, TestHighFDs) {
}
// Set up server and client, and verify we can make a successful call.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
}
// Test that connections are kept alive between calls.
@@ -414,18 +461,17 @@ TEST_P(TestRpc, TestConnectionKeepalive) {
keepalive_time_ms_ = 500;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
SleepFor(MonoDelta::FromMilliseconds(5));
@@ -459,18 +505,17 @@ TEST_P(TestRpc, TestConnectionAlwaysKeepalive) {
keepalive_time_ms_ = -1;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
ReactorMetrics metrics;
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
@@ -496,16 +541,15 @@ TEST_P(TestRpc, TestConnectionAlwaysKeepalive) {
// Test that the metrics on a per connection level work accurately.
TEST_P(TestRpc, TestClientConnectionMetrics) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client with one reactor so that we can grab the metrics from just
// that reactor.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Here we queue a bunch of calls to the server and test that the sender's
@@ -522,15 +566,20 @@ TEST_P(TestRpc, TestClientConnectionMetrics) {
add_req.set_x(rand());
add_req.set_y(rand());
AddResponsePB add_resp;
+ string big_string(8 * 1024 * 1024, 'a');
// Send the calls.
vector<unique_ptr<RpcController>> controllers;
CountDownLatch latch(n_calls);
for (int i = 0; i < n_calls; i++) {
- controllers.emplace_back(new RpcController());
+ unique_ptr<RpcController> rpc(new RpcController());
+ // Attach a big sidecar so that we are less likely to be able to send the
+ // whole RPC in a single write() call without queueing it.
+ int junk;
+ CHECK_OK(rpc->AddOutboundSidecar(RpcSidecar::FromSlice(big_string), &junk));
+ controllers.emplace_back(std::move(rpc));
p.AsyncRequest(GenericCalculatorService::kAddMethodName, add_req, &add_resp,
- controllers.back().get(), boost::bind(
- &CountDownLatch::CountDown, boost::ref(latch)));
+ controllers.back().get(), [&latch]() { latch.CountDown(); });
}
auto cleanup = MakeScopedCleanup([&](){
latch.Wait();
@@ -546,15 +595,17 @@ TEST_P(TestRpc, TestClientConnectionMetrics) {
ASSERT_GT(conn.outbound_queue_size(), 0);
#ifdef __linux__
- // Test that the socket statistics are present. We only assert on those that
- // we know to be present on all kernel versions.
- ASSERT_TRUE(conn.has_socket_stats());
- ASSERT_GT(conn.socket_stats().rtt(), 0);
- ASSERT_GT(conn.socket_stats().rttvar(), 0);
- ASSERT_GT(conn.socket_stats().snd_cwnd(), 0);
- ASSERT_GT(conn.socket_stats().send_bytes_per_sec(), 0);
- ASSERT_TRUE(conn.socket_stats().has_send_queue_bytes());
- ASSERT_TRUE(conn.socket_stats().has_receive_queue_bytes());
+ if (!use_unix_socket()) {
+ // Test that the socket statistics are present. We only assert on those that
+ // we know to be present on all kernel versions.
+ ASSERT_TRUE(conn.has_socket_stats());
+ ASSERT_GT(conn.socket_stats().rtt(), 0);
+ ASSERT_GT(conn.socket_stats().rttvar(), 0);
+ ASSERT_GT(conn.socket_stats().snd_cwnd(), 0);
+ ASSERT_GT(conn.socket_stats().send_bytes_per_sec(), 0);
+ ASSERT_TRUE(conn.socket_stats().has_send_queue_bytes());
+ ASSERT_TRUE(conn.socket_stats().has_receive_queue_bytes());
+ }
#endif
// Unblock all of the calls and wait for them to finish.
@@ -580,15 +631,14 @@ TEST_P(TestRpc, TestReopenOutboundConnections) {
n_server_reactor_threads_ = 1;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Verify the initial counters.
@@ -602,7 +652,7 @@ TEST_P(TestRpc, TestReopenOutboundConnections) {
// Run several iterations, just in case.
for (int i = 0; i < 32; ++i) {
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(i + 1, metrics.total_server_connections_);
@@ -623,15 +673,14 @@ TEST_P(TestRpc, TestCredentialsPolicy) {
n_server_reactor_threads_ = 1;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Verify the initial counters.
@@ -644,7 +693,7 @@ TEST_P(TestRpc, TestCredentialsPolicy) {
ASSERT_EQ(0, metrics.total_server_connections_);
// Make an RPC call with ANY_CREDENTIALS policy.
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(1, metrics.total_server_connections_);
@@ -660,7 +709,7 @@ TEST_P(TestRpc, TestCredentialsPolicy) {
// Make an RPC call with PRIMARY_CREDENTIALS policy. Currently open connection
// with ANY_CREDENTIALS policy should be closed and a new one established
// with PRIMARY_CREDENTIALS policy.
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName,
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName,
CredentialsPolicy::PRIMARY_CREDENTIALS));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
@@ -675,7 +724,7 @@ TEST_P(TestRpc, TestCredentialsPolicy) {
// connection with PRIMARY_CREDENTIALS policy should be re-used because
// the ANY_CREDENTIALS policy satisfies the PRIMARY_CREDENTIALS policy which
// the currently open connection has been established with.
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(2, metrics.total_server_connections_);
@@ -696,17 +745,16 @@ TEST_P(TestRpc, TestConnectionNetworkPlane) {
keepalive_time_ms_ = -1;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up clients with default and non-default network planes.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p1(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p1(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
- Proxy p2(client_messenger, server_addr, server_addr.host(),
+ Proxy p2(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
p2.set_network_plane("control-channel");
@@ -722,7 +770,7 @@ TEST_P(TestRpc, TestConnectionNetworkPlane) {
ASSERT_EQ(0, metrics.num_client_connections_);
// Make an RPC call with the default network plane.
- ASSERT_OK(DoTestSyncCall(p1, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p1, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(1, metrics.total_server_connections_);
@@ -733,7 +781,7 @@ TEST_P(TestRpc, TestConnectionNetworkPlane) {
ASSERT_EQ(1, metrics.num_client_connections_);
// Make an RPC call with the non-default network plane.
- ASSERT_OK(DoTestSyncCall(p2, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p2, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(2, metrics.total_server_connections_);
@@ -745,7 +793,7 @@ TEST_P(TestRpc, TestConnectionNetworkPlane) {
// Make an RPC call with the default network plane again and verify that
// there are no new connections.
- ASSERT_OK(DoTestSyncCall(p1, GenericCalculatorService::kAddMethodName));
+ ASSERT_OK(DoTestSyncCall(&p1, GenericCalculatorService::kAddMethodName));
ASSERT_OK(server_messenger_->reactors_[0]->GetMetrics(&metrics));
ASSERT_EQ(0, metrics.total_client_connections_);
ASSERT_EQ(2, metrics.total_server_connections_);
@@ -764,14 +812,13 @@ TEST_P(TestRpc, TestCallLongerThanKeepalive) {
keepalive_time_ms_ = 1000;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Make a call which sleeps longer than the keepalive.
@@ -788,17 +835,16 @@ TEST_P(TestRpc, TestCallLongerThanKeepalive) {
// and verifies that the call succeeds (i.e. the connection is not closed).
TEST_P(TestRpc, TestTCPKeepalive) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
FLAGS_tcp_keepalive_probe_period_s = 1;
FLAGS_tcp_keepalive_retry_period_s = 1;
FLAGS_tcp_keepalive_retry_count = 1;
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Make a call which sleeps for longer than TCP keepalive probe period,
@@ -815,29 +861,50 @@ TEST_P(TestRpc, TestTCPKeepalive) {
// Test that the RpcSidecar transfers the expected messages.
TEST_P(TestRpc, TestRpcSidecar) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, GetParam()));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Test a zero-length sidecar
- DoTestSidecar(p, 0, 0);
+ DoTestSidecar(&p, 0, 0);
// Test some small sidecars
- DoTestSidecar(p, 123, 456);
+ DoTestSidecar(&p, 123, 456);
// Test some larger sidecars to verify that we properly handle the case where
// we can't write the whole response to the socket in a single call.
- DoTestSidecar(p, 3000 * 1024, 2000 * 1024);
+ DoTestSidecar(&p, 3000 * 1024, 2000 * 1024);
- DoTestOutgoingSidecarExpectOK(p, 0, 0);
- DoTestOutgoingSidecarExpectOK(p, 123, 456);
- DoTestOutgoingSidecarExpectOK(p, 3000 * 1024, 2000 * 1024);
+ DoTestOutgoingSidecarExpectOK(&p, 0, 0);
+ DoTestOutgoingSidecarExpectOK(&p, 123, 456);
+ DoTestOutgoingSidecarExpectOK(&p, 3000 * 1024, 2000 * 1024);
+}
+
+// Test sending the maximum number of sidecars, each of them being a single
+// character. This makes sure we handle the limit of IOV_MAX iovecs per sendmsg
+// call.
+TEST_P(TestRpc, TestMaxSmallSidecars) {
+ // Set up server.
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
+
+ // Set up client.
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
+ GenericCalculatorService::static_service_name());
+
+ Random rng(GetRandomSeed32());
+ vector<string> strings(TransferLimits::kMaxSidecars);
+ for (auto& s : strings) {
+ s = RandomString(2, &rng);
+ }
+ ASSERT_OK(DoTestOutgoingSidecar(&p, strings));
}
TEST_P(TestRpc, TestRpcSidecarLimits) {
@@ -881,7 +948,7 @@ TEST_P(TestRpc, TestRpcSidecarLimits) {
// value. This tests the client's ability to send the maximal message.
// The server will reject the message after it has been transferred.
// This test is disabled for TSAN due to high memory requirements.
- std::vector<int64_t> rpc_max_message_values;
+ vector<int64_t> rpc_max_message_values;
rpc_max_message_values.push_back(FLAGS_rpc_max_message_size);
#ifndef THREAD_SANITIZER
rpc_max_message_values.push_back(std::numeric_limits<int64_t>::max());
@@ -891,14 +958,14 @@ TEST_P(TestRpc, TestRpcSidecarLimits) {
FLAGS_rpc_max_message_size = rpc_max_message_size_val;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, GetParam()));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
RpcController controller;
@@ -907,17 +974,18 @@ TEST_P(TestRpc, TestRpcSidecarLimits) {
int idx;
ASSERT_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(Slice(max_string)), &idx));
- PushTwoStringsRequestPB request;
- request.set_sidecar1_idx(idx);
- request.set_sidecar2_idx(idx);
- PushTwoStringsResponsePB resp;
- Status status = p.SyncRequest(GenericCalculatorService::kPushTwoStringsMethodName,
+ PushStringsRequestPB request;
+ request.add_sidecar_indexes(idx);
+ PushStringsResponsePB resp;
+ Status status = p.SyncRequest(GenericCalculatorService::kPushStringsMethodName,
request, &resp, &controller);
ASSERT_TRUE(status.IsNetworkError()) << "Unexpected error: " << status.ToString();
// Remote responds to extra-large payloads by closing the connection.
ASSERT_STR_MATCHES(status.ToString(),
// Linux
"Connection reset by peer"
+ // Linux domain socket
+ "|Broken pipe"
// While reading from socket.
"|recv got EOF from"
// Linux, SSL enabled
@@ -931,26 +999,25 @@ TEST_P(TestRpc, TestRpcSidecarLimits) {
// Test that timeouts are properly handled.
TEST_P(TestRpc, TestCallTimeout) {
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Test a very short timeout - we expect this will time out while the
// call is still trying to connect, or in the send queue. This was triggering ASAN failures
// before.
- NO_FATALS(DoTestExpectTimeout(p, MonoDelta::FromNanoseconds(1)));
+ NO_FATALS(DoTestExpectTimeout(&p, MonoDelta::FromNanoseconds(1)));
// Test a longer timeout - expect this will time out after we send the request,
// but shorter than our threshold for two-stage timeout handling.
- NO_FATALS(DoTestExpectTimeout(p, MonoDelta::FromMilliseconds(200)));
+ NO_FATALS(DoTestExpectTimeout(&p, MonoDelta::FromMilliseconds(200)));
// Test a longer timeout - expect this will trigger the "two-stage timeout"
// code path.
- NO_FATALS(DoTestExpectTimeout(p, MonoDelta::FromMilliseconds(1500)));
+ NO_FATALS(DoTestExpectTimeout(&p, MonoDelta::FromMilliseconds(1500)));
}
// Inject 500ms delay in negotiation, and send a call with a short timeout, followed by
@@ -961,17 +1028,16 @@ TEST_P(TestRpc, TestCallTimeout) {
// was assigned the timeout of the first call on that connection. So, if the first
// call had a short timeout, the later call would also inherit the timed-out negotiation.
TEST_P(TestRpc, TestCallTimeoutDoesntAffectNegotiation) {
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
FLAGS_rpc_negotiation_inject_delay_ms = 500;
- NO_FATALS(DoTestExpectTimeout(p, MonoDelta::FromMilliseconds(50)));
- ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+ NO_FATALS(DoTestExpectTimeout(&p, MonoDelta::FromMilliseconds(50)));
+ ASSERT_OK(DoTestSyncCall(&p, GenericCalculatorService::kAddMethodName));
// Only the second call should have been received by the server, because we
// don't bother sending an already-timed-out call.
@@ -994,39 +1060,44 @@ static void AcceptAndReadForever(Socket* listen_sock) {
}
}
+// Basic test for methods_by_name(). At the time of writing, this isn't used by
+// Kudu, but is used in other projects like Apache Impala.
+TEST_F(TestRpc, TestMethodsByName) {
+ std::unique_ptr<CalculatorService> service(
+ new CalculatorService(metric_entity_, result_tracker_));
+ const auto& methods = service->methods_by_name();
+ ASSERT_EQ(8, methods.size());
+}
+
// Starts a fake listening socket which never actually negotiates.
// Ensures that the client gets a reasonable status code in this case.
TEST_F(TestRpc, TestNegotiationTimeout) {
// Set up a simple socket server which accepts a connection.
- Sockaddr server_addr;
+ Sockaddr server_addr = Sockaddr::Wildcard();
Socket listen_sock;
ASSERT_OK(StartFakeServer(&listen_sock, &server_addr));
// Create another thread to accept the connection on the fake server.
- scoped_refptr<Thread> acceptor_thread;
- ASSERT_OK(Thread::Create("test", "acceptor",
- AcceptAndReadForever, &listen_sock,
- &acceptor_thread));
+ thread acceptor_thread([&listen_sock]() { AcceptAndReadForever(&listen_sock); });
+ SCOPED_CLEANUP({ acceptor_thread.join(); });
// Set up client.
shared_ptr<Messenger> client_messenger;
ASSERT_OK(CreateMessenger("Client", &client_messenger));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
bool is_negotiation_error = false;
NO_FATALS(DoTestExpectTimeout(
- p, MonoDelta::FromMilliseconds(100), false, &is_negotiation_error));
+ &p, MonoDelta::FromMilliseconds(100), false, &is_negotiation_error));
EXPECT_TRUE(is_negotiation_error);
-
- acceptor_thread->Join();
}
// Test that client calls get failed properly when the server they're connected to
// shuts down.
-TEST_F(TestRpc, TestServerShutsDown) {
+TEST_P(TestRpc, TestServerShutsDown) {
// Set up a simple socket server which accepts a connection.
- Sockaddr server_addr;
+ Sockaddr server_addr = bind_addr();
Socket listen_sock;
ASSERT_OK(StartFakeServer(&listen_sock, &server_addr));
@@ -1034,7 +1105,7 @@ TEST_F(TestRpc, TestServerShutsDown) {
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
ASSERT_OK(CreateMessenger("Client", &client_messenger));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Send a call.
@@ -1053,7 +1124,7 @@ TEST_F(TestRpc, TestServerShutsDown) {
for (int i = 0; i < n_calls; i++) {
controllers.emplace_back(new RpcController());
p.AsyncRequest(GenericCalculatorService::kAddMethodName, req, &resp, controllers.back().get(),
- boost::bind(&CountDownLatch::CountDown, boost::ref(latch)));
+ [&latch]() { latch.CountDown(); });
}
// Accept the TCP connection.
@@ -1113,18 +1184,16 @@ TEST_F(TestRpc, TestServerShutsDown) {
// Test handler latency metric.
TEST_P(TestRpc, TestRpcHandlerLatencyMetric) {
-
- const uint64_t sleep_micros = 20 * 1000;
+ constexpr uint64_t sleep_micros = 20 * 1000;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
CalculatorService::static_service_name());
RpcController controller;
@@ -1137,9 +1206,12 @@ TEST_P(TestRpc, TestRpcHandlerLatencyMetric) {
const unordered_map<const MetricPrototype*, scoped_refptr<Metric> > metric_map =
server_messenger_->metric_entity()->UnsafeMetricsMapForTests();
- scoped_refptr<Histogram> latency_histogram = down_cast<Histogram *>(
+ scoped_refptr<Histogram> latency_histogram = down_cast<Histogram*>(
FindOrDie(metric_map,
&METRIC_handler_latency_kudu_rpc_test_CalculatorService_Sleep).get());
+ scoped_refptr<Counter> queue_overflow_rejections = down_cast<Counter*>(
+ FindOrDie(metric_map,
+ &METRIC_queue_overflow_rejections_kudu_rpc_test_CalculatorService_Sleep).get());
LOG(INFO) << "Sleep() min lat: " << latency_histogram->MinValueForTests();
LOG(INFO) << "Sleep() mean lat: " << latency_histogram->MeanValueForTests();
@@ -1147,6 +1219,7 @@ TEST_P(TestRpc, TestRpcHandlerLatencyMetric) {
LOG(INFO) << "Sleep() #calls: " << latency_histogram->TotalCount();
ASSERT_EQ(1, latency_histogram->TotalCount());
+ ASSERT_EQ(0, queue_overflow_rejections->value());
ASSERT_GE(latency_histogram->MaxValueForTests(), sleep_micros);
ASSERT_TRUE(latency_histogram->MinValueForTests() == latency_histogram->MaxValueForTests());
@@ -1163,8 +1236,8 @@ static void DestroyMessengerCallback(shared_ptr<Messenger>* messenger,
TEST_P(TestRpc, TestRpcCallbackDestroysMessenger) {
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, GetParam()));
- Sockaddr bad_addr;
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Sockaddr bad_addr = Sockaddr::Wildcard();
CountDownLatch latch(1);
AddRequestPB req;
@@ -1176,7 +1249,9 @@ TEST_P(TestRpc, TestRpcCallbackDestroysMessenger) {
{
Proxy p(client_messenger, bad_addr, "xxx-host", "xxx-service");
p.AsyncRequest("my-fake-method", req, &resp, &controller,
- boost::bind(&DestroyMessengerCallback, &client_messenger, &latch));
+ [&client_messenger, &latch]() {
+ DestroyMessengerCallback(&client_messenger, &latch);
+ });
}
latch.Wait();
}
@@ -1187,14 +1262,13 @@ TEST_P(TestRpc, TestRpcContextClientDeadline) {
const uint64_t sleep_micros = 20 * 1000;
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
CalculatorService::static_service_name());
SleepRequestPB req;
@@ -1215,14 +1289,13 @@ TEST_P(TestRpc, TestRpcContextClientDeadline) {
// will make the server reject the call.
TEST_P(TestRpc, TestApplicationFeatureFlag) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
CalculatorService::static_service_name());
{ // Supported flag
@@ -1258,14 +1331,13 @@ TEST_P(TestRpc, TestApplicationFeatureFlagUnsupportedServer) {
kSupportedServerRpcFeatureFlags = {};
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServerWithGeneratedCode(&server_addr, enable_ssl()));
// Set up client.
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
CalculatorService::static_service_name());
{ // Required flag
@@ -1294,15 +1366,14 @@ TEST_P(TestRpc, TestApplicationFeatureFlagUnsupportedServer) {
TEST_P(TestRpc, TestCancellation) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
int timeout_ms = 10;
@@ -1313,14 +1384,14 @@ TEST_P(TestRpc, TestCancellation) {
case OutboundCall::ON_OUTBOUND_QUEUE:
case OutboundCall::SENDING:
case OutboundCall::SENT:
- ASSERT_TRUE(DoTestOutgoingSidecar(p, 0, 0).IsAborted());
- ASSERT_TRUE(DoTestOutgoingSidecar(p, 123, 456).IsAborted());
- ASSERT_TRUE(DoTestOutgoingSidecar(p, 3000 * 1024, 2000 * 1024).IsAborted());
- DoTestExpectTimeout(p, MonoDelta::FromMilliseconds(timeout_ms), true);
+ ASSERT_TRUE(DoTestOutgoingSidecar(&p, 0, 0).IsAborted());
+ ASSERT_TRUE(DoTestOutgoingSidecar(&p, 123, 456).IsAborted());
+ ASSERT_TRUE(DoTestOutgoingSidecar(&p, 3000 * 1024, 2000 * 1024).IsAborted());
+ DoTestExpectTimeout(&p, MonoDelta::FromMilliseconds(timeout_ms), true);
break;
case OutboundCall::NEGOTIATION_TIMED_OUT:
case OutboundCall::TIMED_OUT:
- DoTestExpectTimeout(p, MonoDelta::FromMilliseconds(1000));
+ DoTestExpectTimeout(&p, MonoDelta::FromMilliseconds(1000));
break;
case OutboundCall::CANCELLED:
break;
@@ -1338,9 +1409,9 @@ TEST_P(TestRpc, TestCancellation) {
break;
}
case OutboundCall::FINISHED_SUCCESS:
- DoTestOutgoingSidecarExpectOK(p, 0, 0);
- DoTestOutgoingSidecarExpectOK(p, 123, 456);
- DoTestOutgoingSidecarExpectOK(p, 3000 * 1024, 2000 * 1024);
+ DoTestOutgoingSidecarExpectOK(&p, 0, 0);
+ DoTestOutgoingSidecarExpectOK(&p, 123, 456);
+ DoTestOutgoingSidecarExpectOK(&p, 3000 * 1024, 2000 * 1024);
break;
}
}
@@ -1363,21 +1434,20 @@ static void SleepCallback(uint8_t* payload, CountDownLatch* latch) {
// Test to verify that sidecars aren't corrupted when cancelling an async RPC.
TEST_P(TestRpc, TestCancellationAsync) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
RpcController controller;
// The payload to be used during the RPC.
- gscoped_array<uint8_t> payload(new uint8_t[TEST_PAYLOAD_SIZE]);
+ unique_ptr<uint8_t[]> payload(new uint8_t[TEST_PAYLOAD_SIZE]);
// Used to generate sleep time between invoking RPC and requesting cancellation.
Random rand(SeedRandom());
@@ -1398,9 +1468,10 @@ TEST_P(TestRpc, TestCancellationAsync) {
req.set_sidecar_idx(idx);
CountDownLatch latch(1);
+ auto* payload_raw = payload.get();
p.AsyncRequest(GenericCalculatorService::kSleepWithSidecarMethodName,
req, &resp, &controller,
- boost::bind(SleepCallback, payload.get(), &latch));
+ [payload_raw, &latch]() { SleepCallback(payload_raw, &latch); });
// Sleep for a while before cancelling the RPC.
if (i > 0) SleepFor(MonoDelta::FromMicroseconds(rand.Uniform64(i * 30)));
controller.Cancel();
@@ -1427,18 +1498,18 @@ static void SendAndCancelRpcs(Proxy* p, const Slice& slice) {
int i = 0;
while (MonoTime::Now() < end_time) {
controller.Reset();
- PushTwoStringsRequestPB request;
- PushTwoStringsResponsePB resp;
+ PushStringsRequestPB request;
+ PushStringsResponsePB resp;
int idx;
CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(slice), &idx));
- request.set_sidecar1_idx(idx);
+ request.add_sidecar_indexes(idx);
CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(slice), &idx));
- request.set_sidecar2_idx(idx);
+ request.add_sidecar_indexes(idx);
CountDownLatch latch(1);
- p->AsyncRequest(GenericCalculatorService::kPushTwoStringsMethodName,
+ p->AsyncRequest(GenericCalculatorService::kPushStringsMethodName,
request, &resp, &controller,
- boost::bind(&CountDownLatch::CountDown, boost::ref(latch)));
+ [&latch]() { latch.CountDown(); });
if ((i++ % 8) != 0) {
// Sleep for a while before cancelling the RPC.
@@ -1455,15 +1526,14 @@ static void SendAndCancelRpcs(Proxy* p, const Slice& slice) {
// same client to the same server.
TEST_P(TestRpc, TestCancellationMultiThreads) {
// Set up server.
- Sockaddr server_addr;
- bool enable_ssl = GetParam();
- ASSERT_OK(StartTestServer(&server_addr, enable_ssl));
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
// Set up client.
LOG(INFO) << "Connecting to " << server_addr.ToString();
shared_ptr<Messenger> client_messenger;
- ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl));
- Proxy p(client_messenger, server_addr, server_addr.host(),
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
GenericCalculatorService::static_service_name());
// Buffer used for sidecars by SendAndCancelRpcs().
@@ -1471,18 +1541,81 @@ TEST_P(TestRpc, TestCancellationMultiThreads) {
Slice slice(buf);
// Start a bunch of threads which invoke async RPC and cancellation.
- std::vector<scoped_refptr<Thread>> threads;
- for (int i = 0; i < 30; ++i) {
- scoped_refptr<Thread> rpc_thread;
- ASSERT_OK(Thread::Create("test", "rpc", SendAndCancelRpcs, &p, slice, &rpc_thread));
- threads.push_back(rpc_thread);
+ constexpr int kNumThreads = 30;
+ vector<thread> threads;
+ threads.reserve(kNumThreads);
+ for (int i = 0; i < kNumThreads; ++i) {
+ threads.emplace_back([&p, slice]() { SendAndCancelRpcs(&p, slice); });
}
// Wait for all threads to complete.
- for (scoped_refptr<Thread>& rpc_thread : threads) {
- rpc_thread->Join();
+ for (auto& t : threads) {
+ t.join();
}
client_messenger->Shutdown();
}
+
+// Test performance of Ipv4 vs unix sockets.
+TEST_P(TestRpc, TestPerformanceBySocketType) {
+ static constexpr int kNumMb = 1024;
+ static constexpr int kMbPerRpc = 4;
+ static_assert(kNumMb % kMbPerRpc == 0, "total should be a multiple of per-RPC");
+
+ const vector<string> sidecars = { string(kMbPerRpc * 1024 * 1024, 'x') };
+
+ // Set up server.
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
+
+ // Set up client.
+ LOG(INFO) << "Connecting to " << server_addr.ToString();
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
+ GenericCalculatorService::static_service_name());
+
+ for (int i = 0; i < 5; i++) {
+ Stopwatch sw(Stopwatch::ALL_THREADS);
+ sw.start();
+ for (int i = 0; i < kNumMb / kMbPerRpc; i++) {
+ DoTestOutgoingSidecar(&p, sidecars);
+ }
+ sw.stop();
+ LOG(INFO) << strings::Substitute(
+ "Sending $0MB via $1$2 socket: $3",
+ kNumMb,
+ enable_ssl() ? "ssl-enabled " : "",
+ use_unix_socket() ? "unix" : "tcp",
+ sw.elapsed().ToString());
+ }
+}
+
+// Test that call_id is returned in call response and accessible through RpcController.
+TEST_P(TestRpc, TestCallId) {
+ // Set up server.
+ Sockaddr server_addr = bind_addr();
+ ASSERT_OK(StartTestServer(&server_addr, enable_ssl()));
+
+ // Set up client.
+ shared_ptr<Messenger> client_messenger;
+ ASSERT_OK(CreateMessenger("Client", &client_messenger, 1, enable_ssl()));
+ Proxy p(client_messenger, server_addr, kRemoteHostName,
+ GenericCalculatorService::static_service_name());
+
+ for (int i = 0; i < 10; i++) {
+ AddRequestPB req;
+ req.set_x(rand());
+ req.set_y(rand());
+ RpcController controller;
+ controller.set_timeout(MonoDelta::FromMilliseconds(10000));
+
+ AddResponsePB resp;
+ ASSERT_OK(p.SyncRequest(GenericCalculatorService::kAddMethodName,
+ req, &resp, &controller));
+ ASSERT_EQ(req.x() + req.y(), resp.result());
+ ASSERT_EQ(i, controller.call_id());
+ }
+}
+
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/rpc.cc b/be/src/kudu/rpc/rpc.cc
index 862e804..7701924 100644
--- a/be/src/kudu/rpc/rpc.cc
+++ b/be/src/kudu/rpc/rpc.cc
@@ -20,10 +20,9 @@
#include <algorithm>
#include <cmath>
#include <cstdlib>
+#include <functional>
#include <string>
-#include <boost/bind.hpp> // IWYU pragma: keep
-#include <boost/function.hpp>
#include <glog/logging.h>
#include "kudu/gutil/strings/substitute.h"
@@ -51,7 +50,7 @@ void RpcRetrier::DelayedRetry(Rpc* rpc, const Status& why_status) {
// RPC on our behalf.
MonoDelta backoff = ComputeBackoff(attempt_num_++);
messenger_->ScheduleOnReactor(
- boost::bind(&RpcRetrier::DelayedRetryCb, this, rpc, _1), backoff);
+ [this, rpc](const Status& s) { this->DelayedRetryCb(rpc, s); }, backoff);
}
MonoDelta RpcRetrier::ComputeBackoff(int num_attempts) const {
diff --git a/be/src/kudu/rpc/rpc.h b/be/src/kudu/rpc/rpc.h
index 541efa8..747c5b5 100644
--- a/be/src/kudu/rpc/rpc.h
+++ b/be/src/kudu/rpc/rpc.h
@@ -16,11 +16,11 @@
// under the License.
#pragma once
+#include <functional>
#include <memory>
#include <string>
#include <utility>
-#include "kudu/gutil/callback.h"
#include "kudu/gutil/macros.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/rpc_controller.h"
@@ -90,7 +90,7 @@ class ServerPicker : public RefCountedThreadSafe<ServerPicker<Server>> {
public:
virtual ~ServerPicker() {}
- typedef Callback<void(const Status& status, Server* server)> ServerPickedCallback;
+ typedef std::function<void(const Status&, Server*)> ServerPickedCallback;
// Picks the leader among the replicas serving a resource.
// If the leader was found, it calls the callback with Status::OK() and
diff --git a/be/src/kudu/rpc/rpc_context.cc b/be/src/kudu/rpc/rpc_context.cc
index a65c137..1c5e67f 100644
--- a/be/src/kudu/rpc/rpc_context.cc
+++ b/be/src/kudu/rpc/rpc_context.cc
@@ -70,7 +70,7 @@ void RpcContext::SetResultTracker(scoped_refptr<ResultTracker> result_tracker) {
void RpcContext::RespondSuccess() {
if (AreResultsTracked()) {
result_tracker_->RecordCompletionAndRespond(call_->header().request_id(),
- response_pb_.get());
+ response_pb_);
} else {
VLOG(4) << call_->remote_method().service_name() << ": Sending RPC success response for "
<< call_->ToString() << ":" << std::endl << SecureDebugString(*response_pb_);
@@ -85,7 +85,7 @@ void RpcContext::RespondSuccess() {
void RpcContext::RespondNoCache() {
if (AreResultsTracked()) {
result_tracker_->FailAndRespond(call_->header().request_id(),
- response_pb_.get());
+ response_pb_);
} else {
VLOG(4) << call_->remote_method().service_name() << ": Sending RPC failure response for "
<< call_->ToString() << ": " << SecureDebugString(*response_pb_);
@@ -143,6 +143,10 @@ const rpc::RequestIdPB* RpcContext::request_id() const {
return call_->header().has_request_id() ? &call_->header().request_id() : nullptr;
}
+int32_t RpcContext::call_id() const {
+ return call_->call_id();
+}
+
size_t RpcContext::GetTransferSize() const {
return call_->GetTransferSize();
}
@@ -176,11 +180,11 @@ std::string RpcContext::requestor_string() const {
call_->remote_address().ToString();
}
-std::string RpcContext::method_name() const {
+const std::string& RpcContext::method_name() const {
return call_->remote_method().method_name();
}
-std::string RpcContext::service_name() const {
+const std::string& RpcContext::service_name() const {
return call_->remote_method().service_name();
}
diff --git a/be/src/kudu/rpc/rpc_context.h b/be/src/kudu/rpc/rpc_context.h
index b1466ce..6b68ad7 100644
--- a/be/src/kudu/rpc/rpc_context.h
+++ b/be/src/kudu/rpc/rpc_context.h
@@ -16,8 +16,10 @@
// under the License.
#pragma once
-#include <memory>
#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
#include <string>
#include <glog/logging.h>
@@ -195,13 +197,13 @@ class RpcContext {
std::string requestor_string() const;
// Return the name of the RPC service method being called.
- std::string method_name() const;
+ const std::string& method_name() const;
// Return the name of the RPC service being called.
- std::string service_name() const;
+ const std::string& service_name() const;
- const google::protobuf::Message *request_pb() const { return request_pb_.get(); }
- google::protobuf::Message *response_pb() const { return response_pb_.get(); }
+ const google::protobuf::Message* request_pb() const { return request_pb_; }
+ google::protobuf::Message* response_pb() const { return response_pb_; }
// Return an upper bound on the client timeout deadline. This does not
// account for transmission delays between the client and the server.
@@ -226,6 +228,9 @@ class RpcContext {
// Returns this call's request id, if it is set.
const rpc::RequestIdPB* request_id() const;
+ // Returns this call's call_id.
+ int32_t call_id() const;
+
// Returns the size of the transfer buffer that backs 'call_'. If the
// transfer buffer no longer exists (e.g. GetTransferSize() is called after
// DiscardTransfer()), returns 0.
@@ -242,8 +247,8 @@ class RpcContext {
private:
friend class ResultTracker;
InboundCall* const call_;
- const std::unique_ptr<const google::protobuf::Message> request_pb_;
- const std::unique_ptr<google::protobuf::Message> response_pb_;
+ const google::protobuf::Message* const request_pb_;
+ google::protobuf::Message* const response_pb_;
scoped_refptr<ResultTracker> result_tracker_;
};
diff --git a/be/src/kudu/rpc/rpc_controller.cc b/be/src/kudu/rpc/rpc_controller.cc
index 821b881..6730e7f 100644
--- a/be/src/kudu/rpc/rpc_controller.cc
+++ b/be/src/kudu/rpc/rpc_controller.cc
@@ -30,8 +30,6 @@
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/rpc/transfer.h"
-#include "kudu/util/slice.h"
-
using std::unique_ptr;
using strings::Substitute;
@@ -105,6 +103,10 @@ const ErrorStatusPB* RpcController::error_response() const {
return nullptr;
}
+int32_t RpcController::call_id() const {
+ return call_->call_response_->call_id();
+}
+
Status RpcController::GetInboundSidecar(int idx, Slice* sidecar) const {
return call_->call_response_->GetSidecar(idx, sidecar);
}
@@ -146,7 +148,7 @@ Status RpcController::AddOutboundSidecar(unique_ptr<RpcSidecar> car, int* idx) {
if (outbound_sidecars_.size() >= TransferLimits::kMaxSidecars) {
return Status::RuntimeError("All available sidecars already used");
}
- int64_t sidecar_bytes = car->AsSlice().size();
+ int64_t sidecar_bytes = car->TotalSize();
if (outbound_sidecars_total_bytes_ >
TransferLimits::kMaxTotalSidecarBytes - sidecar_bytes) {
return Status::RuntimeError(Substitute("Total size of sidecars $0 would exceed limit $1",
@@ -166,6 +168,20 @@ void RpcController::SetRequestParam(const google::protobuf::Message& req) {
call_->SetRequestPayload(req, std::move(outbound_sidecars_));
}
+void RpcController::FreeOutboundSidecars() {
+ outbound_sidecars_total_bytes_ = 0;
+ call_->FreeSidecars();
+}
+
+std::vector<unique_ptr<RpcSidecar>> RpcController::ReleaseOutboundSidecars() {
+ outbound_sidecars_total_bytes_ = 0;
+ return std::move(outbound_sidecars_);
+}
+
+unique_ptr<RequestPayload> RpcController::ReleaseRequestPayload() {
+ return DCHECK_NOTNULL(call_)->ReleaseRequestPayload();
+}
+
void RpcController::Cancel() {
DCHECK(call_);
DCHECK(messenger_);
diff --git a/be/src/kudu/rpc/rpc_controller.h b/be/src/kudu/rpc/rpc_controller.h
index bcdabf5..49bd527 100644
--- a/be/src/kudu/rpc/rpc_controller.h
+++ b/be/src/kudu/rpc/rpc_controller.h
@@ -42,6 +42,7 @@ class ErrorStatusPB;
class Messenger;
class OutboundCall;
class RequestIdPB;
+class RequestPayload;
class RpcSidecar;
// Authentication credentials policy for outbound RPCs. Some RPC methods
@@ -141,6 +142,21 @@ class RpcController {
// in some cases even when sent to multiple servers, enabling exactly once semantics.
void SetRequestIdPB(std::unique_ptr<RequestIdPB> request_id);
+ // Releases the outbound sidecars added to this controller. This is useful if
+ // callers want to create a request payload for a request.
+ std::vector<std::unique_ptr<RpcSidecar>> ReleaseOutboundSidecars();
+
+ // Frees the outbound sidecars added to this controller. OutboundCalls may
+ // call this before running the user-provided callback to ensure the state is
+ // freed before the callback is run. However, certain usages of OutboundCall
+ // warrants delaying the freeing until during the callback.
+ void FreeOutboundSidecars();
+
+ // Releases the request payload owned by this controller. This is useful if
+ // callers want to reuse a request payload in another attempt of a call, as
+ // it allows callers to transfer ownership to a new outbound call.
+ std::unique_ptr<RequestPayload> ReleaseRequestPayload();
+
// Returns whether a request id has been set on RPC header.
bool has_request_id() const;
@@ -209,6 +225,12 @@ class RpcController {
credentials_policy_ = policy;
}
+ // Returns the call_id of this RPC call.
+ // Should only be called after the call's Response has been received (that is, when
+ // Connection::HandleCallResponse() has been executed over 'call_'), but the
+ // controller has not been Reset().
+ int32_t call_id() const;
+
// Fills the 'sidecar' parameter with the slice pointing to the i-th
// sidecar upon success.
//
@@ -267,6 +289,7 @@ class RpcController {
// Once the call is sent, it is tracked here.
std::shared_ptr<OutboundCall> call_;
+ // Owned by the controller until released and taken by a call.
std::vector<std::unique_ptr<RpcSidecar>> outbound_sidecars_;
// Total size of sidecars in outbound_sidecars_. This is limited to a maximum
diff --git a/be/src/kudu/rpc/rpc_introspection.proto b/be/src/kudu/rpc/rpc_introspection.proto
index 3e4facd..9a81372 100644
--- a/be/src/kudu/rpc/rpc_introspection.proto
+++ b/be/src/kudu/rpc/rpc_introspection.proto
@@ -74,7 +74,34 @@ message SocketStatsPB {
// Calculated sender throughput.
optional int64 send_bytes_per_sec = 13;
-};
+}
+
+// Transport-related information for an RPC connection.
+message TransportDetailsPB {
+
+ // TCP-specific details.
+ message TcpDetails {
+ // Maximum segment size for the packets: this directly maps into the
+ // TCP_MAXSEG socket option.
+ optional int32 max_segment_size = 1;
+ }
+
+ // TLS-specific details.
+ //
+ // NOTE: TLS/SSL doesn't map nicely into a single layer of the TCP/IP or
+ // the OSI model, but intuitively that's something related to the transport.
+ message TlsDetails {
+ // The name of the TLS protocol negotiated to protect the connection
+ // (e.g. TLSv1.3).
+ optional string protocol = 1;
+
+ // Description of the TLS cipher suite used.
+ optional string cipher_suite = 2;
+ }
+
+ optional TcpDetails tcp = 1;
+ optional TlsDetails tls = 2;
+}
// Debugging information about a currently-open RPC connection.
message RpcConnectionPB {
@@ -82,7 +109,7 @@ message RpcConnectionPB {
UNKNOWN = 999;
NEGOTIATING = 0; // Connection is still being negotiated.
OPEN = 1; // Connection is active.
- };
+ }
required string remote_ip = 1;
required StateType state = 2;
@@ -93,6 +120,9 @@ message RpcConnectionPB {
// Information on the actual TCP connection as reported by the kernel.
optional SocketStatsPB socket_stats = 6;
+
+ // Transport-specific details.
+ optional TransportDetailsPB transport_details = 7;
}
message DumpConnectionsRequestPB {
diff --git a/be/src/kudu/rpc/rpc_service.h b/be/src/kudu/rpc/rpc_service.h
index 42decc7..82bbcd0 100644
--- a/be/src/kudu/rpc/rpc_service.h
+++ b/be/src/kudu/rpc/rpc_service.h
@@ -37,9 +37,11 @@ class RpcService : public RefCountedThreadSafe<RpcService> {
// responsible for responding to the client with a failure message.
virtual Status QueueInboundCall(std::unique_ptr<InboundCall> call) = 0;
- virtual RpcMethodInfo* LookupMethod(const RemoteMethod& /*method*/) {
- return nullptr;
- }
+ // Look up RPC method information.
+ //
+ // If this returns nullptr, then certain functionality like
+ // metrics collection will not be performed for this call.
+ virtual RpcMethodInfo* LookupMethod(const RemoteMethod& method) = 0;
};
} // namespace rpc
diff --git a/be/src/kudu/rpc/rpc_sidecar.cc b/be/src/kudu/rpc/rpc_sidecar.cc
index b4de678..da7244d 100644
--- a/be/src/kudu/rpc/rpc_sidecar.cc
+++ b/be/src/kudu/rpc/rpc_sidecar.cc
@@ -17,10 +17,12 @@
#include "kudu/rpc/rpc_sidecar.h"
+#include <algorithm>
#include <cstdint>
#include <memory>
-#include <utility>
+#include <vector>
+#include <boost/container/vector.hpp>
#include <google/protobuf/repeated_field.h>
#include "kudu/gutil/strings/substitute.h"
@@ -29,6 +31,7 @@
#include "kudu/util/status.h"
using std::unique_ptr;
+using std::vector;
namespace kudu {
namespace rpc {
@@ -39,22 +42,44 @@ namespace rpc {
class SliceSidecar : public RpcSidecar {
public:
explicit SliceSidecar(Slice slice) : slice_(slice) { }
- Slice AsSlice() const override { return slice_; }
-
+ void AppendSlices(TransferPayload* payload) const override {
+ payload->push_back(slice_);
+ }
+ size_t TotalSize() const override {
+ return slice_.size();
+ }
private:
const Slice slice_;
};
class FaststringSidecar : public RpcSidecar {
public:
- explicit FaststringSidecar(unique_ptr<faststring> data) : data_(std::move(data)) { }
- Slice AsSlice() const override { return *data_; }
+ explicit FaststringSidecar(faststring data) {
+ data_.emplace_back(std::move(data));
+ }
+ explicit FaststringSidecar(vector<faststring> data) : data_(std::move(data)) { }
+ void AppendSlices(TransferPayload* payload) const override {
+ for (const auto& fs : data_) {
+ payload->push_back(Slice(fs));
+ }
+ }
+ size_t TotalSize() const override {
+ size_t ret = 0;
+ for (const auto& fs : data_) {
+ ret += fs.size();
+ }
+ return ret;
+ }
private:
- const unique_ptr<faststring> data_;
+ vector<faststring> data_;
};
-unique_ptr<RpcSidecar> RpcSidecar::FromFaststring(unique_ptr<faststring> data) {
+unique_ptr<RpcSidecar> RpcSidecar::FromFaststring(faststring data) {
+ return unique_ptr<RpcSidecar>(new FaststringSidecar(std::move(data)));
+}
+
+unique_ptr<RpcSidecar> RpcSidecar::FromFaststrings(vector<faststring> data) {
return unique_ptr<RpcSidecar>(new FaststringSidecar(std::move(data)));
}
@@ -65,8 +90,9 @@ unique_ptr<RpcSidecar> RpcSidecar::FromSlice(Slice slice) {
Status RpcSidecar::ParseSidecars(
const ::google::protobuf::RepeatedField<::google::protobuf::uint32>& offsets,
- Slice buffer, Slice* sidecars) {
- if (offsets.size() == 0) return Status::OK();
+ Slice buffer,
+ SidecarSliceVector* sidecars) {
+ if (offsets.empty()) return Status::OK();
int last = offsets.size() - 1;
if (last >= TransferLimits::kMaxSidecars) {
@@ -81,6 +107,7 @@ Status RpcSidecar::ParseSidecars(
buffer.size(), TransferLimits::kMaxTotalSidecarBytes));
}
+ sidecars->resize(offsets.size());
for (int i = 0; i < last; ++i) {
int64_t cur_offset = offsets.Get(i);
int64_t next_offset = offsets.Get(i + 1);
@@ -96,7 +123,7 @@ Status RpcSidecar::ParseSidecars(
" but ends before that at offset $1.", i, cur_offset, next_offset));
}
- sidecars[i] = Slice(buffer.data() + cur_offset, next_offset - cur_offset);
+ (*sidecars)[i] = Slice(buffer.data() + cur_offset, next_offset - cur_offset);
}
int64_t cur_offset = offsets.Get(last);
@@ -105,7 +132,7 @@ Status RpcSidecar::ParseSidecars(
"starts at offset $1after message ends (message length $2).", last,
cur_offset, buffer.size()));
}
- sidecars[last] = Slice(buffer.data() + cur_offset, buffer.size() - cur_offset);
+ (*sidecars)[last] = Slice(buffer.data() + cur_offset, buffer.size() - cur_offset);
return Status::OK();
}
diff --git a/be/src/kudu/rpc/rpc_sidecar.h b/be/src/kudu/rpc/rpc_sidecar.h
index cf555cb..6cb3449 100644
--- a/be/src/kudu/rpc/rpc_sidecar.h
+++ b/be/src/kudu/rpc/rpc_sidecar.h
@@ -16,11 +16,15 @@
// under the License.
#pragma once
+#include <cstddef>
#include <memory>
+#include <vector>
+#include <boost/container/small_vector.hpp>
#include <google/protobuf/repeated_field.h> // IWYU pragma: keep
#include <google/protobuf/stubs/port.h>
+#include "kudu/rpc/transfer.h"
#include "kudu/util/slice.h"
namespace kudu {
@@ -30,6 +34,8 @@ class faststring;
namespace rpc {
+typedef boost::container::small_vector<Slice, 2> SidecarSliceVector;
+
// An RpcSidecar is a mechanism which allows replies to RPCs to reference blocks of data
// without extra copies. In other words, whenever a protobuf would have a large field
// where additional copies become expensive, one may opt instead to use an RpcSidecar.
@@ -37,9 +43,10 @@ namespace rpc {
// The RpcSidecar saves on an additional copy to/from the protobuf on both the server and
// client side. Both Inbound- and OutboundCall classes accept sidecars to be sent to the
// client and server respectively. They are ignorant of the sidecar's format, requiring
-// only that it can be represented as a Slice. Data is copied from the Slice returned from
-// AsSlice() to the socket that is responding to the original RPC. The slice should remain
-// valid for as long as the call it is attached to takes to complete.
+// only that it can be represented as a series of Slices. Data is concatenated from the
+// Slices produced by AppendSlices() to the socket that is responding to the original
+// RPC. The slices should remain valid for as long as the call it is attached to takes to
+// complete.
//
// In order to distinguish between separate sidecars, whenever a sidecar is added to the
// RPC response on the server side, an index for that sidecar is returned. This index must
@@ -49,20 +56,27 @@ namespace rpc {
// sidecar data through the RpcContext or RpcController interfaces respectively.
class RpcSidecar {
public:
- static std::unique_ptr<RpcSidecar> FromFaststring(std::unique_ptr<faststring> data);
+ static std::unique_ptr<RpcSidecar> FromFaststring(faststring data);
+ static std::unique_ptr<RpcSidecar> FromFaststrings(std::vector<faststring> data);
static std::unique_ptr<RpcSidecar> FromSlice(Slice slice);
// Utility method to parse a series of sidecar slices into 'sidecars' from 'buffer' and
- // a set of offsets. 'sidecars' must have length >= TransferLimits::kMaxSidecars, and
- // will be filled from index 0.
- // TODO(henryr): Consider a vector instead here if there's no perf. impact.
+ // a set of offsets.
static Status ParseSidecars(
const ::google::protobuf::RepeatedField<::google::protobuf::uint32>& offsets,
- Slice buffer, Slice* sidecars);
+ Slice buffer,
+ SidecarSliceVector* sidecars);
+
+ // Append Slice representation of the sidecar's data to the given payload.
+ //
+ // Note that, even if a sidecar appends multiple slices here, the receiver will
+ // see a single concatenated slice on the other end of this call or response.
+ virtual void AppendSlices(TransferPayload* payload) const = 0;
- // Returns a Slice representation of the sidecar's data.
- virtual Slice AsSlice() const = 0;
+ // Return the total size of the slices to be appended.
+ virtual size_t TotalSize() const = 0;
virtual ~RpcSidecar() { }
+
};
} // namespace rpc
diff --git a/be/src/kudu/rpc/rpc_stub-test.cc b/be/src/kudu/rpc/rpc_stub-test.cc
index 8c5a8a1..d446227 100644
--- a/be/src/kudu/rpc/rpc_stub-test.cc
+++ b/be/src/kudu/rpc/rpc_stub-test.cc
@@ -21,6 +21,7 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
+#include <functional>
#include <limits>
#include <memory>
#include <ostream>
@@ -28,9 +29,6 @@
#include <thread>
#include <vector>
-#include <boost/bind.hpp>
-#include <boost/core/ref.hpp>
-#include <boost/function.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <glog/stl_logging.h>
@@ -144,7 +142,7 @@ TEST_F(RpcStubTest, TestBigCallData) {
controllers.emplace_back(new RpcController);
p.EchoAsync(req, resps.back().get(), controllers.back().get(),
- boost::bind(&CountDownLatch::CountDown, boost::ref(latch)));
+ [&latch]() { latch.CountDown(); });
}
latch.Wait();
@@ -317,7 +315,7 @@ TEST_F(RpcStubTest, TestCallWithMissingPBFieldClientSide) {
// Request is missing the 'y' field.
AddResponsePB resp;
Atomic32 callback_count = 0;
- p.AddAsync(req, &resp, &controller, boost::bind(&DoIncrement, &callback_count));
+ p.AddAsync(req, &resp, &controller, [&callback_count]() { DoIncrement(&callback_count); });
while (NoBarrier_Load(&callback_count) == 0) {
SleepFor(MonoDelta::FromMicroseconds(10));
}
@@ -359,7 +357,7 @@ TEST_F(RpcStubTest, TestCallMissingMethod) {
Proxy p(client_messenger_, server_addr_, server_addr_.host(),
CalculatorService::static_service_name());
- Status s = DoTestSyncCall(p, "DoesNotExist");
+ Status s = DoTestSyncCall(&p, "DoesNotExist");
ASSERT_TRUE(s.IsRemoteError()) << "Bad status: " << s.ToString();
ASSERT_STR_CONTAINS(s.ToString(), "with an invalid method name: DoesNotExist");
}
@@ -449,15 +447,17 @@ struct AsyncSleep {
TEST_F(RpcStubTest, TestDontHandleTimedOutCalls) {
CalculatorServiceProxy p(client_messenger_, server_addr_, server_addr_.host());
vector<unique_ptr<AsyncSleep>> sleeps;
+ sleeps.reserve(n_worker_threads_);
// Send enough sleep calls to occupy the worker threads.
for (int i = 0; i < n_worker_threads_; i++) {
unique_ptr<AsyncSleep> sleep(new AsyncSleep);
sleep->rpc.set_timeout(MonoDelta::FromSeconds(1));
sleep->req.set_sleep_micros(1000*1000); // 1sec
+ auto& l = sleep->latch;
p.SleepAsync(sleep->req, &sleep->resp, &sleep->rpc,
- boost::bind(&CountDownLatch::CountDown, &sleep->latch));
- sleeps.push_back(std::move(sleep));
+ [&l]() { l.CountDown(); });
+ sleeps.emplace_back(std::move(sleep));
}
// We asynchronously sent the RPCs above, but the RPCs might still
@@ -585,8 +585,9 @@ TEST_F(RpcStubTest, TestDumpCallsInFlight) {
CalculatorServiceProxy p(client_messenger_, server_addr_, server_addr_.host());
AsyncSleep sleep;
sleep.req.set_sleep_micros(100 * 1000); // 100ms
+ auto& l = sleep.latch;
p.SleepAsync(sleep.req, &sleep.resp, &sleep.rpc,
- boost::bind(&CountDownLatch::CountDown, &sleep.latch));
+ [&l]() { l.CountDown(); });
// Check the running RPC status on the client messenger.
DumpConnectionsRequestPB dump_req;
@@ -634,8 +635,9 @@ TEST_F(RpcStubTest, TestDumpSampledCalls) {
sleeps[1].req.set_sleep_micros(1500 * 1000); // 1500ms
for (auto& sleep : sleeps) {
+ auto& l = sleep.latch;
p.SleepAsync(sleep.req, &sleep.resp, &sleep.rpc,
- boost::bind(&CountDownLatch::CountDown, &sleep.latch));
+ [&l]() { l.CountDown(); });
}
for (auto& sleep : sleeps) {
sleep.latch.Wait();
@@ -693,7 +695,7 @@ TEST_F(RpcStubTest, TestCallbackClearedAfterRunning) {
req.set_y(20);
AddResponsePB resp;
p.AddAsync(req, &resp, &controller,
- boost::bind(MyTestCallback, &latch, my_refptr));
+ [&latch, my_refptr]() { MyTestCallback(&latch, my_refptr); });
latch.Wait();
// The ref count should go back down to 1. However, we need to loop a little
diff --git a/be/src/kudu/rpc/rpc_verification_util.cc b/be/src/kudu/rpc/rpc_verification_util.cc
index c0bf8b1..27c8af4 100644
--- a/be/src/kudu/rpc/rpc_verification_util.cc
+++ b/be/src/kudu/rpc/rpc_verification_util.cc
@@ -26,33 +26,33 @@
namespace kudu {
-using security::VerificationResult;
+using security::TokenVerificationResult;
namespace rpc {
-Status ParseVerificationResult(const VerificationResult& result,
- ErrorStatusPB::RpcErrorCodePB retry_error,
- ErrorStatusPB::RpcErrorCodePB* error) {
+Status ParseTokenVerificationResult(const TokenVerificationResult& result,
+ ErrorStatusPB::RpcErrorCodePB retry_error,
+ ErrorStatusPB::RpcErrorCodePB* error) {
DCHECK(error);
switch (result) {
- case VerificationResult::VALID: return Status::OK();
+ case TokenVerificationResult::VALID: return Status::OK();
- case VerificationResult::INVALID_TOKEN:
- case VerificationResult::INVALID_SIGNATURE:
- case VerificationResult::EXPIRED_TOKEN:
- case VerificationResult::EXPIRED_SIGNING_KEY: {
+ case TokenVerificationResult::INVALID_TOKEN:
+ case TokenVerificationResult::INVALID_SIGNATURE:
+ case TokenVerificationResult::EXPIRED_TOKEN:
+ case TokenVerificationResult::EXPIRED_SIGNING_KEY: {
// These errors indicate the client should get a new token and try again.
*error = retry_error;
break;
}
- case VerificationResult::UNKNOWN_SIGNING_KEY: {
+ case TokenVerificationResult::UNKNOWN_SIGNING_KEY: {
// The server doesn't recognize the signing key. This indicates that the
// server has not been updated with the most recent TSKs, so tell the
// client to try again later.
*error = ErrorStatusPB::ERROR_UNAVAILABLE;
break;
}
- case VerificationResult::INCOMPATIBLE_FEATURE: {
+ case TokenVerificationResult::INCOMPATIBLE_FEATURE: {
// These error types aren't recoverable by having the client get a new token.
*error = ErrorStatusPB::FATAL_UNAUTHORIZED;
break;
@@ -60,7 +60,7 @@ Status ParseVerificationResult(const VerificationResult& result,
default:
LOG(FATAL) << "Unknown verification result: " << static_cast<int>(result);
}
- return Status::NotAuthorized(VerificationResultToString(result));
+ return Status::NotAuthorized(TokenVerificationResultToString(result));
}
} // namespace rpc
diff --git a/be/src/kudu/rpc/rpc_verification_util.h b/be/src/kudu/rpc/rpc_verification_util.h
index 0c15b9c..cc626be 100644
--- a/be/src/kudu/rpc/rpc_verification_util.h
+++ b/be/src/kudu/rpc/rpc_verification_util.h
@@ -23,7 +23,7 @@
namespace kudu {
namespace security {
-enum class VerificationResult;
+enum class TokenVerificationResult;
} // namespace security
namespace rpc {
@@ -33,9 +33,10 @@ namespace rpc {
// otherwise returns non-OK and sets 'error' appropriately.
// 'retry_error' is the error code to be returned to denote that verification
// should be retried after retrieving a new token.
-Status ParseVerificationResult(const security::VerificationResult& result,
- ErrorStatusPB::RpcErrorCodePB retry_error,
- ErrorStatusPB::RpcErrorCodePB* error);
+Status ParseTokenVerificationResult(
+ const security::TokenVerificationResult& result,
+ ErrorStatusPB::RpcErrorCodePB retry_error,
+ ErrorStatusPB::RpcErrorCodePB* error);
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/rpcz_store.cc b/be/src/kudu/rpc/rpcz_store.cc
index 0f5f666..83c62c9 100644
--- a/be/src/kudu/rpc/rpcz_store.cc
+++ b/be/src/kudu/rpc/rpcz_store.cc
@@ -20,6 +20,7 @@
#include <algorithm> // IWYU pragma: keep
#include <array>
#include <cstdint>
+#include <map>
#include <mutex> // for unique_lock
#include <ostream>
#include <string>
@@ -261,17 +262,12 @@ void RpczStore::LogTrace(InboundCall* call) {
return;
}
}
-
- if (PREDICT_FALSE(FLAGS_rpc_dump_all_traces)) {
+ if (duration_ms > FLAGS_rpc_duration_too_long_ms ||
+ PREDICT_FALSE(FLAGS_rpc_dump_all_traces)) {
+ const auto flags = (duration_ms > FLAGS_rpc_duration_too_long_ms)
+ ? Trace::INCLUDE_ALL : Trace::INCLUDE_TIME_DELTAS;
LOG(INFO) << call->ToString() << " took " << duration_ms << "ms. Trace:";
- call->trace()->Dump(&LOG(INFO), true);
- } else if (duration_ms > FLAGS_rpc_duration_too_long_ms) {
- LOG(INFO) << call->ToString() << " took " << duration_ms << "ms. "
- << "Request Metrics: " << call->trace()->MetricsAsJSON() << "\n";
- string s = call->trace()->DumpToString();
- if (!s.empty()) {
- LOG(INFO) << "Trace:\n" << s;
- }
+ call->trace()->Dump(&LOG(INFO), flags);
}
}
diff --git a/be/src/kudu/rpc/rtest.proto b/be/src/kudu/rpc/rtest.proto
index d212cef..852bc6b 100644
--- a/be/src/kudu/rpc/rtest.proto
+++ b/be/src/kudu/rpc/rtest.proto
@@ -76,17 +76,16 @@ message SendTwoStringsResponsePB {
required uint32 sidecar2 = 2;
}
-// Push two strings to the server as part of the request, in sidecars.
-message PushTwoStringsRequestPB {
- required uint32 sidecar1_idx = 1;
- required uint32 sidecar2_idx = 2;
+// Push strings to the server as part of the request, in sidecars.
+message PushStringsRequestPB {
+ repeated uint32 sidecar_indexes = 1;
}
-message PushTwoStringsResponsePB {
- required uint32 size1 = 1;
- required string data1 = 2;
- required uint32 size2 = 3;
- required string data2 = 4;
+// The server responds with the sizes and checksums of the sidecars that
+// it received.
+message PushStringsResponsePB {
+ repeated uint32 sizes = 1;
+ repeated uint32 crcs = 2;
}
message EchoRequestPB {
diff --git a/be/src/kudu/rpc/sasl_common.cc b/be/src/kudu/rpc/sasl_common.cc
index 0107fba..ab0c3b7 100644
--- a/be/src/kudu/rpc/sasl_common.cc
+++ b/be/src/kudu/rpc/sasl_common.cc
@@ -17,6 +17,10 @@
#include "kudu/rpc/sasl_common.h"
+#include <regex.h>
+#include <sasl/sasl.h>
+#include <sasl/saslplug.h>
+
#include <cstdio>
#include <cstring>
#include <limits>
@@ -24,11 +28,7 @@
#include <ostream>
#include <string>
-#include <boost/algorithm/string/predicate.hpp>
#include <glog/logging.h>
-#include <regex.h>
-#include <sasl/sasl.h>
-#include <sasl/saslplug.h>
#include "kudu/gutil/macros.h"
#include "kudu/rpc/constants.h"
@@ -36,6 +36,8 @@
#include "kudu/util/mutex.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/rw_mutex.h"
+#include "kudu/util/stopwatch.h"
+#include "kudu/util/string_case.h"
#if defined(__APPLE__)
// Almost all functions in the SASL API are marked as deprecated
@@ -275,7 +277,6 @@ Status SaslInit(bool kerberos_keytab_provided) {
// Only execute SASL initialization once
static std::once_flag once;
std::call_once(once, DoSaslInit, kerberos_keytab_provided);
- DCHECK_EQ(kerberos_keytab_provided, has_kerberos_keytab);
return sasl_init_status;
}
@@ -315,7 +316,11 @@ static string SaslErrDesc(int status, sasl_conn_t* conn) {
return CleanSaslError(err);
}
-Status WrapSaslCall(sasl_conn_t* conn, const std::function<int()>& call) {
+Status WrapSaslCall(sasl_conn_t* conn,
+ const std::function<int()>& call,
+ const char* description) {
+ DCHECK(description);
+ SCOPED_LOG_SLOW_EXECUTION(WARNING, 250, description);
// In many cases, the GSSAPI SASL plugin will generate a nice error
// message as a message logged at SASL_LOG_FAIL logging level, but then
// return a useless one in sasl_errstring(). So, we set a global thread-local
@@ -373,6 +378,7 @@ uint32_t GetMaxSendBufferSize(sasl_conn_t* sasl_conn) {
}
Status SaslEncode(sasl_conn_t* conn, Slice plaintext, Slice* ciphertext) {
+ static constexpr const char* const kDesc = "SASL encoding";
const char* out;
unsigned out_len;
RETURN_NOT_OK_PREPEND(WrapSaslCall(conn, [&] {
@@ -380,12 +386,13 @@ Status SaslEncode(sasl_conn_t* conn, Slice plaintext, Slice* ciphertext) {
reinterpret_cast<const char*>(plaintext.data()),
plaintext.size(),
&out, &out_len);
- }), "SASL encode failed");
+ }, kDesc), string(kDesc) + " failed");
*ciphertext = Slice(out, out_len);
return Status::OK();
}
Status SaslDecode(sasl_conn_t* conn, Slice ciphertext, Slice* plaintext) {
+ static constexpr const char* const kDesc = "SASL decoding";
const char* out;
unsigned out_len;
RETURN_NOT_OK_PREPEND(WrapSaslCall(conn, [&] {
@@ -393,7 +400,7 @@ Status SaslDecode(sasl_conn_t* conn, Slice ciphertext, Slice* plaintext) {
reinterpret_cast<const char*>(ciphertext.data()),
ciphertext.size(),
&out, &out_len);
- }), "SASL decode failed");
+ }, kDesc), string(kDesc) + " failed");
*plaintext = Slice(out, out_len);
return Status::OK();
}
@@ -433,6 +440,8 @@ sasl_callback_t SaslBuildCallback(int id, int (*proc)(void), void* context) {
Status EnableProtection(sasl_conn_t* sasl_conn,
SaslProtection::Type minimum_protection,
size_t max_recv_buf_size) {
+ static constexpr const char* const kDesc = "setting SASL security properties";
+
sasl_security_properties_t sec_props;
memset(&sec_props, 0, sizeof(sec_props));
sec_props.min_ssf = minimum_protection;
@@ -441,15 +450,15 @@ Status EnableProtection(sasl_conn_t* sasl_conn,
RETURN_NOT_OK_PREPEND(WrapSaslCall(sasl_conn, [&] {
return sasl_setprop(sasl_conn, SASL_SEC_PROPS, &sec_props);
- }), "failed to set SASL security properties");
+ }, kDesc), string(kDesc) + " failed");
return Status::OK();
}
SaslMechanism::Type SaslMechanism::value_of(const string& mech) {
- if (boost::iequals(mech, "PLAIN")) {
+ if (iequals(mech, "PLAIN")) {
return PLAIN;
}
- if (boost::iequals(mech, "GSSAPI")) {
+ if (iequals(mech, "GSSAPI")) {
return GSSAPI;
}
return INVALID;
diff --git a/be/src/kudu/rpc/sasl_common.h b/be/src/kudu/rpc/sasl_common.h
index dcda131..06311e7 100644
--- a/be/src/kudu/rpc/sasl_common.h
+++ b/be/src/kudu/rpc/sasl_common.h
@@ -88,7 +88,9 @@ Status SaslInit(bool kerberos_keytab_provided = false) WARN_UNUSED_RESULT;
Status DisableSaslInitialization() WARN_UNUSED_RESULT;
// Wrap a call into the SASL library. 'call' should be a lambda which
-// returns a SASL error code.
+// returns a SASL error code, 'conn' is the SASL connection to perform
+// the operation with (might be null), and 'description' is be a description
+// of the SASL call (e.g., the name of the SASL library function).
//
// The result is translated into a Status as follows:
//
@@ -98,7 +100,9 @@ Status DisableSaslInitialization() WARN_UNUSED_RESULT;
//
// The Status message is beautified to be more user-friendly compared
// to the underlying sasl_errdetails() call.
-Status WrapSaslCall(sasl_conn_t* conn, const std::function<int()>& call) WARN_UNUSED_RESULT;
+Status WrapSaslCall(sasl_conn_t* conn,
+ const std::function<int()>& call,
+ const char* description) WARN_UNUSED_RESULT;
// Return <ip>;<port> string formatted for SASL library use.
std::string SaslIpPortString(const Sockaddr& addr);
diff --git a/be/src/kudu/rpc/serialization.cc b/be/src/kudu/rpc/serialization.cc
index 28c015b..b845407 100644
--- a/be/src/kudu/rpc/serialization.cc
+++ b/be/src/kudu/rpc/serialization.cc
@@ -19,12 +19,11 @@
#include <limits>
#include <ostream>
-#include <string>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
-#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/message_lite.h>
#include "kudu/gutil/endian.h"
#include "kudu/gutil/port.h"
diff --git a/be/src/kudu/rpc/server_negotiation.cc b/be/src/kudu/rpc/server_negotiation.cc
index 79d9925..bcbebf6 100644
--- a/be/src/kudu/rpc/server_negotiation.cc
+++ b/be/src/kudu/rpc/server_negotiation.cc
@@ -17,9 +17,13 @@
#include "kudu/rpc/server_negotiation.h"
+#include <sasl/sasl.h>
+#include <sys/socket.h>
+
#include <algorithm>
#include <cstdint>
#include <cstdlib>
+#include <functional>
#include <memory>
#include <mutex>
#include <ostream>
@@ -28,9 +32,7 @@
#include <boost/optional/optional.hpp>
#include <gflags/gflags.h>
-#include <gflags/gflags_declare.h>
#include <glog/logging.h>
-#include <sasl/sasl.h>
#include "kudu/gutil/macros.h"
#include "kudu/gutil/map-util.h"
@@ -39,7 +41,6 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/blocking_ops.h"
#include "kudu/rpc/constants.h"
-#include "kudu/rpc/messenger.h"
#include "kudu/rpc/rpc_verification_util.h"
#include "kudu/rpc/serialization.h"
#include "kudu/security/cert.h"
@@ -66,6 +67,7 @@
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif // #if defined(__APPLE__)
+using kudu::security::RpcEncryption;
using std::set;
using std::string;
using std::unique_ptr;
@@ -154,12 +156,15 @@ ServerNegotiation::ServerNegotiation(unique_ptr<Socket> socket,
const security::TlsContext* tls_context,
const security::TokenVerifier* token_verifier,
RpcEncryption encryption,
+ bool encrypt_loopback,
std::string sasl_proto_name)
: socket_(std::move(socket)),
helper_(SaslHelper::SERVER),
tls_context_(tls_context),
+ tls_handshake_(security::TlsHandshakeType::SERVER),
encryption_(encryption),
tls_negotiated_(false),
+ encrypt_loopback_(encrypt_loopback),
token_verifier_(token_verifier),
negotiated_authn_(AuthenticationType::INVALID),
negotiated_mech_(SaslMechanism::INVALID),
@@ -221,8 +226,7 @@ Status ServerNegotiation::Negotiate() {
if (encryption_ != RpcEncryption::DISABLED &&
tls_context_->has_cert() &&
ContainsKey(client_features_, TLS)) {
- RETURN_NOT_OK(tls_context_->InitiateHandshake(security::TlsHandshakeType::SERVER,
- &tls_handshake_));
+ RETURN_NOT_OK(tls_context_->InitiateHandshake(&tls_handshake_));
if (negotiated_authn_ != AuthenticationType::CERTIFICATE) {
// The server does not need to verify the client's certificate unless it's
@@ -234,8 +238,12 @@ Status ServerNegotiation::Negotiate() {
NegotiatePB request;
RETURN_NOT_OK(RecvNegotiatePB(&request, &recv_buf));
Status s = HandleTlsHandshake(request);
- if (s.ok()) break;
- if (!s.IsIncomplete()) return s;
+ if (s.ok()) {
+ break;
+ }
+ if (!s.IsIncomplete()) {
+ return s;
+ }
}
tls_negotiated_ = true;
}
@@ -298,7 +306,7 @@ Status ServerNegotiation::PreflightCheckGSSAPI(const std::string& sasl_proto_nam
// We aren't going to actually send/receive any messages, but
// this makes it easier to reuse the initialization code.
ServerNegotiation server(
- nullptr, nullptr, nullptr, RpcEncryption::OPTIONAL, sasl_proto_name);
+ nullptr, nullptr, nullptr, RpcEncryption::OPTIONAL, false, sasl_proto_name);
Status s = server.EnableGSSAPI();
if (!s.ok()) {
return Status::RuntimeError(s.message());
@@ -315,7 +323,7 @@ Status ServerNegotiation::PreflightCheckGSSAPI(const std::string& sasl_proto_nam
kSaslMechGSSAPI,
"", 0, // Pass a 0-length token.
&server_out, &server_out_len);
- });
+ }, "calling sasl_server_start()");
// We expect 'Incomplete' status to indicate that the first step of negotiation
// was correct.
@@ -333,7 +341,8 @@ Status ServerNegotiation::PreflightCheckGSSAPI(const std::string& sasl_proto_nam
Status ServerNegotiation::RecvNegotiatePB(NegotiatePB* msg, faststring* recv_buf) {
RequestHeader header;
Slice param_buf;
- RETURN_NOT_OK(ReceiveFramedMessageBlocking(socket(), recv_buf, &header, ¶m_buf, deadline_));
+ RETURN_NOT_OK(ReceiveFramedMessageBlocking(
+ socket_.get(), recv_buf, &header, ¶m_buf, deadline_));
Status s = helper_.CheckNegotiateCallId(header.call_id());
if (!s.ok()) {
RETURN_NOT_OK(SendError(ErrorStatusPB::FATAL_INVALID_RPC_HEADER, s));
@@ -359,7 +368,7 @@ Status ServerNegotiation::SendNegotiatePB(const NegotiatePB& msg) {
DCHECK(msg.has_step()) << "message must have a step";
TRACE("Sending $0 NegotiatePB response", NegotiatePB::NegotiateStep_Name(msg.step()));
- return SendFramedMessageBlocking(socket(), header, msg, deadline_);
+ return SendFramedMessageBlocking(socket_.get(), header, msg, deadline_);
}
Status ServerNegotiation::SendError(ErrorStatusPB::RpcErrorCodePB code, const Status& err) {
@@ -376,9 +385,7 @@ Status ServerNegotiation::SendError(ErrorStatusPB::RpcErrorCodePB code, const St
msg.set_message(err.ToString());
TRACE("Sending RPC error: $0: $1", ErrorStatusPB::RpcErrorCodePB_Name(code), err.ToString());
- RETURN_NOT_OK(SendFramedMessageBlocking(socket(), header, msg, deadline_));
-
- return Status::OK();
+ return SendFramedMessageBlocking(socket_.get(), header, msg, deadline_);
}
Status ServerNegotiation::ValidateConnectionHeader(faststring* recv_buf) {
@@ -413,6 +420,8 @@ Status ServerNegotiation::InitSaslServer() {
server_fqdn = default_server_fqdn.c_str();
}
+ static constexpr const char* const kDesc = "creating new SASL server";
+ //static constexpr const char* const kDesc = "create "
RETURN_NOT_OK_PREPEND(WrapSaslCall(nullptr /* no conn */, [&]() {
return sasl_server_new(
// Registered name of the service using SASL. Required.
@@ -430,7 +439,7 @@ Status ServerNegotiation::InitSaslServer() {
// Security flags.
secflags,
&sasl_conn);
- }), "Unable to create new SASL server");
+ }, kDesc), string(kDesc) + " failed");
sasl_conn_.reset(sasl_conn);
return Status::OK();
}
@@ -535,7 +544,7 @@ Status ServerNegotiation::HandleNegotiate(const NegotiatePB& request) {
server_features_.insert(TLS);
// If the remote peer is local, then we allow using TLS for authentication
// without encryption or integrity.
- if (socket_->IsLoopbackConnection() && !FLAGS_rpc_encrypt_loopback_connections) {
+ if (socket_->IsLoopbackConnection() && !encrypt_loopback_) {
server_features_.insert(TLS_AUTHENTICATION_ONLY);
}
}
@@ -590,18 +599,26 @@ Status ServerNegotiation::HandleTlsHandshake(const NegotiatePB& request) {
}
string token;
- Status s = tls_handshake_.Continue(request.tls_handshake(), &token);
-
+ const Status s = tls_handshake_.Continue(request.tls_handshake(), &token);
if (PREDICT_FALSE(!s.IsIncomplete() && !s.ok())) {
RETURN_NOT_OK(SendError(ErrorStatusPB::FATAL_UNAUTHORIZED, s));
return s;
}
+ const bool needs_extra_step = tls_handshake_.NeedsExtraStep(s, token);
+ if (needs_extra_step) {
+ RETURN_NOT_OK(SendTlsHandshake(std::move(token)));
+ }
- // Regardless of whether this is the final handshake roundtrip (in which case
- // Continue would have returned OK), we still need to return a response.
- RETURN_NOT_OK(SendTlsHandshake(std::move(token)));
+ // Check that the handshake step didn't produce an error. It also propagates
+ // any non-OK status.
RETURN_NOT_OK(s);
+ if (!needs_extra_step && !token.empty()) {
+ DCHECK(s.ok());
+ DCHECK(!token.empty());
+ tls_handshake_.StorePendingData(std::move(token));
+ }
+
// TLS handshake is finished.
if (ContainsKey(server_features_, TLS_AUTHENTICATION_ONLY) &&
ContainsKey(client_features_, TLS_AUTHENTICATION_ONLY)) {
@@ -684,7 +701,7 @@ Status ServerNegotiation::AuthenticateByToken(faststring* recv_buf) {
security::TokenPB token;
auto verification_result = token_verifier_->VerifyTokenSignature(pb.authn_token(), &token);
ErrorStatusPB::RpcErrorCodePB error;
- Status s = ParseVerificationResult(verification_result,
+ Status s = ParseTokenVerificationResult(verification_result,
ErrorStatusPB::FATAL_INVALID_AUTHENTICATION_TOKEN, &error);
if (!s.ok()) {
RETURN_NOT_OK(SendError(error, s));
@@ -705,26 +722,26 @@ Status ServerNegotiation::AuthenticateByToken(faststring* recv_buf) {
}
if (PREDICT_FALSE(FLAGS_rpc_inject_invalid_authn_token_ratio > 0)) {
- security::VerificationResult res;
+ security::TokenVerificationResult res;
int sel = rand() % 4;
switch (sel) {
case 0:
- res = security::VerificationResult::INVALID_TOKEN;
+ res = security::TokenVerificationResult::INVALID_TOKEN;
break;
case 1:
- res = security::VerificationResult::INVALID_SIGNATURE;
+ res = security::TokenVerificationResult::INVALID_SIGNATURE;
break;
case 2:
- res = security::VerificationResult::EXPIRED_TOKEN;
+ res = security::TokenVerificationResult::EXPIRED_TOKEN;
break;
case 3:
- res = security::VerificationResult::EXPIRED_SIGNING_KEY;
+ res = security::TokenVerificationResult::EXPIRED_SIGNING_KEY;
break;
default:
LOG(FATAL) << "unreachable";
}
if (kudu::fault_injection::MaybeTrue(FLAGS_rpc_inject_invalid_authn_token_ratio)) {
- Status s = Status::NotAuthorized(VerificationResultToString(res));
+ Status s = Status::NotAuthorized(TokenVerificationResultToString(res));
RETURN_NOT_OK(SendError(ErrorStatusPB::FATAL_INVALID_AUTHENTICATION_TOKEN, s));
return s;
}
@@ -808,7 +825,8 @@ Status ServerNegotiation::HandleSaslInitiate(const NegotiatePB& request) {
const char* server_out = nullptr;
uint32_t server_out_len = 0;
- TRACE("Calling sasl_server_start()");
+ static constexpr const char* const kDesc = "calling sasl_server_start()";
+ TRACE(kDesc);
Status s = WrapSaslCall(sasl_conn_.get(), [&]() {
return sasl_server_start(
@@ -818,7 +836,7 @@ Status ServerNegotiation::HandleSaslInitiate(const NegotiatePB& request) {
request.token().length(), // Client string len.
&server_out, // The output of the SASL library, might not be NULL terminated
&server_out_len); // Output len.
- });
+ }, kDesc);
if (PREDICT_FALSE(!s.ok() && !s.IsIncomplete())) {
RETURN_NOT_OK(SendError(ErrorStatusPB::FATAL_UNAUTHORIZED, s));
@@ -850,9 +868,10 @@ Status ServerNegotiation::HandleSaslResponse(const NegotiatePB& request) {
return s;
}
+ static constexpr const char* const kDesc = "calling sasl_server_step()";
const char* server_out = nullptr;
uint32_t server_out_len = 0;
- TRACE("Calling sasl_server_step()");
+ TRACE(kDesc);
Status s = WrapSaslCall(sasl_conn_.get(), [&]() {
return sasl_server_step(
sasl_conn_.get(), // The SASL connection context created by init()
@@ -860,7 +879,7 @@ Status ServerNegotiation::HandleSaslResponse(const NegotiatePB& request) {
request.token().length(), // Client string len
&server_out, // The output of the SASL library, might not be NULL terminated
&server_out_len); // Output len
- });
+ }, kDesc);
if (s.ok()) {
DCHECK(server_out_len == 0);
@@ -913,15 +932,15 @@ Status ServerNegotiation::SendSaslSuccess() {
}
}
- RETURN_NOT_OK(SendNegotiatePB(response));
- return Status::OK();
+ return SendNegotiatePB(response);
}
Status ServerNegotiation::RecvConnectionContext(faststring* recv_buf) {
TRACE("Waiting for connection context");
RequestHeader header;
Slice param_buf;
- RETURN_NOT_OK(ReceiveFramedMessageBlocking(socket(), recv_buf, &header, ¶m_buf, deadline_));
+ RETURN_NOT_OK(ReceiveFramedMessageBlocking(
+ socket_.get(), recv_buf, &header, ¶m_buf, deadline_));
DCHECK(header.IsInitialized());
if (header.call_id() != kConnectionContextCallId) {
@@ -983,6 +1002,7 @@ int ServerNegotiation::PlainAuthCb(sasl_conn_t* /*conn*/,
}
bool ServerNegotiation::IsTrustedConnection(const Sockaddr& addr) {
+ if (addr.family() == AF_UNIX) return true;
static std::once_flag once;
std::call_once(once, [] {
g_trusted_subnets = new vector<Network>();
diff --git a/be/src/kudu/rpc/server_negotiation.h b/be/src/kudu/rpc/server_negotiation.h
index 2582af1..6f22798 100644
--- a/be/src/kudu/rpc/server_negotiation.h
+++ b/be/src/kudu/rpc/server_negotiation.h
@@ -17,6 +17,8 @@
#pragma once
+#include <sasl/sasl.h>
+
#include <memory>
#include <set>
#include <string>
@@ -25,10 +27,8 @@
#include <boost/optional/optional.hpp>
#include <glog/logging.h>
-#include <sasl/sasl.h>
#include "kudu/gutil/port.h"
-#include "kudu/rpc/messenger.h"
#include "kudu/rpc/negotiation.h"
#include "kudu/rpc/remote_user.h"
#include "kudu/rpc/rpc_header.pb.h"
@@ -65,7 +65,8 @@ class ServerNegotiation {
ServerNegotiation(std::unique_ptr<Socket> socket,
const security::TlsContext* tls_context,
const security::TokenVerifier* token_verifier,
- RpcEncryption encryption,
+ security::RpcEncryption encryption,
+ bool encrypt_loopback,
std::string sasl_proto_name);
// Enable PLAIN authentication.
@@ -213,7 +214,7 @@ class ServerNegotiation {
Status RecvConnectionContext(faststring* recv_buf) WARN_UNUSED_RESULT;
// Returns true if connection is from trusted subnets or local networks.
- bool IsTrustedConnection(const Sockaddr& addr);
+ static bool IsTrustedConnection(const Sockaddr& addr);
// The socket to the remote client.
std::unique_ptr<Socket> socket_;
@@ -227,8 +228,9 @@ class ServerNegotiation {
// TLS state.
const security::TlsContext* tls_context_;
security::TlsHandshake tls_handshake_;
- const RpcEncryption encryption_;
+ const security::RpcEncryption encryption_;
bool tls_negotiated_;
+ bool encrypt_loopback_;
// TSK state.
const security::TokenVerifier* token_verifier_;
diff --git a/be/src/kudu/rpc/service_if.cc b/be/src/kudu/rpc/service_if.cc
index af53452..63c5aec 100644
--- a/be/src/kudu/rpc/service_if.cc
+++ b/be/src/kudu/rpc/service_if.cc
@@ -48,7 +48,6 @@ TAG_FLAG(enable_exactly_once, hidden);
using google::protobuf::Message;
using std::string;
-using std::unique_ptr;
using strings::Substitute;
namespace kudu {
@@ -64,7 +63,7 @@ bool ServiceIf::SupportsFeature(uint32_t feature) const {
return false;
}
-bool ServiceIf::ParseParam(InboundCall *call, google::protobuf::Message *message) {
+bool ServiceIf::ParseParam(InboundCall* call, Message* message) {
Slice param(call->serialized_request());
if (PREDICT_FALSE(!message->ParseFromArray(param.data(), param.size()))) {
string err = Substitute("invalid parameter for call $0: missing fields: $1",
@@ -78,10 +77,10 @@ bool ServiceIf::ParseParam(InboundCall *call, google::protobuf::Message *message
return true;
}
-void ServiceIf::RespondBadMethod(InboundCall *call) {
- Sockaddr local_addr, remote_addr;
-
+void ServiceIf::RespondBadMethod(InboundCall* call) {
+ Sockaddr local_addr;
CHECK_OK(call->connection()->socket()->GetSocketAddress(&local_addr));
+ Sockaddr remote_addr;
CHECK_OK(call->connection()->socket()->GetPeerAddress(&remote_addr));
string err = Substitute("Call on service $0 received at $1 from $2 with an "
"invalid method name: $3",
@@ -97,20 +96,19 @@ void ServiceIf::RespondBadMethod(InboundCall *call) {
GeneratedServiceIf::~GeneratedServiceIf() {
}
-
-void GeneratedServiceIf::Handle(InboundCall *call) {
+void GeneratedServiceIf::Handle(InboundCall* call) {
const RpcMethodInfo* method_info = call->method_info();
if (!method_info) {
RespondBadMethod(call);
return;
}
- unique_ptr<Message> req(method_info->req_prototype->New());
- if (PREDICT_FALSE(!ParseParam(call, req.get()))) {
+ Message* req = method_info->req_prototype->New(call->pb_arena());
+ if (PREDICT_FALSE(!ParseParam(call, req))) {
return;
}
- Message* resp = method_info->resp_prototype->New();
+ Message* resp = method_info->resp_prototype->New(call->pb_arena());
- RpcContext* ctx = new RpcContext(call, req.release(), resp);
+ RpcContext* ctx = new RpcContext(call, req, resp);
if (!method_info->authz_method(ctx->request_pb(), resp, ctx)) {
// The authz_method itself should have responded to the RPC.
return;
@@ -139,7 +137,6 @@ void GeneratedServiceIf::Handle(InboundCall *call) {
method_info->func(ctx->request_pb(), resp, ctx);
}
-
RpcMethodInfo* GeneratedServiceIf::LookupMethod(const RemoteMethod& method) {
DCHECK_EQ(method.service_name(), service_name());
const auto& it = methods_by_name_.find(method.method_name());
@@ -149,6 +146,9 @@ RpcMethodInfo* GeneratedServiceIf::LookupMethod(const RemoteMethod& method) {
return it->second.get();
}
+GeneratedServiceIf::GeneratedServiceIf(const scoped_refptr<ResultTracker>& tracker)
+ : result_tracker_(tracker) {
+}
} // namespace rpc
} // namespace kudu
diff --git a/be/src/kudu/rpc/service_if.h b/be/src/kudu/rpc/service_if.h
index 686580e..185e291 100644
--- a/be/src/kudu/rpc/service_if.h
+++ b/be/src/kudu/rpc/service_if.h
@@ -47,6 +47,7 @@ struct RpcMethodInfo : public RefCountedThreadSafe<RpcMethodInfo> {
std::unique_ptr<google::protobuf::Message> resp_prototype;
scoped_refptr<Histogram> handler_latency_histogram;
+ scoped_refptr<Counter> queue_overflow_rejections;
// Whether we should track this method's result, using ResultTracker.
bool track_result;
@@ -68,9 +69,9 @@ struct RpcMethodInfo : public RefCountedThreadSafe<RpcMethodInfo> {
class ServiceIf {
public:
virtual ~ServiceIf();
- virtual void Handle(InboundCall* incoming) = 0;
+ virtual void Handle(InboundCall* call) = 0;
virtual void Shutdown();
- virtual std::string service_name() const = 0;
+ virtual const std::string& service_name() const = 0;
// The service should return true if it supports the provided application
// specific feature flag.
@@ -80,7 +81,7 @@ class ServiceIf {
//
// If this returns nullptr, then certain functionality like
// metrics collection will not be performed for this call.
- virtual RpcMethodInfo* LookupMethod(const RemoteMethod& method) {
+ virtual RpcMethodInfo* LookupMethod(const RemoteMethod& /*method*/) {
return nullptr;
}
@@ -95,8 +96,8 @@ class ServiceIf {
}
protected:
- bool ParseParam(InboundCall* call, google::protobuf::Message* message);
- void RespondBadMethod(InboundCall* call);
+ static bool ParseParam(InboundCall* call, google::protobuf::Message* message);
+ static void RespondBadMethod(InboundCall* call);
};
@@ -109,7 +110,7 @@ class GeneratedServiceIf : public ServiceIf {
// it on the current thread.
//
// If no such method is found, responds with an error.
- void Handle(InboundCall* incoming) override;
+ void Handle(InboundCall* call) override;
RpcMethodInfo* LookupMethod(const RemoteMethod& method) override;
@@ -118,14 +119,16 @@ class GeneratedServiceIf : public ServiceIf {
const MethodInfoMap& methods_by_name() const { return methods_by_name_; }
protected:
+ explicit GeneratedServiceIf(const scoped_refptr<ResultTracker>& tracker);
+
+ // The result tracker for this service's methods.
+ const scoped_refptr<ResultTracker> result_tracker_;
+
// For each method, stores the relevant information about how to handle the
// call. Methods are inserted by the constructor of the generated subclass.
// After construction, this map is accessed by multiple threads and therefore
// must not be modified.
MethodInfoMap methods_by_name_;
-
- // The result tracker for this service's methods.
- scoped_refptr<ResultTracker> result_tracker_;
};
} // namespace rpc
diff --git a/be/src/kudu/rpc/service_pool.cc b/be/src/kudu/rpc/service_pool.cc
index a12a34d..d1755aa 100644
--- a/be/src/kudu/rpc/service_pool.cc
+++ b/be/src/kudu/rpc/service_pool.cc
@@ -28,6 +28,7 @@
#include <glog/logging.h>
#include "kudu/gutil/basictypes.h"
+#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
@@ -89,8 +90,10 @@ ServicePool::~ServicePool() {
Status ServicePool::Init(int num_threads) {
for (int i = 0; i < num_threads; i++) {
scoped_refptr<kudu::Thread> new_thread;
- CHECK_OK(kudu::Thread::Create("service pool", "rpc worker",
- &ServicePool::RunThread, this, &new_thread));
+ CHECK_OK(kudu::Thread::Create(
+ Substitute("service pool $0", service_->service_name()),
+ "rpc worker",
+ [this]() { this->RunThread(); }, &new_thread));
threads_.push_back(new_thread);
}
return Status::OK();
@@ -126,6 +129,10 @@ void ServicePool::RejectTooBusy(InboundCall* c) {
c->remote_address().ToString(),
service_queue_.max_size());
rpcs_queue_overflow_->Increment();
+ auto* minfo = c->method_info();
+ if (minfo) {
+ minfo->queue_overflow_rejections->Increment();
+ }
KLOG_EVERY_N_SECS(WARNING, 1) << err_msg << THROTTLE_MSG;
c->RespondFailure(ErrorStatusPB::ERROR_SERVER_TOO_BUSY,
Status::ServiceUnavailable(err_msg));
@@ -226,7 +233,7 @@ void ServicePool::RunThread() {
}
}
-const string ServicePool::service_name() const {
+const string& ServicePool::service_name() const {
return service_->service_name();
}
diff --git a/be/src/kudu/rpc/service_pool.h b/be/src/kudu/rpc/service_pool.h
index 3e7229a..986ffbf 100644
--- a/be/src/kudu/rpc/service_pool.h
+++ b/be/src/kudu/rpc/service_pool.h
@@ -24,7 +24,6 @@
#include <vector>
#include "kudu/gutil/macros.h"
-#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/rpc_service.h"
#include "kudu/rpc/service_queue.h"
@@ -43,7 +42,6 @@ namespace rpc {
class InboundCall;
class RemoteMethod;
class ServiceIf;
-
struct RpcMethodInfo;
// A pool of threads that handle new incoming RPC calls.
@@ -74,7 +72,7 @@ class ServicePool : public RpcService {
RpcMethodInfo* LookupMethod(const RemoteMethod& method) override;
- virtual Status QueueInboundCall(std::unique_ptr<InboundCall> call) OVERRIDE;
+ Status QueueInboundCall(std::unique_ptr<InboundCall> call) override;
const Counter* RpcsTimedOutInQueueMetricForTests() const {
return rpcs_timed_out_in_queue_.get();
@@ -88,7 +86,7 @@ class ServicePool : public RpcService {
return rpcs_queue_overflow_.get();
}
- const std::string service_name() const;
+ const std::string& service_name() const;
private:
void RunThread();
diff --git a/be/src/kudu/rpc/transfer.cc b/be/src/kudu/rpc/transfer.cc
index 674a9f6..cdb3b5e 100644
--- a/be/src/kudu/rpc/transfer.cc
+++ b/be/src/kudu/rpc/transfer.cc
@@ -17,14 +17,16 @@
#include "kudu/rpc/transfer.h"
-#include <sys/uio.h>
+#include <sys/socket.h>
#include <algorithm>
+#include <cstddef>
#include <cstdint>
#include <iostream>
#include <limits>
#include <set>
+#include <boost/container/vector.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
@@ -41,9 +43,7 @@ DEFINE_bool(rpc_max_message_size_enable_validation, true,
"This is a test-only flag.");
TAG_FLAG(rpc_max_message_size_enable_validation, unsafe);
-// Hidden in Impala because we require a particular value and override the user specified
-// value anyways, see RpcMgr::Init() and IMPALA-4874.
-DEFINE_int64_hidden(rpc_max_message_size, (50 * 1024 * 1024),
+DEFINE_int64(rpc_max_message_size, (50 * 1024 * 1024),
"The maximum size of a message that any RPC that the server will accept. "
"Must be at least 1MB.");
TAG_FLAG(rpc_max_message_size, advanced);
@@ -85,67 +85,87 @@ using strings::Substitute;
} \
} while (0)
-TransferCallbacks::~TransferCallbacks()
-{}
-
InboundTransfer::InboundTransfer()
- : total_length_(kMsgLengthPrefixLength),
- cur_offset_(0) {
+ : total_length_(0),
+ cur_offset_(0) {
buf_.resize(kMsgLengthPrefixLength);
}
-Status InboundTransfer::ReceiveBuffer(Socket &socket) {
- if (cur_offset_ < kMsgLengthPrefixLength) {
- // receive uint32 length prefix
- int32_t rem = kMsgLengthPrefixLength - cur_offset_;
- int32_t nread;
- Status status = socket.Recv(&buf_[cur_offset_], rem, &nread);
- RETURN_ON_ERROR_OR_SOCKET_NOT_READY(status);
- if (nread == 0) {
- return Status::OK();
- }
- DCHECK_GE(nread, 0);
- cur_offset_ += nread;
+InboundTransfer::InboundTransfer(faststring initial_buf)
+ : buf_(std::move(initial_buf)),
+ total_length_(0),
+ cur_offset_(buf_.size()) {
+ buf_.resize(std::max<size_t>(kMsgLengthPrefixLength, buf_.size()));
+}
+
+Status InboundTransfer::ReceiveBuffer(Socket* socket, faststring* extra_4) {
+ static constexpr int kExtraReadLength = kMsgLengthPrefixLength;
+ if (total_length_ == 0) {
+ // We haven't yet parsed the message length. It's possible that the
+ // length is already available in the buffer passed in the constructor.
if (cur_offset_ < kMsgLengthPrefixLength) {
- // If we still don't have the full length prefix, we can't continue
- // reading yet.
- return Status::OK();
+ // receive uint32 length prefix
+ int32_t rem = kMsgLengthPrefixLength - cur_offset_;
+ int32_t nread;
+ Status status = socket->Recv(&buf_[cur_offset_], rem, &nread);
+ RETURN_ON_ERROR_OR_SOCKET_NOT_READY(status);
+ if (nread == 0) {
+ return Status::OK();
+ }
+ DCHECK_GE(nread, 0);
+ cur_offset_ += nread;
+ if (cur_offset_ < kMsgLengthPrefixLength) {
+ // If we still don't have the full length prefix, we can't continue
+ // reading yet.
+ return Status::OK();
+ }
}
- // Since we only read 'rem' bytes above, we should now have exactly
- // the length prefix in our buffer and no more.
- DCHECK_EQ(cur_offset_, kMsgLengthPrefixLength);
+ // Parse the message length out of the prefix.
+ DCHECK_GE(cur_offset_, kMsgLengthPrefixLength);
// The length prefix doesn't include its own 4 bytes, so we have to
// add that back in.
total_length_ = NetworkByteOrder::Load32(&buf_[0]) + kMsgLengthPrefixLength;
- if (total_length_ > FLAGS_rpc_max_message_size) {
+ if (PREDICT_FALSE(total_length_ > FLAGS_rpc_max_message_size)) {
return Status::NetworkError(Substitute(
"RPC frame had a length of $0, but we only support messages up to $1 bytes "
"long.", total_length_, FLAGS_rpc_max_message_size));
}
- if (total_length_ <= kMsgLengthPrefixLength) {
- return Status::NetworkError(Substitute("RPC frame had invalid length of $0",
- total_length_));
+ if (PREDICT_FALSE(total_length_ <= kMsgLengthPrefixLength)) {
+ return Status::NetworkError(
+ Substitute("RPC frame had invalid length of $0", total_length_));
}
- buf_.resize(total_length_);
+ buf_.resize(total_length_ + kExtraReadLength);
// Fall through to receive the message body, which is likely to be already
// available on the socket.
}
- // receive message body
- int32_t nread;
-
// Socket::Recv() handles at most INT_MAX at a time, so cap the remainder at
// INT_MAX. The message will be split across multiple Recv() calls.
// Note that this is only needed when rpc_max_message_size > INT_MAX, which is
// currently only used for unit tests.
- int32_t rem = std::min(total_length_ - cur_offset_,
+ int32_t rem = std::min(total_length_ - cur_offset_ + kExtraReadLength,
static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
- Status status = socket.Recv(&buf_[cur_offset_], rem, &nread);
+
+ // receive message body
+ int32_t nread;
+ Status status = socket->Recv(&buf_[cur_offset_], rem, &nread);
RETURN_ON_ERROR_OR_SOCKET_NOT_READY(status);
cur_offset_ += nread;
+ // We may have read some extra bytes, in which case we need to trim them off
+ // and write them into the provided buffer.
+ if (cur_offset_ >= total_length_) {
+ int64_t extra_read = cur_offset_ - total_length_;
+ DCHECK_LE(extra_read, kExtraReadLength);
+ DCHECK_GE(extra_read, 0);
+ extra_4->clear();
+ extra_4->append(&buf_[total_length_], extra_read);
+ cur_offset_ = total_length_;
+ buf_.resize(total_length_);
+ }
+
return Status::OK();
}
@@ -154,7 +174,7 @@ bool InboundTransfer::TransferStarted() const {
}
bool InboundTransfer::TransferFinished() const {
- return cur_offset_ == total_length_;
+ return total_length_ > 0 && cur_offset_ == total_length_;
}
string InboundTransfer::StatusAsString() const {
@@ -162,34 +182,26 @@ string InboundTransfer::StatusAsString() const {
}
OutboundTransfer* OutboundTransfer::CreateForCallRequest(int32_t call_id,
- const TransferPayload &payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks) {
- return new OutboundTransfer(call_id, payload, n_payload_slices, callbacks);
+ TransferPayload payload,
+ TransferCallbacks* callbacks) {
+ return new OutboundTransfer(call_id, std::move(payload), callbacks);
}
-OutboundTransfer* OutboundTransfer::CreateForCallResponse(const TransferPayload &payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks) {
- return new OutboundTransfer(kInvalidCallId, payload, n_payload_slices, callbacks);
+OutboundTransfer* OutboundTransfer::CreateForCallResponse(TransferPayload payload,
+ TransferCallbacks* callbacks) {
+ return new OutboundTransfer(kInvalidCallId, std::move(payload), callbacks);
}
OutboundTransfer::OutboundTransfer(int32_t call_id,
- const TransferPayload &payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks)
- : cur_slice_idx_(0),
- cur_offset_in_slice_(0),
- callbacks_(callbacks),
- call_id_(call_id),
- started_(false),
- aborted_(false) {
-
- n_payload_slices_ = n_payload_slices;
- CHECK_LE(n_payload_slices_, payload_slices_.size());
- for (int i = 0; i < n_payload_slices; i++) {
- payload_slices_[i] = payload[i];
- }
+ TransferPayload payload,
+ TransferCallbacks* callbacks)
+ : payload_slices_(std::move(payload)),
+ cur_slice_idx_(0),
+ cur_offset_in_slice_(0),
+ callbacks_(callbacks),
+ call_id_(call_id),
+ started_(false),
+ aborted_(false) {
}
OutboundTransfer::~OutboundTransfer() {
@@ -199,23 +211,23 @@ OutboundTransfer::~OutboundTransfer() {
}
}
-void OutboundTransfer::Abort(const Status &status) {
+void OutboundTransfer::Abort(const Status& status) {
CHECK(!aborted_) << "Already aborted";
CHECK(!TransferFinished()) << "Cannot abort a finished transfer";
callbacks_->NotifyTransferAborted(status);
aborted_ = true;
}
-Status OutboundTransfer::SendBuffer(Socket &socket) {
- CHECK_LT(cur_slice_idx_, n_payload_slices_);
+Status OutboundTransfer::SendBuffer(Socket* socket) {
+ CHECK_LT(cur_slice_idx_, payload_slices_.size());
started_ = true;
- int n_iovecs = n_payload_slices_ - cur_slice_idx_;
+ int n_iovecs = std::min<int>(payload_slices_.size() - cur_slice_idx_, IOV_MAX);
struct iovec iovec[n_iovecs];
{
int offset_in_slice = cur_offset_in_slice_;
for (int i = 0; i < n_iovecs; i++) {
- Slice &slice = payload_slices_[cur_slice_idx_ + i];
+ auto& slice = payload_slices_[cur_slice_idx_ + i];
iovec[i].iov_base = slice.mutable_data() + offset_in_slice;
iovec[i].iov_len = slice.size() - offset_in_slice;
@@ -224,12 +236,12 @@ Status OutboundTransfer::SendBuffer(Socket &socket) {
}
int64_t written;
- Status status = socket.Writev(iovec, n_iovecs, &written);
+ Status status = socket->Writev(iovec, n_iovecs, &written);
RETURN_ON_ERROR_OR_SOCKET_NOT_READY(status);
// Adjust our accounting of current writer position.
- for (int i = cur_slice_idx_; i < n_payload_slices_; i++) {
- Slice &slice = payload_slices_[i];
+ for (int i = cur_slice_idx_; i < payload_slices_.size(); i++) {
+ const auto& slice = payload_slices_[i];
int rem_in_slice = slice.size() - cur_offset_in_slice_;
DCHECK_GE(rem_in_slice, 0);
@@ -245,11 +257,11 @@ Status OutboundTransfer::SendBuffer(Socket &socket) {
}
}
- if (cur_slice_idx_ == n_payload_slices_) {
+ if (cur_slice_idx_ == payload_slices_.size()) {
callbacks_->NotifyTransferFinished();
DCHECK_EQ(0, cur_offset_in_slice_);
} else {
- DCHECK_LT(cur_slice_idx_, n_payload_slices_);
+ DCHECK_LT(cur_slice_idx_, payload_slices_.size());
DCHECK_LT(cur_offset_in_slice_, payload_slices_[cur_slice_idx_].size());
}
@@ -261,7 +273,7 @@ bool OutboundTransfer::TransferStarted() const {
}
bool OutboundTransfer::TransferFinished() const {
- if (cur_slice_idx_ == n_payload_slices_) {
+ if (cur_slice_idx_ == payload_slices_.size()) {
DCHECK_EQ(0, cur_offset_in_slice_); // sanity check
return true;
}
@@ -274,16 +286,16 @@ string OutboundTransfer::HexDump() const {
}
string ret;
- for (int i = 0; i < n_payload_slices_; i++) {
- ret.append(payload_slices_[i].ToDebugString());
+ for (const auto& s : payload_slices_) {
+ ret.append(s.ToDebugString());
}
return ret;
}
-int32_t OutboundTransfer::TotalLength() const {
- int32_t ret = 0;
- for (int i = 0; i < n_payload_slices_; i++) {
- ret += payload_slices_[i].size();
+size_t OutboundTransfer::TotalLength() const {
+ size_t ret = 0;
+ for (const auto& s : payload_slices_) {
+ ret += s.size();
}
return ret;
}
diff --git a/be/src/kudu/rpc/transfer.h b/be/src/kudu/rpc/transfer.h
index 3342c93..9c0a79a 100644
--- a/be/src/kudu/rpc/transfer.h
+++ b/be/src/kudu/rpc/transfer.h
@@ -16,12 +16,12 @@
// under the License.
#pragma once
-#include <array>
+#include <climits>
#include <cstddef>
#include <cstdint>
-#include <limits.h>
#include <string>
+#include <boost/container/small_vector.hpp>
#include <boost/intrusive/list_hook.hpp>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
@@ -45,15 +45,18 @@ struct TransferCallbacks;
class TransferLimits {
public:
enum {
- kMaxSidecars = 10,
- kMaxPayloadSlices = kMaxSidecars + 2, // (header + msg)
+ kMaxSidecars = 10000,
kMaxTotalSidecarBytes = INT_MAX
};
DISALLOW_IMPLICIT_CONSTRUCTORS(TransferLimits);
};
-typedef std::array<Slice, TransferLimits::kMaxPayloadSlices> TransferPayload;
+// To avoid heap allocation in the common case, assume that most transfer
+// payloads will contain 4 or fewer slices (header, body protobuf, and maybe
+// two sidecars). For more complex responses with more slices, a heap allocation
+// is worth the cost.
+typedef boost::container::small_vector<Slice, 4> TransferPayload;
// This class is used internally by the RPC layer to represent an inbound
// transfer in progress.
@@ -65,9 +68,15 @@ class InboundTransfer {
public:
InboundTransfer();
+ explicit InboundTransfer(faststring initial_buf);
- // read from the socket into our buffer
- Status ReceiveBuffer(Socket &socket);
+ // Read from the socket into our buffer.
+ //
+ // If this is the last read of the transfer (i.e. if TransferFinished() is true
+ // after this call returns OK), up to 4 extra bytes may have been read
+ // from the socket and stored in 'extra_4'. In that case, any previous content of
+ // 'extra_4' is replaced by this extra bytes.
+ Status ReceiveBuffer(Socket* socket, faststring* extra_4);
// Return true if any bytes have yet been sent.
bool TransferStarted() const;
@@ -85,10 +94,9 @@ class InboundTransfer {
private:
- Status ProcessInboundHeader();
-
faststring buf_;
+ // 0 indicates not yet set
uint32_t total_length_;
uint32_t cur_offset_;
@@ -118,15 +126,13 @@ class OutboundTransfer : public boost::intrusive::list_base_hook<> {
// Create an outbound transfer for a call request.
static OutboundTransfer* CreateForCallRequest(int32_t call_id,
- const TransferPayload &payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks);
+ TransferPayload payload,
+ TransferCallbacks* callbacks);
// Create an outbound transfer for a call response.
// See above for details.
- static OutboundTransfer* CreateForCallResponse(const TransferPayload &payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks);
+ static OutboundTransfer* CreateForCallResponse(TransferPayload payload,
+ TransferCallbacks* callbacks);
// Destruct the transfer. A transfer object should never be deallocated
// before it has either (a) finished transferring, or (b) been Abort()ed.
@@ -134,10 +140,10 @@ class OutboundTransfer : public boost::intrusive::list_base_hook<> {
// Abort the current transfer, with the given status.
// This triggers TransferCallbacks::NotifyTransferAborted.
- void Abort(const Status &status);
+ void Abort(const Status& status);
// send from our buffers into the sock
- Status SendBuffer(Socket &socket);
+ Status SendBuffer(Socket* socket);
// Return true if any bytes have yet been sent.
bool TransferStarted() const;
@@ -145,8 +151,8 @@ class OutboundTransfer : public boost::intrusive::list_base_hook<> {
// Return true if the entire transfer has been sent.
bool TransferFinished() const;
- // Return the total number of bytes to be sent (including those already sent)
- int32_t TotalLength() const;
+ // Return the total number of bytes to be sent (including those already sent).
+ size_t TotalLength() const;
std::string HexDump() const;
@@ -163,21 +169,18 @@ class OutboundTransfer : public boost::intrusive::list_base_hook<> {
private:
OutboundTransfer(int32_t call_id,
- const TransferPayload& payload,
- size_t n_payload_slices,
- TransferCallbacks *callbacks);
+ TransferPayload payload,
+ TransferCallbacks* callbacks);
- // Slices to send. Uses an array here instead of a vector to avoid an expensive
- // vector construction (improved performance a couple percent).
+ // Slices to send.
TransferPayload payload_slices_;
- size_t n_payload_slices_;
// The current slice that is being sent.
int32_t cur_slice_idx_;
// The number of bytes in the above slice which has already been sent.
int32_t cur_offset_in_slice_;
- TransferCallbacks *callbacks_;
+ TransferCallbacks* callbacks_;
// In the case of outbound calls, the associated call ID.
// In the case of call responses, kInvalidCallId
@@ -196,13 +199,13 @@ class OutboundTransfer : public boost::intrusive::list_base_hook<> {
// Callbacks made after a transfer completes.
struct TransferCallbacks {
public:
- virtual ~TransferCallbacks();
+ virtual ~TransferCallbacks() {}
// The transfer finished successfully.
virtual void NotifyTransferFinished() = 0;
// The transfer was aborted (e.g because the connection died or an error occurred).
- virtual void NotifyTransferAborted(const Status &status) = 0;
+ virtual void NotifyTransferAborted(const Status& status) = 0;
};
} // namespace rpc
diff --git a/be/src/kudu/security/CMakeLists.txt b/be/src/kudu/security/CMakeLists.txt
index 22e7442..e4ddbaa 100644
--- a/be/src/kudu/security/CMakeLists.txt
+++ b/be/src/kudu/security/CMakeLists.txt
@@ -19,9 +19,6 @@
# The top-level CMakeLists sets a ${KRB5_REALM_OVERRIDE} variable which should
# be linked first into all Kudu binaries.
-# Target including all protobuf-generated code.
-add_custom_target(kudu-security-proto-deps)
-
##############################
# krb5_realm_override
##############################
@@ -32,8 +29,6 @@ if(NOT APPLE)
target_link_libraries(krb5_realm_override dl)
endif()
-add_definitions(-DKUDU_HEADERS_USE_SHORT_STATUS_MACROS)
-
##############################
# token_proto
##############################
@@ -49,7 +44,6 @@ ADD_EXPORTABLE_LIBRARY(token_proto
DEPS ${TOKEN_PROTO_LIBS}
NONLINK_DEPS ${TOKEN_PROTO_TGTS})
-add_dependencies(kudu-security-proto-deps ${TOKEN_PROTO_TGTS})
##############################
# security
@@ -76,7 +70,6 @@ set(SECURITY_SRCS
kerberos_util.cc
gssapi.cc
init.cc
- openssl_util.cc
${PORTED_X509_CHECK_HOST_CC}
security_flags.cc
simple_acl.cc
@@ -102,15 +95,6 @@ ADD_EXPORTABLE_LIBRARY(security
SRCS ${SECURITY_SRCS}
DEPS ${SECURITY_LIBS})
-# Since Kudu tests are explicitly disabled, we want to expose some of their sources
-# to Impala using another variable.
-set(SECURITY_TEST_SRCS_FOR_IMPALA test/mini_kdc.cc)
-add_library(security-test-for-impala ${SECURITY_TEST_SRCS_FOR_IMPALA})
-target_link_libraries(security-test-for-impala
- gutil
- kudu_test_util
- kudu_util
- security)
##############################
# mini_kdc
diff --git a/be/src/kudu/security/ca/cert_management-test.cc b/be/src/kudu/security/ca/cert_management-test.cc
index f3e6c35..19c6aa6 100644
--- a/be/src/kudu/security/ca/cert_management-test.cc
+++ b/be/src/kudu/security/ca/cert_management-test.cc
@@ -17,6 +17,7 @@
#include "kudu/security/ca/cert_management.h"
+#include <initializer_list>
#include <string>
#include <utility>
#include <vector>
@@ -28,9 +29,9 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/security/cert.h"
#include "kudu/security/crypto.h"
-#include "kudu/security/openssl_util.h"
#include "kudu/security/security-test-util.h"
#include "kudu/security/test/test_certs.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
diff --git a/be/src/kudu/security/ca/cert_management.cc b/be/src/kudu/security/ca/cert_management.cc
index 7ccc376..26debcb 100644
--- a/be/src/kudu/security/ca/cert_management.cc
+++ b/be/src/kudu/security/ca/cert_management.cc
@@ -17,26 +17,25 @@
#include "kudu/security/ca/cert_management.h"
+#include <openssl/bn.h>
+#include <openssl/obj_mac.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+
#include <algorithm>
#include <cstdio>
+#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <glog/logging.h>
-#include <openssl/conf.h>
-#ifndef OPENSSL_NO_ENGINE
-#include <openssl/engine.h>
-#endif
-#include <openssl/pem.h>
-#include <openssl/x509.h>
-#include <openssl/x509v3.h>
#include "kudu/gutil/strings/substitute.h"
#include "kudu/security/cert.h"
#include "kudu/security/crypto.h"
-#include "kudu/security/openssl_util.h"
-#include "kudu/util/net/socket.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/status.h"
diff --git a/be/src/kudu/security/ca/cert_management.h b/be/src/kudu/security/ca/cert_management.h
index fb2bd0e..356c2ea 100644
--- a/be/src/kudu/security/ca/cert_management.h
+++ b/be/src/kudu/security/ca/cert_management.h
@@ -17,6 +17,11 @@
#pragma once
+#include <openssl/asn1.h>
+#include <openssl/crypto.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+
#include <cstdint>
#include <memory>
#include <string>
@@ -27,8 +32,8 @@
#include "kudu/gutil/macros.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/stringpiece.h"
-#include "kudu/security/openssl_util.h"
#include "kudu/util/locks.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
diff --git a/be/src/kudu/security/cert-test.cc b/be/src/kudu/security/cert-test.cc
index 12205e1..83e5fb9 100644
--- a/be/src/kudu/security/cert-test.cc
+++ b/be/src/kudu/security/cert-test.cc
@@ -15,6 +15,10 @@
// specific language governing permissions and limitations
// under the License.
+#include "kudu/security/cert.h"
+
+#include <openssl/obj_mac.h>
+
#include <string>
#include <thread>
#include <utility>
@@ -25,11 +29,10 @@
#include <gtest/gtest.h>
#include "kudu/gutil/strings/strip.h"
-#include "kudu/security/cert.h"
#include "kudu/security/crypto.h"
-#include "kudu/security/openssl_util.h"
#include "kudu/security/test/test_certs.h"
#include "kudu/util/barrier.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
diff --git a/be/src/kudu/security/cert.cc b/be/src/kudu/security/cert.cc
index b81d263..f6e9c8e 100644
--- a/be/src/kudu/security/cert.cc
+++ b/be/src/kudu/security/cert.cc
@@ -17,6 +17,15 @@
#include "kudu/security/cert.h"
+#include <openssl/asn1.h>
+#include <openssl/crypto.h>
+#include <openssl/obj_mac.h>
+#include <openssl/objects.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+
+#include <functional>
#include <memory>
#include <mutex>
#include <ostream>
@@ -24,14 +33,11 @@
#include <boost/optional/optional.hpp>
#include <glog/logging.h>
-#include <openssl/evp.h>
-#include <openssl/x509.h>
-#include <openssl/x509v3.h>
#include "kudu/gutil/macros.h"
#include "kudu/security/crypto.h"
-#include "kudu/security/openssl_util.h"
-#include "kudu/security/openssl_util_bio.h"
+#include "kudu/util/openssl_util.h"
+#include "kudu/util/openssl_util_bio.h"
#include "kudu/util/status.h"
using std::string;
diff --git a/be/src/kudu/security/cert.h b/be/src/kudu/security/cert.h
index 4629883..4f9bb01 100644
--- a/be/src/kudu/security/cert.h
+++ b/be/src/kudu/security/cert.h
@@ -17,14 +17,15 @@
#pragma once
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+
#include <memory>
#include <string>
#include <vector>
-#include <openssl/asn1.h>
-
#include "kudu/gutil/port.h"
-#include "kudu/security/openssl_util.h"
+#include "kudu/util/openssl_util.h"
typedef struct X509_name_st X509_NAME;
diff --git a/be/src/kudu/security/crypto-test.cc b/be/src/kudu/security/crypto-test.cc
index c1e32df..cfced00 100644
--- a/be/src/kudu/security/crypto-test.cc
+++ b/be/src/kudu/security/crypto-test.cc
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include "kudu/security/crypto.h"
+
#include <cstring>
#include <string>
#include <utility>
@@ -24,10 +26,9 @@
#include "kudu/gutil/strings/strip.h"
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/security/crypto.h"
-#include "kudu/security/openssl_util.h"
#include "kudu/security/test/test_certs.h"
#include "kudu/util/env.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/path_util.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
@@ -185,7 +186,7 @@ TEST_P(CryptoKeySerDesTest, ToAndFromString) {
NO_FATALS(CheckToAndFromString(public_key, format));
}
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
DataFormats, CryptoKeySerDesTest,
::testing::Values(DataFormat::DER, DataFormat::PEM));
diff --git a/be/src/kudu/security/crypto.cc b/be/src/kudu/security/crypto.cc
index 234d193..1e0f07a 100644
--- a/be/src/kudu/security/crypto.cc
+++ b/be/src/kudu/security/crypto.cc
@@ -17,25 +17,24 @@
#include "kudu/security/crypto.h"
-#include <memory>
-#include <ostream>
-#include <string>
-
-#include <glog/logging.h>
-#include <openssl/bio.h>
#include <openssl/bn.h>
+#include <openssl/crypto.h>
#include <openssl/err.h>
-#include <openssl/evp.h>
-#include <openssl/opensslv.h>
-#include <openssl/ossl_typ.h>
#include <openssl/pem.h>
#include <openssl/rand.h>
-#include <openssl/rsa.h>
+#include <openssl/ssl.h>
#include <openssl/x509.h>
+#include <functional>
+#include <memory>
+#include <ostream>
+#include <string>
+
+#include <glog/logging.h>
+
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/security/openssl_util.h"
-#include "kudu/security/openssl_util_bio.h"
+#include "kudu/util/openssl_util.h"
+#include "kudu/util/openssl_util_bio.h"
#include "kudu/util/status.h"
using std::string;
@@ -52,9 +51,7 @@ namespace {
// signature than the rest of the write functions, so we
// have to provide this wrapper.
int PemWritePrivateKey(BIO* bio, EVP_PKEY* key) {
- auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key));
- return PEM_write_bio_RSAPrivateKey(
- bio, rsa.get(), nullptr, nullptr, 0, nullptr, nullptr);
+ return PEM_write_bio_PKCS8PrivateKey(bio, key, nullptr, nullptr, 0, nullptr, nullptr);
}
int PemWritePublicKey(BIO* bio, EVP_PKEY* key) {
diff --git a/be/src/kudu/security/crypto.h b/be/src/kudu/security/crypto.h
index 145c405..db5d24f 100644
--- a/be/src/kudu/security/crypto.h
+++ b/be/src/kudu/security/crypto.h
@@ -17,17 +17,15 @@
#pragma once
+#include <openssl/ssl.h>
+
#include <cstddef>
#include <string>
-#include <openssl/bio.h>
-#include <openssl/rsa.h>
-
#include "kudu/gutil/port.h"
-#include "kudu/security/openssl_util.h"
+#include "kudu/util/openssl_util.h"
// Forward declarations for the OpenSSL typedefs.
-typedef struct rsa_st RSA;
typedef struct bio_st BIO;
namespace kudu {
diff --git a/be/src/kudu/security/init.cc b/be/src/kudu/security/init.cc
index e740fcb..8d59d51 100644
--- a/be/src/kudu/security/init.cc
+++ b/be/src/kudu/security/init.cc
@@ -17,6 +17,8 @@
#include "kudu/security/init.h"
+#include <krb5/krb5.h>
+
#include <algorithm>
#include <cctype>
#include <cstdint>
@@ -29,19 +31,17 @@
#include <ostream>
#include <random>
#include <string>
-#include <type_traits>
-#include <utility>
#include <boost/optional/optional.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
-#include <krb5/krb5.h>
#include "kudu/gutil/macros.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/strings/util.h"
#include "kudu/security/kinit_context.h"
+#include "kudu/util/countdown_latch.h"
#include "kudu/util/flag_tags.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
@@ -73,6 +73,17 @@ DEFINE_bool(use_system_auth_to_local, kDefaultSystemAuthToLocal,
"'kudu/foo.example.com@EXAMPLE' will map to 'kudu'.");
TAG_FLAG(use_system_auth_to_local, advanced);
+DEFINE_string(principal, "kudu/_HOST",
+ "Kerberos principal that this daemon will log in as. The special token "
+ "_HOST will be replaced with the FQDN of the local host.");
+TAG_FLAG(principal, advanced);
+TAG_FLAG(principal, stable);
+
+DEFINE_string(keytab_file, "",
+ "Path to the Kerberos Keytab file for this server. Specifying a "
+ "keytab file will cause the server to kinit, and enable Kerberos "
+ "to be used to authenticate RPC connections.");
+TAG_FLAG(keytab_file, stable);
using std::mt19937;
using std::random_device;
@@ -85,7 +96,7 @@ namespace kudu {
namespace security {
// Global instance of the context used by the kinit/reacquire thread.
-KinitContext* g_kinit_ctx;
+KinitContext* g_kinit_ctx = nullptr;
namespace {
@@ -102,7 +113,7 @@ Status Krb5CallToStatus(krb5_context ctx, krb5_error_code code) {
std::unique_ptr<const char, std::function<void(const char*)>> err_msg(
krb5_get_error_message(ctx, code),
- std::bind(krb5_free_error_message, ctx, std::placeholders::_1));
+ [ctx](const char* msg) { krb5_free_error_message(ctx, msg); });
return Status::RuntimeError(err_msg.get());
}
#define KRB5_RETURN_NOT_OK_PREPEND(call, prepend) \
@@ -151,34 +162,13 @@ Status Krb5UnparseName(krb5_principal princ, string* name) {
return Status::OK();
}
-// Periodically calls DoRenewal().
-void RenewThread() {
- uint32_t failure_retries = 0;
- while (true) {
- // This thread is run immediately after the first Kinit, so sleep first.
- int64_t renew_interval_s = g_kinit_ctx->GetNextRenewInterval(failure_retries);
- if (failure_retries > 0) {
- // Log in the abnormal case where something failed.
- LOG(INFO) << Substitute("Renew thread sleeping after $0 failures for $1s",
- failure_retries, renew_interval_s);
- }
- SleepFor(MonoDelta::FromSeconds(renew_interval_s));
-
- Status s = g_kinit_ctx->DoRenewal();
- WARN_NOT_OK(s, "Kerberos reacquire error: ");
- if (!s.ok()) {
- ++failure_retries;
- } else {
- failure_retries = 0;
- }
- }
-}
} // anonymous namespace
-KinitContext::KinitContext() {}
+KinitContext::KinitContext() : stop_latch_(1) {}
KinitContext::~KinitContext() {
// Free memory associated with these objects.
+ Kdestroy();
if (principal_ != nullptr) krb5_free_principal(g_krb5_ctx, principal_);
if (keytab_ != nullptr) krb5_kt_close(g_krb5_ctx, keytab_);
if (ccache_ != nullptr) krb5_cc_close(g_krb5_ctx, ccache_);
@@ -320,7 +310,19 @@ Status KinitContext::Kinit(const string& keytab_path, const string& principal) {
KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_opt_alloc(g_krb5_ctx, &opts_),
"unable to allocate get_init_creds_opt struct");
- return KinitInternal();
+ RETURN_NOT_OK(KinitInternal());
+
+ // Start the thread to renew and reacquire Kerberos tickets.
+ RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread",
+ [this]() { this->RenewThread(); }, &reacquire_thread_));
+ return Status::OK();
+}
+
+void KinitContext::Kdestroy() {
+ stop_latch_.CountDown();
+ if (reacquire_thread_.get() != nullptr) {
+ reacquire_thread_->Join();
+ }
}
Status KinitContext::KinitInternal() {
@@ -363,28 +365,29 @@ Status KinitContext::KinitInternal() {
return Status::OK();
}
-namespace {
-// 'in_principal' is the user specified principal to use with Kerberos. It may have a token
-// in the string of the form '_HOST', which if present, needs to be replaced with the FQDN of the
-// current host.
-// 'out_principal' has the final principal with which one may Kinit.
-Status GetConfiguredPrincipal(const std::string& in_principal, string* out_principal) {
- *out_principal = in_principal;
- const auto& kHostToken = "_HOST";
- if (in_principal.find(kHostToken) != string::npos) {
- string hostname;
- // Try to fill in either the FQDN or hostname.
- if (!GetFQDN(&hostname).ok()) {
- RETURN_NOT_OK(GetHostname(&hostname));
+// Periodically calls DoRenewal().
+void KinitContext::RenewThread() {
+ uint32_t failure_retries = 0;
+ int64_t renew_interval_s = GetNextRenewInterval(failure_retries);
+ while (!stop_latch_.WaitFor(MonoDelta::FromSeconds(renew_interval_s))) {
+ Status s = DoRenewal();
+ WARN_NOT_OK(s, "Kerberos reacquire error: ");
+ if (!s.ok()) {
+ ++failure_retries;
+ } else {
+ failure_retries = 0;
}
- // Hosts in principal names are canonicalized to lower-case.
- std::transform(hostname.begin(), hostname.end(), hostname.begin(), tolower);
- GlobalReplaceSubstring(kHostToken, hostname, out_principal);
+
+ if (failure_retries > 0) {
+ // Log in the abnormal case where something failed.
+ LOG(INFO) << Substitute("Renew thread sleeping after $0 failures for $1s",
+ failure_retries, renew_interval_s);
+ }
+
+ // This thread is run immediately after the first Kinit, so sleep first.
+ renew_interval_s = GetNextRenewInterval(failure_retries);
}
- return Status::OK();
}
-} // anonymous namespace
-
RWMutex* KerberosReinitLock() {
return g_kerberos_reinit_lock;
@@ -444,6 +447,22 @@ Status MapPrincipalToLocalName(const std::string& principal, std::string* local_
return Status::OK();
}
+Status GetConfiguredPrincipal(const string& in_principal, string* out_principal) {
+ *out_principal = in_principal;
+ static const auto& kHostToken = "_HOST";
+ if (in_principal.find(kHostToken) != string::npos) {
+ string hostname;
+ // Try to fill in either the FQDN or hostname.
+ if (!GetFQDN(&hostname).ok()) {
+ RETURN_NOT_OK(GetHostname(&hostname));
+ }
+ // Hosts in principal names are canonicalized to lower-case.
+ std::transform(hostname.begin(), hostname.end(), hostname.begin(), tolower);
+ GlobalReplaceSubstring(kHostToken, hostname, out_principal);
+ }
+ return Status::OK();
+}
+
boost::optional<string> GetLoggedInPrincipalFromKeytab() {
if (!g_kinit_ctx) return boost::none;
return g_kinit_ctx->principal_str();
@@ -493,13 +512,25 @@ Status InitKerberosForServer(const std::string& raw_principal, const std::string
RETURN_NOT_OK_PREPEND(g_kinit_ctx->Kinit(
keytab_file, configured_principal), "unable to kinit");
- scoped_refptr<Thread> reacquire_thread;
- // Start the reacquire thread.
- RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread", &RenewThread, &reacquire_thread));
-
return Status::OK();
}
+void DestroyKerberosForServer() {
+ if (g_kinit_ctx == nullptr) return;
+
+ delete g_kinit_ctx;
+ g_kinit_ctx = nullptr;
+}
+
+string GetKrb5ConfigFile() {
+ const char* config_file = getenv("KRB5_CONFIG");
+ if (!config_file) {
+ return "/etc/krb5.conf";
+ }
+
+ return string(config_file);
+}
+
} // namespace security
} // namespace kudu
diff --git a/be/src/kudu/security/init.h b/be/src/kudu/security/init.h
index 80074b3..aa8a8b8 100644
--- a/be/src/kudu/security/init.h
+++ b/be/src/kudu/security/init.h
@@ -55,6 +55,9 @@ Status InitKerberosForServer(const std::string& raw_principal,
const std::string& krb5ccname = kKrb5CCName,
bool disable_krb5_replay_cache = true);
+// Destroy Kerberos for a server.
+void DestroyKerberosForServer();
+
// Returns the process lock 'kerberos_reinit_lock'
// This lock is taken in write mode while the ticket is being reacquired, and
// taken in read mode before using the SASL library which might require a ticket.
@@ -86,5 +89,19 @@ Status CanonicalizeKrb5Principal(std::string* principal);
// exist yet, and trying to avoid rebase pain).
Status MapPrincipalToLocalName(const std::string& principal, std::string* local_name);
+// Get the configured principal. 'in_principal' is the user specified principal to use with
+// Kerberos. It may have a token in the string of the form '_HOST', which if present, needs
+// to be replaced with the FQDN of the current host. 'out_principal' has the final principal
+// with which one may Kinit.
+Status GetConfiguredPrincipal(const std::string& in_principal, std::string* out_principal);
+
+// Get the Kerberos config file location. It defaults to /etc/krb5.conf and it
+// can be overridden by the KRB5_CONFIG environment variable. As the Kerberos
+// libraries use the environment variable directly, this is not required
+// normally, but it can be useful if the file needs to be accessed directly
+// (e.g. when starting a Java subprocess, as Java doesn't respect the
+// environment variable).
+std::string GetKrb5ConfigFile();
+
} // namespace security
} // namespace kudu
diff --git a/be/src/kudu/security/kinit_context.h b/be/src/kudu/security/kinit_context.h
index a6c2b41..9a9e644 100644
--- a/be/src/kudu/security/kinit_context.h
+++ b/be/src/kudu/security/kinit_context.h
@@ -21,7 +21,9 @@
#include <krb5/krb5.h>
+#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/status.h"
+#include "kudu/util/thread.h"
namespace kudu {
namespace security {
@@ -63,6 +65,12 @@ class KinitContext {
private:
Status KinitInternal();
+ // Safe stop the renewal thread before destroying KinitContext
+ void Kdestroy();
+
+ // Periodically calls DoRenewal().
+ void RenewThread();
+
// Helper for DoRenewal() that tries to do a renewal. On success, returns OK and sets
// *found_in_cache = true. If there is an error doing the renewal itself, returns an
// error. If the TGT to be renewed was not found in the cache, return OK and set
@@ -79,6 +87,11 @@ class KinitContext {
// This is the time that the current TGT in use expires.
int32_t ticket_end_timestamp_;
+
+ // To stop reacquire_thread_ when process stopping.
+ CountDownLatch stop_latch_;
+ // A thread to renew and reacquire Kerberos credentials.
+ scoped_refptr<Thread> reacquire_thread_;
};
} // namespace security
diff --git a/be/src/kudu/security/security-test-util.cc b/be/src/kudu/security/security-test-util.cc
index 40b0938..b6e6340 100644
--- a/be/src/kudu/security/security-test-util.cc
+++ b/be/src/kudu/security/security-test-util.cc
@@ -29,14 +29,14 @@
#include "kudu/security/tls_context.h"
#include "kudu/util/test_util.h"
+using kudu::security::ca::CaCertRequestGenerator;
+using kudu::security::ca::CertSigner;
+
namespace kudu {
namespace security {
-using ca::CaCertRequestGenerator;
-using ca::CertSigner;
-
Status GenerateSelfSignedCAForTests(PrivateKey* ca_key, Cert* ca_cert) {
- static const int64_t kRootCaCertExpirationSeconds = 24 * 60 * 60;
+ constexpr int64_t kRootCaCertExpirationSeconds = 24 * 60 * 60;
// Create a key for the self-signed CA.
//
// OpenSSL has a concept of "security levels" which, amongst other things,
@@ -48,13 +48,10 @@ Status GenerateSelfSignedCAForTests(PrivateKey* ca_key, Cert* ca_cert) {
// See https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_get_security_level.html
// for more details.
RETURN_NOT_OK(GeneratePrivateKey(1024, ca_key));
-
- CaCertRequestGenerator::Config config = { "test-ca-cn" };
- RETURN_NOT_OK(CertSigner::SelfSignCA(*ca_key,
- config,
- kRootCaCertExpirationSeconds,
- ca_cert));
- return Status::OK();
+ return CertSigner::SelfSignCA(*ca_key,
+ CaCertRequestGenerator::Config{ "test-ca-cn" },
+ kRootCaCertExpirationSeconds,
+ ca_cert);
}
std::ostream& operator<<(std::ostream& o, PkiConfig c) {
diff --git a/be/src/kudu/security/security_flags.cc b/be/src/kudu/security/security_flags.cc
index acdd662..bfd09fb 100644
--- a/be/src/kudu/security/security_flags.cc
+++ b/be/src/kudu/security/security_flags.cc
@@ -26,15 +26,23 @@ namespace security {
// list. These additional ciphers maintain compatibility with RHEL 6.5 and
// below. The DH AES ciphers are not included since we are not configured to
// use DH key agreement.
+// TODO(aserbin): refresh the list to drop RHEL6/CentOS6 ciphers and
+// sync it with https://wiki.mozilla.org/Security/Server_Side_TLS
const char* const SecurityDefaults::SecurityDefaults::kDefaultTlsCiphers =
- "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:"
- "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:"
- "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:"
- "ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:"
- "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:"
- "AES256-GCM-SHA384:AES128-GCM-SHA256:"
- "AES256-SHA256:AES128-SHA256:"
- "AES256-SHA:AES128-SHA";
+ "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:"
+ "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:"
+ "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:"
+ "ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:"
+ "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:"
+ "AES256-GCM-SHA384:AES128-GCM-SHA256:"
+ "AES256-SHA256:AES128-SHA256:"
+ "AES256-SHA:AES128-SHA";
+
+// This is the "modern compatibility" TLSv1.3 cipher list of the Mozilla
+// Security Server Side TLS recommendations, accessed March 2021.
+// https://wiki.mozilla.org/Security/Server_Side_TLS
+const char* const SecurityDefaults::SecurityDefaults::kDefaultTlsCipherSuites =
+ "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256";
const char* const SecurityDefaults::SecurityDefaults::kDefaultTlsMinVersion = "TLSv1";
diff --git a/be/src/kudu/security/security_flags.h b/be/src/kudu/security/security_flags.h
index e64536d..06e6790 100644
--- a/be/src/kudu/security/security_flags.h
+++ b/be/src/kudu/security/security_flags.h
@@ -28,7 +28,11 @@ typedef TriStateFlag RpcAuthentication;
typedef TriStateFlag RpcEncryption;
struct SecurityDefaults {
- static const char* const kDefaultTlsCiphers;
+ // The names for the 'kDefaultTlsCiphers' and 'kDefaultTlsCipherSuites'
+ // constants are confusingly close, but likely 'kDefaultTlsCiphers' is likely
+ // to be removed when obsoleting TLSv1.2 at some point in the future.
+ static const char* const kDefaultTlsCiphers; // pre-TLSv1.3 ciphers
+ static const char* const kDefaultTlsCipherSuites; // TLSv1.3 and later ciphers
static const char* const kDefaultTlsMinVersion;
};
diff --git a/be/src/kudu/security/simple_acl.cc b/be/src/kudu/security/simple_acl.cc
index 09cc6ab..9157bff 100644
--- a/be/src/kudu/security/simple_acl.cc
+++ b/be/src/kudu/security/simple_acl.cc
@@ -51,7 +51,6 @@ Status SimpleAcl::ParseFlag(const string& flag) {
return Status::OK();
... 23756 lines suppressed ...