You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/08/25 22:53:19 UTC
[2/2] incubator-impala git commit: IMPALA-4669: [KRPC] Add kudu_rpc
library to build
IMPALA-4669: [KRPC] Add kudu_rpc library to build
Import FindKRPC.cmake from Apache Kudu.
Add some files to protoc-gen-krpc link to allow it to find symbols now
defined within Impala (without linking all of Impala's libraries).
Change-Id: I33203e95dff07c87a6ec5c7a31b7a583b91849bc
Reviewed-on: http://gerrit.cloudera.org:8080/5719
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/11352619
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/11352619
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/11352619
Branch: refs/heads/master
Commit: 113526198051f6810c84df513d507074856f5e4c
Parents: 48539a1
Author: Henry Robinson <he...@cloudera.com>
Authored: Tue Nov 8 13:55:19 2016 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Aug 25 22:51:41 2017 +0000
----------------------------------------------------------------------
CMakeLists.txt | 4 ++
be/CMakeLists.txt | 8 +++
be/src/common/global-flags.cc | 9 +++
be/src/common/init.cc | 5 +-
be/src/common/logging.cc | 9 +--
be/src/kudu/rpc/CMakeLists.txt | 13 +++-
cmake_modules/FindKRPC.cmake | 117 ++++++++++++++++++++++++++++++++++++
7 files changed, 151 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 31a3fc4..954193e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -320,6 +320,10 @@ if (NOT APPLE)
endif()
###################################################################
+
+# Required for KRPC_GENERATE, which converts protobuf to stubs.
+find_package(KRPC REQUIRED)
+
# KuduClient can use GLOG
add_definitions(-DKUDU_HEADERS_USE_GLOG)
if(NOT $ENV{KUDU_CLIENT_DIR} EQUAL "")
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 3ac4193..7bf9e0c 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -349,9 +349,14 @@ set (IMPALA_LINK_LIBS
Exec
Exprs
GlobalFlags
+ histogram_proto
ImpalaThrift
kudu_util
+ krpc
Rpc
+ rpc_header_proto
+ rpc_introspection_proto
+ pb_util_proto
Runtime
Scheduling
security
@@ -401,10 +406,12 @@ set (IMPALA_DEPENDENCIES
ThriftSaslTransport
openssl_ssl
openssl_crypto
+ crcutil
gutil
glog
gflags
krb5
+ libev
pprof
breakpad_client
hdfs
@@ -512,6 +519,7 @@ add_subdirectory(src/common)
add_subdirectory(src/exec)
add_subdirectory(src/exprs)
add_subdirectory(src/kudu/security)
+add_subdirectory(src/kudu/rpc)
add_subdirectory(src/kudu/util)
add_subdirectory(src/runtime)
add_subdirectory(src/scheduling)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 5d61edc..9404148 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -161,3 +161,12 @@ DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
DEFINE_bool(enable_stats_extrapolation, false,
"If true, uses table statistics computed with COMPUTE STATS "
"to extrapolate the row counts of partitions.");
+
+DEFINE_string(log_filename, "",
+ "Prefix of log filename - "
+ "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]");
+DEFINE_bool(redirect_stdout_stderr, true,
+ "If true, redirects stdout/stderr to INFO/ERROR log.");
+DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per severity "
+ "level. The most recent log files are retained. If set to 0, all log files are "
+ "retained.");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 9734610..6b48bb8 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -62,10 +62,7 @@ DECLARE_string(hostname);
DECLARE_int32(logbufsecs);
DECLARE_int32(max_minidumps);
DECLARE_string(redaction_rules_file);
-
-DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per severity "
- "level. The most recent log files are retained. If set to 0, all log files are "
- "retained.");
+DECLARE_int32(max_log_files);
DEFINE_int32(max_audit_event_log_files, 0, "Maximum number of audit event log files "
"to retain. The most recent audit event log files are retained. If set to 0, "
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/logging.cc
----------------------------------------------------------------------
diff --git a/be/src/common/logging.cc b/be/src/common/logging.cc
index 8025c9f..021af4a 100644
--- a/be/src/common/logging.cc
+++ b/be/src/common/logging.cc
@@ -40,14 +40,9 @@
#include "common/names.h"
-DEFINE_string(log_filename, "",
- "Prefix of log filename - "
- "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]");
-DEFINE_bool(redirect_stdout_stderr, true,
- "If true, redirects stdout/stderr to INFO/ERROR log.");
-
DECLARE_string(redaction_rules_file);
-
+DECLARE_string(log_filename);
+DECLARE_bool(redirect_stdout_stderr);
DECLARE_string(audit_event_log_dir);
using boost::uuids::random_generator;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/kudu/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/kudu/rpc/CMakeLists.txt b/be/src/kudu/rpc/CMakeLists.txt
index 0cfe6e9..643b854 100644
--- a/be/src/kudu/rpc/CMakeLists.txt
+++ b/be/src/kudu/rpc/CMakeLists.txt
@@ -86,14 +86,21 @@ ADD_EXPORTABLE_LIBRARY(krpc
DEPS ${KRPC_LIBS})
### RPC generator tool
-add_executable(protoc-gen-krpc protoc-gen-krpc.cc)
+add_executable(protoc-gen-krpc protoc-gen-krpc.cc
+ # Impala - add stub for kudu::VersionInfo
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../common/kudu_version.cc
+ # Impala - add definition for any flag names shared between Impala / Kudu.
+ # TODO: Consider either removing code that depends on these flags, or namespacing them
+ # somehow.
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../common/global-flags.cc)
target_link_libraries(protoc-gen-krpc
- ${KUDU_BASE_LIBS}
+ ${KUDU_BASE_LIBS}
rpc_header_proto
protoc
protobuf
gutil
- kudu_util)
+ kudu_util
+ )
#### RPC test
PROTOBUF_GENERATE_CPP(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/cmake_modules/FindKRPC.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/FindKRPC.cmake b/cmake_modules/FindKRPC.cmake
new file mode 100644
index 0000000..593edcd
--- /dev/null
+++ b/cmake_modules/FindKRPC.cmake
@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Kudu RPC generator support
+#########
+find_package(Protobuf REQUIRED)
+
+#
+# Generate the KRPC files for a given .proto file.
+#
+function(KRPC_GENERATE SRCS HDRS TGTS)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: KRPC_GENERATE() called without protobuf files")
+ return()
+ endif(NOT ARGN)
+
+ set(options)
+ set(one_value_args SOURCE_ROOT BINARY_ROOT)
+ set(multi_value_args EXTRA_PROTO_PATHS PROTO_FILES)
+ cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+ if(ARG_UNPARSED_ARGUMENTS)
+ message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+ endif()
+ set(${SRCS})
+ set(${HDRS})
+ set(${TGTS})
+
+ set(EXTRA_PROTO_PATH_ARGS)
+ foreach(PP ${ARG_EXTRA_PROTO_PATHS})
+ set(EXTRA_PROTO_PATH_ARGS ${EXTRA_PROTO_PATH_ARGS} --proto_path ${PP})
+ endforeach()
+
+ if("${ARG_SOURCE_ROOT}" STREQUAL "")
+ SET(ARG_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
+ endif()
+ GET_FILENAME_COMPONENT(ARG_SOURCE_ROOT ${ARG_SOURCE_ROOT} ABSOLUTE)
+
+ if("${ARG_BINARY_ROOT}" STREQUAL "")
+ SET(ARG_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}")
+ endif()
+ GET_FILENAME_COMPONENT(ARG_BINARY_ROOT ${ARG_BINARY_ROOT} ABSOLUTE)
+
+ foreach(FIL ${ARG_PROTO_FILES})
+ get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+ get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+ # Ensure that the protobuf file is within the source root.
+ # This is a requirement of protoc.
+ FILE(RELATIVE_PATH PROTO_REL_TO_ROOT "${ARG_SOURCE_ROOT}" "${ABS_FIL}")
+
+ GET_FILENAME_COMPONENT(REL_DIR "${PROTO_REL_TO_ROOT}" PATH)
+
+ if(NOT REL_DIR STREQUAL "")
+ SET(REL_DIR "${REL_DIR}/")
+ endif()
+
+ set(PROTO_CC_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.cc")
+ set(PROTO_H_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.h")
+ set(SERVICE_CC "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.service.cc")
+ set(SERVICE_H "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.service.h")
+ set(PROXY_CC "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.proxy.cc")
+ set(PROXY_H "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.proxy.h")
+ list(APPEND ${SRCS} "${PROTO_CC_OUT}" "${SERVICE_CC}" "${PROXY_CC}")
+ list(APPEND ${HDRS} "${PROTO_H_OUT}" "${SERVICE_H}" "${PROXY_H}")
+
+ get_filename_component(PROTO_LIB_DIR ${PROTOBUF_SHARED_LIBRARY} DIRECTORY)
+
+ add_custom_command(
+ OUTPUT "${SERVICE_CC}" "${SERVICE_H}" "${PROXY_CC}" "${PROXY_H}"
+ "${PROTO_CC_OUT}" "${PROTO_H_OUT}"
+ COMMAND ${CMAKE_COMMAND}
+ -E env "LD_LIBRARY_PATH=${PROTO_LIB_DIR}:$ENV{LD_LIBRARY_PATH}"
+ ${PROTOBUF_PROTOC_EXECUTABLE}
+ ARGS --plugin $<TARGET_FILE:protoc-gen-krpc>
+ --plugin $<TARGET_FILE:protoc-gen-insertions>
+ --cpp_out ${ARG_BINARY_ROOT}
+ --krpc_out ${ARG_BINARY_ROOT}
+ --insertions_out ${ARG_BINARY_ROOT}
+ --proto_path ${ARG_SOURCE_ROOT}
+ # Used to find built-in .proto files (e.g. FileDescriptorProto)
+ --proto_path ${PROTOBUF_INCLUDE_DIR}
+ ${EXTRA_PROTO_PATH_ARGS} ${ABS_FIL}
+ DEPENDS ${ABS_FIL} protoc-gen-krpc protoc-gen-insertions
+ COMMENT "Running C++ protocol buffer compiler with KRPC plugin on ${FIL}"
+ VERBATIM)
+
+ # This custom target enforces that there's just one invocation of protoc
+ # when there are multiple consumers of the generated files. The target name
+ # must be unique; adding parts of the filename helps ensure this.
+ set(TGT_NAME ${REL_DIR}${FIL})
+ string(REPLACE "/" "-" TGT_NAME ${TGT_NAME})
+ add_custom_target(${TGT_NAME}
+ DEPENDS "${SERVICE_CC}" "${SERVICE_H}"
+ "${PROXY_CC}" "${PROXY_H}"
+ "${PROTO_CC_OUT}" "${PROTO_H_OUT}")
+ list(APPEND ${TGTS} "${TGT_NAME}")
+ endforeach()
+
+ set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+ set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+ set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+ set(${TGTS} ${${TGTS}} PARENT_SCOPE)
+endfunction()