You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2015/07/14 21:48:17 UTC
[2/5] hadoop git commit: HDFS-8764. Generate Hadoop RPC stubs from
protobuf definitions. Contributed by Haohui Mai.
HDFS-8764. Generate Hadoop RPC stubs from protobuf definitions. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f34bda72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f34bda72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f34bda72
Branch: refs/heads/8b02d962b291afe4b08c47f0398c1db0709419a1
Commit: f34bda720184cc2a7e3ac48916c693519387539b
Parents: 6416b38
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Jul 13 16:53:13 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Tue Jul 14 12:47:00 2015 -0700
----------------------------------------------------------------------
.../native/libhdfspp/lib/proto/CMakeLists.txt | 46 +++++++++-
.../native/libhdfspp/lib/proto/cpp_helpers.h | 82 +++++++++++++++++
.../libhdfspp/lib/proto/protoc_gen_hrpc.cc | 94 ++++++++++++++++++++
3 files changed, 221 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
index 156a7f4..3f703b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
@@ -18,4 +18,48 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS
${COMMON_PROTO_DIR}/Security.proto
)
-add_library(proto ${PROTO_SRCS} ${PROTO_HDRS})
+add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc)
+target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY})
+
+function(GEN_HRPC SRCS)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: GEN_HRPC() called without any proto files")
+ return()
+ endif()
+
+ if(DEFINED PROTOBUF_IMPORT_DIRS)
+ foreach(DIR ${PROTOBUF_IMPORT_DIRS})
+ get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+ list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+ if(${_contains_already} EQUAL -1)
+ list(APPEND _protobuf_include_path -I ${ABS_PATH})
+ endif()
+ endforeach()
+ endif()
+
+ set(${SRCS})
+
+ foreach(FIL ${ARGN})
+ get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+ get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+ list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl")
+
+ add_custom_command(
+ OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl"
+ COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+ ARGS --plugin=protoc-gen-hrpc=${CMAKE_CURRENT_BINARY_DIR}/protoc-gen-hrpc --hrpc_out=${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+ DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} protoc-gen-hrpc
+ COMMENT "Running HRPC protocol buffer compiler on ${FIL}"
+ VERBATIM )
+ endforeach()
+
+ set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE)
+ set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+endfunction()
+
+gen_hrpc(HRPC_SRCS
+ ${CLIENT_PROTO_DIR}/ClientNamenodeProtocol.proto
+)
+
+add_library(proto ${PROTO_SRCS} ${PROTO_HDRS} ${HRPC_SRCS})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h
new file mode 100644
index 0000000..6f380ad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h
@@ -0,0 +1,82 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+// Based on original Protocol Buffers design by
+// Sanjay Ghemawat, Jeff Dean, and others.
+
+#ifndef LIBHDFSPP_PROTO_CPP_HELPERS_H_
+#define LIBHDFSPP_PROTO_CPP_HELPERS_H_
+
+#include <string>
+
+/**
+ * The functions in this file are derived from the original implementation of
+ *the protobuf library from Google.
+ **/
+
+static inline std::string StripProto(const std::string &str) {
+ static const std::string kExtension = ".proto";
+ if (str.size() >= kExtension.size() &&
+ str.compare(str.size() - kExtension.size(), kExtension.size(),
+ kExtension) == 0) {
+ return str.substr(0, str.size() - kExtension.size());
+ } else {
+ return str;
+ }
+}
+
+static inline std::string ToCamelCase(const std::string &input) {
+ bool cap_next_letter = true;
+ std::string result;
+ // Note: I distrust ctype.h due to locales.
+ for (size_t i = 0; i < input.size(); i++) {
+ if ('a' <= input[i] && input[i] <= 'z') {
+ if (cap_next_letter) {
+ result += input[i] + ('A' - 'a');
+ } else {
+ result += input[i];
+ }
+ cap_next_letter = false;
+ } else if ('A' <= input[i] && input[i] <= 'Z') {
+ // Capital letters are left as-is.
+ result += input[i];
+ cap_next_letter = false;
+ } else if ('0' <= input[i] && input[i] <= '9') {
+ result += input[i];
+ cap_next_letter = true;
+ } else {
+ cap_next_letter = true;
+ }
+ }
+ return result;
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
new file mode 100644
index 0000000..f384e36
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cpp_helpers.h"
+
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/compiler/plugin.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/io/printer.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/stubs/common.h>
+
+using ::google::protobuf::FileDescriptor;
+using ::google::protobuf::MethodDescriptor;
+using ::google::protobuf::ServiceDescriptor;
+using ::google::protobuf::compiler::CodeGenerator;
+using ::google::protobuf::compiler::GeneratorContext;
+using ::google::protobuf::io::Emiter;
+using ::google::protobuf::io::ZeroCopyOutputStream;
+
+class StubGenerator : public CodeGenerator {
+public:
+ virtual bool Generate(const FileDescriptor *file, const std::string &,
+ GeneratorContext *ctx,
+ std::string *error) const override;
+
+private:
+ void EmitService(const ServiceDescriptor *service, Emiter *out) const;
+ void EmitMethod(const MethodDescriptor *method, Emiter *out) const;
+};
+
+bool StubGenerator::Generate(const FileDescriptor *file, const std::string &,
+ GeneratorContext *ctx, std::string *) const {
+ namespace pb = ::google::protobuf;
+ std::unique_ptr<ZeroCopyOutputStream> os(
+ ctx->Open(StripProto(file->name()) + ".hrpc.inl"));
+ Emiter out(os.get(), '$');
+ for (int i = 0; i < file->service_count(); ++i) {
+ const ServiceDescriptor *service = file->service(i);
+ EmitService(service, &out);
+ }
+ return true;
+}
+
+void StubGenerator::EmitService(const ServiceDescriptor *service,
+ Emiter *out) const {
+ out->Emit("\n// GENERATED AUTOMATICALLY. DO NOT MODIFY.\n"
+ "class $service$ {\n"
+ "private:\n"
+ " ::hdfs::RpcEngine *const engine_;\n"
+ "public:\n"
+ " typedef std::function<void(const ::hdfs::Status &)> Callback;\n"
+ " typedef ::google::protobuf::MessageLite Message;\n"
+ " inline $service$(::hdfs::RpcEngine *engine)\n"
+ " : engine_(engine) {}\n",
+ "service", service->name());
+ for (int i = 0; i < service->method_count(); ++i) {
+ const MethodDescriptor *method = service->method(i);
+ EmitMethod(method, out);
+ }
+ out->Emit("};\n");
+}
+
+void StubGenerator::EmitMethod(const MethodDescriptor *method,
+ Emiter *out) const {
+ out->Emit("\n inline void $camel_method$(const Message *req, "
+ "const std::shared_ptr<Message> &resp, "
+ "Callback &&handler) {\n"
+ " engine_->AsyncRpc(\"$method$\", req, resp, std::move(handler));\n"
+ " }\n",
+ "camel_method", ToCamelCase(method->name()),
+ "method", method->name()
+ );
+}
+
+int main(int argc, char *argv[]) {
+ StubGenerator generator;
+ return google::protobuf::compiler::PluginMain(argc, argv, &generator);
+}