You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@brpc.apache.org by zy...@apache.org on 2020/11/23 11:49:26 UTC
[incubator-brpc] branch http_improve created (now 8a4ffd0)
This is an automated email from the ASF dual-hosted git repository.
zychen pushed a change to branch http_improve
in repository https://gitbox.apache.org/repos/asf/incubator-brpc.git.
at 8a4ffd0 Compatibility improvement of protobuf json format and spring http spec
This branch includes the following new commits:
new 8a4ffd0 Compatibility improvement of protobuf json format and spring http spec
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org
For additional commands, e-mail: dev-help@brpc.apache.org
[incubator-brpc] 01/01: Compatibility improvement of protobuf json
format and spring http spec
Posted by zy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zychen pushed a commit to branch http_improve
in repository https://gitbox.apache.org/repos/asf/incubator-brpc.git
commit 8a4ffd01b1e8323d42a36199722c8c63736ca091
Author: Zhangyi Chen <fr...@gmail.com>
AuthorDate: Mon Nov 23 19:46:07 2020 +0800
Compatibility improvement of protobuf json format and spring http spec
---
src/brpc/policy/http_rpc_protocol.cpp | 7 ++-
src/brpc/policy/http_rpc_protocol.h | 1 +
src/json2pb/json_to_pb.cpp | 105 ++++++++++++++++++++++++++++---
src/json2pb/pb_to_json.cpp | 2 +-
test/brpc_http_rpc_protocol_unittest.cpp | 37 +++++++++++
test/brpc_protobuf_json_unittest.cpp | 19 ++++--
6 files changed, 157 insertions(+), 14 deletions(-)
diff --git a/src/brpc/policy/http_rpc_protocol.cpp b/src/brpc/policy/http_rpc_protocol.cpp
index 99579fe..c175f4f 100644
--- a/src/brpc/policy/http_rpc_protocol.cpp
+++ b/src/brpc/policy/http_rpc_protocol.cpp
@@ -21,6 +21,7 @@
#include <json2pb/pb_to_json.h> // ProtoMessageToJson
#include <json2pb/json_to_pb.h> // JsonToProtoMessage
+#include "brpc/policy/http_rpc_protocol.h"
#include "butil/unique_ptr.h" // std::unique_ptr
#include "butil/string_splitter.h" // StringMultiSplitter
#include "butil/string_printf.h"
@@ -110,6 +111,7 @@ CommonStrings::CommonStrings()
, CONTENT_TYPE_TEXT("text/plain")
, CONTENT_TYPE_JSON("application/json")
, CONTENT_TYPE_PROTO("application/proto")
+ , CONTENT_TYPE_SPRING_PROTO("application/x-protobuf")
, ERROR_CODE("x-bd-error-code")
, AUTHORIZATION("authorization")
, ACCEPT_ENCODING("accept-encoding")
@@ -189,6 +191,9 @@ HttpContentType ParseContentType(butil::StringPiece ct, bool* is_grpc_ct) {
} else if (ct.starts_with("proto")) {
type = HTTP_CONTENT_PROTO;
ct.remove_prefix(5);
+ } else if (ct.starts_with("x-protobuf")) {
+ type = HTTP_CONTENT_PROTO;
+ ct.remove_prefix(10);
} else {
return HTTP_CONTENT_OTHERS;
}
@@ -511,7 +516,7 @@ void SerializeHttpRequest(butil::IOBuf* /*not used*/,
opt.bytes_to_base64 = cntl->has_pb_bytes_to_base64();
opt.jsonify_empty_array = cntl->has_pb_jsonify_empty_array();
opt.always_print_primitive_fields = cntl->has_always_print_primitive_fields();
-
+
opt.enum_option = (FLAGS_pb_enum_as_number
? json2pb::OUTPUT_ENUM_BY_NUMBER
: json2pb::OUTPUT_ENUM_BY_NAME);
diff --git a/src/brpc/policy/http_rpc_protocol.h b/src/brpc/policy/http_rpc_protocol.h
index b6f986a..d43ff7e 100644
--- a/src/brpc/policy/http_rpc_protocol.h
+++ b/src/brpc/policy/http_rpc_protocol.h
@@ -37,6 +37,7 @@ struct CommonStrings {
std::string CONTENT_TYPE_TEXT;
std::string CONTENT_TYPE_JSON;
std::string CONTENT_TYPE_PROTO;
+ std::string CONTENT_TYPE_SPRING_PROTO;
std::string ERROR_CODE;
std::string AUTHORIZATION;
std::string ACCEPT_ENCODING;
diff --git a/src/json2pb/json_to_pb.cpp b/src/json2pb/json_to_pb.cpp
index 41da40d..f4face7 100644
--- a/src/json2pb/json_to_pb.cpp
+++ b/src/json2pb/json_to_pb.cpp
@@ -24,6 +24,7 @@
#include <typeinfo>
#include <limits>
#include <google/protobuf/descriptor.h>
+#include "butil/strings/string_number_conversions.h"
#include "json_to_pb.h"
#include "zero_copy_stream_reader.h" // ZeroCopyStreamReader
#include "encode_decode.h"
@@ -207,10 +208,63 @@ inline bool convert_enum_type(const BUTIL_RAPIDJSON_NAMESPACE::Value&item, bool
return true;
}
+inline bool convert_int64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item, bool repeated,
+ google::protobuf::Message* message,
+ const google::protobuf::FieldDescriptor* field,
+ const google::protobuf::Reflection* reflection,
+ std::string* err) {
+
+ int64_t num;
+ if (item.IsInt64()) {
+ if (repeated) {
+ reflection->AddInt64(message, field, item.GetInt64());
+ } else {
+ reflection->SetInt64(message, field, item.GetInt64());
+ }
+ } else if (item.IsString() &&
+ butil::StringToInt64({item.GetString(), item.GetStringLength()},
+ &num)) {
+ if (repeated) {
+ reflection->AddInt64(message, field, num);
+ } else {
+ reflection->SetInt64(message, field, num);
+ }
+ } else {
+ return value_invalid(field, "INT64", item, err);
+ }
+ return true;
+}
+
+inline bool convert_uint64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item,
+ bool repeated,
+ google::protobuf::Message* message,
+ const google::protobuf::FieldDescriptor* field,
+ const google::protobuf::Reflection* reflection,
+ std::string* err) {
+ uint64_t num;
+ if (item.IsUint64()) {
+ if (repeated) {
+ reflection->AddUInt64(message, field, item.GetUint64());
+ } else {
+ reflection->SetUInt64(message, field, item.GetUint64());
+ }
+ } else if (item.IsString() &&
+ butil::StringToUint64({item.GetString(), item.GetStringLength()},
+ &num)) {
+ if (repeated) {
+ reflection->AddUInt64(message, field, num);
+ } else {
+ reflection->SetUInt64(message, field, num);
+ }
+ } else {
+ return value_invalid(field, "UINT64", item, err);
+ }
+ return true;
+}
+
bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
google::protobuf::Message* message,
- const Json2PbOptions& options,
- std::string* err);
+ const Json2PbOptions& options, std::string* err);
//Json value to protobuf convert rules for type:
//Json value type Protobuf type convert rules
@@ -219,9 +273,10 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
//int64 int uint int64 uint64 valid convert is available
//uint64 int uint int64 uint64 valid convert is available
//int uint int64 uint64 float double available
-//"NaN" "Infinity" "-Infinity" float double only "NaN" "Infinity" "-Infinity" is available
+//"NaN" "Infinity" "-Infinity" float double only "NaN" "Infinity" "-Infinity" is available
//int enum valid enum number value is available
-//string enum valid enum name value is available
+//string enum valid enum name value is available
+//string int64 uint64 valid convert is available
//other mismatch type convertion will be regarded as error.
#define J2PCHECKTYPE(value, cpptype, jsontype) ({ \
MatchType match_type = TYPE_MATCH; \
@@ -234,6 +289,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
match_type; \
})
+
static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
const google::protobuf::FieldDescriptor* field,
google::protobuf::Message* message,
@@ -271,15 +327,48 @@ static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
reflection->Set##method(message, field, value.Get##jsontype()); \
} \
break; \
- }
+ } \
+
CASE_FIELD_TYPE(INT32, Int32, Int);
CASE_FIELD_TYPE(UINT32, UInt32, Uint);
CASE_FIELD_TYPE(BOOL, Bool, Bool);
- CASE_FIELD_TYPE(INT64, Int64, Int64);
- CASE_FIELD_TYPE(UINT64, UInt64, Uint64);
#undef CASE_FIELD_TYPE
- case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:
+ case google::protobuf::FieldDescriptor::CPPTYPE_INT64:
+ if (field->is_repeated()) {
+ const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
+ for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size;
+ ++index) {
+ const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index];
+ if (!convert_int64_type(item, true, message, field, reflection,
+ err)) {
+ return false;
+ }
+ }
+ } else if (!convert_int64_type(value, false, message, field, reflection,
+ err)) {
+ return false;
+ }
+ break;
+
+ case google::protobuf::FieldDescriptor::CPPTYPE_UINT64:
+ if (field->is_repeated()) {
+ const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
+ for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size;
+ ++index) {
+ const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index];
+ if (!convert_uint64_type(item, true, message, field, reflection,
+ err)) {
+ return false;
+ }
+ }
+ } else if (!convert_uint64_type(value, false, message, field, reflection,
+ err)) {
+ return false;
+ }
+ break;
+
+ case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:
if (field->is_repeated()) {
const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size; ++index) {
diff --git a/src/json2pb/pb_to_json.cpp b/src/json2pb/pb_to_json.cpp
index 704f963..55578c3 100644
--- a/src/json2pb/pb_to_json.cpp
+++ b/src/json2pb/pb_to_json.cpp
@@ -51,7 +51,7 @@ public:
bool Convert(const google::protobuf::Message& message, Handler& handler);
const std::string& ErrorText() const { return _error; }
-
+
private:
template <typename Handler>
bool _PbFieldToJson(const google::protobuf::Message& message,
diff --git a/test/brpc_http_rpc_protocol_unittest.cpp b/test/brpc_http_rpc_protocol_unittest.cpp
index 15ef84b..3452332 100644
--- a/test/brpc_http_rpc_protocol_unittest.cpp
+++ b/test/brpc_http_rpc_protocol_unittest.cpp
@@ -1443,4 +1443,41 @@ TEST_F(HttpTest, http2_handle_goaway_streams) {
brpc::Join(ids[i]);
}
}
+
+TEST_F(HttpTest, spring_protobuf_content_type) {
+ const int port = 8923;
+ brpc::Server server;
+ EXPECT_EQ(0, server.AddService(&_svc, brpc::SERVER_DOESNT_OWN_SERVICE));
+ EXPECT_EQ(0, server.Start(port, nullptr));
+
+ brpc::Channel channel;
+ brpc::ChannelOptions options;
+ options.protocol = "http";
+ ASSERT_EQ(0, channel.Init(butil::EndPoint(butil::my_ip(), port), &options));
+
+ brpc::Controller cntl;
+ test::EchoRequest req;
+ test::EchoResponse res;
+ req.set_message(EXP_REQUEST);
+ cntl.http_request().set_method(brpc::HTTP_METHOD_POST);
+ cntl.http_request().uri() = "/EchoService/Echo";
+ cntl.http_request().set_content_type("application/x-protobuf");
+ cntl.request_attachment().append(req.SerializeAsString());
+ channel.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr);
+ ASSERT_FALSE(cntl.Failed());
+ ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type());
+ ASSERT_TRUE(res.ParseFromString(cntl.response_attachment().to_string()));
+ ASSERT_EQ(EXP_RESPONSE, res.message());
+
+ brpc::Controller cntl2;
+ test::EchoService_Stub stub(&channel);
+ req.set_message(EXP_REQUEST);
+ res.Clear();
+ cntl2.http_request().set_content_type("application/x-protobuf");
+ stub.Echo(&cntl2, &req, &res, nullptr);
+ ASSERT_FALSE(cntl.Failed());
+ ASSERT_EQ(EXP_RESPONSE, res.message());
+ ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type());
+}
+
} //namespace
diff --git a/test/brpc_protobuf_json_unittest.cpp b/test/brpc_protobuf_json_unittest.cpp
index d7b4d1c..50b8e72 100644
--- a/test/brpc_protobuf_json_unittest.cpp
+++ b/test/brpc_protobuf_json_unittest.cpp
@@ -1308,7 +1308,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_encode_decode_perf_case) {
}
TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
-
+
std::ifstream in("jsonout", std::ios::in);
std::ostringstream tmp;
tmp << in.rdbuf();
@@ -1317,8 +1317,8 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
printf("----------test pb to json performance------------\n\n");
- std::string error;
-
+ std::string error;
+
butil::Timer timer;
bool res;
float avg_time1 = 0;
@@ -1331,7 +1331,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
res = JsonToProtoMessage(info3, &data, option, &error);
timer.stop();
avg_time1 += timer.u_elapsed();
- ASSERT_TRUE(res);
+ ASSERT_TRUE(res) << error;
ProfilerStart("pb_to_json_complex_perf.prof");
for (int i = 0; i < times; i++) {
std::string error1;
@@ -1460,4 +1460,15 @@ TEST_F(ProtobufJsonTest, extension_case) {
ASSERT_EQ("{\"hobby\":\"coding\",\"name\":\"hello\",\"id\":9,\"datadouble\":2.2,\"datafloat\":1.0}", output);
}
+TEST_F(ProtobufJsonTest, string_to_int64) {
+ auto json = R"({"name":"hello", "id":9, "data": "123456", "datadouble":2.2, "datafloat":1.0})";
+ Person person;
+ std::string err;
+ ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person, &err)) << err;
+ ASSERT_EQ(person.data(), 123456);
+ json = R"({"name":"hello", "id":9, "data": "1234567", "datadouble":2.2, "datafloat":1.0})";
+ ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person));
+ ASSERT_EQ(person.data(), 1234567);
+}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org
For additional commands, e-mail: dev-help@brpc.apache.org