You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/02/09 11:53:27 UTC
[apisix] branch master updated: feat(grpc-transcode): support .pb file (#6264)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 6558ba5 feat(grpc-transcode): support .pb file (#6264)
6558ba5 is described below
commit 6558ba519c304324b84a6364ac4f5fd6be79ffa9
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Wed Feb 9 19:53:19 2022 +0800
feat(grpc-transcode): support .pb file (#6264)
---
apisix/plugins/grpc-transcode/proto.lua | 70 +++++++++++---
apisix/plugins/grpc-transcode/util.lua | 13 ++-
docs/en/latest/plugins/grpc-transcode.md | 63 ++++++++++++-
docs/zh/latest/plugins/grpc-transcode.md | 63 ++++++++++++-
rockspec/apisix-master-0.rockspec | 2 +-
t/grpc_server_example | 2 +-
t/plugin/grpc-transcode2.t | 152 +++++++++++++++++++++++++++++++
7 files changed, 343 insertions(+), 22 deletions(-)
diff --git a/apisix/plugins/grpc-transcode/proto.lua b/apisix/plugins/grpc-transcode/proto.lua
index de19be2..e997b55 100644
--- a/apisix/plugins/grpc-transcode/proto.lua
+++ b/apisix/plugins/grpc-transcode/proto.lua
@@ -14,25 +14,23 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local core = require("apisix.core")
-local config_util = require("apisix.core.config_util")
-local pb = require("pb")
-local protoc = require("protoc")
-local pcall = pcall
-local ipairs = ipairs
-local protos
+local core = require("apisix.core")
+local config_util = require("apisix.core.config_util")
+local pb = require("pb")
+local protoc = require("protoc")
+local pcall = pcall
+local ipairs = ipairs
+local decode_base64 = ngx.decode_base64
+local protos
local lrucache_proto = core.lrucache.new({
ttl = 300, count = 100
})
local proto_fake_file = "filename for loaded"
-local function compile_proto(content)
- -- clear pb state
- pb.state(nil)
-
+local function compile_proto_text(content)
protoc.reload()
local _p = protoc.new()
-- the loaded proto won't appears in _p.loaded without a file name after lua-protobuf=0.3.2,
@@ -50,8 +48,6 @@ local function compile_proto(content)
end
local compiled = _p.loaded
- -- fetch pb state
- compiled.pb_state = pb.state(nil)
local index = {}
for _, s in ipairs(compiled[proto_fake_file].service or {}) do
@@ -69,6 +65,54 @@ local function compile_proto(content)
end
+local function compile_proto_bin(content)
+ content = decode_base64(content)
+ if not content then
+ return nil
+ end
+
+ -- pb.load doesn't return err
+ local ok = pb.load(content)
+ if not ok then
+ return nil
+ end
+
+ local index = {}
+ for name, _, methods in pb.services() do
+ local method_index = {}
+ for _, m in ipairs(methods) do
+ method_index[m.name] = m
+ end
+ -- remove the prefix '.'
+ index[name:sub(2)] = method_index
+ end
+
+ local compiled = {}
+ compiled[proto_fake_file] = {}
+ compiled[proto_fake_file].index = index
+
+ return compiled
+end
+
+
+local function compile_proto(content)
+ -- clear pb state
+ pb.state(nil)
+
+ local compiled, err = compile_proto_text(content)
+ if not compiled then
+ compiled = compile_proto_bin(content)
+ if not compiled then
+ return nil, err
+ end
+ end
+
+ -- fetch pb state
+ compiled.pb_state = pb.state(nil)
+ return compiled
+end
+
+
local _M = {
version = 0.1,
compile_proto = compile_proto,
diff --git a/apisix/plugins/grpc-transcode/util.lua b/apisix/plugins/grpc-transcode/util.lua
index 18c9f78..2a7bfb6 100644
--- a/apisix/plugins/grpc-transcode/util.lua
+++ b/apisix/plugins/grpc-transcode/util.lua
@@ -27,23 +27,26 @@ local type = type
local _M = {version = 0.1}
-function _M.find_method(protos, service, method)
- local loaded = protos[proto_fake_file]
- if not loaded or type(loaded) ~= "table" then
+function _M.find_method(proto, service, method)
+ local loaded = proto[proto_fake_file]
+ if type(loaded) ~= "table" then
+ core.log.error("compiled proto not found")
return nil
end
- if not loaded.index[service] or type(loaded.index[service]) ~= "table" then
+ if type(loaded.index[service]) ~= "table" then
+ core.log.error("compiled proto service not found")
return nil
end
local res = loaded.index[service][method]
if not res then
+ core.log.error("compiled proto method not found")
return nil
end
-- restore pb state
- pb.state(protos.pb_state)
+ pb.state(proto.pb_state)
return res
end
diff --git a/docs/en/latest/plugins/grpc-transcode.md b/docs/en/latest/plugins/grpc-transcode.md
index 2f576ef..206169e 100644
--- a/docs/en/latest/plugins/grpc-transcode.md
+++ b/docs/en/latest/plugins/grpc-transcode.md
@@ -29,7 +29,7 @@ HTTP(s) -> APISIX -> gRPC server
#### Attributes
-* `content`: `.proto` file's content.
+* `content`: `.proto` or `.pb` file's content.
#### Add a proto
@@ -52,6 +52,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136
}'
```
+If your `.proto` file contains imports, or you want to combine multiple `.proto` files into a proto,
+you can use `.pb` file to create the proto.
+
+Assumed we have a `.proto` called `proto/helloworld.proto`, which imports another proto file:
+
+```proto
+syntax = "proto3";
+
+package helloworld;
+import "proto/import.proto";
+...
+```
+
+First of all, let's create a `.pb` file from `.proto` files:
+
+```shell
+protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto
+```
+
+The output binary file `proto.pb` will contain both `helloworld.proto` and `import.proto`.
+
+Then we can submit the content of `proto.pb` as the `content` field of the proto.
+
+As the content is binary, we need to encode it in base64 first. Here we use a Python script to do it:
+
+```python
+#!/usr/bin/env python
+# coding: utf-8
+# save this file as upload_pb.py
+import base64
+import sys
+# sudo pip install requests
+import requests
+
+if len(sys.argv) <= 1:
+ print("bad argument")
+ sys.exit(1)
+with open(sys.argv[1], 'rb') as f:
+ content = base64.b64encode(f.read())
+id = sys.argv[2]
+api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it
+
+reqParam = {
+ "content": content,
+}
+resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={
+ "X-API-KEY": api_key,
+})
+print(resp.status_code)
+print(resp.text)
+```
+
+Create proto:
+
+```bash
+chmod +x ./upload_pb.pb
+./upload_pb.py proto.pb 1
+# 200
+# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\ [...]
+```
+
## Attribute List
| Name | Type | Requirement | Default | Valid | Description |
diff --git a/docs/zh/latest/plugins/grpc-transcode.md b/docs/zh/latest/plugins/grpc-transcode.md
index d031943..842072e 100644
--- a/docs/zh/latest/plugins/grpc-transcode.md
+++ b/docs/zh/latest/plugins/grpc-transcode.md
@@ -27,7 +27,7 @@ HTTP(s) -> APISIX -> gRPC server
### 参数
-* `content`: `.proto` 文件的内容
+* `content`: `.proto` 或 `.pb` 文件的内容
### 添加proto
@@ -50,6 +50,67 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136
}'
```
+如果你的 `.proto` 文件包含 import，或者你想把多个 `.proto` 文件合并成一个 proto。
+你可以使用 `.pb` 文件来创建 proto。
+
+假设我们有一个 `.proto` 叫 `proto/helloworld.proto`，它导入了另一个 proto 文件：
+
+```proto
+syntax = "proto3";
+
+package helloworld;
+import "proto/import.proto";
+...
+```
+
+首先，让我们从 `.proto`文件创建一个`.pb`文件。
+
+```shell
+protoc --include_imports --descriptor_set_out=proto.pb proto/helloworld.proto
+```
+
+输出的二进制文件 `proto.pb` 将同时包含 `helloworld.proto` 和 `import.proto`。
+
+然后我们可以将 `proto.pb` 的内容作为 proto 的 `content` 字段提交。
+
+由于内容是二进制的，我们需要先对其进行 base64 编码。这里我们用一个 Python 脚本来做。
+
+```python
+#!/usr/bin/env python
+# coding: utf-8
+# save this file as upload_pb.py
+import base64
+import sys
+# sudo pip install requests
+import requests
+
+if len(sys.argv) <= 1:
+ print("bad argument")
+ sys.exit(1)
+with open(sys.argv[1], 'rb') as f:
+ content = base64.b64encode(f.read())
+id = sys.argv[2]
+api_key = "edd1c9f034335f136f87ad84b625c8f1" # Change it
+
+reqParam = {
+ "content": content,
+}
+resp = requests.put("http://127.0.0.1:9080/apisix/admin/proto/" + id, json=reqParam, headers={
+ "X-API-KEY": api_key,
+})
+print(resp.status_code)
+print(resp.text)
+```
+
+创建proto：
+
+```bash
+chmod +x ./upload_pb.pb
+./upload_pb.py proto.pb 1
+# 200
+# {"node":{"value":{"create_time":1643879753,"update_time":1643883085,"content":"CmgKEnByb3RvL2ltcG9ydC5wcm90bxIDcGtnIhoKBFVzZXISEgoEbmFtZRgBIAEoCVIEbmFtZSIeCghSZXNwb25zZRISCgRib2R5GAEgASgJUgRib2R5QglaBy4vcHJvdG9iBnByb3RvMwq9AQoPcHJvdG8vc3JjLnByb3RvEgpoZWxsb3dvcmxkGhJwcm90by9pbXBvcnQucHJvdG8iPAoHUmVxdWVzdBIdCgR1c2VyGAEgASgLMgkucGtnLlVzZXJSBHVzZXISEgoEYm9keRgCIAEoCVIEYm9keTI5CgpUZXN0SW1wb3J0EisKA1J1bhITLmhlbGxvd29ybGQuUmVxdWVzdBoNLnBrZy5SZXNwb25zZSIAQglaBy4vcHJvdG9iBnByb3RvMw=="},"key":"\ [...]
+```
+
## 参数列表
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec
index 3bc8121..a6f3411 100644
--- a/rockspec/apisix-master-0.rockspec
+++ b/rockspec/apisix-master-0.rockspec
@@ -46,7 +46,7 @@ dependencies = {
"lua-resty-session = 2.24",
"opentracing-openresty = 0.1",
"lua-resty-radixtree = 2.8.1",
- "lua-protobuf = 0.3.3",
+ "api7-lua-protobuf = 0.1.0",
"lua-resty-openidc = 1.7.2-1",
"luafilesystem = 1.7.0-2",
"api7-lua-tinyyaml = 0.4.2",
diff --git a/t/grpc_server_example b/t/grpc_server_example
index f7ee318..5e74be6 160000
--- a/t/grpc_server_example
+++ b/t/grpc_server_example
@@ -1 +1 @@
-Subproject commit f7ee318f701e04bf21bf000baab539f5a8bc7eaa
+Subproject commit 5e74be697f24151648be1712fce0ab2fdd0ec964
diff --git a/t/plugin/grpc-transcode2.t b/t/plugin/grpc-transcode2.t
index 7a48077..93b84b2 100644
--- a/t/plugin/grpc-transcode2.t
+++ b/t/plugin/grpc-transcode2.t
@@ -230,3 +230,155 @@ location /t {
{"message":"Hello world, name: John"}
--- error_log
failed to encode request data to protobuf
+
+
+
+=== TEST 6: set binary rule
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+
+ local content = t.read_file("t/grpc_server_example/proto.pb")
+ local data = {content = ngx.encode_base64(content)}
+ local code, body = t.test('/apisix/admin/proto/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/grpctest",
+ "plugins": {
+ "grpc-transcode": {
+ "proto_id": "1",
+ "service": "helloworld.TestImport",
+ "method": "Run"
+ }
+ },
+ "upstream": {
+ "scheme": "grpc",
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:50051": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 7: hit route
+--- request
+POST /grpctest
+{"body":"world","user":{"name":"Hello"}}
+--- more_headers
+Content-Type: application/json
+--- response_body chomp
+{"body":"Hello world"}
+
+
+
+=== TEST 8: service/method not found
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/service_not_found",
+ "plugins": {
+ "grpc-transcode": {
+ "proto_id": "1",
+ "service": "helloworld.TestImportx",
+ "method": "Run"
+ }
+ },
+ "upstream": {
+ "scheme": "grpc",
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:50051": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/method_not_found",
+ "plugins": {
+ "grpc-transcode": {
+ "proto_id": "1",
+ "service": "helloworld.TestImport",
+ "method": "Runx"
+ }
+ },
+ "upstream": {
+ "scheme": "grpc",
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:50051": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 9: hit route
+--- request
+POST /service_not_found
+{"body":"world","user":{"name":"Hello"}}
+--- more_headers
+Content-Type: application/json
+--- error_log
+Undefined service method
+--- error_code: 503
+
+
+
+=== TEST 10: hit route
+--- request
+POST /method_not_found
+{"body":"world","user":{"name":"Hello"}}
+--- more_headers
+Content-Type: application/json
+--- error_log
+Undefined service method
+--- error_code: 503