You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/12/07 05:48:02 UTC
[apisix] branch master updated: fix: proxy grpc sometimes occurs 502 (#8364)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 5eb0aadd3 fix: proxy grpc sometimes occurs 502 (#8364)
5eb0aadd3 is described below
commit 5eb0aadd3a9c426110ef3c3db4ca036950c45f35
Author: soulbird <zh...@outlook.com>
AuthorDate: Wed Dec 7 13:47:52 2022 +0800
fix: proxy grpc sometimes occurs 502 (#8364)
Co-authored-by: soulbird <zh...@gmail.com>
Fixes https://github.com/apache/apisix/issues/8166
---
apisix/balancer.lua | 5 ++-
ci/centos7-ci.sh | 2 +-
ci/linux_openresty_common_runner.sh | 2 +-
t/grpc_server_example/go.mod | 4 +--
t/grpc_server_example/go.sum | 29 +++++++++++++--
t/grpc_server_example/main.go | 41 +++++++++++++++++++++
t/node/upstream-keepalive-pool.t | 71 +++++++++++++++++++++++++++++++++++++
7 files changed, 144 insertions(+), 10 deletions(-)
diff --git a/apisix/balancer.lua b/apisix/balancer.lua
index 462d04f07..f83653317 100644
--- a/apisix/balancer.lua
+++ b/apisix/balancer.lua
@@ -313,18 +313,17 @@ do
pool_opt.pool_size = size
local scheme = up_conf.scheme
+ local pool = scheme .. "#" .. server.host .. "#" .. server.port
-- other TLS schemes don't use http balancer keepalive
if (scheme == "https" or scheme == "grpcs") then
- local pool = server.host .. "#" .. server.port
local sni = ctx.var.upstream_host
pool = pool .. "#" .. sni
if up_conf.tls and up_conf.tls.client_cert then
pool = pool .. "#" .. up_conf.tls.client_cert
end
-
- pool_opt.pool = pool
end
+ pool_opt.pool = pool
local ok, err = balancer.set_current_peer(server.host, server.port,
pool_opt)
diff --git a/ci/centos7-ci.sh b/ci/centos7-ci.sh
index a9a3799ae..f86221a1a 100755
--- a/ci/centos7-ci.sh
+++ b/ci/centos7-ci.sh
@@ -55,7 +55,7 @@ install_dependencies() {
CGO_ENABLED=0 go build
./grpc_server_example \
- -grpc-address :50051 -grpcs-address :50052 -grpcs-mtls-address :50053 \
+ -grpc-address :50051 -grpcs-address :50052 -grpcs-mtls-address :50053 -grpc-http-address :50054 \
-crt ../certs/apisix.crt -key ../certs/apisix.key -ca ../certs/mtls_ca.crt \
> grpc_server_example.log 2>&1 || (cat grpc_server_example.log && exit 1)&
diff --git a/ci/linux_openresty_common_runner.sh b/ci/linux_openresty_common_runner.sh
index 9d48f8a32..cd4472521 100755
--- a/ci/linux_openresty_common_runner.sh
+++ b/ci/linux_openresty_common_runner.sh
@@ -71,7 +71,7 @@ script() {
set_coredns
./t/grpc_server_example/grpc_server_example \
- -grpc-address :50051 -grpcs-address :50052 -grpcs-mtls-address :50053 \
+ -grpc-address :50051 -grpcs-address :50052 -grpcs-mtls-address :50053 -grpc-http-address :50054 \
-crt ./t/certs/apisix.crt -key ./t/certs/apisix.key -ca ./t/certs/mtls_ca.crt \
&
diff --git a/t/grpc_server_example/go.mod b/t/grpc_server_example/go.mod
index cccb735e9..8d36bae68 100644
--- a/t/grpc_server_example/go.mod
+++ b/t/grpc_server_example/go.mod
@@ -3,7 +3,7 @@ module github.com/api7/grpc_server_example
go 1.11
require (
- github.com/golang/protobuf v1.5.0
+ golang.org/x/net v0.2.0
google.golang.org/grpc v1.32.0
- google.golang.org/protobuf v1.27.1 // indirect
+ google.golang.org/protobuf v1.27.1
)
diff --git a/t/grpc_server_example/go.sum b/t/grpc_server_example/go.sum
index d150a12f6..d9ceb02e6 100644
--- a/t/grpc_server_example/go.sum
+++ b/t/grpc_server_example/go.sum
@@ -18,29 +18,52 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
diff --git a/t/grpc_server_example/main.go b/t/grpc_server_example/main.go
index 4837dda12..fdb8dedd7 100644
--- a/t/grpc_server_example/main.go
+++ b/t/grpc_server_example/main.go
@@ -28,12 +28,16 @@ import (
"crypto/x509"
"flag"
"fmt"
+ "golang.org/x/net/http2"
+ "golang.org/x/net/http2/h2c"
"io"
"io/ioutil"
"log"
"net"
+ "net/http"
"os"
"os/signal"
+ "strings"
"syscall"
"time"
@@ -50,6 +54,7 @@ var (
grpcAddr = ":50051"
grpcsAddr = ":50052"
grpcsMtlsAddr string
+ grpcHTTPAddr string
crtFilePath = "../t/cert/apisix.crt"
keyFilePath = "../t/cert/apisix.key"
@@ -60,6 +65,7 @@ func init() {
flag.StringVar(&grpcAddr, "grpc-address", grpcAddr, "address for grpc")
flag.StringVar(&grpcsAddr, "grpcs-address", grpcsAddr, "address for grpcs")
flag.StringVar(&grpcsMtlsAddr, "grpcs-mtls-address", grpcsMtlsAddr, "address for grpcs in mTLS")
+ flag.StringVar(&grpcHTTPAddr, "grpc-http-address", grpcHTTPAddr, "addresses for http and grpc services at the same time")
flag.StringVar(&crtFilePath, "crt", crtFilePath, "path to certificate")
flag.StringVar(&keyFilePath, "key", keyFilePath, "path to key")
flag.StringVar(&caFilePath, "ca", caFilePath, "path to ca")
@@ -217,6 +223,21 @@ func (s *server) Run(ctx context.Context, in *pb.Request) (*pb.Response, error)
return &pb.Response{Body: in.User.Name + " " + in.Body}, nil
}
+func gRPCAndHTTPFunc(grpcServer *grpc.Server) http.Handler {
+ return h2c.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("hello http"))
+ })
+
+ if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") {
+ grpcServer.ServeHTTP(w, r)
+ } else {
+ mux.ServeHTTP(w, r)
+ }
+ }), &http2.Server{})
+}
+
func main() {
flag.Parse()
@@ -226,9 +247,11 @@ func main() {
log.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
+
reflection.Register(s)
pb.RegisterGreeterServer(s, &server{})
pb.RegisterTestImportServer(s, &server{})
+
if err := s.Serve(lis); err != nil {
log.Fatalf("failed to serve: %v", err)
}
@@ -252,6 +275,24 @@ func main() {
}
}()
+ if grpcHTTPAddr != "" {
+ go func() {
+ lis, err := net.Listen("tcp", grpcHTTPAddr)
+ if err != nil {
+ log.Fatalf("failed to listen: %v", err)
+ }
+ s := grpc.NewServer()
+
+ reflection.Register(s)
+ pb.RegisterGreeterServer(s, &server{})
+ pb.RegisterTestImportServer(s, &server{})
+
+ if err := http.Serve(lis, gRPCAndHTTPFunc(s)); err != nil {
+ log.Fatalf("failed to serve grpc: %v", err)
+ }
+ }()
+ }
+
if grpcsMtlsAddr != "" {
go func() {
lis, err := net.Listen("tcp", grpcsMtlsAddr)
diff --git a/t/node/upstream-keepalive-pool.t b/t/node/upstream-keepalive-pool.t
index 09c96d827..a8a625ad9 100644
--- a/t/node/upstream-keepalive-pool.t
+++ b/t/node/upstream-keepalive-pool.t
@@ -734,3 +734,74 @@ qr/lua balancer: keepalive create pool, .*/
qr/^lua balancer: keepalive create pool, crc32: \S+, size: 4
lua balancer: keepalive create pool, crc32: \S+, size: 8
$/
+
+
+
+=== TEST 16: backend serve http and grpc with the same port
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local test = require("lib.test_admin").test
+ local json = require("toolkit.json")
+
+ local data = {
+ uri = "",
+ upstream = {
+ scheme = "",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:50054"] = 1,
+ },
+ keepalive_pool = {
+ size = 4
+ }
+ }
+ }
+
+ data.uri = "/helloworld.Greeter/SayHello"
+ data.upstream.scheme = "grpc"
+ local code, body = test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+ if code >= 300 then
+ ngx.status = code
+ ngx.print(body)
+ return
+ end
+
+ data.uri = "/hello"
+ data.upstream.scheme = "http"
+ local code, body = test('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+ if code >= 300 then
+ ngx.status = code
+ ngx.print(body)
+ return
+ end
+
+ }
+ }
+--- response_body
+
+
+
+=== TEST 17: hit http
+--- request
+GET /hello
+--- response_body chomp
+hello http
+
+
+
+=== TEST 18: hit grpc
+--- http2
+--- exec
+grpcurl -import-path ./t/grpc_server_example/proto -proto helloworld.proto -plaintext -d '{"name":"apisix"}' 127.0.0.1:1984 helloworld.Greeter.SayHello
+--- response_body
+{
+ "message": "Hello apisix"
+}