You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/01/11 02:30:01 UTC
[apisix] branch master updated: feat(L4): support TLS over TCP upstream (#6030)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new eb0bf87 feat(L4): support TLS over TCP upstream (#6030)
eb0bf87 is described below
commit eb0bf87b3865b4192991f963594bc24b54d9aff9
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Tue Jan 11 10:29:56 2022 +0800
feat(L4): support TLS over TCP upstream (#6030)
---
.github/workflows/centos7-ci.yml | 2 +-
apisix/cli/ngx_tpl.lua | 6 ++
apisix/schema_def.lua | 5 +-
apisix/upstream.lua | 32 +++++++++
docs/en/latest/admin-api.md | 8 ++-
docs/en/latest/stream-proxy.md | 26 ++++++-
docs/zh/latest/admin-api.md | 7 +-
docs/zh/latest/stream-proxy.md | 26 ++++++-
t/APISIX.pm | 12 ++++
t/stream-node/upstream-tls.t | 146 +++++++++++++++++++++++++++++++++++++++
utils/linux-install-openresty.sh | 2 +-
11 files changed, 258 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml
index c136202..bb306f9 100644
--- a/.github/workflows/centos7-ci.yml
+++ b/.github/workflows/centos7-ci.yml
@@ -44,7 +44,7 @@ jobs:
run: |
export VERSION=${{ steps.branch_env.outputs.version }}
sudo gem install --no-document fpm
- git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git
+ git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git
# move codes under build tool
mkdir ./apisix-build-tools/apisix
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index b3fd0d6..d7d5da4 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -148,6 +148,12 @@ stream {
proxy_pass apisix_backend;
+ {% if use_apisix_openresty then %}
+ set $upstream_sni "apisix_backend";
+ proxy_ssl_server_name on;
+ proxy_ssl_name $upstream_sni;
+ {% end %}
+
log_by_lua_block {
apisix.stream_log_phase()
}
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index 62b0836..6300e10 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -452,7 +452,10 @@ local upstream_schema = {
},
scheme = {
default = "http",
- enum = {"grpc", "grpcs", "http", "https"}
+ enum = {"grpc", "grpcs", "http", "https", "tcp", "tls", "udp"},
+ description = "The scheme of the upstream." ..
+ " For L7 proxy, it can be one of grpc/grpcs/http/https." ..
+ " For L4 proxy, it can be one of tcp/tls/udp."
},
labels = labels_def,
discovery_type = {
diff --git a/apisix/upstream.lua b/apisix/upstream.lua
index 4e23fbf..4f2f0fe 100644
--- a/apisix/upstream.lua
+++ b/apisix/upstream.lua
@@ -24,6 +24,7 @@ local error = error
local tostring = tostring
local ipairs = ipairs
local pairs = pairs
+local ngx_var = ngx.var
local is_http = ngx.config.subsystem == "http"
local upstreams
local healthcheck
@@ -39,6 +40,19 @@ else
end
end
+local set_stream_upstream_tls
+if not is_http then
+ local ok, apisix_ngx_stream_upstream = pcall(require, "resty.apisix.stream.upstream")
+ if ok then
+ set_stream_upstream_tls = apisix_ngx_stream_upstream.set_tls
+ else
+ set_stream_upstream_tls = function ()
+ return nil, "need to build APISIX-OpenResty to support TLS over TCP upstream"
+ end
+ end
+end
+
+
local HTTP_CODE_UPSTREAM_UNAVAILABLE = 503
local _M = {}
@@ -286,6 +300,19 @@ function _M.set_by_route(route, api_ctx)
return 503, err
end
+ local scheme = up_conf.scheme
+ if scheme == "tls" then
+ local ok, err = set_stream_upstream_tls()
+ if not ok then
+ return 503, err
+ end
+
+ local sni = apisix_ssl.server_name()
+ if sni then
+ ngx_var.upstream_sni = sni
+ end
+ end
+
return
end
@@ -451,6 +478,11 @@ local function filter_upstream(value, parent)
value.parent = parent
+ if not is_http and value.scheme == "http" then
+ -- For L4 proxy, the default scheme is "tcp"
+ value.scheme = "tcp"
+ end
+
if not value.nodes then
return
end
diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md
index b0fa7f9..017cbc3 100644
--- a/docs/en/latest/admin-api.md
+++ b/docs/en/latest/admin-api.md
@@ -554,7 +554,7 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
|desc |optional|upstream usage scenarios, and more.||
|pass_host |optional| `host` option when the request is sent to the upstream, can be one of [`pass`, `node`, `rewrite`], the default option is `pass`. `pass`: Pass the client's host transparently to the upstream; `node`: Use the host configured in the node of `upstream`; `rewrite`: Use the value of the configuration `upstream_host`.||
|upstream_host |optional|Specify the host of the upstream request. This option is only valid if the `pass_host` is `rewrite`.||
-|scheme |optional |The scheme used when talk with the upstream. The value is one of ['http', 'https', 'grpc', 'grpcs'], default to 'http'.||
+|scheme |optional |The scheme used when talk with the upstream. For L7 proxy, the value is one of ['http', 'https', 'grpc', 'grpcs']. For L7 proxy, the value is one of ['tcp', 'udp', 'tls']. Default to 'http'. See below for more details.||
|labels |optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
|create_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|1602883670|
|update_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|1602883670|
@@ -581,13 +581,15 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
1. When it is `vars_combinations`, the `key` is required. The `key` can be any [Nginx builtin variables](http://nginx.org/en/docs/varindex.html) combinations, such as `$request_uri$remote_addr`.
1. If there is no value for either `hash_on` or `key`, `remote_addr` will be used as key.
+Features below require APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix):
+
+The `scheme` can be set to `tls`, which actually means "TLS over TCP".
+
`tls.client_cert/key` can be used to communicate with upstream via mTLS.
Their formats are the same as SSL's `cert` and `key` fields.
-This feature requires APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix).
`keepalive_pool` allows the upstream to have its separate connection pool.
Its children fields, like `requests`, can be used to configure the upstream keepalive options.
-This feature requires APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix).
**Config Example:**
diff --git a/docs/en/latest/stream-proxy.md b/docs/en/latest/stream-proxy.md
index 078e374..34797ec 100644
--- a/docs/en/latest/stream-proxy.md
+++ b/docs/en/latest/stream-proxy.md
@@ -166,9 +166,9 @@ Let's take another real world example:
Read [Admin API's Stream Route section](./admin-api.md#stream-route) for the complete options list.
-## Accept TLS over TCP
+## Accept TLS over TCP connection
-APISIX can accept TLS over TCP.
+APISIX can accept TLS over TCP connection.
First of all, we need to enable TLS for the TCP address:
@@ -189,7 +189,6 @@ Third, we need to configure a stream route to match and proxy it to the upstream
```shell
curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
- "remote_addr": "127.0.0.1",
"upstream": {
"nodes": {
"127.0.0.1:1995": 1
@@ -215,3 +214,24 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
```
In this case, a connection handshaked with SNI `a.test.com` will be proxied to `127.0.0.1:5991`.
+
+## Proxy to TLS over TCP upstream
+
+APISIX also supports proxying to TLS over TCP upstream.
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "upstream": {
+ "scheme": "tls",
+ "nodes": {
+ "127.0.0.1:1995": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+By setting the `scheme` to "tls", APISIX will do TLS handshake with the upstream.
+
+When the client is also speaking TLS over TCP, the SNI from the client will pass through to the upstream. Otherwise, a dummy SNI "apisix_backend" will be used.
diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md
index 404604e..6257f24 100644
--- a/docs/zh/latest/admin-api.md
+++ b/docs/zh/latest/admin-api.md
@@ -559,7 +559,7 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
| desc | 可选 | 辅助 | 上游服务描述、使用场景等。 | |
| pass_host | 可选 | 枚举 | 请求发给上游时的 host 设置选型。 [`pass`，`node`，`rewrite`] 之一，默认是`pass`。`pass`: 将客户端的 host 透传给上游； `node`: 使用 `upstream` node 中配置的 host； `rewrite`: 使用配置项 `upstream_host` 的值。 | |
| upstream_host | 可选 | 辅助 | 指定上游请求的 host，只在 `pass_host` 配置为 `rewrite` 时有效。 | |
-| scheme | 可选 | 辅助 | 跟上游通信时使用的 scheme。需要是 ['http', 'https', 'grpc', 'grpcs'] 其中的一个，默认是 'http'。 |
+| scheme | 可选 | 辅助 | 跟上游通信时使用的 scheme。对于 7 层代理，需要是 ['http', 'https', 'grpc', 'grpcs'] 其中的一个。对于 4 层代理，需要是 ['tcp', 'udp', 'tls'] 其中的一个。默认是 'http'。细节见下文。 |
| labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} |
| create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
@@ -585,9 +585,12 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
4. 设为 `consumer` 时，`key` 不需要设置。此时哈希算法采用的 `key` 为认证通过的 `consumer_name`。
5. 如果指定的 `hash_on` 和 `key` 获取不到值时，就是用默认值：`remote_addr`。
+以下特性需要 APISIX 运行于 [APISIX-OpenResty](./how-to-build.md#步骤6：为-Apache-APISIX-构建-OpenResty)：
+
+`scheme` 可以设置成 `tls`，表示 "TLS over TCP"。
+
`tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。
他们的格式和 SSL 对象的 `cert` 和 `key` 一样。
-这个特性需要 APISIX 运行于 [APISIX-OpenResty](./how-to-build.md#步骤6：为-Apache-APISIX-构建-OpenResty)。
`keepalive_pool` 允许 upstream 对象有自己单独的连接池。
它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。
diff --git a/docs/zh/latest/stream-proxy.md b/docs/zh/latest/stream-proxy.md
index 9b0d840..396db99 100644
--- a/docs/zh/latest/stream-proxy.md
+++ b/docs/zh/latest/stream-proxy.md
@@ -163,9 +163,9 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
完整的匹配选项列表参见 [Admin API 的 Stream Route](./admin-api.md#stream-route)。
-## 接收 TLS over TCP
+## 接收 TLS over TCP 连接
-APISIX 支持接收 TLS over TCP。
+APISIX 支持接收 TLS over TCP 连接。
首先，我们需要给对应的 TCP 地址启用 TLS：
@@ -186,7 +186,6 @@ mTLS 也是支持的，参考 [保护路由](./mtls.md#保护路由)。
```shell
curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
- "remote_addr": "127.0.0.1",
"upstream": {
"nodes": {
"127.0.0.1:1995": 1
@@ -212,3 +211,24 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
```
在这里，握手时发送 SNI `a.test.com` 的连接会被代理到 `127.0.0.1:5991`。
+
+## 代理到 TLS over TCP 上游
+
+APISIX 还支持代理到 TLS over TCP 上游。
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "upstream": {
+ "scheme": "tls",
+ "nodes": {
+ "127.0.0.1:1995": 1
+ },
+ "type": "roundrobin"
+ }
+}'
+```
+
+通过设置 `scheme` 为 tls，APISIX 将与上游进行 TLS 握手。
+
+当客户端也使用 TLS over TCP，客户端发送的 SNI 将传递给上游。否则，将使用一个假的 SNI "apisix_backend"。
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 23daaa3..6050bd2 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -251,6 +251,8 @@ _EOC_
if ($stream_tls_request) {
# generate a springboard to send tls stream request
$block->set_value("stream_conf_enable", 1);
+ # avoid conflict with stream_enable
+ $block->set_value("stream_enable");
$block->set_value("request", "GET /stream_tls_request");
my $sni = "nil";
@@ -420,7 +422,17 @@ _EOC_
}
proxy_pass apisix_backend;
+_EOC_
+
+ if ($version =~ m/\/apisix-nginx-module/) {
+ $stream_server_config .= <<_EOC_;
+ proxy_ssl_server_name on;
+ proxy_ssl_name \$upstream_sni;
+ set \$upstream_sni "apisix_backend";
+_EOC_
+ }
+ $stream_server_config .= <<_EOC_;
log_by_lua_block {
apisix.stream_log_phase()
}
diff --git a/t/stream-node/upstream-tls.t b/t/stream-node/upstream-tls.t
new file mode 100644
index 0000000..a9fce58
--- /dev/null
+++ b/t/stream-node/upstream-tls.t
@@ -0,0 +1,146 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX;
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+ plan(skip_all => "apisix-nginx-module not installed");
+} else {
+ plan('no_plan');
+}
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!$block->request) {
+ $block->set_value("stream_enable", 1);
+
+ my $stream_config = $block->stream_config // '';
+ $stream_config .= <<_EOC_;
+ server {
+ listen 8765 ssl;
+ ssl_certificate cert/apisix.crt;
+ ssl_certificate_key cert/apisix.key;
+
+ content_by_lua_block {
+ local sock = ngx.req.socket()
+ local data = sock:receive("1")
+ ngx.say("hello ", ngx.var.ssl_server_name)
+ }
+ }
+_EOC_
+
+ $block->set_value("extra_stream_config", $stream_config);
+ }
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set upstream & stream_routes (id: 1)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "scheme": "tls",
+ "nodes": {
+ "localhost:8765": 1
+ },
+ "type": "roundrobin"
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+ local code, body = t('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "remote_addr": "127.0.0.1",
+ "upstream_id": "1"
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 2: hit route
+--- stream_request
+mmm
+--- stream_response
+hello apisix_backend
+
+
+
+=== TEST 3: set ssl
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin")
+
+ local ssl_cert = t.read_file("t/certs/apisix.crt")
+ local ssl_key = t.read_file("t/certs/apisix.key")
+ local data = {
+ cert = ssl_cert, key = ssl_key,
+ sni = "test.com",
+ }
+ local code, body = t.test('/apisix/admin/ssl/1',
+ ngx.HTTP_PUT,
+ core.json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ return
+ end
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 4: hit route
+--- stream_tls_request
+mmm
+--- stream_sni: test.com
+--- response_body
+hello test.com
diff --git a/utils/linux-install-openresty.sh b/utils/linux-install-openresty.sh
index 017ebf5..9a0a4e9 100755
--- a/utils/linux-install-openresty.sh
+++ b/utils/linux-install-openresty.sh
@@ -26,7 +26,7 @@ sudo apt-get update
if [ "$OPENRESTY_VERSION" == "source" ]; then
cd ..
- wget https://raw.githubusercontent.com/api7/apisix-build-tools/v2.6.0/build-apisix-base.sh
+ wget https://raw.githubusercontent.com/api7/apisix-build-tools/v2.7.0/build-apisix-base.sh
chmod +x build-apisix-base.sh
./build-apisix-base.sh latest