You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by me...@apache.org on 2021/04/10 10:55:30 UTC
[apisix] branch master updated: feat: support upstream mTLS (#4005)
This is an automated email from the ASF dual-hosted git repository.
membphis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new e295ebb feat: support upstream mTLS (#4005)
e295ebb is described below
commit e295ebb9e811e505f0c9f6b604dd6a7b25bb0a74
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Sat Apr 10 18:55:20 2021 +0800
feat: support upstream mTLS (#4005)
---
apisix/admin/routes.lua | 4 +-
apisix/admin/services.lua | 4 +-
apisix/admin/upstreams.lua | 77 +----
apisix/http/service.lua | 41 +--
apisix/router.lua | 40 +--
apisix/schema_def.lua | 28 +-
apisix/ssl.lua | 50 +++-
apisix/ssl/router/radixtree_sni.lua | 49 +--
apisix/upstream.lua | 202 ++++++++++---
docs/en/latest/admin-api.md | 6 +
docs/zh/latest/admin-api.md | 6 +
t/APISIX.pm | 10 +
t/config-center-yaml/route-upstream.t | 54 ++++
t/node/upstream-mtls.t | 547 ++++++++++++++++++++++++++++++++++
14 files changed, 867 insertions(+), 251 deletions(-)
diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua
index 28d0272..cad66fc 100644
--- a/apisix/admin/routes.lua
+++ b/apisix/admin/routes.lua
@@ -16,8 +16,8 @@
--
local expr = require("resty.expr.v1")
local core = require("apisix.core")
+local apisix_upstream = require("apisix.upstream")
local schema_plugin = require("apisix.admin.plugins").check_schema
-local upstreams = require("apisix.admin.upstreams")
local utils = require("apisix.admin.utils")
local tostring = tostring
local type = type
@@ -68,7 +68,7 @@ local function check_conf(id, conf, need_id)
local upstream_conf = conf.upstream
if upstream_conf then
- local ok, err = upstreams.check_upstream_conf(upstream_conf)
+ local ok, err = apisix_upstream.check_upstream_conf(upstream_conf)
if not ok then
return nil, {error_msg = err}
end
diff --git a/apisix/admin/services.lua b/apisix/admin/services.lua
index 9ac26aa..4b6e98e 100644
--- a/apisix/admin/services.lua
+++ b/apisix/admin/services.lua
@@ -16,8 +16,8 @@
--
local core = require("apisix.core")
local get_routes = require("apisix.router").http_routes
+local apisix_upstream = require("apisix.upstream")
local schema_plugin = require("apisix.admin.plugins").check_schema
-local upstreams = require("apisix.admin.upstreams")
local utils = require("apisix.admin.utils")
local tostring = tostring
local ipairs = ipairs
@@ -63,7 +63,7 @@ local function check_conf(id, conf, need_id)
local upstream_conf = conf.upstream
if upstream_conf then
- local ok, err = upstreams.check_upstream_conf(upstream_conf)
+ local ok, err = apisix_upstream.check_upstream_conf(upstream_conf)
if not ok then
return nil, {error_msg = err}
end
diff --git a/apisix/admin/upstreams.lua b/apisix/admin/upstreams.lua
index 312bf77..d367ec3 100644
--- a/apisix/admin/upstreams.lua
+++ b/apisix/admin/upstreams.lua
@@ -17,6 +17,7 @@
local core = require("apisix.core")
local get_routes = require("apisix.router").http_routes
local get_services = require("apisix.http.service").services
+local apisix_upstream = require("apisix.upstream")
local utils = require("apisix.admin.utils")
local tostring = tostring
local ipairs = ipairs
@@ -28,77 +29,6 @@ local _M = {
}
-local function get_chash_key_schema(hash_on)
- if not hash_on then
- return nil, "hash_on is nil"
- end
-
- if hash_on == "vars" then
- return core.schema.upstream_hash_vars_schema
- end
-
- if hash_on == "header" or hash_on == "cookie" then
- return core.schema.upstream_hash_header_schema
- end
-
- if hash_on == "consumer" then
- return nil, nil
- end
-
- if hash_on == "vars_combinations" then
- return core.schema.upstream_hash_vars_combinations_schema
- end
-
- return nil, "invalid hash_on type " .. hash_on
-end
-
-
-local function check_upstream_conf(conf)
- local ok, err = core.schema.check(core.schema.upstream, conf)
- if not ok then
- return false, "invalid configuration: " .. err
- end
-
- if conf.pass_host == "node" and conf.nodes and
- core.table.nkeys(conf.nodes) ~= 1
- then
- return false, "only support single node for `node` mode currently"
- end
-
- if conf.pass_host == "rewrite" and
- (conf.upstream_host == nil or conf.upstream_host == "")
- then
- return false, "`upstream_host` can't be empty when `pass_host` is `rewrite`"
- end
-
- if conf.type ~= "chash" then
- return true
- end
-
- if not conf.hash_on then
- conf.hash_on = "vars"
- end
-
- if conf.hash_on ~= "consumer" and not conf.key then
- return false, "missing key"
- end
-
- local key_schema, err = get_chash_key_schema(conf.hash_on)
- if err then
- return false, "type is chash, err: " .. err
- end
-
- if key_schema then
- local ok, err = core.schema.check(key_schema, conf.key)
- if not ok then
- return false, "invalid configuration: " .. err
- end
- end
-
- return true
-end
-
-
local function check_conf(id, conf, need_id)
if not conf then
return nil, {error_msg = "missing configurations"}
@@ -123,7 +53,7 @@ local function check_conf(id, conf, need_id)
core.log.info("schema: ", core.json.delay_encode(core.schema.upstream))
core.log.info("conf : ", core.json.delay_encode(conf))
- local ok, err = check_upstream_conf(conf)
+ local ok, err = apisix_upstream.check_upstream_conf(conf)
if not ok then
return nil, {error_msg = err}
end
@@ -295,8 +225,5 @@ function _M.patch(id, conf, sub_path)
return res.status, res.body
end
--- for routes and services check upstream conf
-_M.check_upstream_conf = check_upstream_conf
-
return _M
diff --git a/apisix/http/service.lua b/apisix/http/service.lua
index 0e36f9a..83bcb9b 100644
--- a/apisix/http/service.lua
+++ b/apisix/http/service.lua
@@ -15,11 +15,10 @@
-- limitations under the License.
--
local core = require("apisix.core")
+local apisix_upstream = require("apisix.upstream")
local plugin_checker = require("apisix.plugin").plugin_checker
-local ipairs = ipairs
local services
local error = error
-local pairs = pairs
local _M = {
@@ -47,43 +46,7 @@ local function filter(service)
return
end
- if not service.value.upstream then
- return
- end
-
- service.value.upstream.parent = service
-
- if not service.value.upstream.nodes then
- return
- end
-
- local nodes = service.value.upstream.nodes
- if core.table.isarray(nodes) then
- for _, node in ipairs(nodes) do
- local host = node.host
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- service.has_domain = true
- break
- end
- end
- else
- local new_nodes = core.table.new(core.table.nkeys(nodes), 0)
- for addr, weight in pairs(nodes) do
- local host, port = core.utils.parse_addr(addr)
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- service.has_domain = true
- end
- local node = {
- host = host,
- port = port,
- weight = weight,
- }
- core.table.insert(new_nodes, node)
- end
- service.value.upstream.nodes = new_nodes
- end
+ apisix_upstream.filter_upstream(service.value.upstream, service)
core.log.info("filter service: ", core.json.delay_encode(service, true))
end
diff --git a/apisix/router.lua b/apisix/router.lua
index 3afb3a4..840f0cd 100644
--- a/apisix/router.lua
+++ b/apisix/router.lua
@@ -16,11 +16,11 @@
--
local require = require
local http_route = require("apisix.http.route")
+local apisix_upstream = require("apisix.upstream")
local core = require("apisix.core")
local plugin_checker = require("apisix.plugin").plugin_checker
local str_lower = string.lower
local error = error
-local pairs = pairs
local ipairs = ipairs
@@ -44,43 +44,7 @@ local function filter(route)
end
end
- if not route.value.upstream then
- return
- end
-
- route.value.upstream.parent = route
-
- if not route.value.upstream.nodes then
- return
- end
-
- local nodes = route.value.upstream.nodes
- if core.table.isarray(nodes) then
- for _, node in ipairs(nodes) do
- local host = node.host
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- route.has_domain = true
- break
- end
- end
- else
- local new_nodes = core.table.new(core.table.nkeys(nodes), 0)
- for addr, weight in pairs(nodes) do
- local host, port = core.utils.parse_addr(addr)
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- route.has_domain = true
- end
- local node = {
- host = host,
- port = port,
- weight = weight,
- }
- core.table.insert(new_nodes, node)
- end
- route.value.upstream.nodes = new_nodes
- end
+ apisix_upstream.filter_upstream(route.value.upstream, route)
core.log.info("filter route: ", core.json.delay_encode(route, true))
end
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index 6e171a7..b60faad 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -322,6 +322,16 @@ local nodes_schema = {
}
+local certificate_scheme = {
+ type = "string", minLength = 128, maxLength = 64*1024
+}
+
+
+local private_key_schema = {
+ type = "string", minLength = 128, maxLength = 64*1024
+}
+
+
local upstream_schema = {
type = "object",
properties = {
@@ -341,6 +351,14 @@ local upstream_schema = {
},
required = {"connect", "send", "read"},
},
+ tls = {
+ type = "object",
+ properties = {
+ client_cert = certificate_scheme,
+ client_key = private_key_schema,
+ },
+ required = {"client_cert", "client_key"},
+ },
type = {
description = "algorithms of load balancing",
type = "string",
@@ -598,16 +616,6 @@ _M.consumer = {
_M.upstream = upstream_schema
-local certificate_scheme = {
- type = "string", minLength = 128, maxLength = 64*1024
-}
-
-
-local private_key_schema = {
- type = "string", minLength = 128, maxLength = 64*1024
-}
-
-
_M.ssl = {
type = "object",
properties = {
diff --git a/apisix/ssl.lua b/apisix/ssl.lua
index 47a4e34..a3b9a96 100644
--- a/apisix/ssl.lua
+++ b/apisix/ssl.lua
@@ -23,6 +23,15 @@ local assert = assert
local type = type
+local cert_cache = core.lrucache.new {
+ ttl = 3600, count = 1024,
+}
+
+local pkey_cache = core.lrucache.new {
+ ttl = 3600, count = 1024,
+}
+
+
local _M = {}
@@ -60,13 +69,13 @@ end
local function decrypt_priv_pkey(iv, key)
local decoded_key = ngx_decode_base64(key)
if not decoded_key then
- core.log.error("base64 decode ssl key failed and skipped. key[", key, "] ")
+ core.log.error("base64 decode ssl key failed. key[", key, "] ")
return nil
end
local decrypted = iv:decrypt(decoded_key)
if not decrypted then
- core.log.error("decrypt ssl key failed and skipped. key[", key, "] ")
+ core.log.error("decrypt ssl key failed. key[", key, "] ")
end
return decrypted
@@ -84,7 +93,6 @@ local function aes_decrypt_pkey(origin)
end
return origin
end
-_M.aes_decrypt_pkey = aes_decrypt_pkey
function _M.validate(cert, key)
@@ -108,4 +116,40 @@ function _M.validate(cert, key)
end
+local function parse_pem_cert(sni, cert)
+ core.log.debug("parsing cert for sni: ", sni)
+
+ local parsed, err = ngx_ssl.parse_pem_cert(cert)
+ return parsed, err
+end
+
+
+function _M.fetch_cert(sni, cert)
+ local parsed_cert, err = cert_cache(cert, nil, parse_pem_cert, sni, cert)
+ if not parsed_cert then
+ return false, err
+ end
+
+ return parsed_cert
+end
+
+
+local function parse_pem_priv_key(sni, pkey)
+ core.log.debug("parsing priv key for sni: ", sni)
+
+ local parsed, err = ngx_ssl.parse_pem_priv_key(aes_decrypt_pkey(pkey))
+ return parsed, err
+end
+
+
+function _M.fetch_pkey(sni, pkey)
+ local parsed_pkey, err = pkey_cache(pkey, nil, parse_pem_priv_key, sni, pkey)
+ if not parsed_pkey then
+ return false, err
+ end
+
+ return parsed_pkey
+end
+
+
return _M
diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua
index 0790627..fe0bf35 100644
--- a/apisix/ssl/router/radixtree_sni.lua
+++ b/apisix/ssl/router/radixtree_sni.lua
@@ -29,37 +29,12 @@ local ssl_certificates
local radixtree_router
local radixtree_router_ver
-local cert_cache = core.lrucache.new {
- ttl = 3600, count = 512,
-}
-
-local pkey_cache = core.lrucache.new {
- ttl = 3600, count = 512,
-}
-
-
local _M = {
version = 0.1,
server_name = ngx_ssl.server_name,
}
-local function parse_pem_cert(sni, cert)
- core.log.debug("parsing cert for sni: ", sni)
-
- local parsed, err = ngx_ssl.parse_pem_cert(cert)
- return parsed, err
-end
-
-
-local function parse_pem_priv_key(sni, pkey)
- core.log.debug("parsing priv key for sni: ", sni)
-
- local parsed, err = ngx_ssl.parse_pem_priv_key(pkey)
- return parsed, err
-end
-
-
local function create_router(ssl_items)
local ssl_items = ssl_items or {}
@@ -82,23 +57,6 @@ local function create_router(ssl_items)
sni = ssl.value.sni:reverse()
end
- -- decrypt private key
- if ssl.value.key then
- local decrypted = apisix_ssl.aes_decrypt_pkey(ssl.value.key)
- if decrypted then
- ssl.value.key = decrypted
- end
- end
-
- if ssl.value.keys then
- for i = 1, #ssl.value.keys do
- local decrypted = apisix_ssl.aes_decrypt_pkey(ssl.value.keys[i])
- if decrypted then
- ssl.value.keys[i] = decrypted
- end
- end
- end
-
idx = idx + 1
route_items[idx] = {
paths = sni,
@@ -133,7 +91,7 @@ local function set_pem_ssl_key(sni, cert, pkey)
return false, "no request found"
end
- local parsed_cert, err = cert_cache(cert, nil, parse_pem_cert, sni, cert)
+ local parsed_cert, err = apisix_ssl.fetch_cert(sni, cert)
if not parsed_cert then
return false, "failed to parse PEM cert: " .. err
end
@@ -143,9 +101,8 @@ local function set_pem_ssl_key(sni, cert, pkey)
return false, "failed to set PEM cert: " .. err
end
- local parsed_pkey, err = pkey_cache(pkey, nil, parse_pem_priv_key, sni,
- pkey)
- if not parsed_pkey then
+ local parsed_pkey, err = apisix_ssl.fetch_pkey(sni, pkey)
+ if not parsed_cert then
return false, "failed to parse PEM priv key: " .. err
end
diff --git a/apisix/upstream.lua b/apisix/upstream.lua
index 7cef761..5bcc679 100644
--- a/apisix/upstream.lua
+++ b/apisix/upstream.lua
@@ -18,6 +18,7 @@ local require = require
local core = require("apisix.core")
local discovery = require("apisix.discovery.init").discovery
local upstream_util = require("apisix.utils.upstream")
+local apisix_ssl = require("apisix.ssl")
local error = error
local tostring = tostring
local ipairs = ipairs
@@ -27,6 +28,17 @@ local upstreams
local healthcheck
+local set_upstream_tls_client_param
+local ok, apisix_ngx_upstream = pcall(require, "resty.apisix.upstream")
+if ok then
+ set_upstream_tls_client_param = apisix_ngx_upstream.set_cert_and_key
+else
+ set_upstream_tls_client_param = function ()
+ return nil, "need to build APISIX-Openresty to support upstream mTLS"
+ end
+end
+
+
local HTTP_CODE_UPSTREAM_UNAVAILABLE = 503
local _M = {}
@@ -279,6 +291,25 @@ function _M.set_by_route(route, api_ctx)
api_ctx.up_checker = checker
end
+ if up_conf.scheme == "https" and up_conf.tls then
+ -- the sni here is just for logging
+ local sni = api_ctx.var.upstream_host
+ local cert, err = apisix_ssl.fetch_cert(sni, up_conf.tls.client_cert)
+ if not ok then
+ return 503, err
+ end
+
+ local key, err = apisix_ssl.fetch_pkey(sni, up_conf.tls.client_key)
+ if not ok then
+ return 503, err
+ end
+
+ local ok, err = set_upstream_tls_client_param(cert, key)
+ if not ok then
+ return 503, err
+ end
+ end
+
return
end
@@ -297,50 +328,149 @@ function _M.check_schema(conf)
end
+local function get_chash_key_schema(hash_on)
+ if not hash_on then
+ return nil, "hash_on is nil"
+ end
+
+ if hash_on == "vars" then
+ return core.schema.upstream_hash_vars_schema
+ end
+
+ if hash_on == "header" or hash_on == "cookie" then
+ return core.schema.upstream_hash_header_schema
+ end
+
+ if hash_on == "consumer" then
+ return nil, nil
+ end
+
+ if hash_on == "vars_combinations" then
+ return core.schema.upstream_hash_vars_combinations_schema
+ end
+
+ return nil, "invalid hash_on type " .. hash_on
+end
+
+
+local function check_upstream_conf(in_dp, conf)
+ if not in_dp then
+ local ok, err = core.schema.check(core.schema.upstream, conf)
+ if not ok then
+ return false, "invalid configuration: " .. err
+ end
+
+ -- encrypt the key in the admin
+ if conf.tls and conf.tls.client_key then
+ conf.tls.client_key = apisix_ssl.aes_encrypt_pkey(conf.tls.client_key)
+ end
+ end
+
+ if conf.pass_host == "node" and conf.nodes and
+ core.table.nkeys(conf.nodes) ~= 1
+ then
+ return false, "only support single node for `node` mode currently"
+ end
+
+ if conf.pass_host == "rewrite" and
+ (conf.upstream_host == nil or conf.upstream_host == "")
+ then
+ return false, "`upstream_host` can't be empty when `pass_host` is `rewrite`"
+ end
+
+ if conf.tls then
+ local cert = conf.tls.client_cert
+ local key = conf.tls.client_key
+ local ok, err = apisix_ssl.validate(cert, key)
+ if not ok then
+ return false, err
+ end
+ end
+
+ if conf.type ~= "chash" then
+ return true
+ end
+
+ if conf.hash_on ~= "consumer" and not conf.key then
+ return false, "missing key"
+ end
+
+ local key_schema, err = get_chash_key_schema(conf.hash_on)
+ if err then
+ return false, "type is chash, err: " .. err
+ end
+
+ if key_schema then
+ local ok, err = core.schema.check(key_schema, conf.key)
+ if not ok then
+ return false, "invalid configuration: " .. err
+ end
+ end
+
+ return true
+end
+
+
+function _M.check_upstream_conf(conf)
+ return check_upstream_conf(false, conf)
+end
+
+
+local function filter_upstream(value, parent)
+ if not value then
+ return
+ end
+
+ value.parent = parent
+
+ if not value.nodes then
+ return
+ end
+
+ local nodes = value.nodes
+ if core.table.isarray(nodes) then
+ for _, node in ipairs(nodes) do
+ local host = node.host
+ if not core.utils.parse_ipv4(host) and
+ not core.utils.parse_ipv6(host) then
+ parent.has_domain = true
+ break
+ end
+ end
+ else
+ local new_nodes = core.table.new(core.table.nkeys(nodes), 0)
+ for addr, weight in pairs(nodes) do
+ local host, port = core.utils.parse_addr(addr)
+ if not core.utils.parse_ipv4(host) and
+ not core.utils.parse_ipv6(host) then
+ parent.has_domain = true
+ end
+ local node = {
+ host = host,
+ port = port,
+ weight = weight,
+ }
+ core.table.insert(new_nodes, node)
+ end
+ value.nodes = new_nodes
+ end
+end
+_M.filter_upstream = filter_upstream
+
+
function _M.init_worker()
local err
upstreams, err = core.config.new("/upstreams", {
automatic = true,
item_schema = core.schema.upstream,
+ -- also check extra fields in the DP side
+ checker = function (item, schema_type)
+ return check_upstream_conf(true, item)
+ end,
filter = function(upstream)
upstream.has_domain = false
- if not upstream.value then
- return
- end
-
- upstream.value.parent = upstream
- if not upstream.value.nodes then
- return
- end
-
- local nodes = upstream.value.nodes
- if core.table.isarray(nodes) then
- for _, node in ipairs(nodes) do
- local host = node.host
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- upstream.has_domain = true
- break
- end
- end
- else
- local new_nodes = core.table.new(core.table.nkeys(nodes), 0)
- for addr, weight in pairs(nodes) do
- local host, port = core.utils.parse_addr(addr)
- if not core.utils.parse_ipv4(host) and
- not core.utils.parse_ipv6(host) then
- upstream.has_domain = true
- end
- local node = {
- host = host,
- port = port,
- weight = weight,
- }
- core.table.insert(new_nodes, node)
- end
- upstream.value.nodes = new_nodes
- end
+ filter_upstream(upstream.value, upstream)
core.log.info("filter upstream: ", core.json.delay_encode(upstream, true))
end,
diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md
index 1634ef7..c0500c6 100644
--- a/docs/en/latest/admin-api.md
+++ b/docs/en/latest/admin-api.md
@@ -543,6 +543,8 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
|labels |optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
|create_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|
|update_time |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|
+|tls.client_cert |optional| Set the client certificate when connecting to TLS upstream, see below for more details|
+|tls.client_key |optional| Set the client priviate key when connecting to TLS upstream, see below for more details|
`type` can be one of:
@@ -560,6 +562,10 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
1. when it is `vars_combinations`, the `key` is required. The `key` can be any [Nginx builtin variables](http://nginx.org/en/docs/varindex.html) combinations, such as `$request_uri$remote_addr`.
1. If there is no value for either `hash_on` or `key`, `remote_addr` will be used as key.
+`tls.client_cert/key` can be used to communicate with upstream via mTLS.
+Their formats are the same as SSL's `cert` and `key` fields.
+This feature requires APISIX to run on [APISIX-OpenResty](../how-to-build.md#6-build-openresty-for-apisix).
+
**Config Example:**
```shell
diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md
index e3ca1c2..8bffa0f 100644
--- a/docs/zh/latest/admin-api.md
+++ b/docs/zh/latest/admin-api.md
@@ -546,6 +546,8 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上
| labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} |
| create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
+| tls.client_cert | 可选 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | |
+| update_time | 可选 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | |
`type` 可以是以下的一种：
@@ -562,6 +564,10 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上
4. 设为 `consumer` 时，`key` 不需要设置。此时哈希算法采用的 `key` 为认证通过的 `consumer_name`。
5. 如果指定的 `hash_on` 和 `key` 获取不到值时，就是用默认值：`remote_addr`。
+`tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。
+他们的格式和 SSL 对象的 `cert` 和 `key` 一样。
+这个特性需要 APISIX 运行于 [APISIX-OpenResty](../how-to-build.md#6-build-openresty-for-apisix)。
+
**upstream 对象 json 配置内容：**
```shell
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 2d72236..2b7324d 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -389,6 +389,10 @@ _EOC_
_EOC_
+ if (defined $block->upstream_server_config) {
+ $http_config .= $block->upstream_server_config;
+ }
+
my $ipv6_fake_server = "";
if (defined $block->listen_ipv6) {
$ipv6_fake_server = "listen \[::1\]:1980;";
@@ -426,7 +430,13 @@ _EOC_
ssl_certificate cert/apisix.crt;
ssl_certificate_key cert/apisix.key;
lua_ssl_trusted_certificate cert/apisix.crt;
+_EOC_
+ if (defined $block->upstream_server_config) {
+ $http_config .= $block->upstream_server_config;
+ }
+
+ $http_config .= <<_EOC_;
server_tokens off;
ssl_certificate_by_lua_block {
diff --git a/t/config-center-yaml/route-upstream.t b/t/config-center-yaml/route-upstream.t
index 42908da..ff50ce2 100644
--- a/t/config-center-yaml/route-upstream.t
+++ b/t/config-center-yaml/route-upstream.t
@@ -158,3 +158,57 @@ GET /get
--- error_code: 200
--- no_error_log
[error]
+
+
+
+=== TEST 6: upstream hash_on (bad)
+--- yaml_config eval: $::yaml_config
+--- apisix_yaml
+routes:
+ -
+ id: 1
+ uri: /get
+ upstream_id: 1
+upstreams:
+ -
+ id: 1
+ nodes:
+ "httpbin.org:80": 1
+ type: chash
+ hash_on: header
+ key: "$aaa"
+#END
+--- request
+GET /get
+--- error_code: 502
+--- error_log
+invalid configuration: failed to match pattern
+
+
+
+=== TEST 7: upstream hash_on (good)
+--- yaml_config eval: $::yaml_config
+--- apisix_yaml
+routes:
+ -
+ id: 1
+ uri: /hello
+ upstream_id: 1
+upstreams:
+ -
+ id: 1
+ nodes:
+ "127.0.0.1:1980": 1
+ "127.0.0.2:1980": 1
+ type: chash
+ hash_on: header
+ key: "test"
+#END
+--- request
+GET /hello
+--- more_headers
+test: one
+--- error_log
+proxy request to 127.0.0.1:1980
+--- no_error_log
+[error]
diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t
new file mode 100644
index 0000000..9c0a49d
--- /dev/null
+++ b/t/node/upstream-mtls.t
@@ -0,0 +1,547 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX;
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+ plan(skip_all => "apisix-nginx-module not installed");
+} else {
+ plan('no_plan');
+}
+
+repeat_each(1);
+log_level('info');
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: tls without key
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ }
+ },
+ uri = "/hello"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: property \"client_key\" is required"}
+
+
+
+=== TEST 2: tls with bad key
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ("AAA"):rep(128),
+ }
+ },
+ uri = "/hello"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to decrypt previous encrypted key"}
+--- error_log
+decrypt ssl key failed
+
+
+
+=== TEST 3: encrypt key by default
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local ssl_key = t.read_file("t/certs/mtls_client.key")
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ },
+ uri = "/hello"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.upstream.tls.client_key == ssl_key)
+
+ -- upstream
+ local data = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ }
+ local code, body = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.tls.client_key == ssl_key)
+
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ },
+ }
+ local code, body = t.test('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/services/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.upstream.tls.client_key == ssl_key)
+ }
+ }
+--- request
+GET /t
+--- response_body
+false
+false
+false
+
+
+
+=== TEST 4: hit
+--- upstream_server_config
+ ssl_client_certificate ../../certs/mtls_ca.crt;
+ ssl_verify_client on;
+--- request
+GET /hello
+--- response_body
+hello world
+
+
+
+=== TEST 5: wrong cert
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/apisix.crt")
+ local ssl_key = t.read_file("t/certs/apisix.key")
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ },
+ uri = "/hello"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 6: hit
+--- upstream_server_config
+ ssl_client_certificate ../../certs/mtls_ca.crt;
+ ssl_verify_client on;
+--- request
+GET /hello
+--- error_code: 400
+--- error_log
+client SSL certificate verify error
+
+
+
+=== TEST 7: clean old data
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ assert(t.test('/apisix/admin/routes/1',
+ ngx.HTTP_DELETE
+ ))
+ assert(t.test('/apisix/admin/services/1',
+ ngx.HTTP_DELETE
+ ))
+ assert(t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_DELETE
+ ))
+ }
+ }
+--- request
+GET /t
+
+
+
+=== TEST 8: don't encrypt key
+--- yaml_config
+apisix:
+ node_listen: 1984
+ admin_key: null
+ ssl:
+ key_encrypt_salt: null
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+ local ssl_key = t.read_file("t/certs/mtls_client.key")
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ },
+ uri = "/hello"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.upstream.tls.client_key == ssl_key)
+
+ -- upstream
+ local data = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ }
+ local code, body = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/upstreams/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.tls.client_key == ssl_key)
+
+ local data = {
+ upstream = {
+ scheme = "https",
+ type = "roundrobin",
+ nodes = {
+ ["127.0.0.1:1983"] = 1,
+ },
+ tls = {
+ client_cert = ssl_cert,
+ client_key = ssl_key,
+ }
+ },
+ }
+ local code, body = t.test('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, res = t.test('/apisix/admin/services/1',
+ ngx.HTTP_GET
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ res = json.decode(res)
+ ngx.say(res.node.value.upstream.tls.client_key == ssl_key)
+ }
+ }
+--- request
+GET /t
+--- response_body
+true
+true
+true
+
+
+
+=== TEST 9: bind upstream
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local data = {
+ upstream_id = 1,
+ uri = "/server_port"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ }
+ }
+--- request
+GET /t
+
+
+
+=== TEST 10: hit
+--- upstream_server_config
+ ssl_client_certificate ../../certs/mtls_ca.crt;
+ ssl_verify_client on;
+--- request
+GET /server_port
+--- response_body chomp
+1983
+
+
+
+=== TEST 11: bind service
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local data = {
+ service_id = 1,
+ uri = "/hello_chunked"
+ }
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(data)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+ }
+ }
+--- request
+GET /t
+
+
+
+=== TEST 12: hit
+--- upstream_server_config
+ ssl_client_certificate ../../certs/mtls_ca.crt;
+ ssl_verify_client on;
+--- request
+GET /hello_chunked
+--- response_body
+hello world