You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/06/14 01:29:06 UTC

[apisix] branch master updated: feat(ssl): support get upstream cert from ssl object (#7221)

This is an automated email from the ASF dual-hosted git repository.

spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git


The following commit(s) were added to refs/heads/master by this push:
     new 45816272a feat(ssl): support get upstream cert from ssl object (#7221)
45816272a is described below

commit 45816272a80d6d2045259d32acdabbec4039ed1e
Author: soulbird <zh...@outlook.com>
AuthorDate: Tue Jun 14 09:29:01 2022 +0800

    feat(ssl): support get upstream cert from ssl object (#7221)
    
    Co-authored-by: soulbird <zh...@gmail.com>
---
 apisix/admin/ssl.lua                |   2 +-
 apisix/init.lua                     |  23 ++++++
 apisix/schema_def.lua               |  40 ++++++++--
 apisix/ssl.lua                      |   4 +
 apisix/ssl/router/radixtree_sni.lua |  18 ++++-
 apisix/upstream.lua                 |  37 ++++++++-
 docs/en/latest/admin-api.md         |   8 +-
 docs/zh/latest/admin-api.md         |   8 +-
 t/admin/ssl.t                       |  73 +++++++++++++++++-
 t/admin/ssl2.t                      |   8 +-
 t/admin/upstream.t                  | 133 +++++++++++++++++++++++++++++++++
 t/node/upstream-mtls.t              | 144 +++++++++++++++++++++++++++++++++++-
 12 files changed, 477 insertions(+), 21 deletions(-)

diff --git a/apisix/admin/ssl.lua b/apisix/admin/ssl.lua
index 341e03004..9a73107c9 100644
--- a/apisix/admin/ssl.lua
+++ b/apisix/admin/ssl.lua
@@ -46,7 +46,7 @@ local function check_conf(id, conf, need_id)
     conf.id = id
 
     core.log.info("schema: ", core.json.delay_encode(core.schema.ssl))
-    core.log.info("conf  : ", core.json.delay_encode(conf))
+    core.log.info("conf: ", core.json.delay_encode(conf))
 
     local ok, err = apisix_ssl.check_ssl_conf(false, conf)
     if not ok then
diff --git a/apisix/init.lua b/apisix/init.lua
index f26dee4e1..af0f22553 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -497,6 +497,29 @@ function _M.http_access_phase()
                                    or route_val.upstream
     end
 
+    if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
+        api_ctx.matched_upstream.tls.client_cert_id then
+
+        local cert_id = api_ctx.matched_upstream.tls.client_cert_id
+        local upstream_ssl = router.router_ssl.get_by_id(cert_id)
+        if not upstream_ssl or upstream_ssl.type ~= "client" then
+            local err  = upstream_ssl and
+                "ssl type should be 'client'" or
+                "ssl id [" .. cert_id .. "] not exits"
+            core.log.error("failed to get ssl cert: ", err)
+
+            if is_http then
+                return core.response.exit(502)
+            end
+
+            return ngx_exit(1)
+        end
+
+        core.log.info("matched ssl: ",
+                  core.json.delay_encode(upstream_ssl, true))
+        api_ctx.upstream_ssl = upstream_ssl
+    end
+
     if enable_websocket then
         api_ctx.var.upstream_upgrade    = api_ctx.var.http_upgrade
         api_ctx.var.upstream_connection = api_ctx.var.http_connection
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index 767c2fa63..7d39b62aa 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -404,6 +404,7 @@ local upstream_schema = {
         tls = {
             type = "object",
             properties = {
+                client_cert_id = id_schema,
                 client_cert = certificate_scheme,
                 client_key = private_key_schema,
                 verify = {
@@ -414,8 +415,17 @@ local upstream_schema = {
                 },
             },
             dependencies = {
-                client_cert = {"client_key"},
-                client_key = {"client_cert"},
+                client_cert = {
+                    required = {"client_key"},
+                    ["not"] = {required = {"client_cert_id"}}
+                },
+                client_key = {
+                    required = {"client_cert"},
+                    ["not"] = {required = {"client_cert_id"}}
+                },
+                client_cert_id = {
+                    ["not"] = {required = {"client_client", "client_key"}}
+                }
             }
         },
         keepalive_pool = {
@@ -504,7 +514,7 @@ local upstream_schema = {
     oneOf = {
         {required = {"type", "nodes"}},
         {required = {"type", "service_name", "discovery_type"}},
-    },
+    }
 }
 
 -- TODO: add more nginx variable support
@@ -722,6 +732,14 @@ _M.ssl = {
     type = "object",
     properties = {
         id = id_schema,
+        type = {
+            description = "ssl certificate type, " ..
+                            "server to server certificate, " ..
+                            "client to client certificate for upstream",
+            type = "string",
+            default = "server",
+            enum = {"server", "client"}
+        },
         cert = certificate_scheme,
         key = private_key_schema,
         sni = {
@@ -772,10 +790,20 @@ _M.ssl = {
         create_time = timestamp_def,
         update_time = timestamp_def
     },
-    oneOf = {
-        {required = {"sni", "key", "cert"}},
-        {required = {"snis", "key", "cert"}}
+    ["if"] = {
+        properties = {
+            type = {
+                enum = {"server"},
+            },
+        },
+    },
+    ["then"] = {
+        oneOf = {
+            {required = {"sni", "key", "cert"}},
+            {required = {"snis", "key", "cert"}}
+        }
     },
+    ["else"] = {required = {"key", "cert"}}
 }
 
 
diff --git a/apisix/ssl.lua b/apisix/ssl.lua
index c0c47aec0..7d48f3085 100644
--- a/apisix/ssl.lua
+++ b/apisix/ssl.lua
@@ -197,6 +197,10 @@ function _M.check_ssl_conf(in_dp, conf)
         return nil, err
     end
 
+    if conf.type == "client" then
+        return true
+    end
+
     local numcerts = conf.certs and #conf.certs or 0
     local numkeys = conf.keys and #conf.keys or 0
     if numcerts ~= numkeys then
diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua
index 70ac0faa3..891d8d21d 100644
--- a/apisix/ssl/router/radixtree_sni.lua
+++ b/apisix/ssl/router/radixtree_sni.lua
@@ -26,6 +26,7 @@ local error            = error
 local str_find         = core.string.find
 local str_gsub         = string.gsub
 local str_lower        = string.lower
+local tostring         = tostring
 local ssl_certificates
 local radixtree_router
 local radixtree_router_ver
@@ -44,7 +45,7 @@ local function create_router(ssl_items)
     local idx = 0
 
     for _, ssl in config_util.iterate_values(ssl_items) do
-        if ssl.value ~= nil and
+        if ssl.value ~= nil and ssl.value.type == "server" and
             (ssl.value.status == nil or ssl.value.status == 1) then  -- compatible with old version
 
             local j = 0
@@ -261,4 +262,19 @@ function _M.init_worker()
 end
 
 
+function _M.get_by_id(ssl_id)
+    local ssl
+    local ssls = core.config.fetch_created_obj("/ssl")
+    if ssls then
+        ssl = ssls:get(tostring(ssl_id))
+    end
+
+    if not ssl then
+        return nil
+    end
+
+    return ssl.value
+end
+
+
 return _M
diff --git a/apisix/upstream.lua b/apisix/upstream.lua
index d314de49d..0162ad813 100644
--- a/apisix/upstream.lua
+++ b/apisix/upstream.lua
@@ -333,14 +333,24 @@ function _M.set_by_route(route, api_ctx)
 
     local scheme = up_conf.scheme
     if (scheme == "https" or scheme == "grpcs") and up_conf.tls then
+
+        local client_cert, client_key
+        if up_conf.tls.client_cert_id then
+            client_cert = api_ctx.upstream_ssl.cert
+            client_key = api_ctx.upstream_ssl.key
+        else
+            client_cert = up_conf.tls.client_cert
+            client_key = up_conf.tls.client_key
+        end
+
         -- the sni here is just for logging
         local sni = api_ctx.var.upstream_host
-        local cert, err = apisix_ssl.fetch_cert(sni, up_conf.tls.client_cert)
+        local cert, err = apisix_ssl.fetch_cert(sni, client_cert)
         if not ok then
             return 503, err
         end
 
-        local key, err = apisix_ssl.fetch_pkey(sni, up_conf.tls.client_key)
+        local key, err = apisix_ssl.fetch_pkey(sni, client_key)
         if not ok then
             return 503, err
         end
@@ -418,6 +428,29 @@ local function check_upstream_conf(in_dp, conf)
             return false, "invalid configuration: " .. err
         end
 
+        local ssl_id = conf.tls and conf.tls.client_cert_id
+        if ssl_id then
+            local key = "/ssl/" .. ssl_id
+            local res, err = core.etcd.get(key)
+            if not res then
+                return nil, "failed to fetch ssl info by "
+                                    .. "ssl id [" .. ssl_id .. "]: " .. err
+            end
+
+            if res.status ~= 200 then
+                return nil, "failed to fetch ssl info by "
+                                    .. "ssl id [" .. ssl_id .. "], "
+                                    .. "response code: " .. res.status
+            end
+            if res.body and res.body.node and
+                res.body.node.value and res.body.node.value.type ~= "client" then
+
+                return nil, "failed to fetch ssl info by "
+                                    .. "ssl id [" .. ssl_id .. "], "
+                                    .. "wrong ssl type"
+            end
+        end
+
         -- encrypt the key in the admin
         if conf.tls and conf.tls.client_key then
             conf.tls.client_key = apisix_ssl.aes_encrypt_pkey(conf.tls.client_key)
diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md
index 9c7bec756..d78eb0d6e 100644
--- a/docs/en/latest/admin-api.md
+++ b/docs/en/latest/admin-api.md
@@ -541,8 +541,9 @@ In addition to the equalization algorithm selections, Upstream also supports pas
 | labels                      | optional                                    | Attributes of the Upstream specified as key-value pairs.                                                                                                                                                                                                                                                                                                                                                                       [...]
 | create_time                 | optional                                    | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically.                                                                                                                                                                                                                                                                                                                      [...]
 | update_time                 | optional                                    | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically.                                                                                                                                                                                                                                                                                                                      [...]
-| tls.client_cert             | optional                                    | Sets the client certificate while connecting to a TLS Upstream.                                                                                                                                                                                                                                                                                                                                                                [...]
-| tls.client_key              | optional                                    | Sets the client private key while connecting to a TLS Upstream.                                                                                                                                                                                                                                                                                                                                                                [...]
+| tls.client_cert             | optional, can't be used with `tls.client_cert_id`       | Sets the client certificate while connecting to a TLS Upstream.                                                                                                                                                                                                                                                                                                                                                    [...]
+| tls.client_key              | optional, can't be used with `tls.client_cert_id`       | Sets the client private key while connecting to a TLS Upstream.                                                                                                                                                                                                                                                                                                                                                    [...]
+| tls.client_cert_id          | optional, can't be used with `tls.client_cert` and `tls.client_key`       | Set the referenced [SSL](#ssl) id.                                                                                                                                                                                                                                                                                                                                                               [...]
 | keepalive_pool.size         | optional                                    | Sets `keepalive` directive dynamically.                                                                                                                                                                                                                                                                                                                                                                                        [...]
 | keepalive_pool.idle_timeout | optional                                    | Sets `keepalive_timeout` directive dynamically.                                                                                                                                                                                                                                                                                                                                                                                [...]
 | keepalive_pool.requests     | optional                                    | Sets `keepalive_requests` directive dynamically.                                                                                                                                                                                                                                                                                                                                                                               [...]
@@ -570,6 +571,8 @@ You can set the `scheme` to `tls`, which means "TLS over TCP".
 
 To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields.
 
+Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`, otherwise the request will be rejected by APISIX. In addition, only `cert` and `key` will be used in the SSL object.
+
 To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields.
 
 Example Configuration:
@@ -789,6 +792,7 @@ Currently, the response is returned from etcd.
 | labels       | False    | Match Rules              | Attributes of the resource specified as key-value pairs.                                                       | {"version":"v2","build":"16","env":"production"} |
 | create_time  | False    | Auxiliary                | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically.         | 1602883670                                       |
 | update_time  | False    | Auxiliary                | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically.         | 1602883670                                       |
+| type         | False    | Auxiliary            | Identifies the type of certificate, default  `server`.                                                                             | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests.     |
 | status       | False    | Auxiliary                | Enables the current SSL. Set to `1` (enabled) by default.                                                      | `1` to enable, `0` to disable                    |
 
 Example Configuration:
diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md
index 7cffc2adb..367c318ac 100644
--- a/docs/zh/latest/admin-api.md
+++ b/docs/zh/latest/admin-api.md
@@ -549,8 +549,9 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外,还支持对上
 | labels         | 可选                               | 匹配规则       | 标识附加属性的键值对                                                                                                                                                                                                                                                                                                                                        | {"version":"v2","build":"16","env":"production"} |
 | create_time    | 可选                               | 辅助           | 单位为秒的 epoch 时间戳,如果不指定则自动创建                                                                                                                                                                                                                                                                                                               | 1602883670                                       |
 | update_time    | 可选                               | 辅助           | 单位为秒的 epoch 时间戳,如果不指定则自动创建                                                                                                                                                                                                                                                                                                               | 1602883670                                       |
-| tls.client_cert    | 可选                               | https 证书           | 设置跟上游通信时的客户端证书,细节见下文                                                                          | |
-| tls.client_key	 | 可选                               | https 证书私钥           | 设置跟上游通信时的客户端私钥,细节见下文                                                                                                                                                                                                                                                                                                              | |
+| tls.client_cert    | 可选,不能和 `tls.client_cert_id` 一起使用 | https 证书           | 设置跟上游通信时的客户端证书,细节见下文                                                                          | |
+| tls.client_key	 | 可选,不能和 `tls.client_cert_id` 一起使用 | https 证书私钥           | 设置跟上游通信时的客户端私钥,细节见下文                                                                                                                                                                                                                                                                                                              | |
+| tls.client_cert_id | 可选,不能和 `tls.client_cert`、`tls.client_key` 一起使用  | SSL           | 设置引用的 ssl id,详见 [SSL](#ssl)                                                                                                                                                                                                                                                                                                              | |
 |keepalive_pool.size  | 可选 | 辅助 | 动态设置 `keepalive` 指令,细节见下文 |
 |keepalive_pool.idle_timeout  | 可选 | 辅助 | 动态设置 `keepalive_timeout` 指令,细节见下文 |
 |keepalive_pool.requests  | 可选 | 辅助 | 动态设置 `keepalive_requests` 指令,细节见下文 |
@@ -578,6 +579,8 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外,还支持对上
 `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。
 他们的格式和 SSL 对象的 `cert` 和 `key` 一样。
 
+`tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用,否则请求会被 APISIX 拒绝。另外,SSL 对象中只有 `cert`和`key` 会被使用。
+
 `keepalive_pool` 允许 upstream 对象有自己单独的连接池。
 它下属的字段,比如 `requests`,可以用了配置上游连接保持的参数。
 这个特性需要 APISIX 运行于 [APISIX-Base](./FAQ.md#如何构建-APISIX-Base-环境?)。
@@ -799,6 +802,7 @@ $ curl http://127.0.0.1:9080/get
 | labels      | 可选   | 匹配规则       | 标识附加属性的键值对                                                                                   | {"version":"v2","build":"16","env":"production"} |
 | create_time | 可选   | 辅助           | 单位为秒的 epoch 时间戳,如果不指定则自动创建                                                          | 1602883670                                       |
 | update_time | 可选   | 辅助           | 单位为秒的 epoch 时间戳,如果不指定则自动创建                                                          | 1602883670                                       |
+| type      | 可选   | 辅助           | 标识证书的类型,缺省为 `server`。                                                                             | `client` 表示证书是客户端证书,APISIX 访问上游时使用;`server` 表示证书是服务端证书,APISIX 验证客户端请求时使用     |
 | status      | 可选   | 辅助           | 是否启用此 SSL,缺省 `1`。                                                                             | `1` 表示启用,`0` 表示禁用                       |
 
 ssl 对象 json 配置内容:
diff --git a/t/admin/ssl.t b/t/admin/ssl.t
index 49d3ec8cd..0232d2110 100644
--- a/t/admin/ssl.t
+++ b/t/admin/ssl.t
@@ -236,7 +236,7 @@ GET /t
 GET /t
 --- error_code: 400
 --- response_body
-{"error_msg":"invalid configuration: value should match only one schema, but matches none"}
+{"error_msg":"invalid configuration: then clause did not match"}
 --- no_error_log
 [error]
 
@@ -535,7 +535,7 @@ passed
 GET /t
 --- error_code: 400
 --- response_body
-{"error_msg":"invalid configuration: value should match only one schema, but matches none"}
+{"error_msg":"invalid configuration: then clause did not match"}
 --- no_error_log
 [error]
 
@@ -771,3 +771,72 @@ GET /t
 passed
 --- no_error_log
 [error]
+
+
+
+=== TEST 20: missing sni information
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local ssl_cert = t.read_file("t/certs/apisix.crt")
+            local ssl_key =  t.read_file("t/certs/apisix.key")
+            local data = {cert = ssl_cert, key = ssl_key}
+
+            local code, body = t.test('/apisix/admin/ssl/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data),
+                [[{
+                    "node": {
+                        "key": "/apisix/ssl/1"
+                    },
+                    "action": "set"
+                }]]
+                )
+
+            ngx.status = code
+            ngx.print(body)
+        }
+    }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: then clause did not match"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 21: type client, missing sni information
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local ssl_cert = t.read_file("t/certs/apisix.crt")
+            local ssl_key =  t.read_file("t/certs/apisix.key")
+            local data = {type = "client", cert = ssl_cert, key = ssl_key}
+
+            local code, body = t.test('/apisix/admin/ssl/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data),
+                [[{
+                    "node": {
+                        "key": "/apisix/ssl/1"
+                    },
+                    "action": "set"
+                }]]
+                )
+
+            ngx.status = code
+            ngx.print(body)
+        }
+    }
+--- request
+GET /t
+--- response_body chomp
+passed
diff --git a/t/admin/ssl2.t b/t/admin/ssl2.t
index 752e25cd7..865652ce2 100644
--- a/t/admin/ssl2.t
+++ b/t/admin/ssl2.t
@@ -71,7 +71,7 @@ __DATA__
         }
     }
 --- response_body
-{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1}}}
+{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1,"type":"server"}}}
 
 
 
@@ -104,7 +104,7 @@ __DATA__
         }
     }
 --- response_body
-{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1}}}
+{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1,"type":"server"}}}
 
 
 
@@ -137,7 +137,7 @@ __DATA__
         }
     }
 --- response_body
-{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1}}}
+{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1,"type":"server"}}}
 
 
 
@@ -172,7 +172,7 @@ __DATA__
         }
     }
 --- response_body
-{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1}}}
+{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1,"type":"server"}}}
 
 
 
diff --git a/t/admin/upstream.t b/t/admin/upstream.t
index 96dcef3c4..16bfb5157 100644
--- a/t/admin/upstream.t
+++ b/t/admin/upstream.t
@@ -627,3 +627,136 @@ GET /t
 {"error_msg":"wrong upstream id, do not need it"}
 --- no_error_log
 [error]
+
+
+
+=== TEST 19: client_cert/client_key and client_cert_id cannot appear at the same time
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local ssl_cert = t.read_file("t/certs/apisix.crt")
+            local ssl_key =  t.read_file("t/certs/apisix.key")
+            local data = {
+                nodes = {
+                    ["127.0.0.1:8080"] = 1
+                },
+                type = "roundrobin",
+                tls = {
+                    client_cert_id = 1,
+                    client_cert = ssl_cert,
+                    client_key = ssl_key
+                }
+            }
+            local code, body = t.test('/apisix/admin/upstreams',
+                 ngx.HTTP_POST,
+                 core.json.encode(data)
+                )
+
+            ngx.status = code
+            ngx.print(body)
+        }
+    }
+--- request
+GET /t
+--- error_code: 400
+--- response_body eval
+qr/{"error_msg":"invalid configuration: property \\\"tls\\\" validation failed: failed to validate dependent schema for \\\"client_cert|client_key\\\": value wasn't supposed to match schema"}/
+--- no_error_log
+[error]
+
+
+
+=== TEST 20: tls.client_cert_id does not exist
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local data = {
+                nodes = {
+                    ["127.0.0.1:8080"] = 1
+                },
+                type = "roundrobin",
+                tls = {
+                    client_cert_id = 9999999
+                }
+            }
+            local code, body = t.test('/apisix/admin/upstreams',
+                 ngx.HTTP_POST,
+                 core.json.encode(data)
+                )
+
+            ngx.status = code
+            ngx.print(body)
+        }
+    }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to fetch ssl info by ssl id [9999999], response code: 404"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 21: tls.client_cert_id exist with wrong ssl type
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local json = require("toolkit.json")
+            local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+            local ssl_key = t.read_file("t/certs/mtls_client.key")
+            local data = {
+                sni = "test.com",
+                cert = ssl_cert,
+                key = ssl_key
+            }
+            local code, body = t.test('/apisix/admin/ssl/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                upstream = {
+                    scheme = "https",
+                    type = "roundrobin",
+                    nodes = {
+                        ["127.0.0.1:1983"] = 1
+                    },
+                    tls = {
+                        client_cert_id = 1
+                    }
+                },
+                uri = "/hello"
+            }
+            local code, body = t.test('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+        }
+    }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to fetch ssl info by ssl id [1], wrong ssl type"}
+--- no_error_log
+[error]
diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t
index 7af6d2e61..c909dbc9a 100644
--- a/t/node/upstream-mtls.t
+++ b/t/node/upstream-mtls.t
@@ -77,7 +77,7 @@ __DATA__
 GET /t
 --- error_code: 400
 --- response_body
-{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: property \"client_key\" is required when \"client_cert\" is set"}
+{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: failed to validate dependent schema for \"client_cert\": property \"client_key\" is required"}
 
 
 
@@ -545,3 +545,145 @@ GET /t
 GET /hello_chunked
 --- response_body
 hello world
+
+
+
+=== TEST 13: get cert by tls.client_cert_id
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local json = require("toolkit.json")
+
+            local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+            local ssl_key = t.read_file("t/certs/mtls_client.key")
+            local data = {
+                type = "client",
+                cert = ssl_cert,
+                key = ssl_key
+            }
+            local code, body = t.test('/apisix/admin/ssl/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+
+            local data = {
+                upstream = {
+                    scheme = "https",
+                    type = "roundrobin",
+                    nodes = {
+                        ["127.0.0.1:1983"] = 1,
+                    },
+                    tls = {
+                        client_cert_id = 1
+                    }
+                },
+                uri = "/hello"
+            }
+            local code, body = t.test('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+        }
+    }
+--- request
+GET /t
+
+
+
+=== TEST 14: hit
+--- upstream_server_config
+    ssl_client_certificate ../../certs/mtls_ca.crt;
+    ssl_verify_client on;
+--- request
+GET /hello
+--- response_body
+hello world
+
+
+
+=== TEST 15: change ssl object type
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local json = require("toolkit.json")
+
+            local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+            local ssl_key = t.read_file("t/certs/mtls_client.key")
+            local data = {
+                type = "server",
+                sni = "test.com",
+                cert = ssl_cert,
+                key = ssl_key
+            }
+            local code, body = t.test('/apisix/admin/ssl/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+        }
+    }
+--- request
+GET /t
+
+
+
+=== TEST 16: hit, ssl object type mismatch
+--- upstream_server_config
+    ssl_client_certificate ../../certs/mtls_ca.crt;
+    ssl_verify_client on;
+--- request
+GET /hello
+--- error_code: 502
+--- error_log
+failed to get ssl cert: ssl type should be 'client'
+
+
+
+=== TEST 17: delete ssl object
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local json = require("toolkit.json")
+
+            local code, body = t.test('/apisix/admin/ssl/1', ngx.HTTP_DELETE)
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(body)
+                return
+            end
+        }
+    }
+--- request
+GET /t
+
+
+
+=== TEST 18: hit, ssl object not exits
+--- upstream_server_config
+    ssl_client_certificate ../../certs/mtls_ca.crt;
+    ssl_verify_client on;
+--- request
+GET /hello
+--- error_code: 502
+--- error_log
+failed to get ssl cert: ssl id [1] not exits