You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by we...@apache.org on 2020/01/10 07:17:26 UTC
[incubator-apisix] branch master updated: feature: chash key
support more flexible ways (#1022)
This is an automated email from the ASF dual-hosted git repository.
wenming pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 9c9be95 feature: chash key support more flexible ways (#1022)
9c9be95 is described below
commit 9c9be959d38c4a1c8471d685843a1057163a7e82
Author: Changwei Hu <10...@qq.com>
AuthorDate: Fri Jan 10 15:17:17 2020 +0800
feature: chash key support more flexible ways (#1022)
upstream add hash_on, chash key support more flexible ways
routes and services admin api add upstream_conf check
---
bin/apisix | 1 +
conf/config.yaml | 1 +
doc/architecture-design-cn.md | 87 ++++++-
doc/architecture-design.md | 85 ++++++-
lua/apisix/admin/routes.lua | 9 +
lua/apisix/admin/services.lua | 9 +
lua/apisix/admin/upstreams.lua | 67 ++++-
lua/apisix/balancer.lua | 31 ++-
lua/apisix/schema_def.lua | 30 ++-
t/APISIX.pm | 1 +
t/admin/routes.t | 131 ++++++++++
t/admin/services.t | 127 ++++++++++
t/admin/upstream.t | 259 +++++++++++++++++++-
t/node/chash-hashon.t | 540 +++++++++++++++++++++++++++++++++++++++++
14 files changed, 1358 insertions(+), 20 deletions(-)
diff --git a/bin/apisix b/bin/apisix
index 2fe82eb..20c0df4 100755
--- a/bin/apisix
+++ b/bin/apisix
@@ -166,6 +166,7 @@ http {
lua_ssl_verify_depth 5;
ssl_session_timeout 86400;
+ underscores_in_headers {* http.underscores_in_headers *};
lua_socket_log_errors off;
resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} ipv6=off;
diff --git a/conf/config.yaml b/conf/config.yaml
index a743896..bac8813 100644
--- a/conf/config.yaml
+++ b/conf/config.yaml
@@ -67,6 +67,7 @@ nginx_config: # config for render the template to genarate n
client_header_timeout: 60s # timeout for reading client request header, then 408 (Request Time-out) error is returned to the client
client_body_timeout: 60s # timeout for reading client request body, then 408 (Request Time-out) error is returned to the client
send_timeout: 10s # timeout for transmitting a response to the client.then the connection is closed
+ underscores_in_headers: "on" # default enables the use of underscores in client request header fields
etcd:
host: "http://127.0.0.1:2379" # etcd address
diff --git a/doc/architecture-design-cn.md b/doc/architecture-design-cn.md
index 948e48a..968401d 100644
--- a/doc/architecture-design-cn.md
+++ b/doc/architecture-design-cn.md
@@ -237,7 +237,8 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上
|------- |-----|------|
|type |必需|`roundrobin` 支持权重的负载，`chash` 一致性哈希，两者是二选一的|
|nodes |必需|哈希表，内部元素的 key 是上游机器地址列表，格式为`地址 + Port`，其中地址部分可以是 IP 也可以是域名，比如 `192.168.1.100:80`、`foo.com:80`等。value 则是节点的权重，特别的，当权重值为 `0` 有特殊含义，通常代表该上游节点失效，永远不希望被选中。|
-|key |必需|该选项只有类型是 `chash` 才有效。根据 `key` 来查找对应的 node `id`，相同的 `key` 在同一个对象中，永远返回相同 id，目前支持的 Nginx 内置变量有 `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`，其中 `arg_***` 是来自URL的请求参数，[Nginx 变量列表](http://nginx.org/en/docs/varindex.html)|
+|hash_on |可选|该选项只有 `type` 是 `chash` 才有效。`hash_on` 支持的类型有 `vars`（Nginx内置变量），`header`（自定义header），`cookie`，`consumer`，默认值为 `vars`|
+|key |必需|该选项只有 `type` 是 `chash` 才有效，需要配合 `hash_on` 来使用，通过 `hash_on` 和 `key` 来查找对应的 node `id`。`hash_on` 设为 `vars` 时，`key` 为必传参数，目前支持的 Nginx 内置变量有 `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`，其中 `arg_***` 是来自URL的请求参数，[Nginx 变量列表](http://nginx.org/en/docs/varindex.html)；`hash_on` 设为 `header` 时, `key` 为必传参数，自定义的 `header name`；`hash_on` 设为 `cookie` 时, `key` 为必传参数， 自定义的 `cookie name`；`hash_on` 设为 `consumer` 时，`key` 不需 [...]
|checks |可选|配置健康检查的参数，详细可参考[health-check](health-check.md)|
|retries |可选|使用底层的 Nginx 重试机制将请求传递给下一个上游，默认 APISIX 会启用重试机制，根据配置的后端节点个数设置重试次数，如果此参数显式被设置将会覆盖系统默认设置的重试次数。|
@@ -334,9 +335,91 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
}
}'
```
-
更多细节可以参考[健康检查的文档](health-check.md)。
+下面是几个使用不同`hash_on`类型的配置示例：
+##### Consumer
+创建一个consumer对象:
+```shell
+curl http://127.0.0.1:9080/apisix/admin/consumers -X PUT -d `
+{
+ "username": "jack",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-jack"
+ }
+ }
+}`
+```
+新建路由，打开`key-auth`插件认证，`upstream`的`hash_on`类型为`consumer`：
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "plugins": {
+ "key-auth": {}
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer"
+ },
+ "uri": "/server_port"
+}'
+```
+测试请求，认证通过后的`consumer_id`将作为负载均衡哈希算法的哈希值：
+```shell
+curl http://127.0.0.1:9080/server_port -H "apikey: auth-jack"
+```
+
+##### Cookie
+新建路由和`Upstream`，`hash_on`类型为`cookie`：
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "uri": "/hash_on_cookie",
+ "upstream": {
+ "key": "sid",
+ "type ": "chash",
+ "hash_on ": "cookie",
+ "nodes ": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+}'
+```
+
+客户端请求携带`Cookie`：
+```shell
+ curl http://127.0.0.1:9080/hash_on_cookie -H "Cookie: sid=3c183a30cffcda1408daf1c61d47b274"
+```
+
+##### Header
+新建路由和`Upstream`，`hash_on`类型为`header`， `key`为`content-type`：
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "uri": "/hash_on_header",
+ "upstream": {
+ "key": "content-type",
+ "type ": "chash",
+ "hash_on ": "header",
+ "nodes ": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+}'
+```
+
+客户端请求携带`content-type`的`header`：
+```shell
+ curl http://127.0.0.1:9080/hash_on_header -H "Content-Type: application/json"
+```
+
[返回目录](#目录)
diff --git a/doc/architecture-design.md b/doc/architecture-design.md
index ac12cbb..4ce536a 100644
--- a/doc/architecture-design.md
+++ b/doc/architecture-design.md
@@ -232,7 +232,8 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
|------- |-----|------|
|type |required|`roundrobin` supports the weight of the load, `chash` consistency hash, pick one of them.|
|nodes |required|Hash table, the key of the internal element is the upstream machine address list, the format is `Address + Port`, where the address part can be IP or domain name, such as `192.168.1.100:80`, `foo.com:80`, etc. Value is the weight of the node. In particular, when the weight value is `0`, it has a special meaning, which usually means that the upstream node is invalid and never wants to be selected.|
-|key |required|This option is only valid if the type is `chash`. Find the corresponding node `id` according to `key`, the same `key` in the same object, always return the same id. For now, it support nginx built-in variables like `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`, `arg_***` is arguments in the request line, [Nginx variables list](http://nginx.org/en/docs/varindex.html)|
+|hash_on |optional|This option is only valid if the `type` is `chash`. Supported types `vars`(Nginx variables), `header`(custom header), `cookie`, `consumer`, the default value is `vars`.|
+|key |required|This option is only valid if the `type` is `chash`. Find the corresponding node `id` according to `hash_on` and `key`. When `hash_on` is set as `vars`, `key` is the required parameter, for now, it support nginx built-in variables like `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`, `arg_***` is arguments in the request line, [Nginx variables list](http://nginx.org/en/docs/varindex.html). When `hash_ [...]
|checks |optional|Configure the parameters of the health check. For details, refer to [health-check](health-check.md).|
|retries |optional|Pass the request to the next upstream using the underlying Nginx retry mechanism, the retry mechanism is enabled by default and set the number of retries according to the number of backend nodes. If `retries` option is explicitly set, it will override the default value.|
@@ -333,6 +334,88 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
More details can be found in [Health Checking Documents](health-check.md).
+Here are some examples of configurations using different `hash_on` types:
+##### Consumer
+Create a consumer object:
+```shell
+curl http://127.0.0.1:9080/apisix/admin/consumers -X PUT -d `
+{
+ "username": "jack",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-jack"
+ }
+ }
+}`
+```
+Create route object and enable `key-auth` plugin authentication:
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "plugins": {
+ "key-auth": {}
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer"
+ },
+ "uri": "/server_port"
+}'
+```
+Test request, the `consumer_id` after authentication is passed will be used as the hash value of the load balancing hash algorithm:
+```shell
+curl http://127.0.0.1:9080/server_port -H "apikey: auth-jack"
+```
+
+##### Cookie
+Create route and upstream object, `hash_on` is `cookie`:
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "uri": "/hash_on_cookie",
+ "upstream": {
+ "key": "sid",
+ "type ": "chash",
+ "hash_on ": "cookie",
+ "nodes ": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+}'
+```
+The client requests with `Cookie`:
+```shell
+ curl http://127.0.0.1:9080/hash_on_cookie -H "Cookie: sid=3c183a30cffcda1408daf1c61d47b274"
+```
+
+##### Header
+Create route and upstream object, `hash_on` is `header`, `key` is `Content-Type`:
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -X PUT -d '
+{
+ "uri": "/hash_on_header",
+ "upstream": {
+ "key": "content-type",
+ "type ": "chash",
+ "hash_on ": "header",
+ "nodes ": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+}'
+```
+
+The client requests with header `Content-Type`:
+```shell
+ curl http://127.0.0.1:9080/hash_on_header -H "Content-Type: application/json"
+```
+
[Back to top](#Table-of-contents)
diff --git a/lua/apisix/admin/routes.lua b/lua/apisix/admin/routes.lua
index 34ab758..3303e8d 100644
--- a/lua/apisix/admin/routes.lua
+++ b/lua/apisix/admin/routes.lua
@@ -16,6 +16,7 @@
--
local core = require("apisix.core")
local schema_plugin = require("apisix.admin.plugins").check_schema
+local upstreams = require("apisix.admin.upstreams")
local tostring = tostring
local type = type
local loadstring = loadstring
@@ -60,6 +61,14 @@ local function check_conf(id, conf, need_id)
.. "allowed"}
end
+ local upstream_conf = conf.upstream
+ if upstream_conf then
+ local ok, err = upstreams.check_upstream_conf(upstream_conf)
+ if not ok then
+ return nil, {error_msg = err}
+ end
+ end
+
local upstream_id = conf.upstream_id
if upstream_id then
local key = "/upstreams/" .. upstream_id
diff --git a/lua/apisix/admin/services.lua b/lua/apisix/admin/services.lua
index c11688f..e26ea41 100644
--- a/lua/apisix/admin/services.lua
+++ b/lua/apisix/admin/services.lua
@@ -17,6 +17,7 @@
local core = require("apisix.core")
local get_routes = require("apisix.router").http_routes
local schema_plugin = require("apisix.admin.plugins").check_schema
+local upstreams = require("apisix.admin.upstreams")
local tostring = tostring
local ipairs = ipairs
local tonumber = tonumber
@@ -58,6 +59,14 @@ local function check_conf(id, conf, need_id)
return nil, {error_msg = "wrong type of service id"}
end
+ local upstream_conf = conf.upstream
+ if upstream_conf then
+ local ok, err = upstreams.check_upstream_conf(upstream_conf)
+ if not ok then
+ return nil, {error_msg = err}
+ end
+ end
+
local upstream_id = conf.upstream_id
if upstream_id then
local key = "/upstreams/" .. upstream_id
diff --git a/lua/apisix/admin/upstreams.lua b/lua/apisix/admin/upstreams.lua
index 1085884..b49c33a 100644
--- a/lua/apisix/admin/upstreams.lua
+++ b/lua/apisix/admin/upstreams.lua
@@ -28,6 +28,60 @@ local _M = {
}
+local function get_chash_key_schema(hash_on)
+ if not hash_on then
+ return nil, "hash_on is nil"
+ end
+
+ if hash_on == "vars" then
+ return core.schema.upstream_hash_vars_schema
+ end
+
+ if hash_on == "header" or hash_on == "cookie" then
+ return core.schema.upstream_hash_header_schema
+ end
+
+ if hash_on == "consumer" then
+ return nil, nil
+ end
+
+ return nil, "invalid hash_on type " .. hash_on
+end
+
+
+local function check_upstream_conf(conf)
+ local ok, err = core.schema.check(core.schema.upstream, conf)
+ if not ok then
+ return false, "invalid configuration: " .. err
+ end
+
+ if conf.type ~= "chash" then
+ return true
+ end
+
+ if not conf.hash_on then
+ conf.hash_on = "vars"
+ end
+
+ if conf.hash_on ~= "consumer" and not conf.key then
+ return false, "missing key"
+ end
+
+ local key_schema, err = get_chash_key_schema(conf.hash_on)
+ if err then
+ return false, "type is chash, err: " .. err
+ end
+
+ if key_schema then
+ local ok, err = core.schema.check(key_schema, conf.key)
+ if not ok then
+ return false, "invalid configuration: " .. err
+ end
+ end
+ return true
+end
+
+
local function check_conf(id, conf, need_id)
if not conf then
return nil, {error_msg = "missing configurations"}
@@ -45,23 +99,17 @@ local function check_conf(id, conf, need_id)
if need_id and conf.id and tostring(conf.id) ~= tostring(id) then
return nil, {error_msg = "wrong upstream id"}
end
-
core.log.info("schema: ", core.json.delay_encode(core.schema.upstream))
core.log.info("conf : ", core.json.delay_encode(conf))
- local ok, err = core.schema.check(core.schema.upstream, conf)
+ local ok, err = check_upstream_conf(conf)
if not ok then
- return nil, {error_msg = "invalid configuration: " .. err}
+ return nil, {error_msg = err}
end
if need_id and not tonumber(id) then
return nil, {error_msg = "wrong type of service id"}
end
-
- if conf.type == "chash" and not conf.key then
- return nil, {error_msg = "missing key"}
- end
-
return need_id and id or true
end
@@ -233,5 +281,8 @@ function _M.patch(id, conf, sub_path)
return res.status, res.body
end
+-- for routes and services check upstream conf
+_M.check_upstream_conf = check_upstream_conf
+
return _M
diff --git a/lua/apisix/balancer.lua b/lua/apisix/balancer.lua
index 1d6fc34..4b3f4bb 100644
--- a/lua/apisix/balancer.lua
+++ b/lua/apisix/balancer.lua
@@ -122,6 +122,33 @@ local function fetch_healthchecker(upstream, healthcheck_parent, version)
end
+local function fetch_chash_hash_key(ctx, upstream)
+ local key = upstream.key
+ local hash_on = upstream.hash_on or "vars"
+ local chash_key
+
+ if hash_on == "consumer" then
+ chash_key = ctx.consumer_id
+ elseif hash_on == "vars" then
+ chash_key = ctx.var[key]
+ elseif hash_on == "header" then
+ chash_key = ctx.var["http_" .. key]
+ elseif hash_on == "cookie" then
+ chash_key = ctx.var["cookie_" .. key]
+ end
+
+ if not chash_key then
+ chash_key = ctx.var["remote_addr"]
+ core.log.warn("chash_key fetch is nil, use default chash_key remote_addr: ", chash_key)
+ end
+ core.log.info("upstream key: ", key)
+ core.log.info("hash_on: ", hash_on)
+ core.log.info("chash_key: ", core.json.delay_encode(chash_key))
+
+ return chash_key
+end
+
+
local function create_server_picker(upstream, checker)
if upstream.type == "roundrobin" then
local up_nodes = fetch_health_nodes(upstream, checker)
@@ -151,11 +178,11 @@ local function create_server_picker(upstream, checker)
end
local picker = resty_chash:new(nodes)
- local key = upstream.key
return {
upstream = upstream,
get = function (ctx)
- local id = picker:find(ctx.var[key])
+ local chash_key = fetch_chash_hash_key(ctx, upstream)
+ local id = picker:find(chash_key)
-- core.log.warn("chash id: ", id, " val: ", servers[id])
return servers[id]
end
diff --git a/lua/apisix/schema_def.lua b/lua/apisix/schema_def.lua
index c4ab627..f5544d9 100644
--- a/lua/apisix/schema_def.lua
+++ b/lua/apisix/schema_def.lua
@@ -259,12 +259,19 @@ local upstream_schema = {
enum = {"chash", "roundrobin"}
},
checks = health_checker,
+ hash_on = {
+ type = "string",
+ default = "vars",
+ enum = {
+ "vars",
+ "header",
+ "cookie",
+ "consumer",
+ },
+ },
key = {
description = "the key of chash for dynamic load balancing",
type = "string",
- pattern = [[^((uri|server_name|server_addr|request_uri|remote_port]]
- .. [[|remote_addr|query_string|host|hostname)]]
- .. [[|arg_[0-9a-zA-z_-]+)$]],
},
desc = {type = "string", maxLength = 256},
id = id_schema
@@ -273,6 +280,23 @@ local upstream_schema = {
additionalProperties = false,
}
+-- TODO: add more nginx variable support
+_M.upstream_hash_vars_schema = {
+ type = "string",
+ pattern = [[^((uri|server_name|server_addr|request_uri|remote_port]]
+ .. [[|remote_addr|query_string|host|hostname)]]
+ .. [[|arg_[0-9a-zA-z_-]+)$]],
+}
+
+-- validates header name, cookie name.
+-- a-z, A-Z, 0-9, '_' and '-' are allowed.
+-- when "underscores_in_headers on", header name allow '_'.
+-- http://nginx.org/en/docs/http/ngx_http_core_module.html#underscores_in_headers
+_M.upstream_hash_header_schema = {
+ type = "string",
+ pattern = [[^[a-zA-Z0-9-_]+$]]
+}
+
_M.route = {
type = "object",
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 9ba1f4a..f7887ab 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -146,6 +146,7 @@ _EOC_
resolver 8.8.8.8 114.114.114.114 ipv6=off;
resolver_timeout 5;
+ underscores_in_headers on;
lua_socket_log_errors off;
upstream apisix_backend {
diff --git a/t/admin/routes.t b/t/admin/routes.t
index 57bf614..51eeffb 100644
--- a/t/admin/routes.t
+++ b/t/admin/routes.t
@@ -1590,3 +1590,134 @@ GET /t
passed
--- no_error_log
[error]
+
+
+
+=== TEST 43: set route(id: 1) and upstream(type:chash, default hash_on: vars, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash"
+ },
+ "desc": "new route",
+ "uri": "/index.html"
+ }]])
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 43: set route(id: 1) and upstream(type:chash, hash_on: header, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on":"header"
+ },
+ "desc": "new route",
+ "uri": "/index.html"
+ }]])
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 44: set route(id: 1) and upstream(type:chash, hash_on: cookie, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on":"cookie"
+ },
+ "desc": "new route",
+ "uri": "/index.html"
+ }]])
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 45: set route(id: 1) and upstream(type:chash, hash_on: consumer, missing key is ok)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "methods": ["GET"],
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on":"consumer"
+ },
+ "desc": "new route",
+ "uri": "/index.html"
+ }]])
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
diff --git a/t/admin/services.t b/t/admin/services.t
index d9e2f9e..c206508 100644
--- a/t/admin/services.t
+++ b/t/admin/services.t
@@ -751,3 +751,130 @@ GET /t
passed
--- no_error_log
[error]
+
+
+
+=== TEST 22: set service(id: 1) and upstream(type:chash, default hash_on: vars, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash"
+ },
+ "desc": "new service"
+ }]])
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 23: set service(id: 1) and upstream(type:chash, hash_on: header, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "header"
+ },
+ "desc": "new service"
+ }]])
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 24: set service(id: 1) and upstream(type:chash, hash_on: cookie, missing key)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "cookie"
+ },
+ "desc": "new service"
+ }]])
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"missing key"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 25: set service(id: 1) and upstream(type:chash, hash_on: consumer, missing key is ok)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer"
+ },
+ "desc": "new service"
+ }]])
+
+ ngx.status = code
+ ngx.say(code .. " " .. body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+200 passed
+--- no_error_log
+[error]
+
diff --git a/t/admin/upstream.t b/t/admin/upstream.t
index 6ca7308..99d5a7f 100644
--- a/t/admin/upstream.t
+++ b/t/admin/upstream.t
@@ -830,7 +830,7 @@ passed
-=== TEST 25: wrong upstream key
+=== TEST 25: wrong upstream key, hash_on default vars
--- config
location /t {
content_by_lua_block {
@@ -856,7 +856,7 @@ passed
GET /t
--- error_code: 400
--- response_body
-{"error_msg":"invalid configuration: property \"key\" validation failed: failed to match pattern \"^((uri|server_name|server_addr|request_uri|remote_port|remote_addr|query_string|host|hostname)|arg_[0-9a-zA-z_-]+)$\" with \"not_support\""}
+{"error_msg":"invalid configuration: failed to match pattern \"^((uri|server_name|server_addr|request_uri|remote_port|remote_addr|query_string|host|hostname)|arg_[0-9a-zA-z_-]+)$\" with \"not_support\""}
--- no_error_log
[error]
@@ -921,7 +921,7 @@ passed
-=== TEST 28: wrong upstream key
+=== TEST 28: wrong upstream key, hash_on default vars
--- config
location /t {
content_by_lua_block {
@@ -947,7 +947,7 @@ passed
GET /t
--- error_code: 400
--- response_body
-{"error_msg":"invalid configuration: property \"key\" validation failed: failed to match pattern \"^((uri|server_name|server_addr|request_uri|remote_port|remote_addr|query_string|host|hostname)|arg_[0-9a-zA-z_-]+)$\" with \"not_support\""}
+{"error_msg":"invalid configuration: failed to match pattern \"^((uri|server_name|server_addr|request_uri|remote_port|remote_addr|query_string|host|hostname)|arg_[0-9a-zA-z_-]+)$\" with \"not_support\""}
--- no_error_log
[error]
@@ -980,3 +980,254 @@ GET /t
passed
--- no_error_log
[error]
+
+
+
+=== TEST 30: type chash, hash_on: vars
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "arg_device_id",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "vars",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 31: type chash, hash_on: header, header name with '_', underscores_in_headers on
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "custom_header",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "header",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 32: type chash, hash_on: header, header name with invalid character
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "$#^@",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "header",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: failed to match pattern \"^[a-zA-Z0-9-_]+$\" with \"$#^@\""}
+--- no_error_log
+[error]
+
+
+
+=== TEST 33: type chash, hash_on: cookie
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "custom_cookie",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "cookie",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 34: type chash, hash_on: cookie, cookie name with invalid character
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "$#^@abc",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "cookie",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: failed to match pattern \"^[a-zA-Z0-9-_]+$\" with \"$#^@abc\""}
+--- no_error_log
+[error]
+
+
+
+=== TEST 35: type chash, hash_on: consumer, don't need upstream key
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 36: type chash, hash_on: consumer, set key but invalid
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer",
+ "key": "invalid-key",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 37: type chash, invalid hash_on type
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/upstreams/1',
+ ngx.HTTP_PUT,
+ [[{
+ "key": "dsadas",
+ "nodes": {
+ "127.0.0.1:8080": 1
+ },
+ "type": "chash",
+ "hash_on": "aabbcc",
+ "desc": "new chash upstream"
+ }]]
+ )
+
+ ngx.status = code
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: property \"hash_on\" validation failed: matches non of the enum values"}
+--- no_error_log
+[error]
+
diff --git a/t/node/chash-hashon.t b/t/node/chash-hashon.t
new file mode 100644
index 0000000..5839ec9
--- /dev/null
+++ b/t/node/chash-hashon.t
@@ -0,0 +1,540 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BEGIN {
+ if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+ $SkipReason = "unavailable for the hup tests";
+
+ } else {
+ $ENV{TEST_NGINX_USE_HUP} = 1;
+ undef $ENV{TEST_NGINX_USE_STAP};
+ }
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+log_level('info');
+no_root_location();
+no_shuffle();
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: add two consumer with username and plugins
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/consumers',
+ ngx.HTTP_PUT,
+ [[{
+ "username": "jack",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-jack"
+ }
+ }
+ }]],
+ [[{
+ "node": {
+ "value": {
+ "username": "jack",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-jack"
+ }
+ }
+ }
+ },
+ "action": "set"
+ }]]
+ )
+
+ if code ~= 200 then
+ ngx.say("create comsume jack failed")
+ return
+ end
+ ngx.say(code .. " " ..body)
+
+ code, body = t('/apisix/admin/consumers',
+ ngx.HTTP_PUT,
+ [[{
+ "username": "tom",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-tom"
+ }
+ }
+ }]],
+ [[{
+ "node": {
+ "value": {
+ "username": "tom",
+ "plugins": {
+ "key-auth": {
+ "key": "auth-tom"
+ }
+ }
+ }
+ },
+ "action": "set"
+ }]]
+ )
+ ngx.say(code .. " " ..body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+200 passed
+200 passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: add key auth plugin, chash hash_on consumer
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "key-auth": {}
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ },
+ "type": "chash",
+ "hash_on": "consumer"
+ },
+ "uri": "/server_port"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: hit routes, hash_on one consumer
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ request_headers["apikey"] = "auth-jack"
+
+ local ports_count = {}
+ for i = 1, 4 do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":4,"port":"1981"}]
+--- grep_error_log eval
+qr/hash_on: consumer|chash_key: "jack"|chash_key: "tom"/
+--- grep_error_log_out
+hash_on: consumer
+chash_key: "jack"
+hash_on: consumer
+chash_key: "jack"
+hash_on: consumer
+chash_key: "jack"
+hash_on: consumer
+chash_key: "jack"
+
+
+
+
+=== TEST 4: hit routes, hash_on two consumer
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ local ports_count = {}
+ for i = 1, 4 do
+ if i%2 == 0 then
+ request_headers["apikey"] = "auth-tom"
+ else
+ request_headers["apikey"] = "auth-jack"
+ end
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":2,"port":"1981"},{"count":2,"port":"1980"}]
+--- grep_error_log eval
+qr/hash_on: consumer|chash_key: "jack"|chash_key: "tom"/
+--- grep_error_log_out
+hash_on: consumer
+chash_key: "jack"
+hash_on: consumer
+chash_key: "tom"
+hash_on: consumer
+chash_key: "jack"
+hash_on: consumer
+chash_key: "tom"
+
+
+
+=== TEST 5: set route(two upstream node, type chash), hash_on header
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/server_port",
+ "upstream": {
+ "key": "custom_header",
+ "type": "chash",
+ "hash_on": "header",
+ "nodes": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 6: hit routes, hash_on custom header
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ request_headers["custom_header"] = "custom-one"
+
+ local ports_count = {}
+ for i = 1, 4 do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":4,"port":"1980"}]
+--- grep_error_log eval
+qr/hash_on: header|chash_key: "custom-one"/
+--- grep_error_log_out
+hash_on: header
+chash_key: "custom-one"
+hash_on: header
+chash_key: "custom-one"
+hash_on: header
+chash_key: "custom-one"
+hash_on: header
+chash_key: "custom-one"
+
+
+
+=== TEST 7: hit routes, hash_on custom header miss, use default
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ request_headers["miss-custom-header"] = "custom-one"
+
+ local ports_count = {}
+ for i = 1, 4 do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":4,"port":"1980"}]
+--- grep_error_log eval
+qr/chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1/
+--- grep_error_log_out
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+
+
+
+=== TEST 8: set route(two upstream node, type chash), hash_on cookie
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/server_port",
+ "upstream": {
+ "key": "custom-cookie",
+ "type": "chash",
+ "hash_on": "cookie",
+ "nodes": {
+ "127.0.0.1:1980": 1,
+ "127.0.0.1:1981": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 9: hit routes, hash_on custom cookie
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ request_headers["Cookie"] = "custom-cookie=cuscookie"
+
+ local ports_count = {}
+ for i = 1, 4 do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":4,"port":"1981"}]
+--- grep_error_log eval
+qr/hash_on: cookie|chash_key: "cuscookie"/
+--- grep_error_log_out
+hash_on: cookie
+chash_key: "cuscookie"
+hash_on: cookie
+chash_key: "cuscookie"
+hash_on: cookie
+chash_key: "cuscookie"
+hash_on: cookie
+chash_key: "cuscookie"
+
+
+
+
+=== TEST 10: hit routes, hash_on custom cookie miss, use default
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port"
+
+ local request_headers = {}
+ request_headers["Cookie"] = "miss-custom-cookie=cuscookie"
+
+ local ports_count = {}
+ for i = 1, 4 do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET", headers = request_headers})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ ports_count[res.body] = (ports_count[res.body] or 0) + 1
+ end
+
+ local ports_arr = {}
+ for port, count in pairs(ports_count) do
+ table.insert(ports_arr, {port = port, count = count})
+ end
+
+ local function cmd(a, b)
+ return a.port > b.port
+ end
+ table.sort(ports_arr, cmd)
+
+ ngx.say(require("cjson").encode(ports_arr))
+ ngx.exit(200)
+ }
+ }
+--- request
+GET /t
+--- response_body
+[{"count":4,"port":"1980"}]
+--- grep_error_log eval
+qr/chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1/
+--- grep_error_log_out
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+chash_key fetch is nil, use default chash_key remote_addr: 127.0.0.1
+