You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2021/07/13 02:02:41 UTC
[apisix] branch master updated: feat: support lua_shared_dict
variable in ngx_tpl.lua loaded via configuration file (#4524)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 131998d feat: support lua_shared_dict variable in ngx_tpl.lua loaded via configuration file (#4524)
131998d is described below
commit 131998df4f44720b7ddf0424f4d381efb689d4ce
Author: Applenice <zx...@gmail.com>
AuthorDate: Tue Jul 13 10:02:34 2021 +0800
feat: support lua_shared_dict variable in ngx_tpl.lua loaded via configuration file (#4524)
---
apisix/balancer/ewma.lua | 6 +-
apisix/cli/ngx_tpl.lua | 40 +++++-----
apisix/core/config_etcd.lua | 2 +-
apisix/plugins/api-breaker.lua | 2 +-
apisix/plugins/authz-keycloak.lua | 6 +-
apisix/plugins/server-info.lua | 4 +-
conf/config-default.yaml | 25 ++++++
t/APISIX.pm | 34 ++++-----
t/cli/test_main.sh | 155 ++++++++++++++++++++++++++++++++++++++
9 files changed, 227 insertions(+), 47 deletions(-)
diff --git a/apisix/balancer/ewma.lua b/apisix/balancer/ewma.lua
index e9926da..c533a01 100644
--- a/apisix/balancer/ewma.lua
+++ b/apisix/balancer/ewma.lua
@@ -20,13 +20,13 @@ local error = error
local DECAY_TIME = 10 -- this value is in seconds
local LOCK_KEY = ":ewma_key"
-local shm_ewma = ngx_shared.balancer_ewma
-local shm_last_touched_at = ngx_shared.balancer_ewma_last_touched_at
+local shm_ewma = ngx_shared["balancer-ewma"]
+local shm_last_touched_at = ngx_shared["balancer-ewma-last-touched-at"]
local lrucache_addr = core.lrucache.new({ttl = 300, count = 1024})
local lrucache_trans_format = core.lrucache.new({ttl = 300, count = 256})
-local ewma_lock, ewma_lock_err = resty_lock:new("balancer_ewma_locks", {timeout = 0, exptime = 0.1})
+local ewma_lock, ewma_lock_err = resty_lock:new("balancer-ewma-locks", {timeout = 0, exptime = 0.1})
local _M = {name = "ewma"}
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 166c7a0..fc8b685 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -66,7 +66,7 @@ stream {
.. [=[{*lua_cpath*};";
lua_socket_log_errors off;
- lua_shared_dict lrucache-lock-stream 10m;
+ lua_shared_dict lrucache-lock-stream {* stream.lua_shared_dict["lrucache-lock-stream"] *};
resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} {% if dns_resolver_valid then %} valid={*dns_resolver_valid*}{% end %};
resolver_timeout {*resolver_timeout*};
@@ -141,31 +141,31 @@ http {
.. [=[$prefix/deps/lib/lua/5.1/?.so;;]=]
.. [=[{*lua_cpath*};";
- lua_shared_dict internal_status 10m;
- lua_shared_dict plugin-limit-req 10m;
- lua_shared_dict plugin-limit-count 10m;
- lua_shared_dict prometheus-metrics 10m;
- lua_shared_dict plugin-limit-conn 10m;
- lua_shared_dict upstream-healthcheck 10m;
- lua_shared_dict worker-events 10m;
- lua_shared_dict lrucache-lock 10m;
- lua_shared_dict balancer_ewma 10m;
- lua_shared_dict balancer_ewma_locks 10m;
- lua_shared_dict balancer_ewma_last_touched_at 10m;
- lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
- lua_shared_dict tracing_buffer 10m; # plugin: skywalking
- lua_shared_dict plugin-api-breaker 10m;
- lua_shared_dict etcd_cluster_health_check 10m; # etcd health check
+ lua_shared_dict internal-status {* http.lua_shared_dict["internal-status"] *};
+ lua_shared_dict plugin-limit-req {* http.lua_shared_dict["plugin-limit-req"] *};
+ lua_shared_dict plugin-limit-count {* http.lua_shared_dict["plugin-limit-count"] *};
+ lua_shared_dict prometheus-metrics {* http.lua_shared_dict["prometheus-metrics"] *};
+ lua_shared_dict plugin-limit-conn {* http.lua_shared_dict["plugin-limit-conn"] *};
+ lua_shared_dict upstream-healthcheck {* http.lua_shared_dict["upstream-healthcheck"] *};
+ lua_shared_dict worker-events {* http.lua_shared_dict["worker-events"] *};
+ lua_shared_dict lrucache-lock {* http.lua_shared_dict["lrucache-lock"] *};
+ lua_shared_dict balancer-ewma {* http.lua_shared_dict["balancer-ewma"] *};
+ lua_shared_dict balancer-ewma-locks {* http.lua_shared_dict["balancer-ewma-locks"] *};
+ lua_shared_dict balancer-ewma-last-touched-at {* http.lua_shared_dict["balancer-ewma-last-touched-at"] *};
+ lua_shared_dict plugin-limit-count-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-count-redis-cluster-slot-lock"] *};
+ lua_shared_dict tracing_buffer {* http.lua_shared_dict.tracing_buffer *}; # plugin: skywalking
+ lua_shared_dict plugin-api-breaker {* http.lua_shared_dict["plugin-api-breaker"] *};
+ lua_shared_dict etcd-cluster-health-check {* http.lua_shared_dict["etcd-cluster-health-check"] *}; # etcd health check
# for openid-connect and authz-keycloak plugin
- lua_shared_dict discovery 1m; # cache for discovery metadata documents
+ lua_shared_dict discovery {* http.lua_shared_dict["discovery"] *}; # cache for discovery metadata documents
# for openid-connect plugin
- lua_shared_dict jwks 1m; # cache for JWKs
- lua_shared_dict introspection 10m; # cache for JWT verification results
+ lua_shared_dict jwks {* http.lua_shared_dict["jwks"] *}; # cache for JWKs
+ lua_shared_dict introspection {* http.lua_shared_dict["introspection"] *}; # cache for JWT verification results
# for authz-keycloak
- lua_shared_dict access_tokens 1m; # cache for service account access tokens
+ lua_shared_dict access-tokens {* http.lua_shared_dict["access-tokens"] *}; # cache for service account access tokens
# for custom shared dict
{% if http.lua_shared_dicts then %}
diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index 38a32e0..f35f232 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -538,7 +538,7 @@ local function _automatic_fetch(premature, self)
if not health_check.conf then
local _, err = health_check.init({
- shm_name = "etcd_cluster_health_check",
+ shm_name = "etcd-cluster-health-check",
fail_timeout = self.health_check_timeout,
max_fails = 3,
retry = true,
diff --git a/apisix/plugins/api-breaker.lua b/apisix/plugins/api-breaker.lua
index 7856373..ba7e17c 100644
--- a/apisix/plugins/api-breaker.lua
+++ b/apisix/plugins/api-breaker.lua
@@ -21,7 +21,7 @@ local ngx = ngx
local math = math
local error = error
-local shared_buffer = ngx.shared['plugin-'.. plugin_name]
+local shared_buffer = ngx.shared["plugin-".. plugin_name]
if not shared_buffer then
error("failed to get ngx.shared dict when load plugin " .. plugin_name)
end
diff --git a/apisix/plugins/authz-keycloak.lua b/apisix/plugins/authz-keycloak.lua
index 60240a5..9a1c16c 100644
--- a/apisix/plugins/authz-keycloak.lua
+++ b/apisix/plugins/authz-keycloak.lua
@@ -337,7 +337,7 @@ local function authz_keycloak_ensure_sa_access_token(conf)
return 500, "Unable to determine token endpoint."
end
- local session = authz_keycloak_cache_get("access_tokens", token_endpoint .. ":"
+ local session = authz_keycloak_cache_get("access-tokens", token_endpoint .. ":"
.. client_id)
if session then
@@ -425,7 +425,7 @@ local function authz_keycloak_ensure_sa_access_token(conf)
json.refresh_expires_in)
end
- authz_keycloak_cache_set("access_tokens",
+ authz_keycloak_cache_set("access-tokens",
token_endpoint .. ":" .. client_id,
core.json.encode(session), ttl)
end
@@ -500,7 +500,7 @@ local function authz_keycloak_ensure_sa_access_token(conf)
+ authz_keycloak_refresh_token_expires_in(conf, json.refresh_expires_in)
end
- authz_keycloak_cache_set("access_tokens", token_endpoint .. ":" .. client_id,
+ authz_keycloak_cache_set("access-tokens", token_endpoint .. ":" .. client_id,
core.json.encode(session), ttl)
end
diff --git a/apisix/plugins/server-info.lua b/apisix/plugins/server-info.lua
index e775c6d..cc195cd 100644
--- a/apisix/plugins/server-info.lua
+++ b/apisix/plugins/server-info.lua
@@ -53,9 +53,9 @@ local attr_schema = {
}
}
-local internal_status = ngx.shared.internal_status
+local internal_status = ngx.shared["internal-status"]
if not internal_status then
- error("lua_shared_dict \"internal_status\" not configured")
+ error("lua_shared_dict \"internal-status\" not configured")
end
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 191d967..0e26b34 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -145,6 +145,10 @@ nginx_config: # config for render the template to generate n
#envs: # allow to get a list of environment variables
# - TEST_ENV
+ stream:
+ lua_shared_dict:
+ lrucache-lock-stream: 10m
+
# As user can add arbitrary configurations in the snippet,
# it is user's responsibility to check the configurations
# don't conflict with APISIX.
@@ -202,6 +206,27 @@ nginx_config: # config for render the template to generate n
# http://nginx.org/en/docs/http/ngx_http_charset_module.html#charset
variables_hash_max_size: 2048 # Sets the maximum size of the variables hash table.
+ lua_shared_dict:
+ internal-status: 10m
+ plugin-limit-req: 10m
+ plugin-limit-count: 10m
+ prometheus-metrics: 10m
+ plugin-limit-conn: 10m
+ upstream-healthcheck: 10m
+ worker-events: 10m
+ lrucache-lock: 10m
+ balancer-ewma: 10m
+ balancer-ewma-locks: 10m
+ balancer-ewma-last-touched-at: 10m
+ plugin-limit-count-redis-cluster-slot-lock: 1m
+ tracing_buffer: 10m
+ plugin-api-breaker: 10m
+ etcd-cluster-health-check: 10m
+ discovery: 1m
+ jwks: 1m
+ introspection: 10m
+ access-tokens: 1m
+
etcd:
host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
- "http://127.0.0.1:2379" # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 8d0f609..418ead0 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -299,7 +299,7 @@ _EOC_
$lua_deps_path
lua_socket_log_errors off;
- lua_shared_dict lrucache-lock-stream 10m;
+ lua_shared_dict lrucache-lock-stream 10m;
upstream apisix_backend {
server 127.0.0.1:1900;
@@ -414,24 +414,24 @@ _EOC_
$http_config .= <<_EOC_;
$lua_deps_path
- lua_shared_dict plugin-limit-req 10m;
- lua_shared_dict plugin-limit-count 10m;
- lua_shared_dict plugin-limit-conn 10m;
- lua_shared_dict prometheus-metrics 10m;
- lua_shared_dict internal_status 10m;
+ lua_shared_dict plugin-limit-req 10m;
+ lua_shared_dict plugin-limit-count 10m;
+ lua_shared_dict plugin-limit-conn 10m;
+ lua_shared_dict prometheus-metrics 10m;
+ lua_shared_dict internal-status 10m;
lua_shared_dict upstream-healthcheck 32m;
- lua_shared_dict worker-events 10m;
- lua_shared_dict lrucache-lock 10m;
- lua_shared_dict balancer_ewma 1m;
- lua_shared_dict balancer_ewma_locks 1m;
- lua_shared_dict balancer_ewma_last_touched_at 1m;
+ lua_shared_dict worker-events 10m;
+ lua_shared_dict lrucache-lock 10m;
+ lua_shared_dict balancer-ewma 1m;
+ lua_shared_dict balancer-ewma-locks 1m;
+ lua_shared_dict balancer-ewma-last-touched-at 1m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
- lua_shared_dict tracing_buffer 10m; # plugin skywalking
- lua_shared_dict access_tokens 1m; # plugin authz-keycloak
- lua_shared_dict discovery 1m; # plugin authz-keycloak
- lua_shared_dict plugin-api-breaker 10m;
- lua_capture_error_log 1m; # plugin error-log-logger
- lua_shared_dict etcd_cluster_health_check 10m; # etcd health check
+ lua_shared_dict tracing_buffer 10m; # plugin skywalking
+ lua_shared_dict access-tokens 1m; # plugin authz-keycloak
+ lua_shared_dict discovery 1m; # plugin authz-keycloak
+ lua_shared_dict plugin-api-breaker 10m;
+ lua_capture_error_log 1m; # plugin error-log-logger
+ lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
proxy_ssl_name \$upstream_host;
proxy_ssl_server_name on;
diff --git a/t/cli/test_main.sh b/t/cli/test_main.sh
index a0444ed..593dc68 100755
--- a/t/cli/test_main.sh
+++ b/t/cli/test_main.sh
@@ -680,3 +680,158 @@ if ! grep "variables_hash_max_size 1024;" conf/nginx.conf > /dev/null; then
fi
echo "passed: found the 'variables_hash_max_size 1024;' in nginx.conf"
+
+# check the stream lua_shared_dict lrucache_lock value
+git checkout conf/config.yaml
+
+echo '
+apisix:
+ stream_proxy:
+ tcp:
+ - addr: 9100
+ tls: true
+ - addr: "127.0.0.1:9101"
+ udp:
+ - 9200
+ - "127.0.0.1:9201"
+nginx_config:
+ stream:
+ lua_shared_dict:
+ lrucache-lock-stream: 20m
+' > conf/config.yaml
+
+make init
+
+if ! grep "lrucache-lock-stream 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'lrucache-lock-stream 20m;' not in nginx.conf"
+ exit 1
+fi
+
+echo "passed: found the 'lrucache-lock-stream 20m;' in nginx.conf"
+
+# check the http lua_shared_dict variables value
+git checkout conf/config.yaml
+
+echo '
+nginx_config:
+ http:
+ lua_shared_dict:
+ internal-status: 20m
+ plugin-limit-req: 20m
+ plugin-limit-count: 20m
+ prometheus-metrics: 20m
+ plugin-limit-conn: 20m
+ upstream-healthcheck: 20m
+ worker-events: 20m
+ lrucache-lock: 20m
+ balancer-ewma: 20m
+ balancer-ewma-locks: 20m
+ balancer-ewma-last-touched-at: 20m
+ plugin-limit-count-redis-cluster-slot-lock: 2m
+ tracing_buffer: 20m
+ plugin-api-breaker: 20m
+ etcd-cluster-health-check: 20m
+ discovery: 2m
+ jwks: 2m
+ introspection: 20m
+ access-tokens: 2m
+' > conf/config.yaml
+
+make init
+
+if ! grep "internal-status 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'internal-status 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "plugin-limit-req 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'plugin-limit-req 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "plugin-limit-count 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'plugin-limit-count 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "prometheus-metrics 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'prometheus-metrics 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "plugin-limit-conn 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'plugin-limit-conn 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "upstream-healthcheck 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'upstream-healthcheck 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "worker-events 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'worker-events 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "lrucache-lock 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'lrucache-lock 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "balancer-ewma 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'balancer-ewma 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "balancer-ewma-locks 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'balancer-ewma-locks 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "balancer-ewma-last-touched-at 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'balancer-ewma-last-touched-at 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "plugin-limit-count-redis-cluster-slot-lock 2m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'plugin-limit-count-redis-cluster-slot-lock 2m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "tracing_buffer 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'tracing_buffer 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "plugin-api-breaker 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'plugin-api-breaker 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "etcd-cluster-health-check 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'etcd-cluster-health-check 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "discovery 2m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'discovery 2m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "jwks 2m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'jwks 2m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "introspection 20m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'introspection 20m;' not in nginx.conf"
+ exit 1
+fi
+
+if ! grep "access-tokens 2m;" conf/nginx.conf > /dev/null; then
+ echo "failed: 'access-tokens 2m;' not in nginx.conf"
+ exit 1
+fi
+
+echo "passed: found the http lua_shared_dict related parameter in nginx.conf"