You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/03/21 00:46:33 UTC
[apisix] branch master updated: feat: improve kubernetes discovery (#6663)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new a47dab7 feat: improve kubernetes discovery (#6663)
a47dab7 is described below
commit a47dab7eecd90998499eae10b477ce7fd8175c28
Author: zhixiongdu <ro...@libssl.com>
AuthorDate: Mon Mar 21 08:46:26 2022 +0800
feat: improve kubernetes discovery (#6663)
---
apisix/cli/ngx_tpl.lua | 4 +
apisix/cli/ops.lua | 6 +
apisix/discovery/kubernetes/informer_factory.lua | 4 +-
apisix/discovery/kubernetes/init.lua | 15 +-
conf/config-default.yaml | 1 +
docs/en/latest/config.json | 3 +-
docs/en/latest/discovery/kubernetes.md | 188 +++++++++++++++++++++++
docs/zh/latest/discovery/kubernetes.md | 106 ++++++++++---
t/APISIX.pm | 1 +
9 files changed, 296 insertions(+), 32 deletions(-)
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 4f1c8c2..b1ec2bb 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -190,6 +190,10 @@ http {
lua_shared_dict balancer-ewma-last-touched-at {* http.lua_shared_dict["balancer-ewma-last-touched-at"] *};
lua_shared_dict etcd-cluster-health-check {* http.lua_shared_dict["etcd-cluster-health-check"] *}; # etcd health check
+ {% if enabled_discoveries["kubernetes"] then %}
+ lua_shared_dict kubernetes {* http.lua_shared_dict["kubernetes"] *};
+ {% end %}
+
{% if enabled_plugins["limit-conn"] then %}
lua_shared_dict plugin-limit-conn {* http.lua_shared_dict["plugin-limit-conn"] *};
{% end %}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index 7208e5c..2844126 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -251,6 +251,11 @@ Please modify "admin_key" in conf/config.yaml .
use_apisix_openresty = false
end
+ local enabled_discoveries = {}
+ for name in pairs(yaml_conf.discovery or {}) do
+ enabled_discoveries[name] = true
+ end
+
local enabled_plugins = {}
for i, name in ipairs(yaml_conf.plugins or {}) do
enabled_plugins[name] = true
@@ -528,6 +533,7 @@ Please modify "admin_key" in conf/config.yaml .
with_module_status = with_module_status,
use_apisix_openresty = use_apisix_openresty,
error_log = {level = "warn"},
+ enabled_discoveries = enabled_discoveries,
enabled_plugins = enabled_plugins,
enabled_stream_plugins = enabled_stream_plugins,
dubbo_upstream_multiplex_count = dubbo_upstream_multiplex_count,
diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua
index 8b50fc3..a03f27a 100644
--- a/apisix/discovery/kubernetes/informer_factory.lua
+++ b/apisix/discovery/kubernetes/informer_factory.lua
@@ -23,8 +23,6 @@ local type = type
local core = require("apisix.core")
local http = require("resty.http")
-local empty_table = {}
-
local function list_query(informer)
local arguments = {
limit = informer.limit,
@@ -81,7 +79,7 @@ local function list(httpc, apiserver, informer)
informer.version = data.metadata.resourceVersion
if informer.on_added then
- for _, item in ipairs(data.items or empty_table) do
+ for _, item in ipairs(data.items or {}) do
informer:on_added(item, "list")
end
end
diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua
index ba83588..a0491be 100644
--- a/apisix/discovery/kubernetes/init.lua
+++ b/apisix/discovery/kubernetes/init.lua
@@ -31,6 +31,7 @@ local local_conf = require("apisix.core.config_local").local_conf()
local informer_factory = require("apisix.discovery.kubernetes.informer_factory")
local endpoint_dict
+
local default_weight
local endpoint_lrucache = core.lrucache.new({
@@ -39,7 +40,6 @@ local endpoint_lrucache = core.lrucache.new({
})
local endpoint_buffer = {}
-local empty_table = {}
local function sort_nodes_cmp(left, right)
if left.host ~= right.host then
@@ -60,10 +60,10 @@ local function on_endpoint_modified(informer, endpoint)
core.table.clear(endpoint_buffer)
local subsets = endpoint.subsets
- for _, subset in ipairs(subsets or empty_table) do
+ for _, subset in ipairs(subsets or {}) do
if subset.addresses then
local addresses = subset.addresses
- for _, port in ipairs(subset.ports or empty_table) do
+ for _, port in ipairs(subset.ports or {}) do
local port_name
if port.name then
port_name = port.name
@@ -166,7 +166,7 @@ local function setup_namespace_selector(conf, informer)
local match = conf.namespace_selector.match
local m, err
for _, v in ipairs(match) do
- m, err = ngx.re.match(namespace, v, "j")
+ m, err = ngx.re.match(namespace, v, "jo")
if m and m[0] == namespace then
return true
end
@@ -324,10 +324,9 @@ end
function _M.init_worker()
- -- TODO: maybe we can read dict name from discovery config
- endpoint_dict = ngx.shared.discovery
+ endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get nginx shared dict: discovery, please check your APISIX version")
+ error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
end
if process.type() ~= "privileged agent" then
@@ -336,7 +335,7 @@ function _M.init_worker()
local discovery_conf = local_conf.discovery.kubernetes
- default_weight = discovery_conf.default_weight or 50
+ default_weight = discovery_conf.default_weight
local apiserver, err = get_apiserver(discovery_conf)
if err then
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 450b6f1..d42ff21 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -264,6 +264,7 @@ nginx_config: # config for render the template to generate n
introspection: 10m
access-tokens: 1m
ext-plugin: 1m
+ kubernetes: 1m
etcd:
host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index 0bf31b8..5c03681 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -206,7 +206,8 @@
"discovery/dns",
"discovery/consul_kv",
"discovery/nacos",
- "discovery/eureka"
+ "discovery/eureka",
+ "discovery/kubernetes"
]
},
{
diff --git a/docs/en/latest/discovery/kubernetes.md b/docs/en/latest/discovery/kubernetes.md
new file mode 100644
index 0000000..0bf7431
--- /dev/null
+++ b/docs/en/latest/discovery/kubernetes.md
@@ -0,0 +1,188 @@
+---
+title: Kubernetes
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Summary
+
+The [_Kubernetes_](https://kubernetes.io/) service discovery [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) real-time changes of [_Endpoints_](https://kubernetes.io/docs/concepts/services-networking/service/) resources,
+then store theirs value into ngx.shared.kubernetes \
+Discovery also provides a query interface in accordance with the [_APISIX Discovery Specification_](https://github.com/apache/apisix/blob/master/docs/en/latest/discovery.md)
+
+## Configuration
+
+A detailed configuration for the kubernetes service discovery is as follows:
+
+```yaml
+discovery:
+ kubernetes:
+ service:
+ # apiserver schema, options [http, https]
+ schema: https #default https
+
+ # apiserver host, options [ipv4, ipv6, domain, environment variable]
+ host: ${KUBERNETES_SERVICE_HOST} #default ${KUBERNETES_SERVICE_HOST}
+
+ # apiserver port, options [port number, environment variable]
+ port: ${KUBERNETES_SERVICE_PORT} #default ${KUBERNETES_SERVICE_PORT}
+
+ client:
+ # serviceaccount token or token_file
+ token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+ #token: |-
+ # eyJhbGciOiJSUzI1NiIsImtpZCI6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEif
+ # 6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEifeyJhbGciOiJSUzI1NiIsImtpZCI
+
+ # kubernetes discovery plugin support use namespace_selector
+ # you can use one of [equal, not_equal, match, not_match] filter namespace
+ namespace_selector:
+ # only save endpoints with namespace equal default
+ equal: default
+
+ # only save endpoints with namespace not equal default
+ #not_equal: default
+
+ # only save endpoints with namespace match one of [default, ^my-[a-z]+$]
+ #match:
+ #- default
+ #- ^my-[a-z]+$
+
+ # only save endpoints with namespace not match one of [default, ^my-[a-z]+$ ]
+ #not_match:
+ #- default
+ #- ^my-[a-z]+$
+
+ # kubernetes discovery plugin support use label_selector
+ # for the expression of label_selector, please refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels
+ label_selector: |-
+ first="a",second="b"
+```
+
+If the kubernetes service discovery runs inside a pod, you can use minimal configuration:
+
+```yaml
+discovery:
+ kubernetes: { }
+```
+
+If the kubernetes service discovery runs outside a pod, you need to create or select a specified [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/),
+then get its token value, and use following configuration:
+
+```yaml
+discovery:
+ kubernetes:
+ service:
+ schema: https
+ host: # enter apiserver host value here
+ port: # enter apiserver port value here
+ client:
+ token: # enter serviceaccount token value here
+ #token_file: # enter file path here
+```
+
+## Interface
+
+the kubernetes service discovery provides a query interface in accordance with the [_APISIX Discovery Specification_](https://github.com/apache/apisix/blob/master/docs/en/latest/discovery.md)
+
+**function:** \
+ nodes(service_name)
+
+**description:** \
+ nodes() function attempts to look up the ngx.shared.kubernetes for nodes corresponding to service_name, \
+ service_name should match pattern: _[namespace]/[name]:[portName]_
+
+ + namespace: The namespace where the kubernetes endpoints is located
+
+ + name: The name of the kubernetes endpoints
+
+ + portName: The portName of the kubernetes endpoints, if there is no portName, use targetPort, port instead
+
+**return value:** \
+ if the kubernetes endpoints value is as follows:
+
+ ```yaml
+ apiVersion: v1
+ kind: Endpoints
+ metadata:
+ name: plat-dev
+ namespace: default
+ subsets:
+ - addresses:
+ - ip: "10.5.10.109"
+ - ip: "10.5.10.110"
+ ports:
+ - port: 3306
+ ```
+
+ a nodes("default/plat-dev:3306") call will get follow result:
+
+ ```
+ {
+ {
+ host="10.5.10.109",
+ port= 3306,
+ weight= 50,
+ },
+ {
+ host="10.5.10.110",
+ port= 3306,
+ weight= 50,
+ },
+ }
+ ```
+
+## Q&A
+
+> Q: Why only support configuration token to access _Kubernetes APIServer_ \
+> A: Usually, we will use three ways to complete the authentication of _Kubernetes APIServer_:
+>
+>+ mTLS
+>+ token
+>+ basic authentication
+>
+> Because lua-resty-http does not currently support mTLS, and basic authentication is not recommended,\
+> So currently only the token authentication method is implemented
+
+---
+
+> Q: APISIX inherits Nginx's multiple process model, does it mean that each nginx worker process will [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) kubernetes endpoints resources \
+> A: The kubernetes service discovery only uses privileged processes to [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) kubernetes endpoints resources, then store theirs value \
+> into ngx.shared.kubernetes, worker processes get results by querying ngx.shared.kubernetes
+
+---
+
+> Q: How to get [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) token value \
+> A: Assume your [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) located in namespace apisix and name is kubernetes-discovery, you can use the following steps to get token value
+>
+> 1. Get secret name: \
+> you can execute the following command, the output of the first column is the secret name we want
+>
+> ```shell
+> kubectl -n apisix get secrets | grep kubernetes-discovery
+> ```
+>
+> 2. Get token value: \
+> assume secret resources name is kubernetes-discovery-token-c64cv, you can execute the following command, the output is the service account token value we want
+>
+> ```shell
+> kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d
+> ```
diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md
index 9ef14ff..3185fed 100644
--- a/docs/zh/latest/discovery/kubernetes.md
+++ b/docs/zh/latest/discovery/kubernetes.md
@@ -1,3 +1,7 @@
+---
+title: Kubernetes
+---
+
<!--
#
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,14 +21,15 @@
#
-->
-# 基于 Kubernetes 的服务发现
+## 基于 Kubernetes 的服务发现
-Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群 v1.endpoints 的实时变化,
-并将其值存储在 ngx.shared.dict 中, 同时遵循 APISIX Discovery 规范提供查询接口
+Kubernetes 服务发现模块以 [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts) 方式监听 [_Kubernetes_](https://kubernetes.io) 集群 [_Endpoints_](https://kubernetes.io/docs/concepts/services-networking/service) 资源的实时变化,
+并将其值存储到 ngx.shared.kubernetes 中 \
+模块同时遵循 [_APISIX Discovery 规范_](https://github.com/apache/apisix/blob/master/docs/zh/latest/discovery.md) 提供了节点查询接口
-# Kubernetes 服务发现插件的配置
+## Kubernetes 服务发现模块的配置
-Kubernetes 服务发现插件的样例配置如下:
+Kubernetes 服务发现模块的完整配置如下:
```yaml
discovery:
@@ -72,14 +77,14 @@ discovery:
first="a",second="b"
```
-如果 Kubernetes 服务插件运行在 Pod 内, 你可以使用最简配置:
+如果 Kubernetes 服务发现模块运行在 Pod 内, 你可以使用最简配置:
```yaml
discovery:
kubernetes: { }
```
-如果 Kubernetes 服务插件运行在 Pod 外, 你需要新建或选取指定的 ServiceAccount, 获取其 Token 值, 并使用如下配置:
+如果 Kubernetes 服务发现模块运行在 Pod 外, 你需要新建或选取指定的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), 获取其 Token 值, 然后使用如下配置:
```yaml
discovery:
@@ -93,28 +98,89 @@ discovery:
#token_file: # enter file path here
```
-# Kubernetes 服务发现插件的使用
+## Kubernetes 服务发现模块的查询接口
+
+Kubernetes 服务发现模块遵循 [_APISIX Discovery 规范_](https://github.com/apache/apisix/blob/master/docs/zh/latest/discovery.md) 提供查询接口
+
+**函数:**
+ nodes(service_name)
+
+**说明:**
+ service_name 必须满足格式: [namespace]/[name]:[portName]
+
+ + namespace: Endpoints 所在的命名空间
+
+ + name: Endpoints 的资源名
+
+ + portName: Endpoints 定义包含的 portName, 如果 Endpoints 没有定义 portName, 请使用 targetPort,Port 代替
+
+**返回值:**
+ 以如下 Endpoints 为例:
-Kubernetes 服务发现插件提供与其他服务发现插件相同的查询接口 -> nodes(service_name) \
-service_name 的 pattern 如下:
-> _[namespace]/[name]:[portName]_
+ ```yaml
+ apiVersion: v1
+ kind: Endpoints
+ metadata:
+ name: plat-dev
+ namespace: default
+ subsets:
+ - addresses:
+ - ip: "10.5.10.109"
+ - ip: "10.5.10.110"
+ ports:
+ - port: 3306
+ ```
-如果 kubernetes Endpoint 没有定义 portName, Kubernetes 服务发现插件会依次使用 targetPort, port 代替
+ nodes("default/plat-dev:3306") 调用会得到如下的返回值:
-# Q&A
+ ```
+ {
+ {
+ host="10.5.10.109",
+ port= 3306,
+ weight= 50,
+ },
+ {
+ host="10.5.10.110",
+ port= 3306,
+ weight= 50,
+ },
+ }
+ ```
-> Q: 为什么只支持配置 token 来访问 Kubernetes ApiServer \
-> A: 通常情况下,我们会使用三种方式与 Kubernetes ApiServer 通信 :
+## Q&A
+
+> Q: 为什么只支持配置 token 来访问 Kubernetes APIServer \
+> A: 一般情况下,我们有三种方式可以完成与 Kubernetes APIServer 的认证:
>
>+ mTLS
>+ token
>+ basic authentication
>
-> 因为 lua-resty-http 目前不支持 mTLS, 以及 basic authentication 不被推荐使用,\
+> 因为 lua-resty-http 目前不支持 mTLS, basic authentication 不被推荐使用,\
> 所以当前只实现了 token 认证方式
--------
+---
+
+> Q: APISIX 继承了 Nginx 的多进程模型, 是否意味着每个 APISIX 工作进程都会监听 Kubernetes Endpoints \
+> A: Kubernetes 服务发现模块只使用特权进程监听 Kubernetes Endpoints, 然后将其值存储\
+> 到 ngx.shared.kubernetes, 工作进程通过查询 ngx.shared.kubernetes 来获取结果
-> Q: APISIX 是多进程模型, 是否意味着每个 APISIX 工作进程都会监听 Kubernetes v1.endpoints \
-> A: Kubernetes 服务发现插件只使用特权进程监听 Kubernetes v1.endpoints, 然后将结果存储\
-> 在 ngx.shared.dict 中, 业务进程是通过查询 ngx.shared.dict 来获取结果的
+---
+
+> Q: 怎样获取指定 ServiceAccount 的 Token 值 \
+> A: 假定你指定的 ServiceAccount 资源名为 “kubernetes-discovery“, 命名空间为 “apisix”, 请按如下步骤获取其 Token 值
+>
+> 1. 获取 _Secret_ 资源名: \
+> 执行以下命令, 输出的第一列内容就是目标 _Secret_ 资源名
+>
+> ```shell
+> kubectl -n apisix get secrets | grep kubernetes-discovery
+> ```
+>
+> 2. 获取 Token 值: \
+> 假定你获取到的 _Secret_ 资源名为 "kubernetes-discovery-token-c64cv", 执行以下命令, 输出内容就是目标 Token 值
+>
+> ```shell
+> kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d
+> ```
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 671ffe3..b9f7089 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -507,6 +507,7 @@ _EOC_
lua_capture_error_log 1m; # plugin error-log-logger
lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
lua_shared_dict ext-plugin 1m;
+ lua_shared_dict kubernetes 1m;
proxy_ssl_name \$upstream_host;
proxy_ssl_server_name on;