You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by mo...@apache.org on 2024/02/23 07:04:51 UTC
(apisix) branch master updated: feat(discovery): support endpointslices in kubernetes discovery (#10916)
This is an automated email from the ASF dual-hosted git repository.
monkeydluffy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 869d09515 feat(discovery): support endpointslices in kubernetes discovery (#10916)
869d09515 is described below
commit 869d095153df75fbe6e3edcf631340965fdd04ab
Author: dongjiang <do...@126.com>
AuthorDate: Fri Feb 23 15:04:46 2024 +0800
feat(discovery): support endpointslices in kubernetes discovery (#10916)
---
apisix/discovery/kubernetes/init.lua | 98 ++++++++-
apisix/discovery/kubernetes/schema.lua | 7 +
docs/en/latest/discovery/kubernetes.md | 8 +-
docs/zh/latest/discovery/kubernetes.md | 8 +-
t/kubernetes/discovery/kubernetes.t | 42 ++++
t/kubernetes/discovery/kubernetes3.t | 388 +++++++++++++++++++++++++++++++++
6 files changed, 542 insertions(+), 9 deletions(-)
diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua
index d16d4f4fc..84c6ddbcb 100644
--- a/apisix/discovery/kubernetes/init.lua
+++ b/apisix/discovery/kubernetes/init.lua
@@ -50,6 +50,69 @@ local function sort_nodes_cmp(left, right)
return left.port < right.port
end
+local function on_endpoint_slices_modified(handle, endpoint)
+ if handle.namespace_selector and
+ not handle:namespace_selector(endpoint.metadata.namespace) then
+ return
+ end
+
+ core.log.debug(core.json.delay_encode(endpoint))
+ core.table.clear(endpoint_buffer)
+
+ local endpointslices = endpoint.endpoints
+ for _, endpointslice in ipairs(endpointslices or {}) do
+ if endpointslice.addresses then
+ local addresses = endpointslices.addresses
+ for _, port in ipairs(endpoint.ports or {}) do
+ local port_name
+ if port.name then
+ port_name = port.name
+ elseif port.targetPort then
+ port_name = tostring(port.targetPort)
+ else
+ port_name = tostring(port.port)
+ end
+
+ if endpointslice.conditions and endpointslice.condition.ready then
+ local nodes = endpoint_buffer[port_name]
+ if nodes == nil then
+ nodes = core.table.new(0, #endpointslices * #addresses)
+ endpoint_buffer[port_name] = nodes
+ end
+
+ for _, address in ipairs(endpointslices.addresses) do
+ core.table.insert(nodes, {
+ host = address.ip,
+ port = port.port,
+ weight = handle.default_weight
+ })
+ end
+ end
+ end
+ end
+ end
+
+ for _, ports in pairs(endpoint_buffer) do
+ for _, nodes in pairs(ports) do
+ core.table.sort(nodes, sort_nodes_cmp)
+ end
+ end
+ local endpoint_key = endpoint.metadata.namespace .. "/" .. endpoint.metadata.name
+ local endpoint_content = core.json.encode(endpoint_buffer, true)
+ local endpoint_version = ngx.crc32_long(endpoint_content)
+
+ local _, err
+ _, err = handle.endpoint_dict:safe_set(endpoint_key .. "#version", endpoint_version)
+ if err then
+ core.log.error("set endpoint version into discovery DICT failed, ", err)
+ return
+ end
+ _, err = handle.endpoint_dict:safe_set(endpoint_key, endpoint_content)
+ if err then
+ core.log.error("set endpoint into discovery DICT failed, ", err)
+ handle.endpoint_dict:delete(endpoint_key .. "#version")
+ end
+end
local function on_endpoint_modified(handle, endpoint)
if handle.namespace_selector and
@@ -367,8 +430,13 @@ local function single_mode_init(conf)
end
local default_weight = conf.default_weight
-
- local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ local endpoints_informer, err
+ if conf.watch_endpoint_slices_schema then
+ endpoints_informer, err = informer_factory.new("discovery.k8s.io", "v1",
+ "EndpointSlice", "endpointslices", "")
+ else
+ endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ end
if err then
error(err)
return
@@ -377,8 +445,13 @@ local function single_mode_init(conf)
setup_namespace_selector(conf, endpoints_informer)
setup_label_selector(conf, endpoints_informer)
- endpoints_informer.on_added = on_endpoint_modified
- endpoints_informer.on_modified = on_endpoint_modified
+ if conf.watch_endpoint_slices_schema then
+ endpoints_informer.on_added = on_endpoint_slices_modified
+ endpoints_informer.on_modified = on_endpoint_slices_modified
+ else
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ end
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
@@ -463,7 +536,13 @@ local function multiple_mode_init(confs)
local default_weight = conf.default_weight
- local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ local endpoints_informer, err
+ if conf.watch_endpoint_slices_schema then
+ endpoints_informer, err = informer_factory.new("discovery.k8s.io", "v1",
+ "EndpointSlice", "endpointslices", "")
+ else
+ endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ end
if err then
error(err)
return
@@ -472,8 +551,13 @@ local function multiple_mode_init(confs)
setup_namespace_selector(conf, endpoints_informer)
setup_label_selector(conf, endpoints_informer)
- endpoints_informer.on_added = on_endpoint_modified
- endpoints_informer.on_modified = on_endpoint_modified
+ if conf.watch_endpoint_slices_schema then
+ endpoints_informer.on_added = on_endpoint_slices_modified
+ endpoints_informer.on_modified = on_endpoint_slices_modified
+ else
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ end
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
diff --git a/apisix/discovery/kubernetes/schema.lua b/apisix/discovery/kubernetes/schema.lua
index 170608f55..18df2a3b7 100644
--- a/apisix/discovery/kubernetes/schema.lua
+++ b/apisix/discovery/kubernetes/schema.lua
@@ -105,6 +105,11 @@ local shared_size_schema = {
default = "1m",
}
+local watch_endpoint_slices_schema = {
+ type = "boolean",
+ default = false,
+}
+
return {
anyOf = {
{
@@ -160,6 +165,7 @@ return {
label_selector = label_selector_schema,
default_weight = default_weight_schema,
shared_size = shared_size_schema,
+ watch_endpoint_slices = watch_endpoint_slices_schema,
},
},
{
@@ -202,6 +208,7 @@ return {
label_selector = label_selector_schema,
default_weight = default_weight_schema,
shared_size = shared_size_schema,
+ watch_endpoint_slices = watch_endpoint_slices_schema,
},
required = { "id", "service", "client" }
},
diff --git a/docs/en/latest/discovery/kubernetes.md b/docs/en/latest/discovery/kubernetes.md
index fb11aac67..5936613bf 100644
--- a/docs/en/latest/discovery/kubernetes.md
+++ b/docs/en/latest/discovery/kubernetes.md
@@ -91,6 +91,9 @@ discovery:
# reserved lua shared memory size,1m memory can store about 1000 pieces of endpoint
shared_size: 1m #default 1m
+
+ # if watch_endpoint_slices setting true, watch apiserver with endpointslices instead of endpoints
+ watch_endpoint_slices: false #default false
```
If the Kubernetes service discovery runs inside a pod, you can use minimal configuration:
@@ -220,6 +223,9 @@ discovery:
# reserved lua shared memory size,1m memory can store about 1000 pieces of endpoint
shared_size: 1m #default 1m
+
+ # if watch_endpoint_slices setting true, watch apiserver with endpointslices instead of endpoints
+ watch_endpoint_slices: false #default false
```
Multi-Kubernetes service discovery does not fill default values for service and client fields, you need to fill them according to the cluster configuration.
@@ -312,7 +318,7 @@ metadata:
name: apisix-test
rules:
- apiGroups: [ "" ]
- resources: [ endpoints ]
+ resources: [ endpoints,endpointslices ]
verbs: [ get,list,watch ]
---
diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md
index c4b751889..574e2c876 100644
--- a/docs/zh/latest/discovery/kubernetes.md
+++ b/docs/zh/latest/discovery/kubernetes.md
@@ -91,6 +91,9 @@ discovery:
# reserved lua shared memory size, 1m memory can store about 1000 pieces of endpoint
shared_size: 1m #default 1m
+
+ # if watch_endpoint_slices setting true, watch apiserver with endpointslices instead of endpoints
+ watch_endpoint_slices: false #default false
```
如果 Kubernetes 服务发现运行在 Pod 内，你可以使用如下最简配置：
@@ -219,6 +222,9 @@ discovery:
# reserved lua shared memory size,1m memory can store about 1000 pieces of endpoint
shared_size: 1m #default 1m
+
+ # if watch_endpoint_slices setting true, watch apiserver with endpointslices instead of endpoints
+ watch_endpoint_slices: false #default false
```
多集群模式 Kubernetes 服务发现没有为 `service` 和 `client` 域填充默认值，你需要根据集群配置情况自行填充。
@@ -310,7 +316,7 @@ metadata:
name: apisix-test
rules:
- apiGroups: [ "" ]
- resources: [ endpoints ]
+ resources: [ endpoints,endpointslices ]
verbs: [ get,list,watch ]
---
diff --git a/t/kubernetes/discovery/kubernetes.t b/t/kubernetes/discovery/kubernetes.t
index 24bf6f426..c4158de21 100644
--- a/t/kubernetes/discovery/kubernetes.t
+++ b/t/kubernetes/discovery/kubernetes.t
@@ -128,6 +128,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"shared_size": "1m",
"default_weight": 50
}
@@ -162,6 +163,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"shared_size": "1m",
"default_weight": 50
}
@@ -198,6 +200,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"shared_size": "2m",
"default_weight": 50
}
@@ -232,6 +235,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"shared_size": "1m",
"default_weight": 33
}
@@ -283,6 +287,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"default_weight": 50,
"shared_size": "1m"
},
@@ -296,6 +301,7 @@ GET /compare
"client": {
"token": "${KUBERNETES_CLIENT_TOKEN}"
},
+ "watch_endpoint_slices": false,
"default_weight": 33,
"shared_size": "2m"
}
@@ -304,3 +310,39 @@ GET /compare
Content-type: application/json
--- response_body
true
+
+
+
+=== TEST 6: set watch_endpoint_slices true and use kubernetes endpointslices api
+--- yaml_config
+apisix:
+ node_listen: 1984
+deployment:
+ role: data_plane
+ role_data_plane:
+ config_provider: yaml
+discovery:
+ kubernetes:
+ client:
+ token: ${KUBERNETES_CLIENT_TOKEN}
+ default_weight: 33
+ watch_endpoint_slices: true
+--- request
+GET /compare
+{
+ "service": {
+ "schema": "https",
+ "host": "${KUBERNETES_SERVICE_HOST}",
+ "port": "${KUBERNETES_SERVICE_PORT}"
+ },
+ "client": {
+ "token": "${KUBERNETES_CLIENT_TOKEN}"
+ },
+ "watch_endpoint_slices": true,
+ "shared_size": "1m",
+ "default_weight": 33
+}
+--- more_headers
+Content-type: application/json
+--- response_body
+true
diff --git a/t/kubernetes/discovery/kubernetes3.t b/t/kubernetes/discovery/kubernetes3.t
new file mode 100644
index 000000000..cebab5ba2
--- /dev/null
+++ b/t/kubernetes/discovery/kubernetes3.t
@@ -0,0 +1,388 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token";
+ our $token_value = eval {`cat $token_file 2>/dev/null`};
+
+ our $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+deployment:
+ role: data_plane
+ role_data_plane:
+ config_provider: yaml
+discovery:
+ kubernetes:
+ - id: first
+ service:
+ host: "127.0.0.1"
+ port: "6443"
+ client:
+ token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"
+ watch_endpoint_slices: true
+ - id: second
+ service:
+ schema: "http",
+ host: "127.0.0.1",
+ port: "6445"
+ client:
+ token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"
+ watch_endpoint_slices: true
+
+_EOC_
+
+ our $scale_ns_c = <<_EOC_;
+[
+ {
+ "op": "replace_endpointslices",
+ "name": "ep",
+ "namespace": "ns-c",
+ "endpoints": [
+ {
+ "addresses": [
+ "10.0.0.1"
+ ],
+ "conditions": {
+ "ready": true,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ }
+ ]
+ "ports": [
+ {
+ "name": "p1",
+ "port": 5001
+ }
+ ]
+ }
+]
+_EOC_
+
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+log_level('warn');
+no_root_location();
+no_shuffle();
+workers(4);
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ my $apisix_yaml = $block->apisix_yaml // <<_EOC_;
+routes: []
+#END
+_EOC_
+
+ $block->set_value("apisix_yaml", $apisix_yaml);
+
+ my $main_config = $block->main_config // <<_EOC_;
+env KUBERNETES_SERVICE_HOST=127.0.0.1;
+env KUBERNETES_SERVICE_PORT=6443;
+env KUBERNETES_CLIENT_TOKEN=$::token_value;
+env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file;
+_EOC_
+
+ $block->set_value("main_config", $main_config);
+
+ my $config = $block->config // <<_EOC_;
+ location /queries {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local d = require("apisix.discovery.kubernetes")
+
+ ngx.sleep(1)
+
+ ngx.req.read_body()
+ local request_body = ngx.req.get_body_data()
+ local queries = core.json.decode(request_body)
+ local response_body = "{"
+ for _,query in ipairs(queries) do
+ local nodes = d.nodes(query)
+ if nodes==nil or #nodes==0 then
+ response_body=response_body.." "..0
+ else
+ response_body=response_body.." "..#nodes
+ end
+ end
+ ngx.say(response_body.." }")
+ }
+ }
+
+ location /operators {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local core = require("apisix.core")
+ local ipairs = ipairs
+
+ ngx.req.read_body()
+ local request_body = ngx.req.get_body_data()
+ local operators = core.json.decode(request_body)
+
+ core.log.info("get body ", request_body)
+ core.log.info("get operators ", #operators)
+ for _, op in ipairs(operators) do
+ local method, path, body
+ local headers = {
+ ["Host"] = "127.0.0.1:6445"
+ }
+
+ if op.op == "replace_endpointslices" then
+ method = "PATCH"
+ path = "/apis/discovery.k8s.io/namespaces/" .. op.namespace .. "/endpointslices/" .. op.name
+ if #op.endpoints == 0 then
+ body = '[{"path":"/endpoints","op":"replace","value":[]}]'
+ else
+ local t = { { op = "replace", path = "/endpoints", value = op.endpoints } }
+ body = core.json.encode(t, true)
+ end
+ headers["Content-Type"] = "application/json-patch+json"
+ end
+
+ if op.op == "replace_labels" then
+ method = "PATCH"
+ path = "/apis/discovery.k8s.io/namespaces/" .. op.namespace .. "/endpointslices/" .. op.name
+ local t = { { op = "replace", path = "/metadata/labels", value = op.labels } }
+ body = core.json.encode(t, true)
+ headers["Content-Type"] = "application/json-patch+json"
+ end
+
+ local httpc = http.new()
+ core.log.info("begin to connect ", "127.0.0.1:6445")
+ local ok, message = httpc:connect({
+ scheme = "http",
+ host = "127.0.0.1",
+ port = 6445,
+ })
+ if not ok then
+ core.log.error("connect 127.0.0.1:6445 failed, message : ", message)
+ ngx.say("FAILED")
+ end
+ local res, err = httpc:request({
+ method = method,
+ path = path,
+ headers = headers,
+ body = body,
+ })
+ if err ~= nil then
+ core.log.err("operator k8s cluster error: ", err)
+ return 500
+ end
+ if res.status ~= 200 and res.status ~= 201 and res.status ~= 409 then
+ return res.status
+ end
+ end
+ ngx.say("DONE")
+ }
+ }
+
+_EOC_
+
+ $block->set_value("config", $config);
+
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: create namespace and endpoints
+--- yaml_config eval: $::yaml_config
+--- request
+POST /operators
+[
+ {
+ "op": "replace_endpointslices",
+ "namespace": "ns-a",
+ "name": "ep",
+ "endpoints": [
+ {
+ "addresses": [
+ "10.0.0.1",
+ "10.0.0.2"
+ ],
+ "conditions": {
+ "ready": true,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ },
+ {
+ "addresses": [
+ "20.0.0.1",
+ "20.0.0.2"
+ ],
+ "conditions": {
+ "ready": false,
+ "serving": false,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ }
+ ],
+ "ports": [
+ {
+ "name": "p",
+ "port": 5001
+ }
+ ]
+ },
+ {
+ "op": "create_namespace",
+ "name": "ns-b"
+ },
+ {
+ "op": "replace_endpointslices",
+ "namespace": "ns-b",
+ "name": "ep",
+ "endpoints": [
+ {
+ "addresses": [
+ "10.0.0.1",
+ "10.0.0.2"
+ ],
+ "conditions": {
+ "ready": true,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ },
+ {
+ "addresses": [
+ "20.0.0.1",
+ "20.0.0.2"
+ ],
+ "conditions": {
+ "ready": false,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ }
+ ],
+ "ports": [
+ {
+ "name": "p",
+ "port": 5002
+ }
+ ]
+ },
+ {
+ "op": "create_namespace",
+ "name": "ns-c"
+ },
+ {
+ "op": "replace_endpointslices",
+ "namespace": "ns-c",
+ "name": "ep",
+ "endpoints": [
+ {
+ "addresses": [
+ "10.0.0.1",
+ "10.0.0.2"
+ ],
+ "conditions": {
+ "ready": true,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ },
+ {
+ "addresses": [
+ "20.0.0.1",
+ "20.0.0.2"
+ ],
+ "conditions": {
+ "ready": true,
+ "serving": true,
+ "terminating": false
+ },
+ "nodeName": "kind-control-plane"
+ }
+ ],
+ "ports": [
+ {
+ "name": "p",
+ "port": 5003
+ }
+ ]
+ }
+]
+--- more_headers
+Content-type: application/json
+
+
+
+=== TEST 2: use default parameters
+--- yaml_config eval: $::yaml_config
+--- request
+GET /queries
+[
+ "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002",
+ "second/ns-a/ep:p1","second/ns-a/ep:p2","second/ns-b/ep:p1","second/ns-b/ep:p2","second/ns-c/ep:5001","second/ns-c/ep:5002"
+]
+--- more_headers
+Content-type: application/json
+--- response_body eval
+qr{ 0 0 2 2 0 0 0 0 2 2 0 0 }
+
+
+
+=== TEST 3: use specify environment parameters
+--- yaml_config
+apisix:
+ node_listen: 1984
+deployment:
+ role: data_plane
+ role_data_plane:
+ config_provider: yaml
+discovery:
+ kubernetes:
+ - id: first
+ service:
+ host: ${KUBERNETES_SERVICE_HOST}
+ port: ${KUBERNETES_SERVICE_PORT}
+ client:
+ token: ${KUBERNETES_CLIENT_TOKEN}
+ watch_endpoint_slices: true
+ - id: second
+ service:
+ schema: "http",
+ host: "127.0.0.1",
+ port: "6445"
+ client:
+ token: ${KUBERNETES_CLIENT_TOKEN}
+ watch_endpoint_slices: true
+
+--- request
+GET /queries
+[
+ "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002",
+ "second/ns-a/ep:p1","second/ns-a/ep:p2","second/ns-b/ep:p1","second/ns-b/ep:p2","second/ns-c/ep:5001","second/ns-c/ep:5002"
+]
+--- more_headers
+Content-type: application/json
+--- response_body eval
+qr{ 0 0 2 2 0 0 0 0 2 2 0 0 }