You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2022/09/11 10:36:28 UTC
[GitHub] [apisix] zhixiongdu027 opened a new pull request, #7895: feat: support multiple kubernets clusters discovery
zhixiongdu027 opened a new pull request, #7895:
URL: https://github.com/apache/apisix/pull/7895
### Description
<!-- Please include a summary of the change and which issue is fixed. -->
<!-- Please also include relevant motivation and context. -->
Fixes # (issue)
#7569 -> -- Support mulitple kubernets cluster
#7169 -> -- Remove possible extra spaces in client.token
### Checklist
- [y] I have explained the need for this PR and the problem it solves
- [y] I have explained the changes or the new features added to this PR
- [y] I have added tests corresponding to this change
- [y] I have updated the documentation to reflect this change
- [y] I have verified that this change is backward compatible (If not, please discuss on the [APISIX mailing list](https://github.com/apache/apisix/tree/master#community) first)
<!--
Note
1. Mark the PR as draft until it's ready to be reviewed.
2. Always add/update tests for any changes unless you have a good reason.
3. Always update the documentation to reflect the changes made in the PR.
4. Make a new commit to resolve conversations instead of `push -f`.
5. To resolve merge conflicts, merge master instead of rebasing.
6. Use "request review" to notify the reviewer after making changes.
7. Only a reviewer can mark a conversation as resolved.
-->
### Todo
Doc ...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969334255
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
sorry i didn't understand what you mean @tzssangglass
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969414888
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
Review Comment:
`"RuntimeException"` does not look like APISIX style logs
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1245129636
A discussion:
In the current implementation, single cluster and multiple clusters are implemented separately in the code.
Can we support only multiple clusters, the original single cluster just has a cluster of multiple clusters.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1256360531
> thanks, Will there be a release for 2.15.x support multiple kubernets clusters discovery ？ because apisix 2.x can't switch to version 3 seamlessly
No, functional PRs are not back-ported.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1249010494
I will asap, it may take three to four days
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r970199621
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
Review Comment:
I think it's best practice to keep things as they are
In the List-Watch process, there are many error messages that may appear in many Kubernetes responses, all of them are in this format (Reason and Message)
It's not necessary to change the raw error message content for "apisix style"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969433292
##########
apisix/cli/ngx_tpl.lua:
##########
@@ -238,8 +238,11 @@ http {
lua_shared_dict balancer-ewma-last-touched-at {* http.lua_shared_dict["balancer-ewma-last-touched-at"] *};
lua_shared_dict etcd-cluster-health-check {* http.lua_shared_dict["etcd-cluster-health-check"] *}; # etcd health check
- {% if enabled_discoveries["kubernetes"] then %}
- lua_shared_dict kubernetes {* http.lua_shared_dict["kubernetes"] *};
+ # for discovery shared dict
+ {% if discovery_shared_dict then %}
+ {% for key, size in pairs(discovery_shared_dict) do %}
+ lua_shared_dict {*key*} {*size*};
+ {% end %}
Review Comment:
We can add test cases to cover here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander merged pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
spacewander merged PR #7895:
URL: https://github.com/apache/apisix/pull/7895
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1246095917
> I mean 1.
>
Maybe this needs more people to join the discussion
> It's persuasive. Can you talk more about `performance loss`.
Additional id detection and table field access are required in the multiple_mode_nodes function, obviously there is a certain performance loss
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969049507
##########
apisix/discovery/kubernetes/schema.lua:
##########
@@ -28,113 +28,228 @@ local port_patterns = {
local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]]
local namespace_regex_pattern = [[^[\x21-\x7e]*$]]
+local shared_pattern = [[^[1-9][0-9]?m$]]
+
return {
- type = "object",
- properties = {
- service = {
+ anyOf = {
+ {
type = "object",
properties = {
- schema = {
- type = "string",
- enum = { "http", "https" },
- default = "https",
+ service = {
+ type = "object",
+ properties = {
+ schema = {
+ type = "string",
+ enum = { "http", "https" },
+ default = "https",
+ },
+ host = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_HOST}",
+ oneOf = host_patterns,
+ },
+ port = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_PORT}",
+ oneOf = port_patterns,
+ },
+ },
+ default = {
+ schema = "https",
+ host = "${KUBERNETES_SERVICE_HOST}",
+ port = "${KUBERNETES_SERVICE_PORT}",
+ }
},
- host = {
- type = "string",
- default = "${KUBERNETES_SERVICE_HOST}",
- oneOf = host_patterns,
+ client = {
+ type = "object",
+ properties = {
+ token = {
+ type = "string",
+ oneOf = {
+ { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
+ { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ },
+ },
+ token_file = {
+ type = "string",
+ pattern = [[^[^\:*?"<>|]*$]],
+ minLength = 1,
+ maxLength = 500,
+ }
+ },
+ default = {
+ token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ },
+ ["if"] = {
+ ["not"] = {
+ anyOf = {
+ { required = { "token" } },
+ { required = { "token_file" } },
+ }
+ }
+ },
+ ["then"] = {
+ properties = {
+ token_file = {
+ default = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ }
+ }
+ }
},
- port = {
- type = "string",
- default = "${KUBERNETES_SERVICE_PORT}",
- oneOf = port_patterns,
+ default_weight = {
+ type = "integer",
+ default = 50,
+ minimum = 0,
},
- },
- default = {
- schema = "https",
- host = "${KUBERNETES_SERVICE_HOST}",
- port = "${KUBERNETES_SERVICE_PORT}",
- }
- },
- client = {
- type = "object",
- properties = {
- token = {
- type = "string",
+ namespace_selector = {
+ type = "object",
+ properties = {
+ equal = {
+ type = "string",
+ pattern = namespace_pattern,
+ },
+ not_equal = {
+ type = "string",
+ pattern = namespace_pattern,
+ },
+ match = {
+ type = "array",
+ items = {
+ type = "string",
+ pattern = namespace_regex_pattern
+ },
+ minItems = 1
+ },
+ not_match = {
+ type = "array",
+ items = {
+ type = "string",
+ pattern = namespace_regex_pattern
+ },
+ minItems = 1
+ },
+ },
oneOf = {
- { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
- { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ { required = {} },
+ { required = { "equal" } },
+ { required = { "not_equal" } },
+ { required = { "match" } },
+ { required = { "not_match" } }
},
},
- token_file = {
+ label_selector = {
+ type = "string",
+ },
+ shared_size = {
type = "string",
- pattern = [[^[^\:*?"<>|]*$]],
- minLength = 1,
- maxLength = 500,
+ pattern = shared_pattern,
+ default = "1m",
}
},
- oneOf = {
- { required = { "token" } },
- { required = { "token_file" } },
- },
- default = {
- token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
- }
},
- default_weight = {
- type = "integer",
- default = 50,
- minimum = 0,
- },
- namespace_selector = {
- type = "object",
- properties = {
- equal = {
- type = "string",
- pattern = namespace_pattern,
- },
- not_equal = {
- type = "string",
- pattern = namespace_pattern,
- },
- match = {
- type = "array",
- items = {
+ {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "object",
+ properties = {
+ id = {
type = "string",
- pattern = namespace_regex_pattern
+ pattern = [[^[a-z0-9]{1,6}$]]
},
- minItems = 1
- },
- not_match = {
- type = "array",
- items = {
+ service = {
Review Comment:
Maybe not a good idea , because service had default value in single mode but no default value in multiple mode .
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969329996
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
we should update `config-default.yaml`, added `id` attribute
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1253166139
> i see lua_shared_dict kubernetes default is 1m , If there are many services in multiple cluster, it may not be enough, but it cannot be set very large. How to evaluate the memory usage of a service?
@wolgod
Please don't worry.
Usually, 1m of memory can store about 1000 endpoints information
And in multi-cluster mode, each cluster uses its own ngx.shared.DICT.
For example, if you define a release cluster and a debug cluster,
Then there will be two ngx.shared.DICT:
[ kubernetes-release , kubernetes-debug ] ,
and you can specify the memory size separately
In addition, you can use namespace_selector and label_selector to filter endpoint information to reduce memory
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969334255
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
sorry i didn't understand what you mean
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969372058
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
Ok ,get your point .
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1245153000
> "Can we support only multiple clusters"
Makes me think it has two meanings.
1: Abandon compatibility with the current single cluster configuration, users need to manually modify the previous configuration after the upgrade, otherwise it cannot be used
2: Compatible with the current single cluster configuration, but modify the code to treat the single cluster as a multi-cluster configuration with only one array element
I think you may mean 2,
However, there is no full benefit, but it brings performance loss.
Because the performance when dealing with multi-cluster configurations is lower than that of single-cluster configurations
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969208048
##########
apisix/discovery/kubernetes/schema.lua:
##########
@@ -28,113 +28,228 @@ local port_patterns = {
local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]]
local namespace_regex_pattern = [[^[\x21-\x7e]*$]]
+local shared_pattern = [[^[1-9][0-9]?m$]]
+
return {
- type = "object",
- properties = {
- service = {
+ anyOf = {
+ {
type = "object",
properties = {
- schema = {
- type = "string",
- enum = { "http", "https" },
- default = "https",
+ service = {
+ type = "object",
+ properties = {
+ schema = {
+ type = "string",
+ enum = { "http", "https" },
+ default = "https",
+ },
+ host = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_HOST}",
+ oneOf = host_patterns,
+ },
+ port = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_PORT}",
+ oneOf = port_patterns,
+ },
+ },
+ default = {
+ schema = "https",
+ host = "${KUBERNETES_SERVICE_HOST}",
+ port = "${KUBERNETES_SERVICE_PORT}",
+ }
},
- host = {
- type = "string",
- default = "${KUBERNETES_SERVICE_HOST}",
- oneOf = host_patterns,
+ client = {
+ type = "object",
+ properties = {
+ token = {
+ type = "string",
+ oneOf = {
+ { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
+ { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ },
+ },
+ token_file = {
+ type = "string",
+ pattern = [[^[^\:*?"<>|]*$]],
+ minLength = 1,
+ maxLength = 500,
+ }
+ },
+ default = {
+ token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ },
+ ["if"] = {
+ ["not"] = {
+ anyOf = {
+ { required = { "token" } },
+ { required = { "token_file" } },
+ }
+ }
+ },
+ ["then"] = {
+ properties = {
+ token_file = {
+ default = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ }
+ }
+ }
},
- port = {
- type = "string",
- default = "${KUBERNETES_SERVICE_PORT}",
- oneOf = port_patterns,
+ default_weight = {
+ type = "integer",
+ default = 50,
+ minimum = 0,
},
- },
- default = {
- schema = "https",
- host = "${KUBERNETES_SERVICE_HOST}",
- port = "${KUBERNETES_SERVICE_PORT}",
- }
- },
- client = {
- type = "object",
- properties = {
- token = {
- type = "string",
+ namespace_selector = {
Review Comment:
ok, I'll try to optimize the schema fully
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969435253
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
Review Comment:
People view APISIX logs from here, and we don't use their style of logs for every new ecology we dock.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
spacewander commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1248990364
@zhixiongdu027
We are going to release a new version next week. Would you like to add a doc before that?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1253449095
@wolgod
This is not a place for discussion, you should create a ISSUE
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
spacewander commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r968472517
##########
t/kubernetes/discovery/kubernetes2.t:
##########
@@ -0,0 +1,671 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token";
+ our $token_value = eval {`cat $token_file 2>/dev/null`};
+
+ our $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
Review Comment:
I found an one-line script to remove them:
`grep -lP 'config_center: yaml\n\s+enable_admin: false' t/*/*.t |xargs -r sed -i -r -z 's/\n\s+enable_admin: false\n/\n/g'`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969358470
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
here: https://github.com/apache/apisix/blob/2a44ba61b25224f585dc67caed7131d248cf9792/conf/config-default.yaml#L307-L336
only single_mode_nodes configuration here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] wolgod commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
wolgod commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1253406620
>
thanks, I wonder if the community will support apisix-seed service discovery for kubernetes discovery in the future?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
spacewander commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969192955
##########
apisix/discovery/kubernetes/schema.lua:
##########
@@ -28,113 +28,228 @@ local port_patterns = {
local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]]
local namespace_regex_pattern = [[^[\x21-\x7e]*$]]
+local shared_pattern = [[^[1-9][0-9]?m$]]
+
return {
- type = "object",
- properties = {
- service = {
+ anyOf = {
+ {
type = "object",
properties = {
- schema = {
- type = "string",
- enum = { "http", "https" },
- default = "https",
+ service = {
+ type = "object",
+ properties = {
+ schema = {
+ type = "string",
+ enum = { "http", "https" },
+ default = "https",
+ },
+ host = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_HOST}",
+ oneOf = host_patterns,
+ },
+ port = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_PORT}",
+ oneOf = port_patterns,
+ },
+ },
+ default = {
+ schema = "https",
+ host = "${KUBERNETES_SERVICE_HOST}",
+ port = "${KUBERNETES_SERVICE_PORT}",
+ }
},
- host = {
- type = "string",
- default = "${KUBERNETES_SERVICE_HOST}",
- oneOf = host_patterns,
+ client = {
+ type = "object",
+ properties = {
+ token = {
+ type = "string",
+ oneOf = {
+ { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
+ { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ },
+ },
+ token_file = {
+ type = "string",
+ pattern = [[^[^\:*?"<>|]*$]],
+ minLength = 1,
+ maxLength = 500,
+ }
+ },
+ default = {
+ token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ },
+ ["if"] = {
+ ["not"] = {
+ anyOf = {
+ { required = { "token" } },
+ { required = { "token_file" } },
+ }
+ }
+ },
+ ["then"] = {
+ properties = {
+ token_file = {
+ default = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ }
+ }
+ }
},
- port = {
- type = "string",
- default = "${KUBERNETES_SERVICE_PORT}",
- oneOf = port_patterns,
+ default_weight = {
+ type = "integer",
+ default = 50,
+ minimum = 0,
},
- },
- default = {
- schema = "https",
- host = "${KUBERNETES_SERVICE_HOST}",
- port = "${KUBERNETES_SERVICE_PORT}",
- }
- },
- client = {
- type = "object",
- properties = {
- token = {
- type = "string",
+ namespace_selector = {
Review Comment:
What about reusing the namespace_selector schema?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tzssangglass commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tzssangglass commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1245212030
> > "Can we support only multiple clusters"
>
> Makes me think it has two meanings.
>
> 1: Abandon compatibility with the current single cluster configuration, users need to manually modify the previous configuration after the upgrade, otherwise it cannot be used
>
> 2: Compatible with the current single cluster configuration, but modify the code to treat the single cluster as a multi-cluster configuration with only one array element
>
> I think you may mean 2,
>
> However, there is no full benefit, but it brings performance loss. Because the performance when dealing with multi-cluster configurations is lower than that of single-cluster configurations
I mean 1.
> but it brings performance loss.
> Because the performance when dealing with multi-cluster configurations is lower than that of single-cluster configurations
It's persuasive. Can you talk more about `performance loss`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969067184
##########
t/kubernetes/discovery/kubernetes2.t:
##########
@@ -0,0 +1,671 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token";
+ our $token_value = eval {`cat $token_file 2>/dev/null`};
+
+ our $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
Review Comment:
seems like remove "enable_admin: false" will cause all test case fail
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] tokers commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
tokers commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969421145
##########
apisix/cli/ops.lua:
##########
@@ -653,36 +653,52 @@ Please modify "admin_key" in conf/config.yaml .
end
end
- -- inject kubernetes discovery environment variable
+ -- inject kubernetes discovery shared dict and environment variable
if enabled_discoveries["kubernetes"] then
- local kubernetes_conf = yaml_conf.discovery["kubernetes"]
+ if not sys_conf["discovery_shared_dict"] then
Review Comment:
Use plural: `discovery_shared_dicts`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] wolgod commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
wolgod commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1253140482
i see lua_shared_dict kubernetes default is 1m , If there are many services in multiple cluster, it may not be enough, but it cannot be set very large. How to evaluate the memory usage of a service?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969416315
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
Review Comment:
But i think is kubernetes style
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] spacewander commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
spacewander commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r968363995
##########
apisix/discovery/kubernetes/schema.lua:
##########
@@ -28,113 +28,228 @@ local port_patterns = {
local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]]
local namespace_regex_pattern = [[^[\x21-\x7e]*$]]
+local shared_pattern = [[^[1-9][0-9]?m$]]
+
return {
- type = "object",
- properties = {
- service = {
+ anyOf = {
+ {
type = "object",
properties = {
- schema = {
- type = "string",
- enum = { "http", "https" },
- default = "https",
+ service = {
+ type = "object",
+ properties = {
+ schema = {
+ type = "string",
+ enum = { "http", "https" },
+ default = "https",
+ },
+ host = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_HOST}",
+ oneOf = host_patterns,
+ },
+ port = {
+ type = "string",
+ default = "${KUBERNETES_SERVICE_PORT}",
+ oneOf = port_patterns,
+ },
+ },
+ default = {
+ schema = "https",
+ host = "${KUBERNETES_SERVICE_HOST}",
+ port = "${KUBERNETES_SERVICE_PORT}",
+ }
},
- host = {
- type = "string",
- default = "${KUBERNETES_SERVICE_HOST}",
- oneOf = host_patterns,
+ client = {
+ type = "object",
+ properties = {
+ token = {
+ type = "string",
+ oneOf = {
+ { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
+ { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ },
+ },
+ token_file = {
+ type = "string",
+ pattern = [[^[^\:*?"<>|]*$]],
+ minLength = 1,
+ maxLength = 500,
+ }
+ },
+ default = {
+ token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ },
+ ["if"] = {
+ ["not"] = {
+ anyOf = {
+ { required = { "token" } },
+ { required = { "token_file" } },
+ }
+ }
+ },
+ ["then"] = {
+ properties = {
+ token_file = {
+ default = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+ }
+ }
+ }
},
- port = {
- type = "string",
- default = "${KUBERNETES_SERVICE_PORT}",
- oneOf = port_patterns,
+ default_weight = {
+ type = "integer",
+ default = 50,
+ minimum = 0,
},
- },
- default = {
- schema = "https",
- host = "${KUBERNETES_SERVICE_HOST}",
- port = "${KUBERNETES_SERVICE_PORT}",
- }
- },
- client = {
- type = "object",
- properties = {
- token = {
- type = "string",
+ namespace_selector = {
+ type = "object",
+ properties = {
+ equal = {
+ type = "string",
+ pattern = namespace_pattern,
+ },
+ not_equal = {
+ type = "string",
+ pattern = namespace_pattern,
+ },
+ match = {
+ type = "array",
+ items = {
+ type = "string",
+ pattern = namespace_regex_pattern
+ },
+ minItems = 1
+ },
+ not_match = {
+ type = "array",
+ items = {
+ type = "string",
+ pattern = namespace_regex_pattern
+ },
+ minItems = 1
+ },
+ },
oneOf = {
- { pattern = [[\${[_A-Za-z]([_A-Za-z0-9]*[_A-Za-z])*}$]] },
- { pattern = [[^[A-Za-z0-9+\/._=-]{0,4096}$]] },
+ { required = {} },
+ { required = { "equal" } },
+ { required = { "not_equal" } },
+ { required = { "match" } },
+ { required = { "not_match" } }
},
},
- token_file = {
+ label_selector = {
+ type = "string",
+ },
+ shared_size = {
type = "string",
- pattern = [[^[^\:*?"<>|]*$]],
- minLength = 1,
- maxLength = 500,
+ pattern = shared_pattern,
+ default = "1m",
}
},
- oneOf = {
- { required = { "token" } },
- { required = { "token_file" } },
- },
- default = {
- token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
- }
},
- default_weight = {
- type = "integer",
- default = 50,
- minimum = 0,
- },
- namespace_selector = {
- type = "object",
- properties = {
- equal = {
- type = "string",
- pattern = namespace_pattern,
- },
- not_equal = {
- type = "string",
- pattern = namespace_pattern,
- },
- match = {
- type = "array",
- items = {
+ {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "object",
+ properties = {
+ id = {
type = "string",
- pattern = namespace_regex_pattern
+ pattern = [[^[a-z0-9]{1,6}$]]
},
- minItems = 1
- },
- not_match = {
- type = "array",
- items = {
+ service = {
Review Comment:
We can save some repeated fields with something like `service = xxx.service`?
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +301,227 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.info("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multi_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
+ "failed to get lua_shared_dict: ngx.shared.",
+ "kubernetes-" .. id,
+ ", please check your APISIX version"
+ })
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multi_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multi_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
+ "failed to get lua_shared_dict: ngx.shared.",
+ "kubernetes-" .. id,
+ ", please check your APISIX version"
+ })
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ core.log.debug("get informer ", core.json.encode(endpoints_informer, true))
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ core.log.debug("start fetch ", id, " ", core.json.encode(item, true), item.kind)
+ start_fetch(item)
+ end
+end
+
+
+local function multi_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.info("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ core.log.info("ctx: ", core.json.encode(ctx, true), match[1])
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.info("id not exis")
Review Comment:
```suggestion
core.log.info("id not exist")
```
##########
t/kubernetes/discovery/kubernetes2.t:
##########
@@ -0,0 +1,671 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+ our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token";
+ our $token_value = eval {`cat $token_file 2>/dev/null`};
+
+ our $yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ config_center: yaml
+ enable_admin: false
Review Comment:
We don't need to disable admin when config_center is yaml. Many existing yaml relative tests have this configuration, but we can remove it from the new test.
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +301,227 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.info("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multi_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
+ "failed to get lua_shared_dict: ngx.shared.",
+ "kubernetes-" .. id,
+ ", please check your APISIX version"
+ })
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multi_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multi_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
Review Comment:
I would recommend using `string.format` which is more common
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +301,227 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.info("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multi_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
+ "failed to get lua_shared_dict: ngx.shared.",
+ "kubernetes-" .. id,
+ ", please check your APISIX version"
+ })
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multi_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multi_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(core.table.concat {
+ "failed to get lua_shared_dict: ngx.shared.",
+ "kubernetes-" .. id,
+ ", please check your APISIX version"
+ })
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ core.log.debug("get informer ", core.json.encode(endpoints_informer, true))
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ core.log.debug("start fetch ", id, " ", core.json.encode(item, true), item.kind)
+ start_fetch(item)
+ end
+end
+
+
+local function multi_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.info("get unexpected upstream service_name: ", service_name)
Review Comment:
We should use `error` level log?
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -29,10 +29,9 @@ local core = require("apisix.core")
local util = require("apisix.cli.util")
local local_conf = require("apisix.core.config_local").local_conf()
local informer_factory = require("apisix.discovery.kubernetes.informer_factory")
+local setmetatable = setmetatable
Review Comment:
Better to move the `setmetatable` together with `pcall` and others.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on a diff in pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on code in PR #7895:
URL: https://github.com/apache/apisix/pull/7895#discussion_r969372058
##########
apisix/discovery/kubernetes/init.lua:
##########
@@ -298,88 +302,217 @@ local function create_endpoint_lrucache(endpoint_key, endpoint_port)
return endpoint[endpoint_port]
end
+
local _M = {
version = "0.0.1"
}
-function _M.nodes(service_name)
- local pattern = "^(.*):(.*)$" -- namespace/name:port_name
- local match = ngx.re.match(service_name, pattern, "jo")
- if not match then
- core.log.info("get unexpected upstream service_name: ", service_name)
- return nil
- end
- local endpoint_key = match[1]
- local endpoint_port = match[2]
- local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
- if not endpoint_version then
- core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
- return nil
- end
+local function start_fetch(handle)
+ local timer_runner
+ timer_runner = function(premature)
+ if premature then
+ return
+ end
- return endpoint_lrucache(service_name, endpoint_version,
- create_endpoint_lrucache, endpoint_key, endpoint_port)
+ local ok, status = pcall(handle.list_watch, handle, handle.apiserver)
+
+ local retry_interval = 0
+ if not ok then
+ core.log.error("list_watch failed, kind: ", handle.kind,
+ ", reason: ", "RuntimeException", ", message : ", status)
+ retry_interval = 40
+ elseif not status then
+ retry_interval = 40
+ end
+
+ ngx.timer.at(retry_interval, timer_runner)
+ end
+ ngx.timer.at(0, timer_runner)
end
-function _M.init_worker()
- endpoint_dict = ngx.shared.kubernetes
+local function single_mode_init(conf)
+ local endpoint_dict = ngx.shared.kubernetes
if not endpoint_dict then
- error("failed to get lua_shared_dict: kubernetes, please check your APISIX version")
+ error("failed to get lua_shared_dict: ngx.shared.kubernetes, " ..
+ "please check your APISIX version")
end
if process.type() ~= "privileged agent" then
+ ctx = endpoint_dict
return
end
- local discovery_conf = local_conf.discovery.kubernetes
-
- default_weight = discovery_conf.default_weight
-
- local apiserver, err = get_apiserver(discovery_conf)
+ local apiserver, err = get_apiserver(conf)
if err then
error(err)
return
end
- local endpoints_informer, err = informer_factory.new("", "v1",
- "Endpoints", "endpoints", "")
+ local default_weight = conf.default_weight
+
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
if err then
error(err)
return
end
- setup_namespace_selector(discovery_conf, endpoints_informer)
- setup_label_selector(discovery_conf, endpoints_informer)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
endpoints_informer.on_added = on_endpoint_modified
endpoints_informer.on_modified = on_endpoint_modified
endpoints_informer.on_deleted = on_endpoint_deleted
endpoints_informer.pre_list = pre_list
endpoints_informer.post_list = post_list
- local timer_runner
- timer_runner = function(premature)
- if premature then
+ ctx = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
+
+ start_fetch(ctx)
+end
+
+
+local function single_mode_nodes(service_name)
+ local pattern = "^(.*):(.*)$" -- namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local endpoint_dict = ctx
+ local endpoint_key = match[1]
+ local endpoint_port = match[2]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+local function multiple_mode_worker_init(confs)
+ for _, conf in ipairs(confs) do
+
+ local id = conf.id
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ ctx[id] = endpoint_dict
+ end
+end
+
+
+local function multiple_mode_init(confs)
+ ctx = core.table.new(#confs, 0)
+
+ if process.type() ~= "privileged agent" then
+ multiple_mode_worker_init(confs)
+ return
+ end
+
+ for _, conf in ipairs(confs) do
+ local id = conf.id
+
+ if ctx[id] then
+ error("duplicate id value")
+ end
+
+ local endpoint_dict = ngx.shared["kubernetes-" .. id]
+ if not endpoint_dict then
+ error(string.format("failed to get lua_shared_dict: ngx.shared.kubernetes-%s, ", id) ..
+ "please check your APISIX version")
+ end
+
+ local apiserver, err = get_apiserver(conf)
+ if err then
+ error(err)
return
end
- local ok, status = pcall(endpoints_informer.list_watch, endpoints_informer, apiserver)
+ local default_weight = conf.default_weight
- local retry_interval = 0
- if not ok then
- core.log.error("list_watch failed, kind: ", endpoints_informer.kind,
- ", reason: ", "RuntimeException", ", message : ", status)
- retry_interval = 40
- elseif not status then
- retry_interval = 40
+ local endpoints_informer, err = informer_factory.new("", "v1", "Endpoints", "endpoints", "")
+ if err then
+ error(err)
+ return
end
- ngx.timer.at(retry_interval, timer_runner)
+ setup_namespace_selector(conf, endpoints_informer)
+ setup_label_selector(conf, endpoints_informer)
+
+ endpoints_informer.on_added = on_endpoint_modified
+ endpoints_informer.on_modified = on_endpoint_modified
+ endpoints_informer.on_deleted = on_endpoint_deleted
+ endpoints_informer.pre_list = pre_list
+ endpoints_informer.post_list = post_list
+
+ ctx[id] = setmetatable({
+ endpoint_dict = endpoint_dict,
+ apiserver = apiserver,
+ default_weight = default_weight
+ }, { __index = endpoints_informer })
end
- ngx.timer.at(0, timer_runner)
+ for id, item in pairs(ctx) do
+ start_fetch(item)
+ end
+end
+
+
+local function multiple_mode_nodes(service_name)
+ local pattern = "^(.*)/(.*/.*):(.*)$" -- id/namespace/name:port_name
+ local match = ngx.re.match(service_name, pattern, "jo")
+ if not match then
+ core.log.error("get unexpected upstream service_name: ", service_name)
+ return nil
+ end
+
+ local id = match[1]
+ local endpoint_dict = ctx[id]
+ if not endpoint_dict then
+ core.log.error("id not exist")
+ return nil
+ end
+
+ local endpoint_key = match[2]
+ local endpoint_port = match[3]
+ local endpoint_version = endpoint_dict:get_stale(endpoint_key .. "#version")
+ if not endpoint_version then
+ core.log.info("get empty endpoint version from discovery DICT ", endpoint_key)
+ return nil
+ end
+
+ return endpoint_lrucache(service_name, endpoint_version,
+ create_endpoint_lrucache, endpoint_dict, endpoint_key, endpoint_port)
+end
+
+
+function _M.init_worker()
+ local discovery_conf = local_conf.discovery.kubernetes
Review Comment:
Ok ,get your point .
This part will be submitted together with the documentation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] zhixiongdu027 commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
zhixiongdu027 commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1242947194
@spacewander @tokers @tzssangglass
Hello:
I think the code part is already ready for review,
The documentation part will be submitted later
Please help for review ,Many TKS.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [apisix] wolgod commented on pull request #7895: feat: support multiple kubernets clusters discovery
Posted by GitBox <gi...@apache.org>.
wolgod commented on PR #7895:
URL: https://github.com/apache/apisix/pull/7895#issuecomment-1256128640
thanks, Will there be a release for 2.15.x support multiple kubernets clusters discovery ？ because apisix 2.x can't switch to version 3 seamlessly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org