You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by as...@apache.org on 2025/05/21 06:44:12 UTC
(apisix) branch master updated: feat: add healthcheck for sync configuration (#12200)
This is an automated email from the ASF dual-hosted git repository.
ashishtiwari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new e430a2bd6 feat: add healthcheck for sync configuration (#12200)
e430a2bd6 is described below
commit e430a2bd6bffa6719e30f149af1dc3833d7e39b9
Author: Ashish Tiwari <as...@gmail.com>
AuthorDate: Wed May 21 12:14:06 2025 +0530
feat: add healthcheck for sync configuration (#12200)
---
apisix/cli/config.lua | 1 +
apisix/cli/ngx_tpl.lua | 20 ++++++
apisix/cli/ops.lua | 8 +++
apisix/core/config_etcd.lua | 24 +++++++-
apisix/core/config_yaml.lua | 21 ++++++-
apisix/init.lua | 56 +++++++++++++++++
conf/config.yaml.example | 5 +-
docs/en/latest/config.json | 4 ++
docs/en/latest/status-api.md | 78 ++++++++++++++++++++++++
docs/zh/latest/config.json | 4 ++
docs/zh/latest/status-api.md | 78 ++++++++++++++++++++++++
t/APISIX.pm | 14 +++++
t/admin/standalone-healthcheck.t | 128 +++++++++++++++++++++++++++++++++++++++
t/cli/test_status_api.sh | 74 ++++++++++++++++++++++
14 files changed, 511 insertions(+), 4 deletions(-)
diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua
index d7b9e0a76..835a0b02c 100644
--- a/apisix/cli/config.lua
+++ b/apisix/cli/config.lua
@@ -96,6 +96,7 @@ local _M = {
lua_shared_dict = {
["prometheus-metrics"] = "15m",
["standalone-config"] = "10m",
+ ["status-report"] = "1m",
}
},
stream = {
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 78f89c544..d5d03bad0 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -70,6 +70,9 @@ lua {
{% if standalone_with_admin_api then %}
lua_shared_dict standalone-config {* meta.lua_shared_dict["standalone-config"] *};
{% end %}
+ {% if status then %}
+ lua_shared_dict status-report {* meta.lua_shared_dict["status-report"] *};
+ {% end %}
}
{% if enabled_stream_plugins["prometheus"] and not enable_http then %}
@@ -550,6 +553,23 @@ http {
}
{% end %}
+ {% if status then %}
+ server {
+ listen {* status_server_addr *} enable_process=privileged_agent;
+ access_log off;
+ location /status {
+ content_by_lua_block {
+ apisix.status()
+ }
+ }
+ location /status/ready {
+ content_by_lua_block {
+ apisix.status_ready()
+ }
+ }
+ }
+ {% end %}
+
{% if enabled_plugins["prometheus"] and prometheus_server_addr then %}
server {
{% if use_apisix_base then %}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index 3724a3056..3b2e555b5 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -343,6 +343,13 @@ Please modify "admin_key" in conf/config.yaml .
9180, port)
end
+ local status_server_addr
+ if yaml_conf.apisix.status then
+ status_server_addr = validate_and_get_listen_addr("status port", "127.0.0.1",
+ yaml_conf.apisix.status.ip, 7085,
+ yaml_conf.apisix.status.port)
+ end
+
local control_server_addr
if yaml_conf.apisix.enable_control then
if not yaml_conf.apisix.control then
@@ -568,6 +575,7 @@ Please modify "admin_key" in conf/config.yaml .
enabled_plugins = enabled_plugins,
enabled_stream_plugins = enabled_stream_plugins,
dubbo_upstream_multiplex_count = dubbo_upstream_multiplex_count,
+ status_server_addr = status_server_addr,
tcp_enable_ssl = tcp_enable_ssl,
admin_server_addr = admin_server_addr,
control_server_addr = control_server_addr,
diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index 1868481ab..6b5f1c99d 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -29,10 +29,12 @@ local core_str = require("apisix.core.string")
local new_tab = require("table.new")
local inspect = require("inspect")
local errlog = require("ngx.errlog")
+local process = require("ngx.process")
local log_level = errlog.get_sys_filter_level()
local NGX_INFO = ngx.INFO
local check_schema = require("apisix.core.schema").check
local exiting = ngx.worker.exiting
+local worker_id = ngx.worker.id
local insert_tab = table.insert
local type = type
local ipairs = ipairs
@@ -40,6 +42,7 @@ local setmetatable = setmetatable
local ngx_sleep = require("apisix.core.utils").sleep
local ngx_timer_at = ngx.timer.at
local ngx_time = ngx.time
+local ngx = ngx
local sub_str = string.sub
local tostring = tostring
local tonumber = tonumber
@@ -65,6 +68,7 @@ local err_etcd_grpc_engine_timeout = "context deadline exceeded"
local err_etcd_grpc_ngx_timeout = "timeout"
local err_etcd_unhealthy_all = "has no healthy etcd endpoint available"
local health_check_shm_name = "etcd-cluster-health-check"
+local status_report_shared_dict_name = "status-report"
if not is_http then
health_check_shm_name = health_check_shm_name .. "-stream"
end
@@ -482,6 +486,23 @@ local function short_key(self, str)
end
+local function sync_status_to_shdict(status)
+ local local_conf = config_local.local_conf()
+ if not local_conf.apisix.status then
+ return
+ end
+ if process.type() ~= "worker" then
+ return
+ end
+ local status_shdict = ngx.shared[status_report_shared_dict_name]
+ if not status_shdict then
+ return
+ end
+ local id = worker_id()
+ status_shdict:set(id, status)
+end
+
+
local function load_full_data(self, dir_res, headers)
local err
local changed = false
@@ -587,6 +608,7 @@ local function load_full_data(self, dir_res, headers)
end
self.need_reload = false
+ sync_status_to_shdict(true)
end
@@ -961,7 +983,6 @@ function _M.new(key, opts)
if not health_check_timeout or health_check_timeout < 0 then
health_check_timeout = 10
end
-
local automatic = opts and opts.automatic
local item_schema = opts and opts.item_schema
local filter_fun = opts and opts.filter
@@ -1128,6 +1149,7 @@ end
function _M.init_worker()
+ sync_status_to_shdict(false)
local local_conf, err = config_local.local_conf()
if not local_conf then
return nil, err
diff --git a/apisix/core/config_yaml.lua b/apisix/core/config_yaml.lua
index 428cdfc0d..b6a589d3b 100644
--- a/apisix/core/config_yaml.lua
+++ b/apisix/core/config_yaml.lua
@@ -45,10 +45,12 @@ local pcall = pcall
local io = io
local ngx = ngx
local re_find = ngx.re.find
+local process = require("ngx.process")
+local worker_id = ngx.worker.id
local apisix_yaml_path = profile:yaml_path("apisix")
local created_obj = {}
local shared_dict
-
+local status_report_shared_dict_name = "status-report"
local _M = {
version = 0.2,
@@ -72,6 +74,20 @@ local mt = {
local apisix_yaml
local apisix_yaml_mtime
+local function sync_status_to_shdict(status)
+ if process.type() ~= "worker" then
+ return
+ end
+ local status_shdict = ngx.shared[status_report_shared_dict_name]
+ if not status_shdict then
+ return
+ end
+ local id = worker_id()
+ log.info("sync status to shared dict, id: ", id, " status: ", status)
+ status_shdict:set(id, status)
+end
+
+
local function update_config(table, conf_version)
if not table then
log.error("failed update config: empty table")
@@ -85,6 +101,7 @@ local function update_config(table, conf_version)
end
apisix_yaml = table
+ sync_status_to_shdict(true)
apisix_yaml_mtime = conf_version
end
_M._update_config = update_config
@@ -359,7 +376,6 @@ local function _automatic_fetch(premature, self)
log.error("failed to decode config from shared dict: ", err)
goto SKIP_SHARED_DICT
end
-
_M._update_config(config)
log.info("config loaded from shared dict")
@@ -516,6 +532,7 @@ end
function _M.init_worker()
+ sync_status_to_shdict(false)
if is_use_admin_api() then
apisix_yaml = {}
apisix_yaml_mtime = 0
diff --git a/apisix/init.lua b/apisix/init.lua
index c0d91b463..bd3879a72 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -58,6 +58,7 @@ local str_sub = string.sub
local tonumber = tonumber
local type = type
local pairs = pairs
+local tostring = tostring
local ngx_re_match = ngx.re.match
local control_api_router
@@ -868,6 +869,61 @@ local function healthcheck_passive(api_ctx)
end
+function _M.status()
+ core.response.exit(200, core.json.encode({ status = "ok" }))
+end
+
+function _M.status_ready()
+ local local_conf = core.config.local_conf()
+ local role = core.table.try_read_attr(local_conf, "deployment", "role")
+ local provider = core.table.try_read_attr(local_conf, "deployment", "role_" ..
+ role, "config_provider")
+ if provider == "yaml" or provider == "etcd" then
+ local status_shdict = ngx.shared["status-report"]
+ local ids = status_shdict:get_keys()
+ local error
+ local worker_count = ngx.worker.count()
+ if #ids ~= worker_count then
+ core.log.warn("worker count: ", worker_count, " but status report count: ", #ids)
+ error = "worker count: " .. ngx.worker.count() ..
+ " but status report count: " .. #ids
+ end
+ if error then
+ core.response.exit(503, core.json.encode({
+ status = "error",
+ error = error
+ }))
+ return
+ end
+ for _, id in ipairs(ids) do
+ local ready = status_shdict:get(id)
+ if not ready then
+ core.log.warn("worker id: ", id, " has not received configuration")
+ error = "worker id: " .. id ..
+ " has not received configuration"
+ break
+ end
+ end
+
+ if error then
+ core.response.exit(503, core.json.encode({
+ status = "error",
+ error = error
+ }))
+ return
+ end
+
+ core.response.exit(200, core.json.encode({ status = "ok" }))
+ return
+ end
+
+ core.response.exit(503, core.json.encode({
+ status = "error",
+ message = "unknown config provider: " .. tostring(provider)
+ }), { ["Content-Type"] = "application/json" })
+end
+
+
function _M.http_log_phase()
local api_ctx = common_phase("log")
if not api_ctx then
diff --git a/conf/config.yaml.example b/conf/config.yaml.example
index 176f0f510..0b980a3b4 100644
--- a/conf/config.yaml.example
+++ b/conf/config.yaml.example
@@ -132,7 +132,10 @@ apisix:
events: # Event distribution module configuration
module: lua-resty-events # Sets the name of the events module used.
# Supported module: lua-resty-worker-events and lua-resty-events
-
+# status: # When enabled, APISIX will provide `/status` and `/status/ready` endpoints
+ # ip: 127.0.0.1 # /status endpoint will return 200 status code if APISIX has successfully started and running correctly
+ # port: 7085 # /status/ready endpoint will return 503 status code if any of the workers do not receive config from etcd
+ # or (standalone mode) the config isn't loaded yet either via file or Admin API.
nginx_config: # Config for render the template to generate nginx.conf
# user: root # Set the execution user of the worker process. This is only
# effective if the master process runs with super-user privileges.
diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index bdbdf7785..8c708f7ef 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -251,6 +251,10 @@
{
"type": "doc",
"id": "control-api"
+ },
+ {
+ "type": "doc",
+ "id": "status-api"
}
]
},
diff --git a/docs/en/latest/status-api.md b/docs/en/latest/status-api.md
new file mode 100644
index 000000000..6171496bc
--- /dev/null
+++ b/docs/en/latest/status-api.md
@@ -0,0 +1,78 @@
+---
+title: Status API
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+In Apache APISIX, the status API is used to:
+
+* Check if APISIX has successfully started and running correctly.
+* Check if all of the workers have received and loaded the configuration.
+
+To change the default endpoint (`127.0.0.1:7085`) of the Status API server, change the `ip` and `port` in the `status` section in your configuration file (`conf/config.yaml`):
+
+```yaml
+apisix:
+ status:
+ ip: "127.0.0.1"
+ port: 7085
+```
+
+This API can be used to perform readiness probes on APISIX before APISIX starts receiving user requests.
+
+### GET /status
+
+Returns a JSON reporting the status of APISIX workers. If APISIX is not running, the request will error out while establishing TCP connection. Otherwise this endpoint will always return ok if request reaches a running worker.
+
+```json
+{
+ "status": "ok"
+}
+```
+
+### GET /status/ready
+
+Returns `ok` when all workers have loaded the configuration, otherwise returns the specific error with `503` error code. Below are specific examples.
+
+When all workers have loaded the configuration:
+
+```json
+{
+ "status": "ok"
+}
+```
+
+When 1 workers has't been initialised:
+
+```json
+{
+ "status": "error",
+ "error": "worker count: 16 but status report count: 15"
+}
+```
+
+When a particular worker hasn't loaded the configuration:
+
+```json
+{
+ "error": "worker id: 9 has not received configuration",
+ "status": "error"
+}
+```
diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json
index b463a59fd..437e53be8 100644
--- a/docs/zh/latest/config.json
+++ b/docs/zh/latest/config.json
@@ -221,6 +221,10 @@
{
"type": "doc",
"id": "control-api"
+ },
+ {
+ "type": "doc",
+ "id": "status-api"
}
]
},
diff --git a/docs/zh/latest/status-api.md b/docs/zh/latest/status-api.md
new file mode 100644
index 000000000..7eae237f5
--- /dev/null
+++ b/docs/zh/latest/status-api.md
@@ -0,0 +1,78 @@
+---
+title: Status API
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+在 Apache APISIX 中，Status API 用于：
+
+* 检查 APISIX 是否已成功启动并正确运行
+* 检查所有 workers 是否已收到配置并加载
+
+要更改 Status API 服务器的默认端点（`127.0.0.1:7085`），请更改配置文件（`conf/config.yaml`）中 `status` 部分中的 `ip` 和 `port`：
+
+```yaml
+apisix:
+ status:
+ ip: "127.0.0.1"
+ port: 7085
+```
+
+此 API 可用于在 APISIX 开始接收用户请求之前对 APISIX 执行就绪探测。
+
+### GET /status
+
+返回报告 APISIX 工作人员状态的 JSON。如果 APISIX 未运行，建立 TCP 连接时请求将报错。否则，如果请求到达正在运行的 worker，此端点将始终返回 ok。
+
+```json
+{
+ "status": "ok"
+}
+```
+
+### GET /status/ready
+
+当所有 worker 都已加载配置时，返回 `ok`；否则，返回特定错误，错误代码为 `503`。以下是具体示例。
+
+当所有 worker 都已加载配置时：
+
+```json
+{
+ "status": "ok"
+}
+```
+
+当 1 个 workers 尚未初始化时：
+
+```json
+{
+ "status": "error",
+ "error": "worker count: 16 but status report count: 15"
+}
+```
+
+当特定 worker 尚未加载配置时：
+
+```json
+{
+ "error": "worker id: 9 has not received configuration",
+ "status": "error"
+}
+```
diff --git a/t/APISIX.pm b/t/APISIX.pm
index fbf2cf47d..69f4b6fb1 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -276,6 +276,7 @@ thread_pool grpc-client-nginx-module threads=1;
lua {
lua_shared_dict prometheus-metrics 15m;
lua_shared_dict standalone-config 10m;
+ lua_shared_dict status-report 1m;
}
_EOC_
}
@@ -717,6 +718,19 @@ _EOC_
_EOC_
$http_config .= <<_EOC_;
+ server {
+ listen 7085;
+ location /status/ready {
+ content_by_lua_block {
+ apisix.status_ready()
+ }
+ }
+ location /status {
+ content_by_lua_block {
+ apisix.status()
+ }
+ }
+ }
server {
listen unix:$apisix_home/t/servroot/logs/worker_events.sock;
access_log off;
diff --git a/t/admin/standalone-healthcheck.t b/t/admin/standalone-healthcheck.t
new file mode 100644
index 000000000..50d5a0e83
--- /dev/null
+++ b/t/admin/standalone-healthcheck.t
@@ -0,0 +1,128 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BEGIN {
+ # restarts cause the memory cache to be emptied, don't do this
+ $ENV{TEST_NGINX_FORCE_RESTART_ON_TEST} = 0;
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+use_hup();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->yaml_config) {
+ $block->set_value("yaml_config", <<'EOF');
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: yaml
+ admin:
+ admin_key:
+ - name: admin
+ key: edd1c9f034335f136f87ad84b625c8f1
+ role: admin
+EOF
+ }
+
+ if (!defined $block->no_error_log) {
+ $block->set_value("no_error_log", "");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: test
+--- timeout: 15
+--- max_size: 204800
+--- exec
+cd t && pnpm test admin/standalone.spec.ts 2>&1
+--- no_error_log
+failed to execute the script with status
+--- response_body eval
+qr/PASS admin\/standalone.spec.ts/
+
+
+
+=== TEST 2: send /healthcheck should fail because config is not loaded yet
+--- init_by_lua_block
+ require "resty.core"
+ apisix = require("apisix")
+ local shared_dict = ngx.shared["standalone-config"]
+ shared_dict:delete("config")
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local healthcheck_uri = "http://127.0.0.1:7085" .. "/status/ready"
+ local httpc = http.new()
+ local res, _ = httpc:request_uri(healthcheck_uri, {method = "GET", keepalive = false})
+ ngx.status = res.status
+ }
+ }
+--- request
+GET /t
+--- error_code: 503
+
+
+
+=== TEST 3: configure route and send /healthcheck should pass
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/configs',
+ ngx.HTTP_PUT,
+ [[{"routes":[{"id":"r1","uri":"/r1","upstream":{"nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"plugins":{"proxy-rewrite":{"uri":"/hello"}}}]}]],
+ nil,
+ {
+ ["X-API-KEY"] = "edd1c9f034335f136f87ad84b625c8f1"
+ }
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ local code, body = t('/apisix/admin/configs',
+ ngx.HTTP_PUT,
+ [[{"routes":[{"id":"r1","uri":"/r1","upstream":{"nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"plugins":{"proxy-rewrite":{"uri":"/hello"}}}]}]],
+ nil,
+ {
+ ["X-API-KEY"] = "edd1c9f034335f136f87ad84b625c8f1"
+ }
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.sleep(1)
+ local http = require("resty.http")
+ local healthcheck_uri = "http://127.0.0.1:7085" .. "/status/ready"
+ local httpc = http.new()
+ local res, _ = httpc:request_uri(healthcheck_uri, {method = "GET", keepalive = false})
+ ngx.status = res.status
+ }
+ }
+--- request
+GET /t
+--- error_code: 200
diff --git a/t/cli/test_status_api.sh b/t/cli/test_status_api.sh
new file mode 100755
index 000000000..ea703c1b2
--- /dev/null
+++ b/t/cli/test_status_api.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+git checkout conf/config.yaml
+
+
+echo '
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ etcd:
+ host:
+ - "http://127.0.0.1:23790"
+ - "http://127.0.0.1:23791"
+ - "http://127.0.0.1:23792"
+ prefix: /apisix
+nginx_config:
+ error_log_level: info
+apisix:
+ status:
+ ip: 127.0.0.1
+ port: 7085
+' > conf/config.yaml
+
+# create 3 node etcd cluster in docker
+ETCD_NAME_0=etcd0
+ETCD_NAME_1=etcd1
+ETCD_NAME_2=etcd2
+docker compose -f ./t/cli/docker-compose-etcd-cluster.yaml up -d
+
+make run
+
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:7085/status | grep 200 \
+|| (echo "failed: status api didn't return 200"; exit 1)
+
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:7085/status/ready | grep 200 \
+|| (echo "failed: status/ready api didn't return 200"; exit 1)
+
+# stop two etcd endpoints but status api should return 200 as all workers are synced
+docker stop ${ETCD_NAME_0}
+docker stop ${ETCD_NAME_1}
+
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:7085/status | grep 200 \
+|| (echo "failed: status api didn't return 200"; exit 1)
+
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:7085/status/ready | grep 200 \
+|| (echo "failed: status/ready api didn't return 200"; exit 1)
+
+docker stop ${ETCD_NAME_2}
+
+echo "/status/ready returns 200 even when etcd endpoints are down as all workers are synced"
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:7085/status/ready | grep 200 \
+|| (echo "failed: status/ready api didn't return 200"; exit 1)
+
+docker compose -f ./t/cli/docker-compose-etcd-cluster.yaml down