You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by "Revolyssup (via GitHub)" <gi...@apache.org> on 2025/10/13 10:43:16 UTC
[PR] fix: port conflict in worker process for prometheus port [apisix]
Revolyssup opened a new pull request, #12667:
URL: https://github.com/apache/apisix/pull/12667
In a previous change, the prometheus server was moved from priviliged agent to worker which sometimes during restarts causes port conflict issues. This PR fixes it by enabling port reuse for prometheus server address.
## Reproduction steps
1. Run the following script
```bash
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
DEFAULT_ETCD_IMAGE_NAME="bitnamilegacy/etcd"
DEFAULT_ETCD_IMAGE_TAG="3.5.7"
DEFAULT_APISIX_IMAGE_NAME="apache/apisix"
DEFAULT_APISIX_IMAGE_TAG="3.14.0-ubuntu"
DEFAULT_ETCD_LISTEN_PORT=2379
DEFAULT_APISIX_PORT=9180
DEFAULT_ETCD_NAME="etcd-quickstart"
DEFAULT_APP_NAME="apisix-quickstart"
DEFAULT_NET_NAME="apisix-quickstart-net"
usage() {
echo "Runs a Docker based Apache APISIX."
echo
echo "See the document for more information:"
echo " https://docs.api7.ai/apisix/getting-started"
exit 0
}
echo_fail() {
printf "\e[31m✘ \e[0m$@\n"
}
echo_pass() {
printf "\e[32m✔ \e[0m$@\n"
}
echo_warning() {
printf "\e[33m⚠ $@\e[0m\n"
}
ensure_docker() {
{
docker ps -q >/dev/null 2>&1
} || {
return 1
}
}
ensure_curl() {
{
curl -h >/dev/null 2>&1
} || {
return 1
}
}
install_apisix() {
echo "Installing APISIX with the quickstart options."
echo ""
echo "Creating bridge network ${DEFAULT_NET_NAME}."
docker network create -d bridge $DEFAULT_NET_NAME && echo_pass "network ${DEFAULT_NET_NAME} created" || {
echo_fail "Create network failed!"
return 1
}
echo ""
echo "Starting the container ${DEFAULT_ETCD_NAME}."
docker run -d \
--name ${DEFAULT_ETCD_NAME} \
--network=$DEFAULT_NET_NAME \
-e ALLOW_NONE_AUTHENTICATION=yes \
-e ETCD_ADVERTISE_CLIENT_URLS=http://${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT} \
${DEFAULT_ETCD_IMAGE_NAME}:${DEFAULT_ETCD_IMAGE_TAG} && echo_pass "etcd is listening on ${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT}" || {
echo_fail "Start etcd failed!"
return 1
}
echo ""
APISIX_DEPLOYMENT_ETCD_HOST="[\"http://${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT}\"]"
echo "Starting the container ${DEFAULT_APP_NAME}."
docker run -d \
--name ${DEFAULT_APP_NAME} \
--network=$DEFAULT_NET_NAME \
-p9080:9080 -p9180:9180 -p9443:9443/tcp -p9443:9443/udp -p9090:9092 -p9100:9100 -p9091:9091 \
-e APISIX_DEPLOYMENT_ETCD_HOST=${APISIX_DEPLOYMENT_ETCD_HOST} \
${DEFAULT_APISIX_IMAGE_NAME}:${DEFAULT_APISIX_IMAGE_TAG} && validate_apisix && sleep 2 || {
echo_fail "Start APISIX failed!"
return 1
}
docker exec ${DEFAULT_APP_NAME} /bin/bash -c "echo '
apisix:
enable_control: true
control:
ip: "0.0.0.0"
port: 9092
deployment:
role: traditional
role_traditional:
config_provider: etcd
admin:
admin_key_required: false
allow_admin:
- 0.0.0.0/0
plugin_attr:
prometheus:
export_addr:
ip: 0.0.0.0
port: 9091
' > /usr/local/apisix/conf/config.yaml"
docker exec ${DEFAULT_APP_NAME} apisix reload >>/dev/null 2>&1
echo_warning "WARNING: The Admin API key is currently disabled. You should turn on admin_key_required and set a strong Admin API key in production for security."
echo ""
}
destroy_apisix() {
echo "Destroying existing ${DEFAULT_APP_NAME} container, if any."
echo ""
docker rm -f $DEFAULT_APP_NAME >>/dev/null 2>&1
docker rm -f $DEFAULT_ETCD_NAME >>/dev/null 2>&1
docker network rm $DEFAULT_NET_NAME >>/dev/null 2>&1
sleep 2
}
validate_apisix() {
local rv=0
retry 30 curl "http://localhost:${DEFAULT_APISIX_PORT}/apisix/admin/services" >>/dev/null 2>&1 && echo_pass "APISIX is up" || rv=$?
}
main() {
ensure_docker || {
echo_fail "Docker is not available, please install it first"
exit 1
}
ensure_curl || {
echo_fail "curl is not available, please install it first"
exit 1
}
destroy_apisix
install_apisix || {
exit 1
}
echo_pass "APISIX is ready!"
}
main "$@"
```
1. Send request
```bash
curl -i "http://127.0.0.1:9180/apisix/admin/routes" -X PUT -d '
{
"id": "getting-started-ip",
"uri": "/ip",
"upstream": {
"type": "roundrobin",
"nodes": {
"httpbin.org:80": 1
}
}
}'
HTTP/1.1 403 Forbidden
Server: openresty
Date: Mon, 13 Oct 2025 10:15:20 GMT
Content-Type: text/html; charset=utf-8
Content-Length: 225
Connection: keep-alive
<html>
<head><title>403 Forbidden</title></head>
<body>
<center><h1>403 Forbidden</h1></center>
<hr><center>openresty</center>
<p><em>Powered by <a href="https://apisix.apache.org/">APISIX</a>.</em></p></body>
</html>
```
Result: Request will be forbidden. To confirm check error log and you will see the log
`bind() to 0.0.0.0:9091 failed (98: Address already in use)`
## Fix
The above script is modified to update the code in apisix container with the fix
```bash
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
DEFAULT_ETCD_IMAGE_NAME="bitnamilegacy/etcd"
DEFAULT_ETCD_IMAGE_TAG="3.5.7"
DEFAULT_APISIX_IMAGE_NAME="apache/apisix"
DEFAULT_APISIX_IMAGE_TAG="3.14.0-ubuntu"
DEFAULT_ETCD_LISTEN_PORT=2379
DEFAULT_APISIX_PORT=9180
DEFAULT_ETCD_NAME="etcd-quickstart"
DEFAULT_APP_NAME="apisix-quickstart"
DEFAULT_NET_NAME="apisix-quickstart-net"
usage() {
echo "Runs a Docker based Apache APISIX."
echo
echo "See the document for more information:"
echo " https://docs.api7.ai/apisix/getting-started"
exit 0
}
echo_fail() {
printf "\e[31m✘ \e[0m$@\n"
}
echo_pass() {
printf "\e[32m✔ \e[0m$@\n"
}
echo_warning() {
printf "\e[33m⚠ $@\e[0m\n"
}
ensure_docker() {
{
docker ps -q >/dev/null 2>&1
} || {
return 1
}
}
ensure_curl() {
{
curl -h >/dev/null 2>&1
} || {
return 1
}
}
install_apisix() {
echo "Installing APISIX with the quickstart options."
echo ""
echo "Creating bridge network ${DEFAULT_NET_NAME}."
docker network create -d bridge $DEFAULT_NET_NAME && echo_pass "network ${DEFAULT_NET_NAME} created" || {
echo_fail "Create network failed!"
return 1
}
echo ""
echo "Starting the container ${DEFAULT_ETCD_NAME}."
docker run -d \
--name ${DEFAULT_ETCD_NAME} \
--network=$DEFAULT_NET_NAME \
-e ALLOW_NONE_AUTHENTICATION=yes \
-e ETCD_ADVERTISE_CLIENT_URLS=http://${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT} \
${DEFAULT_ETCD_IMAGE_NAME}:${DEFAULT_ETCD_IMAGE_TAG} && echo_pass "etcd is listening on ${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT}" || {
echo_fail "Start etcd failed!"
return 1
}
echo ""
APISIX_DEPLOYMENT_ETCD_HOST="[\"http://${DEFAULT_ETCD_NAME}:${DEFAULT_ETCD_LISTEN_PORT}\"]"
echo "Starting the container ${DEFAULT_APP_NAME}."
docker run -d \
--name ${DEFAULT_APP_NAME} \
--network=$DEFAULT_NET_NAME \
-p9080:9080 -p9180:9180 -p9443:9443/tcp -p9443:9443/udp -p9090:9092 -p9100:9100 -p9091:9091 \
-e APISIX_DEPLOYMENT_ETCD_HOST=${APISIX_DEPLOYMENT_ETCD_HOST} \
${DEFAULT_APISIX_IMAGE_NAME}:${DEFAULT_APISIX_IMAGE_TAG} && validate_apisix && sleep 2 || {
echo_fail "Start APISIX failed!"
return 1
}
#fix
docker exec ${DEFAULT_APP_NAME} sed -i 's/listen {\* prometheus_server_addr \*};/listen {\* prometheus_server_addr \*} reuseport so_keepalive=off;/g' /usr/local/apisix/apisix/cli/ngx_tpl.lua
docker exec ${DEFAULT_APP_NAME} apisix reload >>/dev/null 2>&1
docker exec ${DEFAULT_APP_NAME} /bin/bash -c "echo '
apisix:
enable_control: true
control:
ip: "0.0.0.0"
port: 9092
deployment:
role: traditional
role_traditional:
config_provider: etcd
admin:
admin_key_required: false
allow_admin:
- 0.0.0.0/0
plugin_attr:
prometheus:
export_addr:
ip: 0.0.0.0
port: 9091
' > /usr/local/apisix/conf/config.yaml"
docker exec ${DEFAULT_APP_NAME} apisix reload >>/dev/null 2>&1
echo_warning "WARNING: The Admin API key is currently disabled. You should turn on admin_key_required and set a strong Admin API key in production for security."
echo ""
}
destroy_apisix() {
echo "Destroying existing ${DEFAULT_APP_NAME} container, if any."
echo ""
docker rm -f $DEFAULT_APP_NAME >>/dev/null 2>&1
docker rm -f $DEFAULT_ETCD_NAME >>/dev/null 2>&1
docker network rm $DEFAULT_NET_NAME >>/dev/null 2>&1
sleep 2
}
validate_apisix() {
local rv=0
retry 30 curl "http://localhost:${DEFAULT_APISIX_PORT}/apisix/admin/services" >>/dev/null 2>&1 && echo_pass "APISIX is up" || rv=$?
}
main() {
ensure_docker || {
echo_fail "Docker is not available, please install it first"
exit 1
}
ensure_curl || {
echo_fail "curl is not available, please install it first"
exit 1
}
destroy_apisix
install_apisix || {
exit 1
}
echo_pass "APISIX is ready!"
}
main "$@"
```
1. Send curl request
```bash
curl -i "http://127.0.0.1:9180/apisix/admin/routes" -X PUT -d '
{
"id": "getting-started-ip",
"uri": "/ip",
"upstream": {
"type": "roundrobin",
"nodes": {
"httpbin.org:80": 1
}
}
}'
HTTP/1.1 201 Created
Date: Mon, 13 Oct 2025 10:33:51 GMT
Content-Type: application/json
Transfer-Encoding: chunked
Connection: keep-alive
Server: APISIX/3.14.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true
Access-Control-Expose-Headers: *
Access-Control-Max-Age: 3600
X-API-VERSION: v3
{"key":"/apisix/routes/getting-started-ip","value":{"create_time":1760351631,"upstream":{"type":"roundrobin","nodes":{"httpbin.org:80":1}},"uri":"/ip","update_time":1760351631,"id":"getting-started-ip"}}
```
Result: The request passes
### Checklist
- [ ] I have explained the need for this PR and the problem it solves
- [ ] I have explained the changes or the new features added to this PR
- [ ] I have added tests corresponding to this change
- [ ] I have updated the documentation to reflect this change
- [ ] I have verified that this change is backward compatible (If not, please discuss on the [APISIX mailing list](https://github.com/apache/apisix/tree/master#community) first)
<!--
Note
1. Mark the PR as draft until it's ready to be reviewed.
2. Always add/update tests for any changes unless you have a good reason.
3. Always update the documentation to reflect the changes made in the PR.
4. Make a new commit to resolve conversations instead of `push -f`.
5. To resolve merge conflicts, merge master instead of rebasing.
6. Use "request review" to notify the reviewer after making changes.
7. Only a reviewer can mark a conversation as resolved.
-->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix: port conflict in worker process for prometheus port [apisix]
Posted by "Revolyssup (via GitHub)" <gi...@apache.org>.
Revolyssup commented on code in PR #12667:
URL: https://github.com/apache/apisix/pull/12667#discussion_r2428010631
##########
apisix/cli/ngx_tpl.lua:
##########
@@ -105,7 +105,7 @@ http {
}
server {
- listen {* prometheus_server_addr *};
+ listen {* prometheus_server_addr *} reuseport so_keepalive=off;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix: port conflict in worker process for prometheus port [apisix]
Posted by "Revolyssup (via GitHub)" <gi...@apache.org>.
Revolyssup merged PR #12667:
URL: https://github.com/apache/apisix/pull/12667
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix: port conflict in worker process for prometheus port [apisix]
Posted by "membphis (via GitHub)" <gi...@apache.org>.
membphis commented on code in PR #12667:
URL: https://github.com/apache/apisix/pull/12667#discussion_r2427970925
##########
apisix/cli/ngx_tpl.lua:
##########
@@ -105,7 +105,7 @@ http {
}
server {
- listen {* prometheus_server_addr *};
+ listen {* prometheus_server_addr *} reuseport so_keepalive=off;
Review Comment:
`listen {* prometheus_server_addr *} reuseport;`
it should be enough
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix: port conflict in worker process for prometheus port [apisix]
Posted by "bzp2010 (via GitHub)" <gi...@apache.org>.
bzp2010 commented on code in PR #12667:
URL: https://github.com/apache/apisix/pull/12667#discussion_r2427732254
##########
apisix/cli/ngx_tpl.lua:
##########
@@ -105,7 +105,7 @@ http {
}
server {
- listen {* prometheus_server_addr *};
+ listen {* prometheus_server_addr *} reuseport so_keepalive=off;
Review Comment:
Why disable keepalive?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org