You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@httpd.apache.org by Henry GALVEZ <he...@intm.fr> on 2022/03/07 15:29:49 UTC
[users@httpd] Problems with load balancer, possible proxy module bug
Hi to all
We have a host that evaluates access with a Balancer and a HealthCheck.
In version 2.4.43, if the HealthCheck's status is 1 and after a certain time it returns to 0, everything goes back to normal.
But from version 2.4.46 until version 2.4.52, when HealthCheck returns to status 0, it continues to throw the following error: "All Workers Are In Error State" and forces the restart of the HTTPD service.
Configuration example:
```config
#################################################################
## httpd.conf ##
##################################################################
ServerRoot /usr/local/apache2
LoadModule autoindex_module /usr/local/apache2/modules/mod_autoindex.so
LoadModule unique_id_module /usr/local/apache2/modules/mod_unique_id.so
#LoadModule mpm_worker_module /usr/local/apache2/modules/mod_mpm_worker.so
LoadModule authn_core_module /usr/local/apache2/modules/mod_authn_core.so
LoadModule authz_core_module /usr/local/apache2/modules/mod_authz_core.so
LoadModule auth_basic_module /usr/local/apache2/modules/mod_auth_basic.so
LoadModule authn_file_module /usr/local/apache2/modules/mod_authn_file.so
LoadModule authz_user_module /usr/local/apache2/modules/mod_authz_user.so
LoadModule mpm_event_module /usr/local/apache2/modules/mod_mpm_event.so
LoadModule headers_module /usr/local/apache2/modules/mod_headers.so
LoadModule authz_host_module /usr/local/apache2/modules/mod_authz_host.so
LoadModule deflate_module /usr/local/apache2/modules/mod_deflate.so
LoadModule setenvif_module /usr/local/apache2/modules/mod_setenvif.so
LoadModule userdir_module /usr/local/apache2/modules/mod_userdir.so
LoadModule dir_module /usr/local/apache2/modules/mod_dir.so
LoadModule expires_module /usr/local/apache2/modules/mod_expires.so
LoadModule rewrite_module /usr/local/apache2/modules/mod_rewrite.so
LoadModule alias_module /usr/local/apache2/modules/mod_alias.so
LoadModule mime_module /usr/local/apache2/modules/mod_mime.so
LoadModule status_module /usr/local/apache2/modules/mod_status.so
LoadModule proxy_module /usr/local/apache2/modules/mod_proxy.so
LoadModule proxy_http_module /usr/local/apache2/modules/mod_proxy_http.so
LoadModule cache_module /usr/local/apache2/modules/mod_cache.so
LoadModule proxy_ajp_module /usr/local/apache2/modules/mod_proxy_ajp.so
LoadModule ssl_module /usr/local/apache2/modules/mod_ssl.so
LoadModule proxy_balancer_module /usr/local/apache2/modules/mod_proxy_balancer.so
LoadModule slotmem_shm_module /usr/local/apache2/modules/mod_slotmem_shm.so
LoadModule socache_shmcb_module /usr/local/apache2/modules/mod_socache_shmcb.so
LoadModule lbmethod_byrequests_module /usr/local/apache2/modules/mod_lbmethod_byrequests.so
LoadModule info_module /usr/local/apache2/modules/mod_info.so
LoadModule proxy_hcheck_module /usr/local/apache2/modules/mod_proxy_hcheck.so
LoadModule unixd_module /usr/local/apache2/modules/mod_unixd.so
LoadModule log_config_module /usr/local/apache2/modules/mod_log_config.so
LoadModule watchdog_module /usr/local/apache2/modules/mod_watchdog.so
<IfModule proxy_hcheck_module>
LogLevel proxy_hcheck:trace8 proxy:trace8
</IfModule>
Timeout 1800
KeepAlive on
MaxRequestWorkers 8192
ServerLimit 128
ThreadsPerChild 64
MaxRequestsPerChild 1000
StartServers 3
MinSpareThreads 5
MaxSpareThreads 250
MaxKeepAliveRequests 250
KeepAliveTimeout 5
AsyncRequestWorkerFactor 2
userdir disabled root
TypesConfig /usr/local/apache2/conf/mime.types
AddEncoding x-compress Z
AddEncoding x-gzip gz tgz
############################################
# Definitions
############################################
Listen ######:443
ServerName ######
ServerTokens Prod
DeflateFilterNote ratio
# Serveur HTTPD
DocumentRoot /usr/local/apache2/htdocs/
DirectoryIndex index.html index.htm index.jsp
# Filtering accessLog to delete images, CSS and JavaScript
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla|css|js|pdf|ico)$ dontlog
RequestHeader setifempty Correlation-ID %{UNIQUE_ID}e
SetEnvIf Correlation-ID "(.*)" MyCorrelationId=$1
Header setifempty Correlation-ID "%{MyCorrelationId}e"
# Recovery of the original applicant only (X-Forwarded-For = host1, host2, host3... If we pass through several layers)
RequestHeader setIfEmpty X-Forwarded-For "%{REMOTE_ADDR}s"
RequestHeader edit X-Forwarded-For ^([^,]*),?.*$ $1
# Logs management
LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" (%{ratio}n) \"%{User-Agent}i\" %D %{Correlation-ID}i %H" combined
ErrorLogFormat "[%t] [%m:%l] [pid %P:tid %T] [client\ %a] [referer\ %{Referer}i] [correlation_id\ %{Correlation-ID}i] %M -BT: %{error==}B - %{oserror==}B"
PidFile /usr/local/apache2/httpd.pid
Mutex file:/usr/local/apache2 default
DefaultRuntimeDir /usr/local/apache2
ScoreBoardFile /usr/local/apache2/logs/httpd.scoreboard
CustomLog "|| /usr/local/apache2/bin/rotatelogs -l /usr/local/apache2/logs/access_log.%Y-%m-%d-%H_%M_%S 3600" combined env=!dontlog
ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l /usr/local/apache2/logs/error_log.%Y-%m-%d-%H_%M_%S 60"
LogLevel debug
# Rewrite to redirect root URLs to the application URL
RewriteEngine on
RewriteRule "^/services/param/([0-9]+)/date/(.*)" "/services/"
RewriteCond %{HTTPS} !=on
RewriteRule ^/(.*) [https://%{HTTP_HOST}/$1]https://%{HTTP_HOST}/$1 [R,L]
# Parametering for SSL
SSLRandomSeed startup builtin
AddType application/x-x509-ca-cert .crt
AddType application/x-pkcs7-crl .crl
SSLPassPhraseDialog builtin
SSLSessionCache "shmcb:/usr/local/apache2/logs/ssl_scache(512000)"
Mutex default
SSLEngine on
SSLCipherSuite ECDHE-#########-SHA256
# Active only TLS v1.1 & v1.2
SSLProtocol All -SSLv3
SSLCertificateFile /usr/local/apache2/secret/cert_pki_ent_HP.pem
SSLCertificateKeyFile /usr/local/apache2/secret/cert_pki_ent_HP.key
SSLCertificateChainFile /usr/local/apache2/secret/cert_pki_ent_HP.cer
SSLHonorCipherOrder off
SSLProxyEngine on
SetEnvIf User-Agent ".*MSIE.*" \
nokeepalive ssl-unclean-shutdown \
downgrade-1.0 force-response-1.0
Customlog "||/usr/local/apache2/bin/rotatelogs -l //usr/local/apache2/logs/ssl_request_log.%Y-%m-%d-%H_%M_%S 3600" "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"
# Must be outside <location></location>
Options +FollowSymLinks
############################################
# Customer Cache Treatment
############################################
RequestHeader edit "If-None-Match" "^\"(.*)-gzip\"$" "\"$1\""
Header edit "ETag" "^\"(.*[^g][^z][^i][^p])\"$" "\"$1-gzip\""
ExpiresActive On
ExpiresDefault "access plus 0 minutes"
ExpiresByType text/html "access plus 0 minutes"
ExpiresByType image/gif "access plus 24 hours"
ExpiresByType image/jpeg "access plus 24 hours"
ExpiresByType image/png "access plus 24 hours"
ExpiresByType text/css "access plus 24 hours"
ExpiresByType text/javascript "access plus 24 hours"
ExpiresByType application/javascript "access plus 24 hours"
ExpiresByType image/svg+xml "access plus 24 hours"
<Location />
SetOutputFilter DEFLATE
BrowserMatch ^Mozilla/4 gzip-only-text/html
BrowserMatch ^Mozilla/4\.0[678] no-gzip
BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla)$ no-gzip dont-vary
Header append Vary User-Agent env=!dont-vary
</Location>
# Rewrite to redirect root URLs to the application URL
Header edit Set-Cookie "^(API_JSESSIONID=.*); .*" "$1; domain=.example.com; HttpOnly; secure"
Header edit Set-Cookie (XSRF-TOKEN=.*);.* "$1; path=/; domain=.example.com; secure"
ExtendedStatus On
# The server returns Server: Apache
ServerTokens productOnly
# Returns an error code 405 (unauthorized method) to the client (mod_proxy)
TraceEnable off
ServerSignature off
HostnameLookups off
# Security Restriction: Cache Apache and prevents the use of trace mode
RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
RewriteRule .* - [F]
# Parameters for validation Geomobile Client Certificate
SSLVerifyDepth 2
SSLCACertificateFile /usr/local/apache2/secret/chain_bundle.pem
SSLVerifyClient none
<VirtualHost *:443>
SSLEngine on
ServerName api.example.com
ServerAlias api.example.com
<IfModule proxy_hcheck_module>
LogLevel proxy_hcheck:trace8 proxy:trace8
</IfModule>
# Server HTTPD
DocumentRoot /usr/local/apache2/htdocs/test
SetEnvIf Origin "http(s)?://(www\.)?(admin.example.com:443|home.example.com:443|warnings.example.com:443|files.example.com:443|hook.example.com:443|optional.example.com:443|review.example.com:443|api.example.com:443|cluster.example.com:54657|cluster.example.com:54656|cluster.example.com:54658|cluster.example.com:54655|cluster.example.com:54659|cluster.example.com:54660|cluster.example.com:54661)$" AccessControlAllowOrigin=$0
Header set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin
Header set Access-Control-Allow-Credentials "true"
# CSRF
RequestHeader unset Access-Control-Request-Headers
Header set Access-Control-Allow-Headers "X-XSRF-TOKEN, content-type"
Header set Access-Control-Expose-Headers Circuit-Breakers-On-Error
CustomLog "|| /usr/local/apache2/bin/rotatelogs -p '/usr/local/apache2/outils/MANAGE_LOGS.sh' -l //usr/local/apache2/logs/access_test_log.%Y-%m-%d-%H_%M_%S 3600" combined env=!dontlog
ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l //usr/local/apache2/logs/error_test_log.%Y-%m-%d-%H_%M_%S 60"
ProxyHCExpr reqok {%{REQUEST_STATUS} =~ /^[2]/}
<Proxy balancer://application-files/>
BalancerMember http://server-one:9382 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9382 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>
<Proxy balancer://application-db/>
BalancerMember http://server-one:9385 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9385 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>
<Proxy balancer://application-jobs/>
BalancerMember http://server-one:9384 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9384 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>
# Balancer manager
ProxyPass /balancer-manager !
<Location /balancer-manager>
SetHandler balancer-manager
Require host server-one.example.com server-two.example.com server-one server-two
</Location>
ProxyPass /config.api.json !
ProxyPass /config-extra.api.json !
ProxyPass /version.json !
ProxyPassMatch /management.* !
ProxyPass /services/ balancer://application-files/services/
ProxyPassReverse /services/ balancer://application-files/services/
ProxyPass /db_services/ balancer://application-db/db_services/
ProxyPassReverse /db_services/ balancer://application-db/db_services/
ProxyPass /jobs/ balancer://application-jobs/jobs/
ProxyPassReverse /jobs/ balancer://application-jobs/jobs/
</VirtualHost>
```
In the logs I can't find the reason stopping the performance of the HealthCheck and the Balancer.
Could someone help me find the possible source of the error or how to extract more information to report it as a bug, plis.
Thank in advance.