You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@httpd.apache.org by Herb Burnswell <he...@gmail.com> on 2022/06/01 00:08:12 UTC
[users@httpd] Strange 400 response codes
Hi,
We have an issue that I'd like to get some guidance on how to investigate
further. We have a Tomcat application that is fronted by 3 HTTPD proxies
(Apache/2.4.34) running mod_proxy_balancer. What we see in the HTTPD
access logs are 400 response codes that include entries like:
preview.example.com 10.24.3.10 "-" - - [31/May/2022:15:16:30 -0700] "GET
/BOTTOMS/shorts/c/0144 HTTP/1.1" 400 278 "
https://www.example.com/my-account/view" "Mozilla/5.0 (Linux; Android 12;
SM-N975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.78
Mobile Safari/537.36" "-" 293 9697 5006
preview.example.com 10.24.3.10 "-" - - [31/May/2022:15:35:13 -0700] "POST
/checkout/multi/payment-method/add HTTP/1.1" 400 278 "
https://www.example.com/checkout/multi/payment-method/add" "Mozilla/5.0
(Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/101.0.4951.64 Safari/537.36" "-" 115 9435 0
*/etc/httpd/conf/httpd.conf:*
# ------------------------------------------
# Listen Port
# ------------------------------------------
Listen 127.0.0.1:80
Listen 443
# ------------------------------------------
# Load Modules
# ------------------------------------------
LoadModule ssl_module modules/mod_ssl.so
LoadModule systemd_module modules/mod_systemd.so
LoadModule unixd_module modules/mod_unixd.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule logio_module modules/mod_logio.so
LoadModule mpm_event_module modules/mod_mpm_event.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
LoadModule status_module modules/mod_status.so
LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so
LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so
LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule headers_module modules/mod_headers.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule alias_module modules/mod_alias.so
LoadModule dir_module modules/mod_dir.so
LoadModule mime_module modules/mod_mime.so
LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so
LoadModule watchdog_module modules/mod_watchdog.so
# ------------------------------------------
# Run As
# ------------------------------------------
User apache
Group apache
# ------------------------------------------
# Server Admin
# ------------------------------------------
ServerAdmin root@localhost
ServerTokens ProductOnly
Include conf.d/*.conf
# ------------------------------------------
# Doc Root
# ------------------------------------------
DocumentRoot /var/www/html
# ------------------------------------------
# Logs
# ------------------------------------------
ErrorLog "logs/error_log"
LogLevel warn
LogFormat "%v %h \"%{BALANCER_WORKER_NAME}e\" %l %u %t \"%r\" %>s %b
\"%{Referer}i\" \"%{User-Agent}i\" \"%{JSESSIONID}C\" %D %I %O" combinedio
LogFormat "%v \"%{X-Forwarded-For}i\" \"%{BALANCER_WORKER_NAME}e\" %l %u %t
\"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{JSESSIONID}C\" %D %I
%O" proxy
SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
CustomLog "logs/access_log" combinedio env=!forwarded
CustomLog "logs/access_log" proxy env=forwarded
# ------------------------------------------
# SSL
# ------------------------------------------
SSLSessionCache "shmcb:logs/session-cache(512000)"
SSLStaplingCache "shmcb:logs/stapling-cache(160000)"
# ------------------------------------------------
# Virtual Hosts
# ------------------------------------------------
<VirtualHost 127.0.0.1:80>
<Location "/serverstatus">
SetHandler server-status
</Location>
ErrorLog /dev/null
CustomLog /dev/null common
</VirtualHost>
*/etc/httpd/conf.d/www.example.com.conf:*
<VirtualHost *:443>
ServerName example.com
ServerAlias www.example.com
SSLEngine on
SSLProxyEngine on
SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
SSLHonorCipherOrder On
SSLCompression off
SSLUseStapling on
SSLSessionTickets Off
SSLCertificateFile /etc/pki/tls/certs/file.crt
SSLCertificateKeyFile /etc/pki/tls/certs/file.key
Header always set Strict-Transport-Security "max-age=63072000;
includeSubDomains"
Header always set X-Frame-Options SAMEORIGIN
Header add Set-Cookie "ROUTEID=.%{BALANCER_WORKER_ROUTE}e; path=/"
env=BALANCER_ROUTE_CHANGED
Alias "/balancermanager_com" /var/www/html/balancermanager_com
<Location /balancermanager_com>
SetHandler balancer-manager
Order Deny,Allow
Deny from all
Allow from 10.1.1.56
</Location>
<Directory /var/www/html/maintenance>
Require all granted
</Directory>
ProxyHCExpr site_up {hc('body') !~ /ok/}
<Proxy balancer://storefront-com>
BalancerMember https://app410.example.com:8443 route=app410 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app411.example.com:8443 route=app411 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app413.example.com:8443 route=app413 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app414.example.com:8443 route=app414
keepalive=On ttl=90 timeout=60 hcmethod=GET hcexpr=site_up
hcuri=/healthcheck hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app415.example.com:8443 route=app415 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app416.example.com:8443 route=app416 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app417.example.com:8443 route=app417 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app418.example.com:8443 route=app418 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
BalancerMember https://app419.example.com:8443 route=app419 keepalive=On
ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
hcinterval=10 hcpasses=2 hcfails=2
ProxySet lbmethod=bybusyness
</Proxy>
RewriteEngine On
ErrorDocument 503 /maintenance/us/index.html
RewriteCond /var/www/html/maintenance/us/enabled -f
RewriteCond %{REQUEST_URI} !=/maintenance/us/index.html
RewriteRule ^ - [R=503,L]
RewriteCond /var/www/html/maintenance/us/enabled !-f
RewriteRule ^/maintenance/us/index.html$ / [R,L]
ProxyRequests Off
ProxyPreserveHost On
ProxyBadHeader Ignore
ProxyPassMatch .*\.php$ !
ProxyPassMatch .*\.asp$ !
ProxyPassMatch .*\.pl$ !
ProxyPassMatch .*\.pm$ !
ProxyPassMatch .*\.rb$ !
ProxyPassMatch .*\.py$ !
ProxyPass /maintenance !
Alias "/favicon.ico" /var/www/html/favicon.ico
ProxyPass "/" balancer://storefront-com/ stickysession=ROUTEID
ProxyPassReverse "/" balancer://storefront-com/ stickysession=ROUTEID
</VirtualHost>
What appears to be happening is that the requests are being 'addressed' by
the alphabetically first *.conf file that is in /etc/httpd/conf.d.
Previously we had another config file that alphabetically preceded the
preview.example.com.conf config and the log entries referenced it as %v.
But after turning the original alphabetically first config off, the log
entries reference the now first preview.example.com config. But as shown
above, the referrer in all log entries is: https://www.example.com.
Can anyone recommend how we can understand what might be the issue here?
Thanks in advance,
HB
[users@httpd] Re: Strange 400 response codes
Posted by Herbert Burnswell <he...@gmail.com>.
Hi All,
We are still experiencing issues with receiving 'Bad Request' responses as
mentioned in my original post. With no replies to my original post, I'm
not sure if that means it is an unknown issue without suggestions or not.
The fact that the error in the log comes from whichever httpd config is
first alphabetically leads me to believe that httpd is 'confused' with the
request. Is it possible that the request coming from the backend Tomcat is
malformed, or would that be a different error than the 400? Any
guidance would be greatly appreciated, as we are stuck at the moment.
Thanks,
HB
On Tue, May 31, 2022 at 5:08 PM Herb Burnswell <he...@gmail.com>
wrote:
> Hi,
>
> We have an issue that I'd like to get some guidance on how to investigate
> further. We have a Tomcat application that is fronted by 3 HTTPD proxies
> (Apache/2.4.34) running mod_proxy_balancer. What we see in the HTTPD
> access logs are 400 response codes that include entries like:
>
>
>
> preview.example.com 10.24.3.10 "-" - - [31/May/2022:15:16:30 -0700] "GET
> /BOTTOMS/shorts/c/0144 HTTP/1.1" 400 278 "
> https://www.example.com/my-account/view" "Mozilla/5.0 (Linux; Android 12;
> SM-N975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.78
> Mobile Safari/537.36" "-" 293 9697 5006
>
> preview.example.com 10.24.3.10 "-" - - [31/May/2022:15:35:13 -0700] "POST
> /checkout/multi/payment-method/add HTTP/1.1" 400 278 "
> https://www.example.com/checkout/multi/payment-method/add" "Mozilla/5.0
> (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)
> Chrome/101.0.4951.64 Safari/537.36" "-" 115 9435 0
>
>
>
>
>
> */etc/httpd/conf/httpd.conf:*
>
> # ------------------------------------------
> # Listen Port
> # ------------------------------------------
>
> Listen 127.0.0.1:80
> Listen 443
>
> # ------------------------------------------
> # Load Modules
> # ------------------------------------------
>
> LoadModule ssl_module modules/mod_ssl.so
> LoadModule systemd_module modules/mod_systemd.so
> LoadModule unixd_module modules/mod_unixd.so
> LoadModule authz_core_module modules/mod_authz_core.so
> LoadModule access_compat_module modules/mod_access_compat.so
> LoadModule proxy_module modules/mod_proxy.so
> LoadModule proxy_http_module modules/mod_proxy_http.so
> LoadModule log_config_module modules/mod_log_config.so
> LoadModule logio_module modules/mod_logio.so
> LoadModule mpm_event_module modules/mod_mpm_event.so
> LoadModule rewrite_module modules/mod_rewrite.so
> LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
> LoadModule status_module modules/mod_status.so
> LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
> LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so
> LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so
> LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so
> LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
> LoadModule headers_module modules/mod_headers.so
> LoadModule setenvif_module modules/mod_setenvif.so
> LoadModule alias_module modules/mod_alias.so
> LoadModule dir_module modules/mod_dir.so
> LoadModule mime_module modules/mod_mime.so
> LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so
> LoadModule watchdog_module modules/mod_watchdog.so
>
> # ------------------------------------------
> # Run As
> # ------------------------------------------
>
> User apache
> Group apache
>
> # ------------------------------------------
> # Server Admin
> # ------------------------------------------
>
> ServerAdmin root@localhost
> ServerTokens ProductOnly
>
> Include conf.d/*.conf
>
> # ------------------------------------------
> # Doc Root
> # ------------------------------------------
>
> DocumentRoot /var/www/html
>
> # ------------------------------------------
> # Logs
> # ------------------------------------------
>
> ErrorLog "logs/error_log"
> LogLevel warn
>
> LogFormat "%v %h \"%{BALANCER_WORKER_NAME}e\" %l %u %t \"%r\" %>s %b
> \"%{Referer}i\" \"%{User-Agent}i\" \"%{JSESSIONID}C\" %D %I %O" combinedio
> LogFormat "%v \"%{X-Forwarded-For}i\" \"%{BALANCER_WORKER_NAME}e\" %l %u
> %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{JSESSIONID}C\" %D
> %I %O" proxy
>
> SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
>
> CustomLog "logs/access_log" combinedio env=!forwarded
> CustomLog "logs/access_log" proxy env=forwarded
>
> # ------------------------------------------
> # SSL
> # ------------------------------------------
>
> SSLSessionCache "shmcb:logs/session-cache(512000)"
> SSLStaplingCache "shmcb:logs/stapling-cache(160000)"
>
> # ------------------------------------------------
> # Virtual Hosts
> # ------------------------------------------------
>
> <VirtualHost 127.0.0.1:80>
>
> <Location "/serverstatus">
>
> SetHandler server-status
>
> </Location>
>
> ErrorLog /dev/null
> CustomLog /dev/null common
>
> </VirtualHost>
>
>
>
>
> */etc/httpd/conf.d/www.example.com.conf:*
>
>
>
>
> <VirtualHost *:443>
>
> ServerName example.com
> ServerAlias www.example.com
>
> SSLEngine on
> SSLProxyEngine on
> SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
> SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
> SSLHonorCipherOrder On
> SSLCompression off
> SSLUseStapling on
> SSLSessionTickets Off
> SSLCertificateFile /etc/pki/tls/certs/file.crt
> SSLCertificateKeyFile /etc/pki/tls/certs/file.key
>
> Header always set Strict-Transport-Security "max-age=63072000;
> includeSubDomains"
> Header always set X-Frame-Options SAMEORIGIN
>
> Header add Set-Cookie "ROUTEID=.%{BALANCER_WORKER_ROUTE}e; path=/"
> env=BALANCER_ROUTE_CHANGED
>
> Alias "/balancermanager_com" /var/www/html/balancermanager_com
>
> <Location /balancermanager_com>
>
> SetHandler balancer-manager
> Order Deny,Allow
> Deny from all
> Allow from 10.1.1.56
>
> </Location>
>
> <Directory /var/www/html/maintenance>
>
> Require all granted
>
> </Directory>
>
> ProxyHCExpr site_up {hc('body') !~ /ok/}
>
> <Proxy balancer://storefront-com>
>
> BalancerMember https://app410.example.com:8443 route=app410 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app411.example.com:8443 route=app411 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app413.example.com:8443 route=app413 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app414.example.com:8443 route=app414
> keepalive=On ttl=90 timeout=60 hcmethod=GET hcexpr=site_up
> hcuri=/healthcheck hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app415.example.com:8443 route=app415 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app416.example.com:8443 route=app416 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app417.example.com:8443 route=app417 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app418.example.com:8443 route=app418 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> BalancerMember https://app419.example.com:8443 route=app419 keepalive=On
> ttl=90 timeout=60 hcmethod=GET hcexpr=site_up hcuri=/healthcheck
> hcinterval=10 hcpasses=2 hcfails=2
> ProxySet lbmethod=bybusyness
>
> </Proxy>
>
> RewriteEngine On
>
> ErrorDocument 503 /maintenance/us/index.html
> RewriteCond /var/www/html/maintenance/us/enabled -f
> RewriteCond %{REQUEST_URI} !=/maintenance/us/index.html
> RewriteRule ^ - [R=503,L]
>
> RewriteCond /var/www/html/maintenance/us/enabled !-f
> RewriteRule ^/maintenance/us/index.html$ / [R,L]
>
> ProxyRequests Off
> ProxyPreserveHost On
> ProxyBadHeader Ignore
> ProxyPassMatch .*\.php$ !
> ProxyPassMatch .*\.asp$ !
> ProxyPassMatch .*\.pl$ !
> ProxyPassMatch .*\.pm$ !
> ProxyPassMatch .*\.rb$ !
> ProxyPassMatch .*\.py$ !
> ProxyPass /maintenance !
> Alias "/favicon.ico" /var/www/html/favicon.ico
> ProxyPass "/" balancer://storefront-com/ stickysession=ROUTEID
> ProxyPassReverse "/" balancer://storefront-com/ stickysession=ROUTEID
>
> </VirtualHost>
>
>
> What appears to be happening is that the requests are being 'addressed' by
> the alphabetically first *.conf file that is in /etc/httpd/conf.d.
> Previously we had another config file that alphabetically preceded the
> preview.example.com.conf config and the log entries referenced it as %v.
> But after turning the original alphabetically first config off, the log
> entries reference the now first preview.example.com config. But as shown
> above, the referrer in all log entries is: https://www.example.com.
>
> Can anyone recommend how we can understand what might be the issue here?
>
> Thanks in advance,
>
> HB
>