You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@httpd.apache.org by Robert Rigg <ri...@gmail.com> on 2013/08/30 20:54:33 UTC
[users@httpd] Fwd: Segmentation Fault when httpd child processes reach MaxRequestsPerChild
All httpd child processes Segmentation fault near MaxRequestsPerChild,
seemingly when it is time for the process to recycle.
MaxRequestsPerChild currently set to 10000. I am also not able to
reproduce this on a test system. Child process exits clean when attached
with gdb. Child process also exits clean on stops/restarts.
[notice] child pid 14054 exit signal Segmentation fault (11)
#Info:
RHEL 6.4
Server version: Apache/2.2.15 (Unix)
> Server built: Apr 29 2013 04:13:12
> Server's Module Magic Number: 20051115:25
> Server loaded: APR 1.3.9, APR-Util 1.3.9
> Compiled using: APR 1.3.9, APR-Util 1.3.9
> Architecture: 64-bit
> Server MPM: Prefork
> threaded: no
> forked: yes (variable process count)
> Server compiled with....
> -D APACHE_MPM_DIR="server/mpm/prefork"
> -D APR_HAS_SENDFILE
> -D APR_HAS_MMAP
> -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
> -D APR_USE_SYSVSEM_SERIALIZE
> -D APR_USE_PTHREAD_SERIALIZE
> -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
> -D APR_HAS_OTHER_CHILD
> -D AP_HAVE_RELIABLE_PIPED_LOGS
> -D DYNAMIC_MODULE_LIMIT=128
> -D HTTPD_ROOT="/etc/httpd"
> -D SUEXEC_BIN="/usr/sbin/suexec"
> -D DEFAULT_PIDLOG="run/httpd.pid"
> -D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
> -D DEFAULT_LOCKFILE="logs/accept.lock"
> -D DEFAULT_ERRORLOG="logs/error_log"
> -D AP_TYPES_CONFIG_FILE="conf/mime.types"
> -D SERVER_CONFIG_FILE="conf/httpd.conf"
#
[debug] prefork.c(1018): AcceptMutex: sysvsem (default: sysvsem)
#gdb bt sample:
Program terminated with signal 11, Segmentation fault.
> #0 __pthread_mutex_lock (mutex=0x0) at pthread_mutex_lock.c:50
> 50 unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
#strace sample:
*Filtered
open("/etc/pki/nssdb/pkcs11.txt", O_RDONLY) = 23
>
> close(23) = 0
> open("/etc/ld.so.cache", O_RDONLY) = 23
> close(23) = 0
> open("/usr/lib64/libnsssysinit.so", O_RDONLY) = 23
> close(23) = 0
> ...
> open("/etc/pki/nssdb/cert9.db", O_RDONLY) = 23
> fcntl(23, F_GETFD) = 0
> fcntl(23, F_SETFD, FD_CLOEXEC) = 0
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848,
> st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0,
> st_blksize=4096, st_blocks=24, st_size=9216,
> st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
> , st_ctime=2013/08/20-09:59:34}) = 0
> lseek(23, 0, SEEK_SET) = 0
> read(23, "SQLite format 3\0\4\0\1\1\0@ \0\0\0\1\0\0\0\0"...,
> 100) = 100
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET,
> start=1073741824, len=1}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET,
> start=1073741826, len=510}) = 0
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET,
> start=1073741824, len=1}) = 0
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848,
> st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0,
> st_blksize=4096, st_blocks=24, st_size=9216,
> st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
> , st_ctime=2013/08/20-09:59:34}) = 0
> lseek(23, 0, SEEK_SET) = 0
> read(23, "SQLite format 3\0\4\0\1\1\0@ \0\0\0\1\0\0\0\0"...,
> 1024) = 1024
> lseek(23, 3072, SEEK_SET) = 3072
> read(23,
> "\r\0\0\0\1\0J\0\0J\3\315\0\0\0\0\0\0\0\0\0\0\0\2073\1\7\27\37\37\1\216"...,
> 1024) = 1024
> lseek(23, 5120, SEEK_SET) = 5120
> read(23,
> "\r\0\0\0\5\2\302\0\3\212\3\275\3D\3\4\2\302\0\0\0\0\0\0\0\0\0\0\0\0\0\0"...,
> 1024) = 1024
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0,
> len=0}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET,
> start=1073741824, len=1}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET,
> start=1073741826, len=510}) = 0
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET,
> start=1073741824, len=1}) = 0
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848,
> st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0,
> st_blksize=4096, st_blocks=24, st_size=9216,
> st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
> , st_ctime=2013/08/20-09:59:34}) = 0
>
> ...
>
> rt_sigprocmask(SIG_BLOCK, [INT], NULL, 8) = 0
> rt_sigaction(SIGINT, {0x7f8dc4ec0e50, [], SA_RESTORER,
> 0x7f8dc33f0920}, {0x7f8db3507f0c, ~[ILL ABRT BUS FPE KILL SEGV USR2 STOP
> XCPU XFSZ SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x7f8dc33f0920}, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [INT], NULL, 8) = 0
> munmap(0x7f8dae20f000, 2178616) = 0
> munmap(0x7f8daeced000, 2129488) = 0
> munmap(0x7f8daead9000, 2176864) = 0
> munmap(0x7f8dae89c000, 2343352) = 0
> munmap(0x7f8dae627000, 2574912) = 0
> munmap(0x7f8dae423000, 2109528) = 0
> munmap(0x7f8daeef5000, 2142192) = 0
> munmap(0x7f8daf100000, 2125616) = 0
> munmap(0x7f8daf307000, 2128512) = 0
> munmap(0x7f8daf50f000, 2107408) = 0
> munmap(0x7f8daf712000, 2103968) = 0
> munmap(0x7f8daf914000, 2111904) = 0
> munmap(0x7f8dafb18000, 2162656) = 0
> munmap(0x7f8dafd28000, 2140264) = 0
> munmap(0x7f8db0911000, 2116672) = 0
> munmap(0x7f8db044f000, 2099376) = 0
> munmap(0x7f8db0650000, 2885912) = 0
> munmap(0x7f8db0b16000, 2123736) = 0
> munmap(0x7f8db0d1d000, 2357704) = 0
> munmap(0x7f8db11ec000, 2119816) = 0
> munmap(0x7f8db13f2000, 2125296) = 0
> munmap(0x7f8db15f9000, 2198440) = 0
> munmap(0x7f8db467b000, 2289928) = 0
> munmap(0x7f8db48ab000, 2232272) = 0
> munmap(0x7f8db4e50000, 2149768) = 0
> munmap(0x7f8db4acc000, 3684744) = 0
> munmap(0x7f8db5290000, 2142072) = 0
> munmap(0x7f8db505d000, 2303840) = 0
> munmap(0x7f8db549b000, 4278992) = 0
> munmap(0x7f8db58b0000, 2148288) = 0
> munmap(0x7f8db5abd000, 2131392) = 0
> munmap(0x7f8db6d48000, 2440032) = 0
> munmap(0x7f8db6aab000, 2737840) = 0
> munmap(0x7f8db655d000, 2165496) = 0
> munmap(0x7f8db676e000, 3394936) = 0
> munmap(0x7f8db6337000, 2250552) = 0
> munmap(0x7f8db60e7000, 2423648) = 0
> munmap(0x7f8db5ec9000, 2217576) = 0
> munmap(0x7f8db5cc6000, 2106112) = 0
> munmap(0x7f8db6f9c000, 3939304) = 0
> munmap(0x7f8db735e000, 2268048) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826,
> len=510}) = 0
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> access("/etc/pki/nssdb/cert9.db-journal", F_OK)
> = -1 ENOENT (No such file or directory)
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644,
> st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24,
> st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17,
> st_ctime=2013/08/20-09:59:34}) = 0
> lseek(23, 24, SEEK_SET) = 24
> read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826,
> len=510}) = 0
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such
> file or directory)
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644,
> st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24,
> st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17,
> st_ctime=2013/08/20-09:59:34}) = 0
> lseek(23, 24, SEEK_SET) = 24
> read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
> brk(0x7f8dc6191000) = 0x7f8dc6191000
> munmap(0x7f8db7e3e000, 2161288) = 0
> munmap(0x7f8db7bea000, 2438600) = 0
> munmap(0x7f8db79b8000, 2301736) = 0
> munmap(0x7f8db7588000, 2257152) = 0
> close(8) = 0
> close(7) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826,
> len=510}) = 0
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824,
> len=1}) = 0
> access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such
> file or directory)
> fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644,
> st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24,
> st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17,
> st_ctime=2013/08/20-09:59:34}) = 0
> lseek(23, 24, SEEK_SET) = 24
> read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
> fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
> --- SIGSEGV (Segmentation fault) @ 0 (0) ---
> chdir("/apache2/apache-dumps") = 0
> rt_sigaction(SIGSEGV, {SIG_DFL, [], SA_RESTORER|SA_INTERRUPT,
> 0x7f8dc3760500}, {SIG_DFL, [], SA_RESTORER|SA_RESETHAND, 0x7f8dc3760500},
> 8) = 0
> kill(26317, SIGSEGV) = 0
> rt_sigreturn(0x66cd) = 140246846382624
> --- SIGSEGV (Segmentation fault) @ 0 (0) ---
>
>
Re: [users@httpd] Fwd: Segmentation Fault when httpd child processes
reach MaxRequestsPerChild
Posted by Robert Rigg <ri...@gmail.com>.
I am not especially experienced with gdb. However, I do know that the
debuginfo packages are not installed on the production system. Is there
something specific that I could gather that might help?
(gdb) bt full
> #0 0x00007f07d58bf220 in pthread_mutex_lock () from /lib64/libpthread.so.0
> No symbol table info available.
> #1 0x00007f07d1c20fe9 in ?? ()
> No symbol table info available.
> #2 0x00007f07d84b4640 in ?? ()
> No symbol table info available.
> #3 0x00007f07d24b5fd8 in ?? ()
> No symbol table info available.
> #4 0x0000000000000800 in ?? ()
> No symbol table info available.
> #5 0x00007f07d84b4640 in ?? ()
> No symbol table info available.
> #6 0x0000000000000003 in ?? ()
> No symbol table info available.
> #7 0x000000000000001c in ?? ()
> No symbol table info available.
> #8 0x00007f07d8623790 in ?? ()
> No symbol table info available.
> #9 0x00007f07d84b4640 in ?? ()
> No symbol table info available.
> #10 0x00007f07d8502d60 in ?? ()
> No symbol table info available.
> #11 0x00007fff64096260 in ?? ()
> No symbol table info available.
> #12 0x0000000000000003 in ?? ()
> No symbol table info available.
> #13 0x000000000000001c in ?? ()
> No symbol table info available.
> #14 0x00007f07d900d3f0 in ?? ()
> No symbol table info available.
> #15 0x00007f07d24b633c in ?? ()
> No symbol table info available.
> #16 0x00007f07d84b42c0 in ?? ()
> No symbol table info available.
> #17 0x0000000000000000 in ?? ()
> No symbol table info available.
Re: [users@httpd] Fwd: Segmentation Fault when httpd child processes
reach MaxRequestsPerChild
Posted by Eric Covener <co...@gmail.com>.
>
> #gdb bt sample:
>
>> Program terminated with signal 11, Segmentation fault.
>> #0 __pthread_mutex_lock (mutex=0x0) at pthread_mutex_lock.c:50
>> 50 unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
>
is there a complete backtrace?
---------------------------------------------------------------------
To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
For additional commands, e-mail: users-help@httpd.apache.org