You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@couchdb.apache.org by Alexey Loshkarev <el...@gmail.com> on 2011/04/27 16:22:26 UTC
high IO wait and unresponsible views after couchdb restart
Hi folks.
I'm experiencing strange problem with CouchDB.
I have some large databases (about ~30-40 GB with ~2.5M documents in
it). There also (sure) indexes. Index files are 300-700 MB after
compaction.
Sometimes (it's inreproducible), after restarting couchdb, couchdb
view engine hangs for a while (tens of minutes, sometimes). It means,
no view request may be done. Documents are accessible ok, without a
problems, but view calls hangs.
Also, there is high disc activity.
beam.smp IO Wait is 100% with 3-5 MB/s reads (almost no writes).
There are no messages in couchdb.log
There are no processes at status page of futon.
Replications, if i start them, works with no problems.
View index files are not growing, so I can't guess, wich view is affected.
The only problem is unresponsive view engine.
After a while view engine starts reponse as nothing was happens and
database continue works nice.
CouchDB version is 1.0.2
Erlang - 13.2.4
Gentoo Linux
I tried to strace those process, here are pieces of log:
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428933, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274036736) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428935, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274032640) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428937, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274028544) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428939, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274024448) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428941, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274020352) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428943, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274016256) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428945, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274012160) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428947, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274008064) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428949, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36274003968) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e1e5588, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428951, NULL) = -1 EAGAIN
(Resource temporarily unavailable)
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
............
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428351, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275220480) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428353, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275216384) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428355, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275212288) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428357, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275208192) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428359, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275204096) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428361, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275200000) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428363, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275195904) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428365, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275191808) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428367, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275187712) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428369, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275183616) = 1
futex(0x7fdc7e1e55cc, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fdc7e1e55c8,
{FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7fdc7e29cbec, FUTEX_WAIT_PRIVATE, 7428371, NULL) = 0
futex(0x7fdc7e29cba8, FUTEX_WAKE_PRIVATE, 1) = 0
pread(68, "\0", 1, 36275179520) = 1
--
----------------
Best regards
Alexey Loshkarev
mailto:elf2001@gmail.com