You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2009/04/09 23:37:24 UTC
svn commit: r763816 - in /couchdb/trunk:
share/www/script/test/attachments.js src/couchdb/couch_db.erl
src/couchdb/couch_file.erl src/couchdb/couch_stats_collector.erl
src/couchdb/couch_stream.erl
Author: damien
Date: Thu Apr 9 21:37:23 2009
New Revision: 763816
URL: http://svn.apache.org/viewvc?rev=763816&view=rev
Log:
Fix for attachment sparseness bug COUCHDB-220 by giving each attachment it's own stream and calling set_min_buffer instead of ensure_buffer. Also fixed spurious couch_file crash messages by putting the statistics decrement code into a seperate monitoring process.
Modified:
couchdb/trunk/share/www/script/test/attachments.js
couchdb/trunk/src/couchdb/couch_db.erl
couchdb/trunk/src/couchdb/couch_file.erl
couchdb/trunk/src/couchdb/couch_stats_collector.erl
couchdb/trunk/src/couchdb/couch_stream.erl
Modified: couchdb/trunk/share/www/script/test/attachments.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/attachments.js?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/attachments.js (original)
+++ couchdb/trunk/share/www/script/test/attachments.js Thu Apr 9 21:37:23 2009
@@ -142,5 +142,38 @@
var xhr = CouchDB.request("GET", "/test_suite_db/bin_doc4/attachment.txt");
T(xhr.status == 200);
T(xhr.responseText == "This is a string");
+
+
+ // Attachment sparseness COUCHDB-220
+
+ var docs = []
+ for (var i = 0; i < 5; i++) {
+ var doc = {
+ _id: (i).toString(),
+ _attachments:{
+ "foo.txt": {
+ content_type:"text/plain",
+ data: "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ="
+ }
+ }
+ }
+ docs.push(doc)
+ }
+
+ db.bulkSave(docs);
+
+ var before = db.info().disk_size;
+
+ // Compact it.
+ T(db.compact().ok);
+ T(db.last_req.status == 202);
+ // compaction isn't instantaneous, loop until done
+ while (db.info().compact_running) {};
+
+ var after = db.info().disk_size;
+
+ // Compaction should reduce the database slightly, but not
+ // orders of magnitude (unless attachments introduce sparseness)
+ T(after > before * 0.1, "before: " + before + " after: " + after);
};
Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Thu Apr 9 21:37:23 2009
@@ -548,34 +548,7 @@
doc_flush_binaries(Doc, Fd) ->
- % calc size of binaries to write out
- Bins = Doc#doc.attachments,
- PreAllocSize = lists:foldl(
- fun(BinValue, SizeAcc) ->
- case BinValue of
- {_Key, {_Type, {Fd0, _StreamPointer, _Len}}} when Fd0 == Fd ->
- % already written to our file, nothing to write
- SizeAcc;
- {_Key, {_Type, {_OtherFd, _StreamPointer, Len}}} ->
- % written to a different file
- SizeAcc + Len;
- {_Key, {_Type, Bin}} when is_binary(Bin) ->
- % we have a new binary to write
- SizeAcc + size(Bin);
- {_Key, {_Type, {Fun, undefined}}} when is_function(Fun) ->
- % function without a known length
- % we'll have to alloc as we go with this one, for now, nothing
- SizeAcc;
- {_Key, {_Type, {Fun, Len}}} when is_function(Fun) ->
- % function to yield binary data with known length
- SizeAcc + Len
- end
- end,
- 0, Bins),
-
- {ok, OutputStream} = couch_stream:open(Fd),
- ok = couch_stream:ensure_buffer(OutputStream, PreAllocSize),
- NewBins = lists:map(
+ NewAttachments = lists:map(
fun({Key, {Type, BinValue}}) ->
NewBinValue =
case BinValue of
@@ -585,6 +558,8 @@
{OtherFd, StreamPointer, Len} ->
% written to a different file (or a closed file
% instance, which will cause an error)
+ {ok, OutputStream} = couch_stream:open(Fd),
+ ok = couch_stream:set_min_buffer(OutputStream, Len),
{ok, {NewStreamPointer, Len}, _EndSp} =
couch_stream:foldl(OtherFd, StreamPointer, Len,
fun(Bin, {BeginPointer, SizeAcc}) ->
@@ -597,31 +572,38 @@
end
end,
{{0,0}, 0}),
+ couch_stream:close(OutputStream),
{Fd, NewStreamPointer, Len};
Bin when is_binary(Bin) ->
+ {ok, OutputStream} = couch_stream:open(Fd),
+ ok = couch_stream:set_min_buffer(OutputStream, size(Bin)),
{ok, StreamPointer} = couch_stream:write(OutputStream, Bin),
+ couch_stream:close(OutputStream),
{Fd, StreamPointer, size(Bin)};
{StreamFun, undefined} when is_function(StreamFun) ->
% max_attachment_chunk_size control the max we buffer in memory
MaxChunkSize = list_to_integer(couch_config:get("couchdb",
"max_attachment_chunk_size","4294967296")),
+ {ok, OutputStream} = couch_stream:open(Fd),
WriterFun = make_writer_fun(OutputStream),
% StreamFun(MaxChunkSize, WriterFun)
% will call our WriterFun
% once for each chunk of the attachment.
{ok, {TotalLength, NewStreamPointer}} =
StreamFun(MaxChunkSize, WriterFun, {0, nil}),
+ couch_stream:close(OutputStream),
{Fd, NewStreamPointer, TotalLength};
{Fun, Len} when is_function(Fun) ->
+ {ok, OutputStream} = couch_stream:open(Fd),
+ ok = couch_stream:set_min_buffer(OutputStream, Len),
{ok, StreamPointer} =
write_streamed_attachment(OutputStream, Fun, Len, nil),
+ couch_stream:close(OutputStream),
{Fd, StreamPointer, Len}
end,
{Key, {Type, NewBinValue}}
- end, Bins),
-
- {ok, _FinalPos} = couch_stream:close(OutputStream),
- Doc#doc{attachments = NewBins}.
+ end, Doc#doc.attachments),
+ Doc#doc{attachments = NewAttachments}.
make_writer_fun(Stream) ->
Modified: couchdb/trunk/src/couchdb/couch_file.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_file.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_file.erl (original)
+++ couchdb/trunk/src/couchdb/couch_file.erl Thu Apr 9 21:37:23 2009
@@ -269,7 +269,6 @@
% server functions
init({Filepath, Options, ReturnPid, Ref}) ->
- process_flag(trap_exit, true),
case lists:member(create, Options) of
true ->
filelib:ensure_dir(Filepath),
@@ -285,14 +284,14 @@
true ->
{ok, 0} = file:position(Fd, 0),
ok = file:truncate(Fd),
- catch couch_stats_collector:increment({couchdb, open_os_files}),
+ track_stats(),
{ok, Fd};
false ->
ok = file:close(Fd),
init_status_error(ReturnPid, Ref, file_exists)
end;
false ->
- catch couch_stats_collector:increment({couchdb, open_os_files}),
+ track_stats(),
{ok, Fd}
end;
Error ->
@@ -304,7 +303,7 @@
{ok, Fd_Read} ->
{ok, Fd} = file:open(Filepath, [read, write, raw, binary]),
ok = file:close(Fd_Read),
- catch couch_stats_collector:increment({couchdb, open_os_files}),
+ track_stats(),
{ok, Fd};
Error ->
init_status_error(ReturnPid, Ref, Error)
@@ -312,10 +311,21 @@
end.
-terminate(_Reason, _Fd) ->
- catch couch_stats_collector:decrement({couchdb, open_os_files}),
+terminate(_Reason, _Fd) ->
ok.
+track_stats() ->
+ try couch_stats_collector:increment({couchdb, open_os_files}) of
+ ok ->
+ Self = self(),
+ spawn(
+ fun() ->
+ erlang:monitor(process, Self),
+ receive {'DOWN', _, _, _, _} -> ok end,
+ couch_stats_collector:decrement({couchdb, open_os_files})
+ end)
+ catch _ -> ok
+ end.
handle_call({pread, Pos, Bytes}, _From, Fd) ->
{reply, file:pread(Fd, Pos, Bytes), Fd};
@@ -349,5 +359,5 @@
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-handle_info({'EXIT', _, Reason}, Fd) ->
- {stop, Reason, Fd}.
+handle_info(foo, _Fd) ->
+ ok.
Modified: couchdb/trunk/src/couchdb/couch_stats_collector.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stats_collector.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_stats_collector.erl (original)
+++ couchdb/trunk/src/couchdb/couch_stats_collector.erl Thu Apr 9 21:37:23 2009
@@ -56,13 +56,17 @@
increment({Module, list_to_atom(integer_to_list(Key))});
increment(Key) ->
case catch ets:update_counter(?HIT_COUNTER_TABLE, Key, 1) of
- {'EXIT', {badarg, _}} -> ets:insert(?HIT_COUNTER_TABLE, {Key, 1});
+ {'EXIT', {badarg, _}} ->
+ true = ets:insert(?HIT_COUNTER_TABLE, {Key, 1}),
+ ok;
_ -> ok
end.
decrement(Key) ->
case catch ets:update_counter(?HIT_COUNTER_TABLE, Key, -1) of
- {'EXIT', {badarg, _}} -> ets:insert(?HIT_COUNTER_TABLE, {Key, -1});
+ {'EXIT', {badarg, _}} ->
+ true = ets:insert(?HIT_COUNTER_TABLE, {Key, -1}),
+ ok;
_ -> ok
end.
Modified: couchdb/trunk/src/couchdb/couch_stream.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stream.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_stream.erl (original)
+++ couchdb/trunk/src/couchdb/couch_stream.erl Thu Apr 9 21:37:23 2009
@@ -38,6 +38,7 @@
current_pos = 0,
bytes_remaining = 0,
next_alloc = 0,
+ first_alloc = 0,
min_alloc = 16#00010000
}).
Re: svn commit: r763816 - in /couchdb/trunk: share/www/script/test/attachments.js src/couchdb/couch_db.erl src/couchdb/couch_file.erl src/couchdb/couch_stats_collector.erl src/couchdb/couch_stream.erl
Posted by Damien Katz <da...@apache.org>.
No.
-Damien
On Apr 11, 2009, at 11:07 AM, Jan Lehnardt wrote:
>
> On 9 Apr 2009, at 22:37, damien@apache.org wrote:
>
>> Author: damien
>> Date: Thu Apr 9 21:37:23 2009
>> New Revision: 763816
>>
>> URL: http://svn.apache.org/viewvc?rev=763816&view=rev
>> Log:
>> Fix for attachment sparseness bug COUCHDB-220 by giving each
>> attachment it's own stream and calling set_min_buffer instead of
>> ensure_buffer. Also fixed spurious couch_file crash messages by
>> putting the statistics decrement code into a seperate monitoring
>> process.
>
> Does this break the file format?
>
> Cheers
> Jan
> --
>
Re: svn commit: r763816 - in /couchdb/trunk: share/www/script/test/attachments.js src/couchdb/couch_db.erl src/couchdb/couch_file.erl src/couchdb/couch_stats_collector.erl src/couchdb/couch_stream.erl
Posted by Jan Lehnardt <ja...@apache.org>.
On 9 Apr 2009, at 22:37, damien@apache.org wrote:
> Author: damien
> Date: Thu Apr 9 21:37:23 2009
> New Revision: 763816
>
> URL: http://svn.apache.org/viewvc?rev=763816&view=rev
> Log:
> Fix for attachment sparseness bug COUCHDB-220 by giving each
> attachment it's own stream and calling set_min_buffer instead of
> ensure_buffer. Also fixed spurious couch_file crash messages by
> putting the statistics decrement code into a seperate monitoring
> process.
Does this break the file format?
Cheers
Jan
--