You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by fd...@apache.org on 2011/11/29 15:13:28 UTC
[1/3] git commit: Make couch_file:flush/1 calls automatic
Updated Branches:
refs/heads/COUCHDB-1342 62dde7e9f -> 4f7336ed3
Make couch_file:flush/1 calls automatic
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/4f7336ed
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/4f7336ed
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/4f7336ed
Branch: refs/heads/COUCHDB-1342
Commit: 4f7336ed30c0a34823cb332190b57a608725a21c
Parents: 0b89f5a
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Tue Nov 29 00:12:10 2011 +0000
Committer: Filipe David Borba Manana <fd...@apache.org>
Committed: Tue Nov 29 13:58:43 2011 +0000
----------------------------------------------------------------------
src/couch_mrview/src/couch_mrview_index.erl | 3 +-
src/couch_mrview/src/couch_mrview_updater.erl | 8 +---
src/couch_mrview/src/couch_mrview_util.erl | 1 -
src/couchdb/couch_db_updater.erl | 9 +----
src/couchdb/couch_file.erl | 37 +++++++++++++++----
test/etap/010-file-basics.t | 6 ---
test/etap/011-file-headers.t | 7 ----
test/etap/020-btree-basics.t | 10 -----
test/etap/021-btree-reductions.t | 1 -
test/etap/050-stream.t | 1 -
10 files changed, 33 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 21d38d1..1e7b7a3 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -128,8 +128,7 @@ finish_update(State) ->
commit(State) ->
Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
- ok = couch_file:write_header(State#mrst.fd, Header),
- ok = couch_file:flush(State#mrst.fd).
+ ok = couch_file:write_header(State#mrst.fd, Header).
compact(Db, State, Opts) ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index ef6f2e7..3014664 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -55,8 +55,7 @@ start_update(Partial, State, NumChanges) ->
purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
#mrst{
id_btree=IdBtree,
- views=Views,
- fd=Fd
+ views=Views
} = State,
Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
@@ -88,7 +87,6 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
end,
Views2 = lists:map(RemKeysFun, Views),
- ok = couch_file:flush(Fd),
{ok, State#mrst{
id_btree=IdBtree2,
views=Views2,
@@ -231,8 +229,7 @@ insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) ->
write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
#mrst{
id_btree=IdBtree,
- first_build=FirstBuild,
- fd=Fd
+ first_build=FirstBuild
} = State,
{ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
@@ -248,7 +245,6 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}
end,
- ok = couch_file:flush(Fd),
State#mrst{
views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs),
update_seq=UpdateSeq,
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 065a2cd..a13b154 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -546,7 +546,6 @@ delete_file(FName) ->
reset_index(Db, Fd, #mrst{sig=Sig}=State) ->
ok = couch_file:truncate(Fd, 0),
ok = couch_file:write_header(Fd, {Sig, nil}),
- ok = couch_file:flush(Fd),
init_state(Db, Fd, reset_state(State), nil).
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl
index 03a04aa..a4d4202 100644
--- a/src/couchdb/couch_db_updater.erl
+++ b/src/couchdb/couch_db_updater.erl
@@ -659,8 +659,7 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) ->
fulldocinfo_by_id_btree = DocInfoByIdBTree,
docinfo_by_seq_btree = DocInfoBySeqBTree,
update_seq = LastSeq,
- revs_limit = RevsLimit,
- fd = Fd
+ revs_limit = RevsLimit
} = Db,
Ids = [Id || [{_Client, #doc{id=Id}}|_] <- DocsList],
% lookup up the old documents, if they exist.
@@ -695,7 +694,6 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) ->
fulldocinfo_by_id_btree = DocInfoByIdBTree2,
docinfo_by_seq_btree = DocInfoBySeqBTree2,
update_seq = NewSeq},
- couch_file:flush(Fd),
% Check if we just updated any design documents, and update the validation
% funs if we did.
@@ -787,7 +785,6 @@ commit_data(Db, _) ->
end,
ok = couch_file:write_header(Fd, Header),
- ok = couch_file:flush(Fd),
case lists:member(after_header, FsyncOptions) of
true -> ok = couch_file:sync(Fd);
@@ -838,7 +835,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
InfoBySeq = lists:usort(fun(#doc_info{id=A}, #doc_info{id=B}) -> A =< B end,
InfoBySeq0),
Ids = [Id || #doc_info{id=Id} <- InfoBySeq],
- ok = couch_file:flush(DestFd),
LookupResults = couch_btree:lookup(Db#db.fulldocinfo_by_id_btree, Ids),
NewFullDocInfos1 = lists:map(
@@ -888,8 +884,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
copy_compact(Db, NewDb0, Retry) ->
FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header],
NewDb = NewDb0#db{fsync_options=FsyncOptions},
- ok = couch_file:flush(Db#db.fd),
- ok = couch_file:flush(NewDb#db.fd),
TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq),
BufferSize = list_to_integer(
couch_config:get("database_compaction", "doc_buffer_size", "524288")),
@@ -955,7 +949,6 @@ copy_compact(Db, NewDb0, Retry) ->
NewDb4 = NewDb3
end,
- ok = couch_file:flush(NewDb4#db.fd),
commit_data(NewDb4#db{update_seq=Db#db.update_seq}).
start_copy_compact(#db{name=Name,filepath=Filepath,header=#db_header{purge_seq=PurgeSeq}}=Db) ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_file.erl b/src/couchdb/couch_file.erl
index 3979eb7..bb69004 100644
--- a/src/couchdb/couch_file.erl
+++ b/src/couchdb/couch_file.erl
@@ -132,6 +132,16 @@ pread_binary(Fd, Pos) ->
pread_iolist(Fd, Pos) ->
+ case do_read_iolist(Fd, Pos) of
+ eof ->
+ flush(Fd),
+ do_read_iolist(Fd, Pos);
+ Else ->
+ Else
+ end.
+
+
+do_read_iolist(Fd, Pos) ->
case gen_server:call(Fd, {pread_iolist, Pos}, infinity) of
{ok, _IoList} = Ok ->
Ok;
@@ -260,7 +270,8 @@ write_header(Fd, Data) ->
Md5 = couch_util:md5(Bin),
% now we assemble the final header binary and write to disk
FinalBin = <<Md5/binary, Bin/binary>>,
- ok = gen_server:call(Fd, {write_header, FinalBin}, infinity).
+ ok = gen_server:call(Fd, {write_header, FinalBin}, infinity),
+ ok = flush(Fd).
% server functions
@@ -472,8 +483,12 @@ read_raw_iolist_int(ReadFd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE
read_raw_iolist_int(ReadFd, Pos, Len) ->
BlockOffset = Pos rem ?SIZE_BLOCK,
TotalBytes = calculate_total_read_len(BlockOffset, Len),
- {ok, <<RawBin:TotalBytes/binary>>} = file:pread(ReadFd, Pos, TotalBytes),
- {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}.
+ case file:pread(ReadFd, Pos, TotalBytes) of
+ {ok, <<RawBin:TotalBytes/binary>>} ->
+ {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes};
+ eof ->
+ throw(eof)
+ end.
-spec extract_md5(iolist()) -> {binary(), iolist()}.
extract_md5(FullIoList) ->
@@ -649,7 +664,13 @@ write_header_blocks(Fd, Eof, Header) ->
reader_loop(Fd) ->
receive
{read, Pos, From} ->
- read_iolist(Fd, Pos, From),
+ Result = try
+ read_iolist(Fd, Pos)
+ catch
+ throw:eof ->
+ eof
+ end,
+ gen_server:reply(From, Result),
reader_loop(Fd);
{find_header, Eof, From} ->
gen_server:reply(From, find_header(Fd, Eof div ?SIZE_BLOCK)),
@@ -660,9 +681,9 @@ reader_loop(Fd) ->
end.
--compile({inline, [read_iolist/3]}).
+-compile({inline, [read_iolist/2]}).
-read_iolist(Fd, Pos, From) ->
+read_iolist(Fd, Pos) ->
{RawData, NextPos} = try
% up to 8Kbs of read ahead
read_raw_iolist_int(Fd, Pos, 2 * ?SIZE_BLOCK - (Pos rem ?SIZE_BLOCK))
@@ -676,8 +697,8 @@ read_iolist(Fd, Pos, From) ->
1 ->
{Md5, IoList} = extract_md5(
maybe_read_more_iolist(RestRawData, 16 + Len, NextPos, Fd)),
- gen_server:reply(From, {ok, IoList, Md5});
+ {ok, IoList, Md5};
0 ->
IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, Fd),
- gen_server:reply(From, {ok, IoList})
+ {ok, IoList}
end.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/010-file-basics.t
----------------------------------------------------------------------
diff --git a/test/etap/010-file-basics.t b/test/etap/010-file-basics.t
index a1b9a98..fb1b29e 100755
--- a/test/etap/010-file-basics.t
+++ b/test/etap/010-file-basics.t
@@ -57,7 +57,6 @@ test() ->
?etap_match(couch_file:append_binary(Fd, <<"fancy!">>), {ok, Size, _},
"Appending a binary returns the current file size."),
- ok = couch_file:flush(Fd),
etap:is({ok, foo}, couch_file:pread_term(Fd, 0),
"Reading the first term returns what we wrote: foo"),
@@ -71,30 +70,25 @@ test() ->
),
{ok, BinPos, _} = couch_file:append_binary(Fd, <<131,100,0,3,102,111,111>>),
- ok = couch_file:flush(Fd),
etap:is({ok, foo}, couch_file:pread_term(Fd, BinPos),
"Reading a term from a written binary term representation succeeds."),
BigBin = list_to_binary(lists:duplicate(100000, 0)),
{ok, BigBinPos, _} = couch_file:append_binary(Fd, BigBin),
- ok = couch_file:flush(Fd),
etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos),
"Reading a large term from a written representation succeeds."),
ok = couch_file:write_header(Fd, hello),
- ok = couch_file:flush(Fd),
etap:is({ok, hello}, couch_file:read_header(Fd),
"Reading a header succeeds."),
{ok, BigBinPos2, _} = couch_file:append_binary(Fd, BigBin),
- ok = couch_file:flush(Fd),
etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos2),
"Reading a large term from a written representation succeeds 2."),
% append_binary == append_iolist?
% Possible bug in pread_iolist or iolist() -> append_binary
{ok, IOLPos, _} = couch_file:append_binary(Fd, ["foo", $m, <<"bam">>]),
- ok = couch_file:flush(Fd),
{ok, IoList} = couch_file:pread_iolist(Fd, IOLPos),
etap:is(<<"foombam">>, iolist_to_binary(IoList),
"Reading an results in a binary form of the written iolist()"),
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/011-file-headers.t
----------------------------------------------------------------------
diff --git a/test/etap/011-file-headers.t b/test/etap/011-file-headers.t
index 82c028f..a26b032 100755
--- a/test/etap/011-file-headers.t
+++ b/test/etap/011-file-headers.t
@@ -45,7 +45,6 @@ test() ->
etap:is_greater(Size1, 0,
"Writing a header allocates space in the file."),
- ok = couch_file:flush(Fd),
etap:is({ok, {<<"some_data">>, 32}}, couch_file:read_header(Fd),
"Reading the header returns what we wrote."),
@@ -56,7 +55,6 @@ test() ->
etap:is_greater(Size2, Size1,
"Writing a second header allocates more space."),
- ok = couch_file:flush(Fd),
etap:is({ok, [foo, <<"more">>]}, couch_file:read_header(Fd),
"Reading the second header does not return the first header."),
@@ -71,7 +69,6 @@ test() ->
"Rewriting the same second header returns the same second size."),
couch_file:write_header(Fd, erlang:make_tuple(5000, <<"CouchDB">>)),
- ok = couch_file:flush(Fd),
etap:is(
couch_file:read_header(Fd),
{ok, erlang:make_tuple(5000, <<"CouchDB">>)},
@@ -85,7 +82,6 @@ test() ->
% Destroy the 0x1 byte that marks a header
check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
- ok = couch_file:flush(CouchFd),
etap:isnt(Expect, couch_file:read_header(CouchFd),
"Should return a different header before corruption."),
file:pwrite(RawFd, HeaderPos, <<0>>),
@@ -95,7 +91,6 @@ test() ->
% Corrupt the size.
check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
- ok = couch_file:flush(CouchFd),
etap:isnt(Expect, couch_file:read_header(CouchFd),
"Should return a different header before corruption."),
% +1 for 0x1 byte marker
@@ -106,7 +101,6 @@ test() ->
% Corrupt the MD5 signature
check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
- ok = couch_file:flush(CouchFd),
etap:isnt(Expect, couch_file:read_header(CouchFd),
"Should return a different header before corruption."),
% +5 = +1 for 0x1 byte and +4 for term size.
@@ -117,7 +111,6 @@ test() ->
% Corrupt the data
check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
- ok = couch_file:flush(CouchFd),
etap:isnt(Expect, couch_file:read_header(CouchFd),
"Should return a different header before corruption."),
% +21 = +1 for 0x1 byte, +4 for term size and +16 for MD5 sig
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/020-btree-basics.t
----------------------------------------------------------------------
diff --git a/test/etap/020-btree-basics.t b/test/etap/020-btree-basics.t
index 372b494..e65aeb2 100755
--- a/test/etap/020-btree-basics.t
+++ b/test/etap/020-btree-basics.t
@@ -97,7 +97,6 @@ test_kvs(KeyValues) ->
"After removing all keys btree size is 0."),
{Btree4, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) ->
- ok = couch_file:flush(Fd),
{ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []),
case couch_btree:size(BtAcc2) > PrevSize of
true ->
@@ -107,7 +106,6 @@ test_kvs(KeyValues) ->
end,
{BtAcc2, couch_btree:size(BtAcc2)}
end, {Btree3, couch_btree:size(Btree3)}, KeyValues),
- ok = couch_file:flush(Fd),
etap:ok(test_btree(Btree4, KeyValues),
"Adding all keys one at a time returns a complete btree."),
@@ -115,7 +113,6 @@ test_kvs(KeyValues) ->
"Non empty btrees have a size > 0."),
{Btree5, _} = lists:foldl(fun({K, _}, {BtAcc, PrevSize}) ->
- ok = couch_file:flush(Fd),
{ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]),
case couch_btree:size(BtAcc2) < PrevSize of
true ->
@@ -125,7 +122,6 @@ test_kvs(KeyValues) ->
end,
{BtAcc2, couch_btree:size(BtAcc2)}
end, {Btree4, couch_btree:size(Btree4)}, KeyValues),
- ok = couch_file:flush(Fd),
etap:ok(test_btree(Btree5, []),
"Removing all keys one at a time returns an empty btree."),
@@ -134,7 +130,6 @@ test_kvs(KeyValues) ->
KeyValuesRev = lists:reverse(KeyValues),
{Btree6, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) ->
- ok = couch_file:flush(Fd),
{ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []),
case couch_btree:size(BtAcc2) > PrevSize of
true ->
@@ -177,7 +172,6 @@ test_kvs(KeyValues) ->
true.
test_btree(#btree{fd = Fd} = Btree, KeyValues) ->
- ok = couch_file:flush(Fd),
ok = test_key_access(Btree, KeyValues),
ok = test_lookup_access(Btree, KeyValues),
ok = test_final_reductions(Btree, KeyValues),
@@ -185,7 +179,6 @@ test_btree(#btree{fd = Fd} = Btree, KeyValues) ->
true.
test_add_remove(#btree{fd = Fd} = Btree, OutKeyValues, RemainingKeyValues) ->
- ok = couch_file:flush(Fd),
Btree2 = lists:foldl(fun({K, _}, BtAcc) ->
{ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]),
BtAcc2
@@ -207,13 +200,11 @@ test_key_access(#btree{fd = Fd} = Btree, List) ->
end,
Length = length(List),
Sorted = lists:sort(List),
- ok = couch_file:flush(Fd),
{ok, _, {[], Length}} = couch_btree:foldl(Btree, FoldFun, {Sorted, 0}),
{ok, _, {[], Length}} = couch_btree:fold(Btree, FoldFun, {Sorted, 0}, [{dir, rev}]),
ok.
test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) ->
- ok = couch_file:flush(Fd),
FoldFun = fun({Key, Value}, {Key, Value}) -> {stop, true} end,
lists:foreach(fun({Key, Value}) ->
[{ok, {Key, Value}}] = couch_btree:lookup(Btree, [Key]),
@@ -221,7 +212,6 @@ test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) ->
end, KeyValues).
test_final_reductions(#btree{fd = Fd} = Btree, KeyValues) ->
- ok = couch_file:flush(Fd),
KVLen = length(KeyValues),
FoldLFun = fun(_X, LeadingReds, Acc) ->
CountToStart = KVLen div 3 + Acc,
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/021-btree-reductions.t
----------------------------------------------------------------------
diff --git a/test/etap/021-btree-reductions.t b/test/etap/021-btree-reductions.t
index e597772..331e49a 100755
--- a/test/etap/021-btree-reductions.t
+++ b/test/etap/021-btree-reductions.t
@@ -47,7 +47,6 @@ test()->
end, {"odd", []}, lists:seq(1, rows())),
{ok, Btree2} = couch_btree:add_remove(Btree, EvenOddKVs, []),
- ok = couch_file:flush(Fd),
GroupFun = fun({K1, _}, {K2, _}) -> K1 == K2 end,
FoldFun = fun(GroupedKey, Unreduced, Acc) ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/050-stream.t
----------------------------------------------------------------------
diff --git a/test/etap/050-stream.t b/test/etap/050-stream.t
index 622079a..de0dfad 100755
--- a/test/etap/050-stream.t
+++ b/test/etap/050-stream.t
@@ -26,7 +26,6 @@ main(_) ->
ok.
read_all(Fd, PosList) ->
- ok = couch_file:flush(Fd),
Data = couch_stream:foldl(Fd, PosList, fun(Bin, Acc) -> [Bin, Acc] end, []),
iolist_to_binary(Data).