You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/04/04 21:32:56 UTC
[01/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca [Forced Update!]
Repository: couchdb-couch
Updated Branches:
refs/heads/COUCHDB-3288-remove-public-db-record b424a8895 -> c515bcae9 (forced update)
Make couch_btree:chunkify/1 prefer fewer chunks
This changes couch_btree:chunkify/1 to produce fewer larger chunks
rather than creating chunks of even-ish size.
COUCHDB-3298
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/8556adbb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/8556adbb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/8556adbb
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 8556adbb98e79a09ec254967ee6acf3bef8d1fb6
Parents: bbbd532
Author: Paul J. Davis <pa...@gmail.com>
Authored: Sat Feb 11 15:26:26 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Sat Feb 11 15:26:26 2017 -0600
----------------------------------------------------------------------
src/couch_btree.erl | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/8556adbb/src/couch_btree.erl
----------------------------------------------------------------------
diff --git a/src/couch_btree.erl b/src/couch_btree.erl
index e9ff7f5..8f2395c 100644
--- a/src/couch_btree.erl
+++ b/src/couch_btree.erl
@@ -342,11 +342,9 @@ complete_root(Bt, KPs) ->
% it's probably really inefficient.
chunkify(InList) ->
- BaseChunkSize = get_chunk_size(),
+ ChunkThreshold = get_chunk_size(),
case ?term_size(InList) of
- Size when Size > BaseChunkSize ->
- NumberOfChunksLikely = ((Size div BaseChunkSize) + 1),
- ChunkThreshold = Size div NumberOfChunksLikely,
+ Size when Size > ChunkThreshold ->
chunkify(InList, ChunkThreshold, [], 0, []);
_Else ->
[InList]
[20/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Fix compaction daemon tests
There was a race condition in the compaction daemon tests where the
compaction daemon would think a compaction had finished but the test
would grab file size info before the compaction had swapped out which
breaks the assertion that fragmentation was reduced.
There was also a slight error in calculating the fragmentation in the
test suite by using the wrong size value.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/f09e3210
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/f09e3210
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/f09e3210
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: f09e3210158feb3e193809e8fff24c0df0171c2e
Parents: 016e1aa
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu Feb 2 12:33:55 2017 -0600
Committer: Jan Lehnardt <ja...@apache.org>
Committed: Thu Mar 23 14:23:30 2017 +0100
----------------------------------------------------------------------
test/couchdb_compaction_daemon_tests.erl | 119 +++++++++++++++++---------
1 file changed, 79 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f09e3210/test/couchdb_compaction_daemon_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_compaction_daemon_tests.erl b/test/couchdb_compaction_daemon_tests.erl
index 4c35fb7..20879a7 100644
--- a/test/couchdb_compaction_daemon_tests.erl
+++ b/test/couchdb_compaction_daemon_tests.erl
@@ -26,6 +26,10 @@ start() ->
Ctx.
setup() ->
+ ok = meck:new(couch_db_updater, [passthrough]),
+ ok = meck:new(couch_mrview_compactor, [passthrough]),
+ ok = meck:new(couch_compaction_daemon, [passthrough]),
+
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
create_design_doc(Db),
@@ -40,6 +44,11 @@ teardown(DbName) ->
end,
Configs),
couch_server:delete(DbName, [?ADMIN_CTX]),
+
+ (catch meck:unload(couch_compaction_daemon)),
+ (catch meck:unload(couch_mrview_compactor)),
+ (catch meck:unload(couch_db_updater)),
+
ok.
@@ -66,6 +75,8 @@ should_compact_by_default_rule(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
populate(DbName, 70, 70, 200 * 1024),
+ CompactionMonitor = spawn_compaction_monitor(DbName),
+
{_, DbFileSize} = get_db_frag(DbName),
{_, ViewFileSize} = get_view_frag(DbName),
@@ -75,8 +86,7 @@ should_compact_by_default_rule(DbName) ->
false)
end),
- wait_compaction_started(DbName),
- wait_compaction_finished(DbName),
+ wait_for_compaction(CompactionMonitor),
with_config_change(DbName, fun() ->
ok = config:delete("compactions", "_default", false)
@@ -100,6 +110,8 @@ should_compact_by_dbname_rule(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
populate(DbName, 70, 70, 200 * 1024),
+ CompactionMonitor = spawn_compaction_monitor(DbName),
+
{_, DbFileSize} = get_db_frag(DbName),
{_, ViewFileSize} = get_view_frag(DbName),
@@ -109,8 +121,7 @@ should_compact_by_dbname_rule(DbName) ->
false)
end),
- wait_compaction_started(DbName),
- wait_compaction_finished(DbName),
+ wait_for_compaction(CompactionMonitor),
with_config_change(DbName, fun() ->
ok = config:delete("compactions", ?b2l(DbName), false)
@@ -190,7 +201,7 @@ get_db_frag(DbName) ->
{ok, Info} = couch_db:get_db_info(Db),
couch_db:close(Db),
FileSize = get_size(file, Info),
- DataSize = get_size(external, Info),
+ DataSize = get_size(active, Info),
{round((FileSize - DataSize) / FileSize * 100), FileSize}.
get_view_frag(DbName) ->
@@ -198,49 +209,77 @@ get_view_frag(DbName) ->
{ok, Info} = couch_mrview:get_info(Db, <<"_design/foo">>),
couch_db:close(Db),
FileSize = get_size(file, Info),
- DataSize = get_size(external, Info),
+ DataSize = get_size(active, Info),
{round((FileSize - DataSize) / FileSize * 100), FileSize}.
get_size(Kind, Info) ->
couch_util:get_nested_json_value({Info}, [sizes, Kind]).
-wait_compaction_started(DbName) ->
- WaitFun = fun() ->
- case is_compaction_running(DbName) of
- false -> wait;
- true -> ok
- end
- end,
- case test_util:wait(WaitFun, ?TIMEOUT) of
- timeout ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Compaction starting timeout"}]});
- _ ->
- ok
- end.
-
-wait_compaction_finished(DbName) ->
- WaitFun = fun() ->
- case is_compaction_running(DbName) of
- true -> wait;
- false -> ok
- end
+spawn_compaction_monitor(DbName) ->
+ TestPid = self(),
+ {Pid, Ref} = spawn_monitor(fun() ->
+ DaemonPid = whereis(couch_compaction_daemon),
+ DbPid = couch_util:with_db(DbName, fun(Db) ->
+ couch_db:get_pid(Db)
+ end),
+ {ok, ViewPid} = couch_index_server:get_index(couch_mrview_index,
+ DbName, <<"_design/foo">>),
+ TestPid ! {self(), started},
+ receive
+ {TestPid, go} -> ok
+ after ?TIMEOUT ->
+ erlang:error(timeout)
+ end,
+ meck:wait(
+ 1,
+ couch_compaction_daemon,
+ handle_cast,
+ [{config_update, '_', '_'}, '_'],
+ DaemonPid,
+ ?TIMEOUT
+ ),
+ meck:wait(
+ 1,
+ couch_db_updater,
+ handle_cast,
+ [{compact_done, '_'}, '_'],
+ DbPid,
+ ?TIMEOUT
+ ),
+ meck:wait(
+ 1,
+ couch_mrview_compactor,
+ swap_compacted,
+ ['_', '_'],
+ ViewPid,
+ ?TIMEOUT
+ )
+ end),
+ receive
+ {Pid, started} -> ok;
+ {'DOWN', Ref, _, _, _} -> erlang:error(monitor_failure)
+ after ?TIMEOUT ->
+ erlang:error({assertion_failed, [
+ {module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Compaction starting timeout"}
+ ]})
end,
- case test_util:wait(WaitFun, ?TIMEOUT) of
- timeout ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Compaction timeout"}]});
- _ ->
- ok
+ {Pid, Ref}.
+
+wait_for_compaction({Pid, Ref}) ->
+ Pid ! {self(), go},
+ receive
+ {'DOWN', Ref, _, _, normal} -> ok;
+ {'DOWN', Ref, _, _, Other} -> erlang:error(Other)
+ after ?TIMEOUT ->
+ erlang:error({assertion_failed, [
+ {module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Compaction finishing timeout"}
+ ]})
end.
-is_compaction_running(_DbName) ->
- couch_compaction_daemon:in_progress() /= [].
-
is_idle(DbName) ->
{ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
Monitors = couch_db:monitored_by(Db),
[12/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge branch '70794-reduce-sum-errors'
Closes #229
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/cb3b35a7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/cb3b35a7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/cb3b35a7
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: cb3b35a736ccbb6cd43a9487108d1ab63197d064
Parents: 9d8be06 9f9c482
Author: Tony Sun <to...@cloudant.com>
Authored: Tue Mar 7 19:41:17 2017 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Tue Mar 7 19:41:17 2017 -0800
----------------------------------------------------------------------
src/couch_query_servers.erl | 66 ++++++++++++++++++++++++++++++++++------
1 file changed, 56 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[03/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Add sys db callbacks only once per db open
Currently we are adding the sys db callbacks for every request instead of
once while opening the db. This fix will ensure that we will only add the
sys db call backs once while opening the db.
COUCHDB-3307
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/8fc85c86
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/8fc85c86
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/8fc85c86
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 8fc85c8653dcec640717d8a51ea29a28194cbe4a
Parents: 66292db
Author: brkolla <bk...@cloudant.com>
Authored: Sun Feb 19 16:33:39 2017 -0500
Committer: brkolla <bk...@cloudant.com>
Committed: Tue Feb 28 11:40:00 2017 -0500
----------------------------------------------------------------------
src/couch_server.erl | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/8fc85c86/src/couch_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_server.erl b/src/couch_server.erl
index 59bffa5..893b957 100644
--- a/src/couch_server.erl
+++ b/src/couch_server.erl
@@ -71,18 +71,17 @@ sup_start_link() ->
open(DbName, Options0) ->
- Options = maybe_add_sys_db_callbacks(DbName, Options0),
- Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}),
+ Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}),
case ets:lookup(couch_dbs, DbName) of
- [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked ->
+ [#db{fd=Fd, fd_monitor=Lock, options=Options} = Db] when Lock =/= locked ->
update_lru(DbName, Options),
{ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
_ ->
+ Options = maybe_add_sys_db_callbacks(DbName, Options0),
Timeout = couch_util:get_value(timeout, Options, infinity),
Create = couch_util:get_value(create_if_missing, Options, false),
case gen_server:call(couch_server, {open, DbName, Options}, Timeout) of
{ok, #db{fd=Fd} = Db} ->
- update_lru(DbName, Options),
{ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
{not_found, no_db_file} when Create ->
couch_log:warning("creating missing database: ~s", [DbName]),
[07/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Efficiently bypass vhost handling if there are none
COUCHDB-3318
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/f706bb87
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/f706bb87
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/f706bb87
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: f706bb87be71006875c2fbaed3a14a18c79396f8
Parents: 38d5180
Author: Robert Newson <rn...@apache.org>
Authored: Mon Mar 6 12:05:33 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Mon Mar 6 12:05:33 2017 +0000
----------------------------------------------------------------------
src/couch_httpd_vhost.erl | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f706bb87/src/couch_httpd_vhost.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_vhost.erl b/src/couch_httpd_vhost.erl
index 91a2476..f23f41d 100644
--- a/src/couch_httpd_vhost.erl
+++ b/src/couch_httpd_vhost.erl
@@ -96,6 +96,14 @@ get_state() ->
%% @doc Try to find a rule matching current Host heade. some rule is
%% found it rewrite the Mochiweb Request else it return current Request.
dispatch_host(MochiReq) ->
+ case vhost_enabled() of
+ true ->
+ dispatch_host_int(MochiReq);
+ false ->
+ MochiReq
+ end.
+
+dispatch_host_int(MochiReq) ->
#vhosts_state{
vhost_globals = VHostGlobals,
vhosts = VHosts,
@@ -398,3 +406,14 @@ load_conf() ->
"redirect_vhost_handler", DefaultVHostFun)),
{VHostGlobals, VHosts, Fun}.
+
+%% cheaply determine if there are any virtual hosts
+%% configured at all.
+vhost_enabled() ->
+ case {config:get("httpd", "vhost_global_handlers"),
+ config:get("vhosts")} of
+ {undefined, []} ->
+ false;
+ _ ->
+ true
+ end.
[06/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Fix `badarith` error in couch_db:get_db_info call
When folding we account for a previous `null`, `undefined`, or a number. However
btree:size/1 returns 0, `nil` or a number.
Switched `undefined` to `nil`. Couldn't find where btree:size would ever return
`undefined`, it seems we meant to use `nil` instead.
COUCHDB-3316
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/63ef3374
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/63ef3374
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/63ef3374
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 63ef337475a266379c7208f52f827980e69d1d1b
Parents: 0080f15
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Thu Mar 2 21:24:52 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Thu Mar 2 21:24:52 2017 -0500
----------------------------------------------------------------------
src/couch_db.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/63ef3374/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index 8005e6d..d01a3e0 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -391,7 +391,7 @@ active_size(#db{}=Db, #size_info{}=SI) ->
case couch_btree:size(T) of
_ when Acc == null ->
null;
- undefined ->
+ nil ->
null;
Size ->
Acc + Size
[13/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Fix initial accumulator value for builtin_sum_rows
This a fixup for https://github.com/apache/couchdb-couch/pull/229.
The initial value for accumulator suppose to be 0. Since the result of
sum and count is integer. The problem was discovered by dialyzer.
COUCHDB-3305
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/1f152daa
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/1f152daa
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/1f152daa
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 1f152daaac23dc96132139fb532786f77becdd64
Parents: cb3b35a
Author: ILYA Khlopotov <ii...@apache.org>
Authored: Wed Mar 8 12:24:46 2017 -0800
Committer: ILYA Khlopotov <ii...@apache.org>
Committed: Wed Mar 8 12:24:46 2017 -0800
----------------------------------------------------------------------
src/couch_query_servers.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1f152daa/src/couch_query_servers.erl
----------------------------------------------------------------------
diff --git a/src/couch_query_servers.erl b/src/couch_query_servers.erl
index ddc079e..63b0e39 100644
--- a/src/couch_query_servers.erl
+++ b/src/couch_query_servers.erl
@@ -142,13 +142,13 @@ os_rereduce(Lang, OsRedSrcs, KVs) ->
builtin_reduce(_Re, [], _KVs, Acc) ->
{ok, lists:reverse(Acc)};
builtin_reduce(Re, [<<"_sum",_/binary>>|BuiltinReds], KVs, Acc) ->
- Sum = builtin_sum_rows(KVs, []),
+ Sum = builtin_sum_rows(KVs, 0),
builtin_reduce(Re, BuiltinReds, KVs, [Sum|Acc]);
builtin_reduce(reduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) ->
Count = length(KVs),
builtin_reduce(reduce, BuiltinReds, KVs, [Count|Acc]);
builtin_reduce(rereduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) ->
- Count = builtin_sum_rows(KVs, []),
+ Count = builtin_sum_rows(KVs, 0),
builtin_reduce(rereduce, BuiltinReds, KVs, [Count|Acc]);
builtin_reduce(Re, [<<"_stats",_/binary>>|BuiltinReds], KVs, Acc) ->
Stats = builtin_stats(Re, KVs),
[15/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Add sys_dbs to the LRU
COUCHDB-3325
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/92c25a98
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/92c25a98
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/92c25a98
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 92c25a98666e59ca497fa5732a81e771ff52e07d
Parents: 5685f17
Author: Robert Newson <rn...@apache.org>
Authored: Tue Mar 14 17:40:21 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Tue Mar 14 23:59:11 2017 +0000
----------------------------------------------------------------------
src/couch_db_updater.erl | 7 +---
src/couch_file.erl | 22 +++---------
src/couch_lru.erl | 5 +--
src/couch_server.erl | 75 ++++++++++++++-------------------------
test/couchdb_views_tests.erl | 2 --
5 files changed, 35 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl
index 7872635..270fffe 100644
--- a/src/couch_db_updater.erl
+++ b/src/couch_db_updater.erl
@@ -61,12 +61,7 @@ init({DbName, Filepath, Fd, Options}) ->
end
end,
Db = init_db(DbName, Filepath, Fd, Header, Options),
- case lists:member(sys_db, Options) of
- false ->
- couch_stats_process_tracker:track([couchdb, open_databases]);
- true ->
- ok
- end,
+ couch_stats_process_tracker:track([couchdb, open_databases]),
% we don't load validation funs here because the fabric query is liable to
% race conditions. Instead see couch_db:validate_doc_update, which loads
% them lazily
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couch_file.erl b/src/couch_file.erl
index d40c525..d7b176e 100644
--- a/src/couch_file.erl
+++ b/src/couch_file.erl
@@ -362,7 +362,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
{ok, 0} = file:position(Fd, 0),
ok = file:truncate(Fd),
ok = file:sync(Fd),
- maybe_track_open_os_files(Options),
+ couch_stats_process_tracker:track([couchdb, open_os_files]),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
{ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}};
false ->
@@ -370,7 +370,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
init_status_error(ReturnPid, Ref, {error, eexist})
end;
false ->
- maybe_track_open_os_files(Options),
+ couch_stats_process_tracker:track([couchdb, open_os_files]),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
{ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}}
end;
@@ -385,7 +385,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
%% Save Fd in process dictionary for debugging purposes
put(couch_file_fd, {Fd, Filepath}),
ok = file:close(Fd_Read),
- maybe_track_open_os_files(Options),
+ couch_stats_process_tracker:track([couchdb, open_os_files]),
{ok, Eof} = file:position(Fd, eof),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
{ok, #file{fd=Fd, eof=Eof, is_sys=IsSys, pread_limit=Limit}};
@@ -402,14 +402,6 @@ file_open_options(Options) ->
[append]
end.
-maybe_track_open_os_files(Options) ->
- case not lists:member(sys_db, Options) of
- true ->
- couch_stats_process_tracker:track([couchdb, open_os_files]);
- false ->
- ok
- end.
-
terminate(_Reason, #file{fd = nil}) ->
ok;
terminate(_Reason, #file{fd = Fd}) ->
@@ -676,13 +668,7 @@ split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) ->
split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]).
-% System dbs aren't monitored by couch_stats_process_tracker
-is_idle(#file{is_sys=true}) ->
- case process_info(self(), monitored_by) of
- {monitored_by, []} -> true;
- _ -> false
- end;
-is_idle(#file{is_sys=false}) ->
+is_idle(#file{}) ->
Tracker = whereis(couch_stats_process_tracker),
case process_info(self(), monitored_by) of
{monitored_by, []} -> true;
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_lru.erl
----------------------------------------------------------------------
diff --git a/src/couch_lru.erl b/src/couch_lru.erl
index d58eb69..84d7bee 100644
--- a/src/couch_lru.erl
+++ b/src/couch_lru.erl
@@ -34,13 +34,14 @@ update(DbName, {Tree0, Dict0}) ->
{Tree0, Dict0}
end.
+%% Attempt to close the oldest idle database.
close({Tree, _} = Cache) ->
close_int(gb_trees:next(gb_trees:iterator(Tree)), Cache).
%% internals
close_int(none, _) ->
- erlang:error(all_dbs_active);
+ false;
close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
true ->
@@ -49,7 +50,7 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)};
+ {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}};
false ->
true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_server.erl b/src/couch_server.erl
index 893b957..1272e7a 100644
--- a/src/couch_server.erl
+++ b/src/couch_server.erl
@@ -73,8 +73,8 @@ sup_start_link() ->
open(DbName, Options0) ->
Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}),
case ets:lookup(couch_dbs, DbName) of
- [#db{fd=Fd, fd_monitor=Lock, options=Options} = Db] when Lock =/= locked ->
- update_lru(DbName, Options),
+ [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked ->
+ update_lru(DbName),
{ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
_ ->
Options = maybe_add_sys_db_callbacks(DbName, Options0),
@@ -91,11 +91,8 @@ open(DbName, Options0) ->
end
end.
-update_lru(DbName, Options) ->
- case lists:member(sys_db, Options) of
- false -> gen_server:cast(couch_server, {update_lru, DbName});
- true -> ok
- end.
+update_lru(DbName) ->
+ gen_server:cast(couch_server, {update_lru, DbName}).
close_lru() ->
gen_server:call(couch_server, close_lru).
@@ -271,20 +268,15 @@ all_databases(Fun, Acc0) ->
{ok, FinalAcc}.
-make_room(Server, Options) ->
- case lists:member(sys_db, Options) of
- false -> maybe_close_lru_db(Server);
- true -> {ok, Server}
- end.
-
maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server)
when NumOpen < MaxOpen ->
{ok, Server};
maybe_close_lru_db(#server{lru=Lru}=Server) ->
- try
- {ok, db_closed(Server#server{lru = couch_lru:close(Lru)}, [])}
- catch error:all_dbs_active ->
- {error, all_dbs_active}
+ case couch_lru:close(Lru) of
+ false ->
+ {ok, Server};
+ {true, NewLru} ->
+ maybe_close_lru_db(db_closed(Server#server{lru = NewLru}))
end.
open_async(Server, From, DbName, Filepath, Options) ->
@@ -316,13 +308,14 @@ open_async(Server, From, DbName, Filepath, Options) ->
options = Options
}),
true = ets:insert(couch_dbs_pid_to_name, {Opener, DbName}),
- db_opened(Server, Options).
+ db_opened(Server).
handle_call(close_lru, _From, #server{lru=Lru} = Server) ->
- try
- {reply, ok, db_closed(Server#server{lru = couch_lru:close(Lru)}, [])}
- catch error:all_dbs_active ->
- {reply, {error, all_dbs_active}, Server}
+ case couch_lru:close(Lru) of
+ false ->
+ {reply, ok, Server};
+ {true, NewLru} ->
+ {reply, ok, db_closed(Server#server{lru = NewLru})}
end;
handle_call(open_dbs_count, _From, Server) ->
{reply, Server#server.dbs_open, Server};
@@ -355,12 +348,7 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) ->
end,
true = ets:insert(couch_dbs, Db),
true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}),
- Lru = case couch_db:is_system_db(Db) of
- false ->
- couch_lru:insert(DbName, Server#server.lru);
- true ->
- Server#server.lru
- end,
+ Lru = couch_lru:insert(DbName, Server#server.lru),
{reply, ok, Server#server{lru = Lru}}
end;
handle_call({open_result, T0, DbName, {error, eexist}}, From, Server) ->
@@ -382,7 +370,7 @@ handle_call({open_result, _T0, DbName, Error}, {FromPid, _Tag}, Server) ->
_ ->
Server
end,
- {reply, ok, db_closed(NewServer, Db#db.options)}
+ {reply, ok, db_closed(NewServer)}
end;
handle_call({open, DbName, Options}, From, Server) ->
case ets:lookup(couch_dbs, DbName) of
@@ -390,7 +378,7 @@ handle_call({open, DbName, Options}, From, Server) ->
DbNameList = binary_to_list(DbName),
case check_dbname(Server, DbNameList) of
ok ->
- case make_room(Server, Options) of
+ case maybe_close_lru_db(Server) of
{ok, Server2} ->
Filepath = get_full_filename(Server, DbNameList),
{noreply, open_async(Server2, From, DbName, Filepath, Options)};
@@ -418,7 +406,7 @@ handle_call({create, DbName, Options}, From, Server) ->
ok ->
case ets:lookup(couch_dbs, DbName) of
[] ->
- case make_room(Server, Options) of
+ case maybe_close_lru_db(Server) of
{ok, Server2} ->
{noreply, open_async(Server2, From, DbName, Filepath,
[create | Options])};
@@ -454,12 +442,12 @@ handle_call({delete, DbName, Options}, _From, Server) ->
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
[gen_server:reply(F, not_found) || F <- Froms],
- db_closed(Server, Db#db.options);
+ db_closed(Server);
[#db{main_pid=Pid} = Db] ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- db_closed(Server, Db#db.options)
+ db_closed(Server)
end,
%% Delete any leftover compaction files. If we don't do this a
@@ -490,10 +478,7 @@ handle_call({db_updated, #db{}=Db}, _From, Server0) ->
Server = try ets:lookup_element(couch_dbs, DbName, #db.instance_start_time) of
StartTime ->
true = ets:insert(couch_dbs, Db),
- Lru = case couch_db:is_system_db(Db) of
- false -> couch_lru:update(DbName, Server0#server.lru);
- true -> Server0#server.lru
- end,
+ Lru = couch_lru:update(DbName, Server0#server.lru),
Server0#server{lru = Lru};
_ ->
Server0
@@ -532,7 +517,7 @@ handle_info({'EXIT', Pid, Reason}, Server) ->
end,
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
- {noreply, db_closed(Server, Db#db.options)};
+ {noreply, db_closed(Server)};
[] ->
{noreply, Server}
end;
@@ -542,17 +527,11 @@ handle_info(restart_config_listener, State) ->
handle_info(Info, Server) ->
{stop, {unknown_message, Info}, Server}.
-db_opened(Server, Options) ->
- case lists:member(sys_db, Options) of
- false -> Server#server{dbs_open=Server#server.dbs_open + 1};
- true -> Server
- end.
+db_opened(Server) ->
+ Server#server{dbs_open=Server#server.dbs_open + 1}.
-db_closed(Server, Options) ->
- case lists:member(sys_db, Options) of
- false -> Server#server{dbs_open=Server#server.dbs_open - 1};
- true -> Server
- end.
+db_closed(Server) ->
+ Server#server{dbs_open=Server#server.dbs_open - 1}.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/test/couchdb_views_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_views_tests.erl b/test/couchdb_views_tests.erl
index 7b04e85..f1fddfc 100644
--- a/test/couchdb_views_tests.erl
+++ b/test/couchdb_views_tests.erl
@@ -372,7 +372,6 @@ couchdb_1283() ->
MonRef = erlang:monitor(process, CPid),
Writer3 = spawn_writer(Db3#db.name),
?assert(is_process_alive(Writer3)),
- ?assertEqual({error, all_dbs_active}, get_writer_status(Writer3)),
?assert(is_process_alive(Writer1)),
?assert(is_process_alive(Writer2)),
@@ -391,7 +390,6 @@ couchdb_1283() ->
{reason, "Failure compacting view group"}]})
end,
- ?assertEqual(ok, writer_try_again(Writer3)),
?assertEqual(ok, get_writer_status(Writer3)),
?assert(is_process_alive(Writer1)),
[26/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Remove public access to the db record
This completes the removal of public access to the db record from the
couch application. The large majority of which is removing direct access
to the #db.name, #db.main_pid, and #db.update_seq fields.
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/c515bcae
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/c515bcae
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/c515bcae
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: c515bcae97a9eaa7d515a4620ba0e44a3f1fa2ef
Parents: 552a29d
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 1 15:15:09 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:32:27 2017 -0500
----------------------------------------------------------------------
include/couch_db.hrl | 27 -------------
src/couch_auth_cache.erl | 17 +++++---
src/couch_changes.erl | 56 ++++++++++++++-------------
src/couch_compaction_daemon.erl | 4 +-
src/couch_db.erl | 55 ++++++++++++++++++++++++++
src/couch_db_int.hrl | 38 ++++++++++++++++++
src/couch_db_plugin.erl | 6 ++-
src/couch_db_updater.erl | 1 +
src/couch_httpd_db.erl | 12 +++---
src/couch_users_db.erl | 8 ++--
src/couch_util.erl | 15 +++++--
test/couch_auth_cache_tests.erl | 2 +-
test/couch_changes_tests.erl | 2 +-
test/couch_db_plugin_tests.erl | 13 ++++---
test/couch_server_tests.erl | 11 ++++--
test/couchdb_compaction_daemon_tests.erl | 2 +-
test/couchdb_views_tests.erl | 21 +++++-----
17 files changed, 192 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/include/couch_db.hrl
----------------------------------------------------------------------
diff --git a/include/couch_db.hrl b/include/couch_db.hrl
index e7cd85d..5abb316 100644
--- a/include/couch_db.hrl
+++ b/include/couch_db.hrl
@@ -128,33 +128,6 @@
handler
}).
--record(db, {
- main_pid = nil,
- compactor_pid = nil,
- instance_start_time, % number of microsecs since jan 1 1970 as a binary string
- fd,
- fd_monitor,
- header = couch_db_header:new(),
- committed_update_seq,
- id_tree,
- seq_tree,
- local_tree,
- update_seq,
- name,
- filepath,
- validate_doc_funs = undefined,
- security = [],
- security_ptr = nil,
- user_ctx = #user_ctx{},
- waiting_delayed_commit = nil,
- revs_limit = 1000,
- fsync_options = [],
- options = [],
- compression,
- before_doc_update = nil, % nil | fun(Doc, Db) -> NewDoc
- after_doc_read = nil % nil | fun(Doc, Db) -> NewDoc
-}).
-
-record(view_fold_helper_funs, {
reduce_count,
passed_end,
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_auth_cache.erl
----------------------------------------------------------------------
diff --git a/src/couch_auth_cache.erl b/src/couch_auth_cache.erl
index 9b00a9d..54a6794 100644
--- a/src/couch_auth_cache.erl
+++ b/src/couch_auth_cache.erl
@@ -289,8 +289,9 @@ reinit_cache(#state{db_mon_ref = Ref, closed = Closed} = State) ->
true = ets:insert(?STATE, {auth_db_name, AuthDbName}),
AuthDb = open_auth_db(),
true = ets:insert(?STATE, {auth_db, AuthDb}),
+ DbPid = couch_db:get_pid(AuthDb),
NewState#state{closed = [Ref|Closed],
- db_mon_ref = erlang:monitor(process, AuthDb#db.main_pid)}.
+ db_mon_ref = erlang:monitor(process, DbPid)}.
add_cache_entry(_, _, _, #state{max_cache_size = 0} = State) ->
@@ -331,13 +332,15 @@ refresh_entries(AuthDb) ->
nil ->
ok;
AuthDb2 ->
- case AuthDb2#db.update_seq > AuthDb#db.update_seq of
+ AuthDbSeq = couch_db:get_update_seq(AuthDb),
+ AuthDb2Seq = couch_db:get_update_seq(AuthDb2),
+ case AuthDb2Seq > AuthDbSeq of
true ->
{ok, _, _} = couch_db:enum_docs_since(
AuthDb2,
- AuthDb#db.update_seq,
+ AuthDbSeq,
fun(DocInfo, _, _) -> refresh_entry(AuthDb2, DocInfo) end,
- AuthDb#db.update_seq,
+ AuthDbSeq,
[]
),
true = ets:insert(?STATE, {auth_db, AuthDb2});
@@ -395,7 +398,9 @@ cache_needs_refresh() ->
nil ->
false;
AuthDb2 ->
- AuthDb2#db.update_seq > AuthDb#db.update_seq
+ AuthDbSeq = couch_db:get_update_seq(AuthDb),
+ AuthDb2Seq = couch_db:get_update_seq(AuthDb2),
+ AuthDb2Seq > AuthDbSeq
end
end,
false
@@ -416,7 +421,7 @@ exec_if_auth_db(Fun) ->
exec_if_auth_db(Fun, DefRes) ->
case ets:lookup(?STATE, auth_db) of
- [{auth_db, #db{} = AuthDb}] ->
+ [{auth_db, AuthDb}] ->
Fun(AuthDb);
_ ->
DefRes
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_changes.erl b/src/couch_changes.erl
index 52ff39d..ea7f65c 100644
--- a/src/couch_changes.erl
+++ b/src/couch_changes.erl
@@ -78,9 +78,10 @@ handle_changes(Args1, Req, Db0, Type) ->
_ ->
{false, undefined, undefined}
end,
+ DbName = couch_db:name(Db0),
{StartListenerFun, View} = if UseViewChanges ->
{ok, {_, View0, _}, _, _} = couch_mrview_util:get_view(
- Db0#db.name, DDocName, ViewName, #mrargs{}),
+ DbName, DDocName, ViewName, #mrargs{}),
case View0#mrview.seq_btree of
#btree{} ->
ok;
@@ -89,14 +90,14 @@ handle_changes(Args1, Req, Db0, Type) ->
end,
SNFun = fun() ->
couch_event:link_listener(
- ?MODULE, handle_view_event, {self(), DDocName}, [{dbname, Db0#db.name}]
+ ?MODULE, handle_view_event, {self(), DDocName}, [{dbname, DbName}]
)
end,
{SNFun, View0};
true ->
SNFun = fun() ->
couch_event:link_listener(
- ?MODULE, handle_db_event, self(), [{dbname, Db0#db.name}]
+ ?MODULE, handle_db_event, self(), [{dbname, DbName}]
)
end,
{SNFun, undefined}
@@ -111,7 +112,7 @@ handle_changes(Args1, Req, Db0, Type) ->
end,
View2 = if UseViewChanges ->
{ok, {_, View1, _}, _, _} = couch_mrview_util:get_view(
- Db0#db.name, DDocName, ViewName, #mrargs{}),
+ DbName, DDocName, ViewName, #mrargs{}),
View1;
true ->
undefined
@@ -219,11 +220,11 @@ configure_filter("_view", Style, Req, Db) ->
catch _:_ ->
view
end,
- case Db#db.id_tree of
- undefined ->
+ case couch_db:is_clustered(Db) of
+ true ->
DIR = fabric_util:doc_id_and_rev(DDoc),
{fetch, FilterType, Style, DIR, VName};
- _ ->
+ false ->
{FilterType, Style, DDoc, VName}
end;
[] ->
@@ -242,11 +243,11 @@ configure_filter(FilterName, Style, Req, Db) ->
[DName, FName] ->
{ok, DDoc} = open_ddoc(Db, <<"_design/", DName/binary>>),
check_member_exists(DDoc, [<<"filters">>, FName]),
- case Db#db.id_tree of
- undefined ->
+ case couch_db:is_clustered(Db) of
+ true ->
DIR = fabric_util:doc_id_and_rev(DDoc),
{fetch, custom, Style, Req, DIR, FName};
- _ ->
+ false->
{custom, Style, Req, DDoc, FName}
end;
@@ -395,15 +396,19 @@ check_fields(_Fields) ->
throw({bad_request, "Selector error: fields must be JSON array"}).
-open_ddoc(#db{name=DbName, id_tree=undefined}, DDocId) ->
- case ddoc_cache:open_doc(mem3:dbname(DbName), DDocId) of
- {ok, _} = Resp -> Resp;
- Else -> throw(Else)
- end;
open_ddoc(Db, DDocId) ->
- case couch_db:open_doc(Db, DDocId, [ejson_body]) of
- {ok, _} = Resp -> Resp;
- Else -> throw(Else)
+ DbName = couch_db:name(Db),
+ case couch_db:is_clustered(Db) of
+ true ->
+ case ddoc_cache:open_doc(mem3:dbname(DbName), DDocId) of
+ {ok, _} = Resp -> Resp;
+ Else -> throw(Else)
+ end;
+ false ->
+ case couch_db:open_doc(Db, DDocId, [ejson_body]) of
+ {ok, _} = Resp -> Resp;
+ Else -> throw(Else)
+ end
end.
@@ -566,7 +571,7 @@ can_optimize(_, _) ->
send_changes_doc_ids(Db, StartSeq, Dir, Fun, Acc0, {doc_ids, _Style, DocIds}) ->
- Lookups = couch_btree:lookup(Db#db.id_tree, DocIds),
+ Lookups = couch_db:get_full_doc_infos(Db, DocIds),
FullInfos = lists:foldl(fun
({ok, FDI}, Acc) -> [FDI | Acc];
(not_found, Acc) -> Acc
@@ -575,11 +580,9 @@ send_changes_doc_ids(Db, StartSeq, Dir, Fun, Acc0, {doc_ids, _Style, DocIds}) ->
send_changes_design_docs(Db, StartSeq, Dir, Fun, Acc0, {design_docs, _Style}) ->
- FoldFun = fun(FullDocInfo, _, Acc) ->
- {ok, [FullDocInfo | Acc]}
- end,
+ FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end,
KeyOpts = [{start_key, <<"_design/">>}, {end_key_gt, <<"_design0">>}],
- {ok, _, FullInfos} = couch_btree:fold(Db#db.id_tree, FoldFun, [], KeyOpts),
+ {ok, FullInfos} = couch_db:fold_docs(Db, FoldFun, [], KeyOpts),
send_lookup_changes(FullInfos, StartSeq, Dir, Db, Fun, Acc0).
@@ -640,8 +643,8 @@ keep_sending_changes(Args, Acc0, FirstRound) ->
true ->
case wait_updated(Timeout, TimeoutFun, UserAcc2) of
{updated, UserAcc4} ->
- DbOptions1 = [{user_ctx, Db#db.user_ctx} | DbOptions],
- case couch_db:open(Db#db.name, DbOptions1) of
+ DbOptions1 = [{user_ctx, couch_db:get_user_ctx(Db)} | DbOptions],
+ case couch_db:open(couch_db:name(Db), DbOptions1) of
{ok, Db2} ->
keep_sending_changes(
Args#changes_args{limit=NewLimit},
@@ -665,7 +668,8 @@ keep_sending_changes(Args, Acc0, FirstRound) ->
maybe_refresh_view(_, undefined, undefined) ->
undefined;
maybe_refresh_view(Db, DDocName, ViewName) ->
- {ok, {_, View, _}, _, _} = couch_mrview_util:get_view(Db#db.name, DDocName, ViewName, #mrargs{}),
+ DbName = couch_db:name(Db),
+ {ok, {_, View, _}, _, _} = couch_mrview_util:get_view(DbName, DDocName, ViewName, #mrargs{}),
View.
end_sending_changes(Callback, UserAcc, EndSeq, ResponseType) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_compaction_daemon.erl
----------------------------------------------------------------------
diff --git a/src/couch_compaction_daemon.erl b/src/couch_compaction_daemon.erl
index 8f95eb2..f3b646d 100644
--- a/src/couch_compaction_daemon.erl
+++ b/src/couch_compaction_daemon.erl
@@ -319,7 +319,7 @@ can_db_compact(#config{db_frag = Threshold} = Config, Db) ->
{Frag, SpaceRequired} = frag(DbInfo),
couch_log:debug("Fragmentation for database `~s` is ~p%, estimated"
" space for compaction is ~p bytes.",
- [Db#db.name, Frag, SpaceRequired]),
+ [couch_db:name(Db), Frag, SpaceRequired]),
case check_frag(Threshold, Frag) of
false ->
false;
@@ -332,7 +332,7 @@ can_db_compact(#config{db_frag = Threshold} = Config, Db) ->
couch_log:warning("Compaction daemon - skipping database `~s` "
"compaction: the estimated necessary disk space is about ~p"
" bytes but the currently available disk space is ~p bytes.",
- [Db#db.name, SpaceRequired, Free]),
+ [couch_db:name(Db), SpaceRequired, Free]),
false
end
end
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index 3a29a3d..75fc730 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -22,6 +22,9 @@
incref/1,
decref/1,
+ clustered_db/2,
+ clustered_db/3,
+
monitor/1,
monitored_by/1,
is_idle/1,
@@ -32,21 +35,28 @@
name/1,
compression/1,
+ get_after_doc_read_fun/1,
+ get_before_doc_update_fun/1,
get_committed_update_seq/1,
get_compacted_seq/1,
+ get_compactor_pid/1,
get_db_info/1,
get_doc_count/1,
get_epochs/1,
+ get_filepath/1,
get_instance_start_time/1,
get_last_purged/1,
get_pid/1,
get_revs_limit/1,
get_security/1,
get_update_seq/1,
+ get_user_ctx/1,
get_uuid/1,
get_purge_seq/1,
+ is_db/1,
is_system_db/1,
+ is_clustered/1,
increment_update_seq/1,
set_revs_limit/2,
@@ -80,6 +90,8 @@
with_stream/3,
+ fold_docs/4,
+ fold_local_docs/4,
enum_docs/4,
enum_docs_reduce_to_count/1,
@@ -113,6 +125,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include("couch_db_int.hrl").
-define(DBNAME_REGEX,
"^[a-z][a-z0-9\\_\\$()\\+\\-\\/]*" % use the stock CouchDB regex
@@ -187,6 +200,12 @@ reopen(#db{main_pid = Pid, fd = Fd, fd_monitor = OldRef, user_ctx = UserCtx}) ->
{ok, NewDb#db{user_ctx = UserCtx, fd_monitor = NewRef}}
end.
+clustered_db(DbName, UserCtx) ->
+ clustered_db(DbName, UserCtx, []).
+
+clustered_db(DbName, UserCtx, SecProps) ->
+ {ok, #db{name = DbName, user_ctx = UserCtx, security = SecProps}}.
+
incref(#db{fd = Fd} = Db) ->
Ref = erlang:monitor(process, Fd),
{ok, Db#db{fd_monitor = Ref}}.
@@ -195,9 +214,19 @@ decref(#db{fd_monitor = Monitor}) ->
erlang:demonitor(Monitor, [flush]),
ok.
+is_db(#db{}) ->
+ true;
+is_db(_) ->
+ false.
+
is_system_db(#db{options = Options}) ->
lists:member(sys_db, Options).
+is_clustered(#db{main_pid = nil}) ->
+ true;
+is_clustered(#db{}) ->
+ false.
+
ensure_full_commit(#db{main_pid=Pid, instance_start_time=StartTime}) ->
ok = gen_server:call(Pid, full_commit, infinity),
{ok, StartTime}.
@@ -378,12 +407,21 @@ increment_update_seq(#db{main_pid=Pid}) ->
purge_docs(#db{main_pid=Pid}, IdsRevs) ->
gen_server:call(Pid, {purge_docs, IdsRevs}).
+get_after_doc_read_fun(#db{after_doc_read = Fun}) ->
+ Fun.
+
+get_before_doc_update_fun(#db{before_doc_update = Fun}) ->
+ Fun.
+
get_committed_update_seq(#db{committed_update_seq=Seq}) ->
Seq.
get_update_seq(#db{update_seq=Seq})->
Seq.
+get_user_ctx(#db{user_ctx = UserCtx}) ->
+ UserCtx.
+
get_purge_seq(#db{}=Db) ->
couch_db_header:purge_seq(Db#db.header).
@@ -410,12 +448,18 @@ get_epochs(#db{}=Db) ->
validate_epochs(Epochs),
Epochs.
+get_filepath(#db{filepath = FilePath}) ->
+ FilePath.
+
get_instance_start_time(#db{instance_start_time = IST}) ->
IST.
get_compacted_seq(#db{}=Db) ->
couch_db_header:compacted_seq(Db#db.header).
+get_compactor_pid(#db{compactor_pid = Pid}) ->
+ Pid.
+
get_db_info(Db) ->
#db{fd=Fd,
header=Header,
@@ -1365,6 +1409,17 @@ enum_docs_since(Db, SinceSeq, InFun, Acc, Options) ->
[{start_key, SinceSeq + 1} | Options]),
{ok, enum_docs_since_reduce_to_count(LastReduction), AccOut}.
+
+fold_docs(Db, InFun, InAcc, Opts) ->
+ Wrapper = fun(FDI, _, Acc) -> InFun(FDI, Acc) end,
+ {ok, _, AccOut} = couch_btree:fold(Db#db.id_tree, Wrapper, InAcc, Opts),
+ {ok, AccOut}.
+
+fold_local_docs(Db, InFun, InAcc, Opts) ->
+ Wrapper = fun(FDI, _, Acc) -> InFun(FDI, Acc) end,
+ {ok, _, AccOut} = couch_btree:fold(Db#db.local_tree, Wrapper, InAcc, Opts),
+ {ok, AccOut}.
+
enum_docs(Db, InFun, InAcc, Options0) ->
{NS, Options} = extract_namespace(Options0),
enum_docs(Db, NS, InFun, InAcc, Options).
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_db_int.hrl
----------------------------------------------------------------------
diff --git a/src/couch_db_int.hrl b/src/couch_db_int.hrl
new file mode 100644
index 0000000..fc739b7
--- /dev/null
+++ b/src/couch_db_int.hrl
@@ -0,0 +1,38 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(db, {
+ main_pid = nil,
+ compactor_pid = nil,
+ instance_start_time, % number of microsecs since jan 1 1970 as a binary string
+ fd,
+ fd_monitor,
+ header = couch_db_header:new(),
+ committed_update_seq,
+ id_tree,
+ seq_tree,
+ local_tree,
+ update_seq,
+ name,
+ filepath,
+ validate_doc_funs = undefined,
+ security = [],
+ security_ptr = nil,
+ user_ctx = #user_ctx{},
+ waiting_delayed_commit = nil,
+ revs_limit = 1000,
+ fsync_options = [],
+ options = [],
+ compression,
+ before_doc_update = nil, % nil | fun(Doc, Db) -> NewDoc
+ after_doc_read = nil % nil | fun(Doc, Db) -> NewDoc
+}).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_db_plugin.erl
----------------------------------------------------------------------
diff --git a/src/couch_db_plugin.erl b/src/couch_db_plugin.erl
index 774e9e0..740b812 100644
--- a/src/couch_db_plugin.erl
+++ b/src/couch_db_plugin.erl
@@ -32,13 +32,15 @@
validate_dbname(DbName, Normalized, Default) ->
maybe_handle(validate_dbname, [DbName, Normalized], Default).
-before_doc_update(#db{before_doc_update = Fun} = Db, Doc0) ->
+before_doc_update(Db, Doc0) ->
+ Fun = couch_db:get_before_doc_update_fun(Db),
case with_pipe(before_doc_update, [Doc0, Db]) of
[Doc1, _Db] when is_function(Fun) -> Fun(Doc1, Db);
[Doc1, _Db] -> Doc1
end.
-after_doc_read(#db{after_doc_read = Fun} = Db, Doc0) ->
+after_doc_read(Db, Doc0) ->
+ Fun = couch_db:get_after_doc_read_fun(Db),
case with_pipe(after_doc_read, [Doc0, Db]) of
[Doc1, _Db] when is_function(Fun) -> Fun(Doc1, Db);
[Doc1, _Db] -> Doc1
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl
index 270fffe..8f6fc35 100644
--- a/src/couch_db_updater.erl
+++ b/src/couch_db_updater.erl
@@ -20,6 +20,7 @@
-export([init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3,handle_info/2]).
-include_lib("couch/include/couch_db.hrl").
+-include("couch_db_int.hrl").
-record(comp_header, {
db_header,
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_httpd_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_db.erl b/src/couch_httpd_db.erl
index e1af1bf..1198a67 100644
--- a/src/couch_httpd_db.erl
+++ b/src/couch_httpd_db.erl
@@ -70,7 +70,8 @@ handle_changes_req(#httpd{method='GET'}=Req, Db, ChangesArgs, ChangesFun) ->
handle_changes_req(#httpd{}=Req, _Db, _ChangesArgs, _ChangesFun) ->
couch_httpd:send_method_not_allowed(Req, "GET,HEAD,POST").
-handle_changes_req1(Req, #db{name=DbName}=Db, ChangesArgs, ChangesFun) ->
+handle_changes_req1(Req, Db, ChangesArgs, ChangesFun) ->
+ DbName = couch_db:name(Db),
AuthDbName = ?l2b(config:get("couch_httpd_auth", "authentication_db")),
case AuthDbName of
DbName ->
@@ -287,7 +288,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_ensure_full_commit">>]}=Req, Db) -
RequiredSeq > CommittedSeq ->
couch_db:ensure_full_commit(Db);
true ->
- {ok, Db#db.instance_start_time}
+ {ok, couch_db:get_instance_start_time(Db)}
end
end,
send_json(Req, 201, {[
@@ -733,7 +734,8 @@ update_doc_result_to_json(DocId, Error) ->
update_doc(Req, Db, DocId, #doc{deleted=false}=Doc) ->
- Loc = absolute_uri(Req, "/" ++ couch_util:url_encode(Db#db.name) ++ "/" ++ couch_util:url_encode(DocId)),
+ DbName = couch_db:name(Db),
+ Loc = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName) ++ "/" ++ couch_util:url_encode(DocId)),
update_doc(Req, Db, DocId, Doc, [{"Location", Loc}]);
update_doc(Req, Db, DocId, Doc) ->
update_doc(Req, Db, DocId, Doc, []).
@@ -1033,7 +1035,7 @@ db_attachment_req(#httpd{method=Method,mochi_req=MochiReq}=Req, Db, DocId, FileN
[];
_ ->
[{"Location", absolute_uri(Req, "/" ++
- couch_util:url_encode(Db#db.name) ++ "/" ++
+ couch_util:url_encode(couch_db:name(Db)) ++ "/" ++
couch_util:url_encode(DocId) ++ "/" ++
couch_util:url_encode(FileName)
)}]
@@ -1145,7 +1147,7 @@ parse_changes_query(Req, Db) ->
{"descending", "true"} ->
Args#changes_args{dir=rev};
{"since", "now"} ->
- UpdateSeq = couch_util:with_db(Db#db.name, fun(WDb) ->
+ UpdateSeq = couch_util:with_db(couch_db:name(Db), fun(WDb) ->
couch_db:get_update_seq(WDb)
end),
Args#changes_args{since=UpdateSeq};
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_users_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_users_db.erl b/src/couch_users_db.erl
index 6f7b9af..c7b41f1 100644
--- a/src/couch_users_db.erl
+++ b/src/couch_users_db.erl
@@ -39,8 +39,8 @@
% -> 404 // Not Found
% Else
% -> save_doc
-before_doc_update(Doc, #db{user_ctx = UserCtx} = Db) ->
- #user_ctx{name=Name} = UserCtx,
+before_doc_update(Doc, Db) ->
+ #user_ctx{name=Name} = couch_db:get_user_ctx(Db),
DocName = get_doc_name(Doc),
case (catch couch_db:check_is_admin(Db)) of
ok ->
@@ -108,8 +108,8 @@ after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, Db) ->
throw({forbidden,
<<"Only administrators can view design docs in the users database.">>})
end;
-after_doc_read(Doc, #db{user_ctx = UserCtx} = Db) ->
- #user_ctx{name=Name} = UserCtx,
+after_doc_read(Doc, Db) ->
+ #user_ctx{name=Name} = couch_db:get_user_ctx(Db),
DocName = get_doc_name(Doc),
case (catch couch_db:check_is_admin(Db)) of
ok ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/src/couch_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_util.erl b/src/couch_util.erl
index 6001ae2..d688c12 100644
--- a/src/couch_util.erl
+++ b/src/couch_util.erl
@@ -198,7 +198,9 @@ json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) ->
json_apply_field({Key, NewValue}, [], Acc) ->
{[{Key, NewValue}|Acc]}.
-json_user_ctx(#db{name=ShardName, user_ctx=Ctx}) ->
+json_user_ctx(Db) ->
+ ShardName = couch_db:name(Db),
+ Ctx = couch_db:get_user_ctx(Db),
{[{<<"db">>, mem3:dbname(ShardName)},
{<<"name">>,Ctx#user_ctx.name},
{<<"roles">>,Ctx#user_ctx.roles}]}.
@@ -455,9 +457,7 @@ encode_doc_id(Id) ->
url_encode(Id).
-with_db(Db, Fun) when is_record(Db, db) ->
- Fun(Db);
-with_db(DbName, Fun) ->
+with_db(DbName, Fun) when is_binary(DbName) ->
case couch_db:open_int(DbName, [?ADMIN_CTX]) of
{ok, Db} ->
try
@@ -467,6 +467,13 @@ with_db(DbName, Fun) ->
end;
Else ->
throw(Else)
+ end;
+with_db(Db, Fun) ->
+ case couch_db:is_db(Db) of
+ true ->
+ Fun(Db);
+ false ->
+ erlang:error({invalid_db, Db})
end.
rfc1123_date() ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couch_auth_cache_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_auth_cache_tests.erl b/test/couch_auth_cache_tests.erl
index 76179de..08aecd1 100644
--- a/test/couch_auth_cache_tests.erl
+++ b/test/couch_auth_cache_tests.erl
@@ -265,7 +265,7 @@ hash_password(Password) ->
shutdown_db(DbName) ->
{ok, AuthDb} = couch_db:open_int(DbName, [?ADMIN_CTX]),
ok = couch_db:close(AuthDb),
- couch_util:shutdown_sync(AuthDb#db.main_pid),
+ couch_util:shutdown_sync(couch_db:get_pid(AuthDb)),
ok = timer:sleep(1000).
get_doc_rev(DbName, UserName) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couch_changes_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_changes_tests.erl b/test/couch_changes_tests.erl
index 3c0e5f6..494d90f 100644
--- a/test/couch_changes_tests.erl
+++ b/test/couch_changes_tests.erl
@@ -645,7 +645,7 @@ should_filter_by_user_ctx({DbName, _}) ->
]}),
ChArgs = #changes_args{filter = "app/valid"},
UserCtx = #user_ctx{name = <<"doc3">>, roles = []},
- DbRec = #db{name = DbName, user_ctx = UserCtx},
+ {ok, DbRec} = couch_db:clustered_db(DbName, UserCtx),
Req = {json_req, {[{
<<"userCtx">>, couch_util:json_user_ctx(DbRec)
}]}},
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couch_db_plugin_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_db_plugin_tests.erl b/test/couch_db_plugin_tests.erl
index ea9b230..94dd3df 100644
--- a/test/couch_db_plugin_tests.erl
+++ b/test/couch_db_plugin_tests.erl
@@ -43,6 +43,7 @@ data_providers() -> [].
data_subscriptions() -> [].
processes() -> [].
notify(_, _, _) -> ok.
+fake_db() -> element(2, couch_db:clustered_db(fake, totes_fake)).
setup() ->
couch_tests:setup([
@@ -133,33 +134,33 @@ validate_dbname_pass() ->
before_doc_update_match() ->
?assertMatch(
{true, [before_doc_update, doc]},
- couch_db_plugin:before_doc_update(#db{}, {true, [doc]})).
+ couch_db_plugin:before_doc_update(fake_db(), {true, [doc]})).
before_doc_update_no_match() ->
?assertMatch(
{false, [doc]},
- couch_db_plugin:before_doc_update(#db{}, {false, [doc]})).
+ couch_db_plugin:before_doc_update(fake_db(), {false, [doc]})).
before_doc_update_throw() ->
?assertThrow(
before_doc_update,
- couch_db_plugin:before_doc_update(#db{}, {fail, [doc]})).
+ couch_db_plugin:before_doc_update(fake_db(), {fail, [doc]})).
after_doc_read_match() ->
?assertMatch(
{true, [after_doc_read, doc]},
- couch_db_plugin:after_doc_read(#db{}, {true, [doc]})).
+ couch_db_plugin:after_doc_read(fake_db(), {true, [doc]})).
after_doc_read_no_match() ->
?assertMatch(
{false, [doc]},
- couch_db_plugin:after_doc_read(#db{}, {false, [doc]})).
+ couch_db_plugin:after_doc_read(fake_db(), {false, [doc]})).
after_doc_read_throw() ->
?assertThrow(
after_doc_read,
- couch_db_plugin:after_doc_read(#db{}, {fail, [doc]})).
+ couch_db_plugin:after_doc_read(fake_db(), {fail, [doc]})).
validate_docid_match() ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couch_server_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_server_tests.erl b/test/couch_server_tests.erl
index c8f8381..4fd7ff2 100644
--- a/test/couch_server_tests.erl
+++ b/test/couch_server_tests.erl
@@ -32,8 +32,7 @@ setup(_) ->
setup().
teardown(Db) ->
- (catch couch_db:close(Db)),
- (catch file:delete(Db#db.filepath)).
+ (catch couch_db:close(Db)).
teardown(rename, Db) ->
config:set("couchdb", "enable_database_recovery", "false", false),
@@ -61,7 +60,9 @@ make_test_case(Mod, Funs) ->
{foreachx, fun setup/1, fun teardown/2, [{Mod, Fun} || Fun <- Funs]}
}.
-should_rename_on_delete(_, #db{filepath = Origin, name = DbName}) ->
+should_rename_on_delete(_, Db) ->
+ DbName = couch_db:name(Db),
+ Origin = couch_db:get_filepath(Db),
?_test(begin
?assert(filelib:is_regular(Origin)),
?assertMatch(ok, couch_server:delete(DbName, [])),
@@ -74,7 +75,9 @@ should_rename_on_delete(_, #db{filepath = Origin, name = DbName}) ->
?assert(filelib:is_regular(Renamed))
end).
-should_delete(_, #db{filepath = Origin, name = DbName}) ->
+should_delete(_, Db) ->
+ DbName = couch_db:name(Db),
+ Origin = couch_db:get_filepath(Db),
?_test(begin
?assert(filelib:is_regular(Origin)),
?assertMatch(ok, couch_server:delete(DbName, [])),
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couchdb_compaction_daemon_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_compaction_daemon_tests.erl b/test/couchdb_compaction_daemon_tests.erl
index 25d9b13..70e505e 100644
--- a/test/couchdb_compaction_daemon_tests.erl
+++ b/test/couchdb_compaction_daemon_tests.erl
@@ -182,7 +182,7 @@ update(DbName) ->
lists:foreach(fun(_) ->
Doc = couch_doc:from_json_obj({[{<<"_id">>, couch_uuids:new()}]}),
{ok, _} = couch_db:update_docs(Db, [Doc]),
- query_view(Db#db.name)
+ query_view(couch_db:name(Db))
end, lists:seq(1, 200)),
couch_db:close(Db).
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/c515bcae/test/couchdb_views_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_views_tests.erl b/test/couchdb_views_tests.erl
index f1fddfc..02e9d72 100644
--- a/test/couchdb_views_tests.erl
+++ b/test/couchdb_views_tests.erl
@@ -340,7 +340,7 @@ couchdb_1283() ->
]}),
{ok, _} = couch_db:update_doc(MDb1, DDoc, []),
ok = populate_db(MDb1, 100, 100),
- query_view(MDb1#db.name, "foo", "foo"),
+ query_view(couch_db:name(MDb1), "foo", "foo"),
ok = couch_db:close(MDb1),
{ok, Db1} = couch_db:create(?tempdb(), [?ADMIN_CTX]),
@@ -350,8 +350,8 @@ couchdb_1283() ->
{ok, Db3} = couch_db:create(?tempdb(), [?ADMIN_CTX]),
ok = couch_db:close(Db3),
- Writer1 = spawn_writer(Db1#db.name),
- Writer2 = spawn_writer(Db2#db.name),
+ Writer1 = spawn_writer(couch_db:name(Db1)),
+ Writer2 = spawn_writer(couch_db:name(Db2)),
?assert(is_process_alive(Writer1)),
?assert(is_process_alive(Writer2)),
@@ -361,16 +361,16 @@ couchdb_1283() ->
%% Below we do exactly the same as couch_mrview:compact holds inside
%% because we need have access to compaction Pid, not a Ref.
- %% {ok, MonRef} = couch_mrview:compact(MDb1#db.name, <<"_design/foo">>,
+ %% {ok, MonRef} = couch_mrview:compact(MDb1, <<"_design/foo">>,
%% [monitor]),
{ok, Pid} = couch_index_server:get_index(
- couch_mrview_index, MDb1#db.name, <<"_design/foo">>),
+ couch_mrview_index, couch_db:name(MDb1), <<"_design/foo">>),
{ok, CPid} = gen_server:call(Pid, compact),
%% By suspending compaction process we ensure that compaction won't get
%% finished too early to make get_writer_status assertion fail.
erlang:suspend_process(CPid),
MonRef = erlang:monitor(process, CPid),
- Writer3 = spawn_writer(Db3#db.name),
+ Writer3 = spawn_writer(couch_db:name(Db3)),
?assert(is_process_alive(Writer3)),
?assert(is_process_alive(Writer1)),
@@ -526,7 +526,8 @@ view_cleanup(DbName) ->
count_users(DbName) ->
{ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
- {monitored_by, Monitors} = erlang:process_info(Db#db.main_pid, monitored_by),
+ DbPid = couch_db:get_pid(Db),
+ {monitored_by, Monitors} = erlang:process_info(DbPid, monitored_by),
CouchFiles = [P || P <- Monitors, couch_file:process_info(P) =/= undefined],
ok = couch_db:close(Db),
length(lists:usort(Monitors) -- [self() | CouchFiles]).
@@ -552,7 +553,8 @@ restore_backup_db_file(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
- exit(Db#db.main_pid, shutdown),
+ DbPid = couch_db:get_pid(Db),
+ exit(DbPid, shutdown),
DbFile = filename:join([DbDir, ?b2l(DbName) ++ ".couch"]),
ok = file:delete(DbFile),
@@ -573,7 +575,8 @@ wait_db_compact_done(_DbName, 0) ->
wait_db_compact_done(DbName, N) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
- case is_pid(Db#db.compactor_pid) of
+ CompactorPid = couch_db:get_compactor_pid(Db),
+ case is_pid(CompactorPid) of
false ->
ok;
true ->
[17/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Allow limiting maximum document body size
Configuration is via the `couchdb.max_document_size`. In the past that
was implemented as a maximum http request body size and this finally implements
it by actually checking a document's body size.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/7e48bda4
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/7e48bda4
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/7e48bda4
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 7e48bda4459cc8e4dbb8bd86966792f533571d83
Parents: 2d984b5
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Mon Mar 13 02:33:57 2017 -0400
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Mar 15 22:33:22 2017 -0400
----------------------------------------------------------------------
src/couch_doc.erl | 14 ++++++++++++--
src/couch_httpd_db.erl | 8 ++++----
test/couch_doc_json_tests.erl | 21 ++++++++++++++++++---
3 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/7e48bda4/src/couch_doc.erl
----------------------------------------------------------------------
diff --git a/src/couch_doc.erl b/src/couch_doc.erl
index a913eee..381ad4b 100644
--- a/src/couch_doc.erl
+++ b/src/couch_doc.erl
@@ -13,7 +13,7 @@
-module(couch_doc).
-export([to_doc_info/1,to_doc_info_path/1,parse_rev/1,parse_revs/1,rev_to_str/1,revs_to_strs/1]).
--export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]).
+-export([from_json_obj/1, from_json_obj_validate/1, to_json_obj/2,has_stubs/1, merge_stubs/2]).
-export([validate_docid/1, get_validate_doc_fun/1]).
-export([doc_from_multi_part_stream/2, doc_from_multi_part_stream/3]).
-export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
@@ -124,6 +124,16 @@ doc_to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds},
++ to_json_attachments(Doc#doc.atts, Options)
}.
+from_json_obj_validate(EJson) ->
+ MaxSize = config:get_integer("couchdb", "max_document_size", 4294967296),
+ Doc = from_json_obj(EJson),
+ case erlang:external_size(Doc#doc.body) =< MaxSize of
+ true ->
+ Doc;
+ false ->
+ throw({request_entity_too_large, Doc#doc.id})
+ end.
+
from_json_obj({Props}) ->
transfer_fields(Props, #doc{body=[]});
@@ -414,7 +424,7 @@ doc_from_multi_part_stream(ContentType, DataFun, Ref) ->
{{started_open_doc_revs, NewRef}, Parser, _ParserRef} ->
restart_open_doc_revs(Parser, Ref, NewRef);
{{doc_bytes, Ref, DocBytes}, Parser, ParserRef} ->
- Doc = from_json_obj(?JSON_DECODE(DocBytes)),
+ Doc = from_json_obj_validate(?JSON_DECODE(DocBytes)),
erlang:put(mochiweb_request_recv, true),
% we'll send the Parser process ID to the remote nodes so they can
% retrieve their own copies of the attachment data
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/7e48bda4/src/couch_httpd_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_db.erl b/src/couch_httpd_db.erl
index 3793a06..e1af1bf 100644
--- a/src/couch_httpd_db.erl
+++ b/src/couch_httpd_db.erl
@@ -256,7 +256,7 @@ db_req(#httpd{method='GET',path_parts=[_DbName]}=Req, Db) ->
db_req(#httpd{method='POST',path_parts=[_DbName]}=Req, Db) ->
couch_httpd:validate_ctype(Req, "application/json"),
- Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)),
+ Doc = couch_doc:from_json_obj_validate(couch_httpd:json_body(Req)),
validate_attachment_names(Doc),
Doc2 = case Doc#doc.id of
<<"">> ->
@@ -319,7 +319,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) ->
true ->
Docs = lists:map(
fun({ObjProps} = JsonObj) ->
- Doc = couch_doc:from_json_obj(JsonObj),
+ Doc = couch_doc:from_json_obj_validate(JsonObj),
validate_attachment_names(Doc),
Id = case Doc#doc.id of
<<>> -> couch_uuids:new();
@@ -353,7 +353,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) ->
end;
false ->
Docs = lists:map(fun(JsonObj) ->
- Doc = couch_doc:from_json_obj(JsonObj),
+ Doc = couch_doc:from_json_obj_validate(JsonObj),
validate_attachment_names(Doc),
Doc
end, DocsArray),
@@ -809,7 +809,7 @@ couch_doc_from_req(Req, DocId, #doc{revs=Revs}=Doc) ->
end,
Doc#doc{id=DocId, revs=Revs2};
couch_doc_from_req(Req, DocId, Json) ->
- couch_doc_from_req(Req, DocId, couch_doc:from_json_obj(Json)).
+ couch_doc_from_req(Req, DocId, couch_doc:from_json_obj_validate(Json)).
% Useful for debugging
% couch_doc_open(Db, DocId) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/7e48bda4/test/couch_doc_json_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_doc_json_tests.erl b/test/couch_doc_json_tests.erl
index 9003d06..ce099d1 100644
--- a/test/couch_doc_json_tests.erl
+++ b/test/couch_doc_json_tests.erl
@@ -38,6 +38,8 @@ mock(couch_log) ->
ok;
mock(config) ->
meck:new(config, [passthrough]),
+ meck:expect(config, get_integer,
+ fun("couchdb", "max_document_size", 4294967296) -> 1024 end),
meck:expect(config, get, fun(_, _) -> undefined end),
meck:expect(config, get, fun(_, _, Default) -> Default end),
ok.
@@ -165,7 +167,7 @@ from_json_success_cases() ->
],
lists:map(
fun({EJson, Expect, Msg}) ->
- {Msg, ?_assertMatch(Expect, couch_doc:from_json_obj(EJson))}
+ {Msg, ?_assertMatch(Expect, couch_doc:from_json_obj_validate(EJson))}
end,
Cases).
@@ -230,16 +232,29 @@ from_json_error_cases() ->
{[{<<"_something">>, 5}]},
{doc_validation, <<"Bad special document member: _something">>},
"Underscore prefix fields are reserved."
+ },
+ {
+ fun() ->
+ {[
+ {<<"_id">>, <<"large_doc">>},
+ {<<"x">> , << <<"x">> || _ <- lists:seq(1,1025) >>}
+ ]}
+ end,
+ {request_entity_too_large, <<"large_doc">>},
+ "Document too large."
}
],
lists:map(fun
+ ({Fun, Expect, Msg}) when is_function(Fun, 0) ->
+ Error = (catch couch_doc:from_json_obj_validate(Fun())),
+ {Msg, ?_assertMatch(Expect, Error)};
({EJson, Expect, Msg}) ->
- Error = (catch couch_doc:from_json_obj(EJson)),
+ Error = (catch couch_doc:from_json_obj_validate(EJson)),
{Msg, ?_assertMatch(Expect, Error)};
({EJson, Msg}) ->
try
- couch_doc:from_json_obj(EJson),
+ couch_doc:from_json_obj_validate(EJson),
{"Conversion failed to raise an exception", ?_assert(false)}
catch
_:_ -> {Msg, ?_assert(true)}
[09/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Rename max_document_size to max_http_request_size
`max_document_size` is implemented as `max_http_request_size`. There was no
real check for document size. In some cases the implementation was close enough
of a proxy (PUT-ing and GET-ing single docs), but in some edge cases, like
_bulk_docs requests the discrepancy between request size and document size
could be rather large.
The section was changed accordingly from `couchdb` to `httpd`. `httpd` was
chosen as it applies to both clustered as well as local interface.
There is a parallel effort to implement an actual max_document_size check. The
set of commit should be merged close enough together to allow for a backwards
compatible transition.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/0420f404
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/0420f404
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/0420f404
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 0420f404a7ba2c5c5964f2ae5a87cda1c2178387
Parents: f99e30c
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 19:03:32 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 19:03:32 2017 -0500
----------------------------------------------------------------------
src/couch_httpd.erl | 5 ++---
src/couch_httpd_external.erl | 4 ++--
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/0420f404/src/couch_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd.erl b/src/couch_httpd.erl
index 8e7dfeb..380b73f 100644
--- a/src/couch_httpd.erl
+++ b/src/couch_httpd.erl
@@ -451,7 +451,7 @@ validate_ctype(Req, Ctype) ->
check_max_request_length(Req) ->
Len = list_to_integer(header_value(Req, "Content-Length", "0")),
- MaxLen = config:get_integer("couchdb", "max_document_size", 4294967296),
+ MaxLen = config:get_integer("httpd", "max_http_request_size", 4294967296),
case Len > MaxLen of
true ->
exit({body_too_large, Len});
@@ -576,8 +576,7 @@ body_length(#httpd{mochi_req=MochiReq}) ->
MochiReq:get(body_length).
body(#httpd{mochi_req=MochiReq, req_body=undefined}) ->
- MaxSize = list_to_integer(
- config:get("couchdb", "max_document_size", "4294967296")),
+ MaxSize = config:get_integer("httpd", "max_http_request_size", 4294967296),
MochiReq:recv_body(MaxSize);
body(#httpd{req_body=ReqBody}) ->
ReqBody.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/0420f404/src/couch_httpd_external.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_external.erl b/src/couch_httpd_external.erl
index 626a3fb..1f2f1e8 100644
--- a/src/couch_httpd_external.erl
+++ b/src/couch_httpd_external.erl
@@ -63,8 +63,8 @@ json_req_obj(#httpd{mochi_req=Req,
}, Db, DocId) ->
Body = case ReqBody of
undefined ->
- MaxSize = list_to_integer(
- config:get("couchdb", "max_document_size", "4294967296")),
+ MaxSize = config:get_integer("httpd", "max_http_request_size",
+ 4294967296),
Req:recv_body(MaxSize);
Else -> Else
end,
[18/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge branch 'couchdb-2992'
Closes #235
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/c50a8cda
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/c50a8cda
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/c50a8cda
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: c50a8cda6b0b669dbc987ebd76362620077822fe
Parents: 2d984b5 7e48bda
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Mar 15 22:34:25 2017 -0400
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Mar 15 22:34:25 2017 -0400
----------------------------------------------------------------------
src/couch_doc.erl | 14 ++++++++++++--
src/couch_httpd_db.erl | 8 ++++----
test/couch_doc_json_tests.erl | 21 ++++++++++++++++++---
3 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[05/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge branch 'COUCHDB-3298-improve-couch-btree-chunkify'
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/0080f15e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/0080f15e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/0080f15e
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 0080f15e9bb287eeeaca8479c5a3a5d35306bbcf
Parents: 38d5180 ff9fb71
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Mar 1 15:15:46 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 1 15:15:46 2017 -0600
----------------------------------------------------------------------
src/couch_btree.erl | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[02/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Ensure multi-item chunks in couch_btree:chunkify/1
If the last element of a chunk has a huge reduction it was possible to
return a btree node that had a single key. This prevents the edge case
by forcing it into the previous chunk. Without this we can end up with a
case where a path in the tree can extend for many levels with only a
single key in each node.
COUCHDB-3298
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/ff9fb711
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/ff9fb711
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/ff9fb711
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: ff9fb7112ee5250af01e1b38c8cfa9caed152ae7
Parents: 8556adb
Author: Paul J. Davis <pa...@gmail.com>
Authored: Sat Feb 11 15:29:14 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Sat Feb 11 15:29:14 2017 -0600
----------------------------------------------------------------------
src/couch_btree.erl | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/ff9fb711/src/couch_btree.erl
----------------------------------------------------------------------
diff --git a/src/couch_btree.erl b/src/couch_btree.erl
index 8f2395c..adbc92b 100644
--- a/src/couch_btree.erl
+++ b/src/couch_btree.erl
@@ -352,6 +352,9 @@ chunkify(InList) ->
chunkify([], _ChunkThreshold, [], 0, OutputChunks) ->
lists:reverse(OutputChunks);
+chunkify([], _ChunkThreshold, [Item], _OutListSize, [PrevChunk | RestChunks]) ->
+ NewPrevChunk = PrevChunk ++ [Item],
+ lists:reverse(RestChunks, [NewPrevChunk]);
chunkify([], _ChunkThreshold, OutList, _OutListSize, OutputChunks) ->
lists:reverse([lists:reverse(OutList) | OutputChunks]);
chunkify([InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) ->
[08/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge remote-tracking branch 'cloudant/3318-bypass-vhosts'
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/f99e30c3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/f99e30c3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/f99e30c3
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: f99e30c38997e33538493204d5d58ea1a0f62de8
Parents: 63ef337 f706bb8
Author: Robert Newson <rn...@apache.org>
Authored: Mon Mar 6 14:06:09 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Mon Mar 6 14:06:09 2017 +0000
----------------------------------------------------------------------
src/couch_httpd_vhost.erl | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
[25/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Add a test helper for creating fake db records
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/552a29d1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/552a29d1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/552a29d1
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 552a29d1e107014dab6c209f2f91a5c33af89728
Parents: 73c273f
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 1 15:15:09 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:32:27 2017 -0500
----------------------------------------------------------------------
src/test_util.erl | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/552a29d1/src/test_util.erl
----------------------------------------------------------------------
diff --git a/src/test_util.erl b/src/test_util.erl
index 3c4d170..b5bb232 100644
--- a/src/test_util.erl
+++ b/src/test_util.erl
@@ -13,6 +13,8 @@
-module(test_util).
-include_lib("couch/include/couch_eunit.hrl").
+-include("couch_db.hrl").
+-include("couch_db_int.hrl").
-export([init_code_path/0]).
-export([source_file/1, build_file/1]).
@@ -229,6 +231,16 @@ stop(#test_context{mocked = Mocked, started = Apps}) ->
meck:unload(Mocked),
stop_applications(Apps).
+fake_db(Fields) ->
+ Indexes = lists:zip(
+ record_info(fields, db),
+ lists:seq(2, record_info(size, db))
+ ),
+ lists:foldl(fun({FieldName, Value}, Acc) ->
+ Idx = couch_util:get_value(FieldName, Indexes),
+ setelement(Idx, Acc, Value)
+ end, #db{}, Fields).
+
now_us() ->
{MegaSecs, Secs, MicroSecs} = now(),
(MegaSecs * 1000000 + Secs) * 1000000 + MicroSecs.
[14/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge remote branch 'cloudant:70794-fix-initial-acc-for-builtin_sum_rows'
This closes #234
Signed-off-by: ILYA Khlopotov <ii...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/5685f172
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/5685f172
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/5685f172
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 5685f172d232700412e9c34dba0407c5acebe8f8
Parents: cb3b35a 1f152da
Author: ILYA Khlopotov <ii...@apache.org>
Authored: Wed Mar 8 13:22:30 2017 -0800
Committer: ILYA Khlopotov <ii...@apache.org>
Committed: Wed Mar 8 13:22:30 2017 -0800
----------------------------------------------------------------------
src/couch_query_servers.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[21/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Remove PSE references from compaction test
I accidentally included part of the PSE work when I extracted this
patch. Should just be the minor tweak to access the updater pid
directly.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/21c8d37a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/21c8d37a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/21c8d37a
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 21c8d37ac6ee1a7fed1de1f54f95a4d3cd9f5248
Parents: f09e321
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu Mar 23 14:55:48 2017 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Mar 23 14:55:48 2017 -0500
----------------------------------------------------------------------
test/couchdb_compaction_daemon_tests.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/21c8d37a/test/couchdb_compaction_daemon_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_compaction_daemon_tests.erl b/test/couchdb_compaction_daemon_tests.erl
index 20879a7..25d9b13 100644
--- a/test/couchdb_compaction_daemon_tests.erl
+++ b/test/couchdb_compaction_daemon_tests.erl
@@ -220,7 +220,7 @@ spawn_compaction_monitor(DbName) ->
{Pid, Ref} = spawn_monitor(fun() ->
DaemonPid = whereis(couch_compaction_daemon),
DbPid = couch_util:with_db(DbName, fun(Db) ->
- couch_db:get_pid(Db)
+ Db#db.main_pid
end),
{ok, ViewPid} = couch_index_server:get_index(couch_mrview_index,
DbName, <<"_design/foo">>),
@@ -257,7 +257,7 @@ spawn_compaction_monitor(DbName) ->
end),
receive
{Pid, started} -> ok;
- {'DOWN', Ref, _, _, _} -> erlang:error(monitor_failure)
+ {'DOWN', Ref, _, _, Reason} -> erlang:error({monitor_failure, Reason})
after ?TIMEOUT ->
erlang:error({assertion_failed, [
{module, ?MODULE},
[16/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
fix compiler and dialyzer warnings
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/2d984b59
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/2d984b59
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/2d984b59
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 2d984b59ed6886a179fd500efe9780a0d0ad62e3
Parents: 92c25a9
Author: Robert Newson <rn...@apache.org>
Authored: Wed Mar 15 18:13:09 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Mar 15 18:15:03 2017 +0000
----------------------------------------------------------------------
src/couch_server.erl | 28 ++++++++++------------------
1 file changed, 10 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/2d984b59/src/couch_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_server.erl b/src/couch_server.erl
index 1272e7a..1152300 100644
--- a/src/couch_server.erl
+++ b/src/couch_server.erl
@@ -359,7 +359,7 @@ handle_call({open_result, _T0, DbName, Error}, {FromPid, _Tag}, Server) ->
[] ->
% db was deleted during async open
{reply, ok, Server};
- [#db{fd=ReqType, compactor_pid=Froms}=Db] ->
+ [#db{fd=ReqType, compactor_pid=Froms}] ->
[gen_server:reply(From, Error) || From <- Froms],
couch_log:info("open_result error ~p for ~s", [Error, DbName]),
true = ets:delete(couch_dbs, DbName),
@@ -378,13 +378,9 @@ handle_call({open, DbName, Options}, From, Server) ->
DbNameList = binary_to_list(DbName),
case check_dbname(Server, DbNameList) of
ok ->
- case maybe_close_lru_db(Server) of
- {ok, Server2} ->
- Filepath = get_full_filename(Server, DbNameList),
- {noreply, open_async(Server2, From, DbName, Filepath, Options)};
- CloseError ->
- {reply, CloseError, Server}
- end;
+ {ok, Server2} = maybe_close_lru_db(Server),
+ Filepath = get_full_filename(Server, DbNameList),
+ {noreply, open_async(Server2, From, DbName, Filepath, Options)};
Error ->
{reply, Error, Server}
end;
@@ -406,13 +402,9 @@ handle_call({create, DbName, Options}, From, Server) ->
ok ->
case ets:lookup(couch_dbs, DbName) of
[] ->
- case maybe_close_lru_db(Server) of
- {ok, Server2} ->
- {noreply, open_async(Server2, From, DbName, Filepath,
- [create | Options])};
- CloseError ->
- {reply, CloseError, Server}
- end;
+ {ok, Server2} = maybe_close_lru_db(Server),
+ {noreply, open_async(Server2, From, DbName, Filepath,
+ [create | Options])};
[#db{fd=open}=Db] ->
% We're trying to create a database while someone is in
% the middle of trying to open it. We allow one creator
@@ -436,14 +428,14 @@ handle_call({delete, DbName, Options}, _From, Server) ->
Server2 =
case ets:lookup(couch_dbs, DbName) of
[] -> Server;
- [#db{main_pid=Pid, compactor_pid=Froms} = Db] when is_list(Froms) ->
+ [#db{main_pid=Pid, compactor_pid=Froms}] when is_list(Froms) ->
% icky hack of field values - compactor_pid used to store clients
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
[gen_server:reply(F, not_found) || F <- Froms],
db_closed(Server);
- [#db{main_pid=Pid} = Db] ->
+ [#db{main_pid=Pid}] ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
@@ -502,7 +494,7 @@ handle_info({'EXIT', _Pid, config_change}, Server) ->
handle_info({'EXIT', Pid, Reason}, Server) ->
case ets:lookup(couch_dbs_pid_to_name, Pid) of
[{Pid, DbName}] ->
- [#db{compactor_pid=Froms}=Db] = ets:lookup(couch_dbs, DbName),
+ [#db{compactor_pid=Froms}] = ets:lookup(couch_dbs, DbName),
if Reason /= snappy_nif_not_loaded -> ok; true ->
Msg = io_lib:format("To open the database `~s`, Apache CouchDB "
"must be built with Erlang OTP R13B04 or higher.", [DbName]),
[11/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Return error row instead of crashing
When input is invalid for built in reducers, we return an error row
instead of crashing. Thanks to davisp for providing the elegant
solution.
COUCHDB-3305
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/9f9c4822
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/9f9c4822
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/9f9c4822
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 9f9c48221a96452f36790aa8dd52c557d65b9cde
Parents: 9d8be06
Author: Tony Sun <to...@cloudant.com>
Authored: Tue Feb 21 14:18:49 2017 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Tue Mar 7 19:31:20 2017 -0800
----------------------------------------------------------------------
src/couch_query_servers.erl | 66 ++++++++++++++++++++++++++++++++++------
1 file changed, 56 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/9f9c4822/src/couch_query_servers.erl
----------------------------------------------------------------------
diff --git a/src/couch_query_servers.erl b/src/couch_query_servers.erl
index 92d9e24..ddc079e 100644
--- a/src/couch_query_servers.erl
+++ b/src/couch_query_servers.erl
@@ -27,7 +27,7 @@
-include_lib("couch/include/couch_db.hrl").
-define(SUMERROR, <<"The _sum function requires that map values be numbers, "
- "arrays of numbers, or objects, not '~p'. Objects cannot be mixed with other "
+ "arrays of numbers, or objects. Objects cannot be mixed with other "
"data structures. Objects can be arbitrarily nested, provided that the values "
"for all fields are themselves numbers, arrays of numbers, or objects.">>).
@@ -142,25 +142,34 @@ os_rereduce(Lang, OsRedSrcs, KVs) ->
builtin_reduce(_Re, [], _KVs, Acc) ->
{ok, lists:reverse(Acc)};
builtin_reduce(Re, [<<"_sum",_/binary>>|BuiltinReds], KVs, Acc) ->
- Sum = builtin_sum_rows(KVs),
+ Sum = builtin_sum_rows(KVs, []),
builtin_reduce(Re, BuiltinReds, KVs, [Sum|Acc]);
builtin_reduce(reduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) ->
Count = length(KVs),
builtin_reduce(reduce, BuiltinReds, KVs, [Count|Acc]);
builtin_reduce(rereduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) ->
- Count = builtin_sum_rows(KVs),
+ Count = builtin_sum_rows(KVs, []),
builtin_reduce(rereduce, BuiltinReds, KVs, [Count|Acc]);
builtin_reduce(Re, [<<"_stats",_/binary>>|BuiltinReds], KVs, Acc) ->
Stats = builtin_stats(Re, KVs),
builtin_reduce(Re, BuiltinReds, KVs, [Stats|Acc]).
-builtin_sum_rows(KVs) ->
- lists:foldl(fun([_Key, Value], Acc) -> sum_values(Value, Acc) end, 0, KVs).
-sum_values({Props}, 0) ->
- {Props};
-sum_values({Props}, {AccProps}) ->
- {sum_objects(lists:sort(Props), lists:sort(AccProps))};
+builtin_sum_rows([], Acc) ->
+ Acc;
+builtin_sum_rows([[_Key, Value] | RestKVs], Acc) ->
+ try sum_values(Value, Acc) of
+ NewAcc ->
+ builtin_sum_rows(RestKVs, NewAcc)
+ catch
+ throw:{builtin_reduce_error, Obj} ->
+ Obj;
+ throw:{invalid_value, Reason, Cause} ->
+ {[{<<"error">>, <<"builtin_reduce_error">>},
+ {<<"reason">>, Reason}, {<<"caused_by">>, Cause}]}
+ end.
+
+
sum_values(Value, Acc) when is_number(Value), is_number(Acc) ->
Acc + Value;
sum_values(Value, Acc) when is_list(Value), is_list(Acc) ->
@@ -169,6 +178,19 @@ sum_values(Value, Acc) when is_number(Value), is_list(Acc) ->
sum_arrays(Acc, [Value]);
sum_values(Value, Acc) when is_list(Value), is_number(Acc) ->
sum_arrays([Acc], Value);
+sum_values({Props}, Acc) ->
+ case lists:keyfind(<<"error">>, 1, Props) of
+ {<<"error">>, <<"builtin_reduce_error">>} ->
+ throw({builtin_reduce_error, {Props}});
+ false ->
+ ok
+ end,
+ case Acc of
+ 0 ->
+ {Props};
+ {AccProps} ->
+ {sum_objects(lists:sort(Props), lists:sort(AccProps))}
+ end;
sum_values(Else, _Acc) ->
throw_sum_error(Else).
@@ -490,7 +512,7 @@ ret_os_process(Proc) ->
ok.
throw_sum_error(Else) ->
- throw({invalid_value, iolist_to_binary(io_lib:format(?SUMERROR, [Else]))}).
+ throw({invalid_value, ?SUMERROR, Else}).
throw_stat_error(Else) ->
throw({invalid_value, iolist_to_binary(io_lib:format(?STATERROR, [Else]))}).
@@ -499,6 +521,17 @@ throw_stat_error(Else) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+builtin_sum_rows_negative_test() ->
+ A = [{[{<<"a">>, 1}]}, {[{<<"a">>, 2}]}, {[{<<"a">>, 3}]}],
+ E = {[{<<"error">>, <<"builtin_reduce_error">>}]},
+ ?assertEqual(E, builtin_sum_rows([["K", E]], [])),
+ % The below case is where the value is invalid, but no error because
+ % it's only one document.
+ ?assertEqual(A, builtin_sum_rows([["K", A]], [])),
+ {Result} = builtin_sum_rows([["K", A]], [1, 2, 3]),
+ ?assertEqual({<<"error">>, <<"builtin_reduce_error">>},
+ lists:keyfind(<<"error">>, 1, Result)).
+
sum_values_test() ->
?assertEqual(3, sum_values(1, 2)),
?assertEqual([2,4,6], sum_values(1, [1,4,6])),
@@ -512,6 +545,19 @@ sum_values_test() ->
?assertEqual(Z, sum_values(X, Y)),
?assertEqual(Z, sum_values(Y, X)).
+sum_values_negative_test() ->
+ % invalid value
+ A = [{[{<<"a">>, 1}]}, {[{<<"a">>, 2}]}, {[{<<"a">>, 3}]}],
+ B = ["error 1", "error 2"],
+ C = [<<"error 3">>, <<"error 4">>],
+ KV = {[{<<"error">>, <<"builtin_reduce_error">>},
+ {<<"reason">>, ?SUMERROR}, {<<"caused_by">>, <<"some cause">>}]},
+ ?assertThrow({invalid_value, _, _}, sum_values(A, [1, 2, 3])),
+ ?assertThrow({invalid_value, _, _}, sum_values(A, 0)),
+ ?assertThrow({invalid_value, _, _}, sum_values(B, [1, 2])),
+ ?assertThrow({invalid_value, _, _}, sum_values(C, [0])),
+ ?assertThrow({builtin_reduce_error, KV}, sum_values(KV, [0])).
+
stat_values_test() ->
?assertEqual({1, 2, 0, 1, 1}, stat_values(1, 0)),
?assertEqual({11, 2, 1, 10, 101}, stat_values(1, 10)),
[24/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Update couch_server to not use the db record
This removes introspection of the #db record by couch_server. While its
required for the pluggable storage engine upgrade, its also nice to
remove the hacky overloading of #db record fields for couch_server
logic.
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/73c273ff
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/73c273ff
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/73c273ff
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 73c273ff93807cb845ccacd769d0ab7e1030b69d
Parents: e1491f1
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Feb 3 10:20:30 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:32:27 2017 -0500
----------------------------------------------------------------------
src/couch_db.erl | 23 +++++++++
src/couch_lru.erl | 14 +++---
src/couch_server.erl | 109 ++++++++++++++++++++++--------------------
src/couch_server_int.hrl | 23 +++++++++
4 files changed, 110 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index 1f68200..3a29a3d 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -19,6 +19,9 @@
reopen/1,
close/1,
+ incref/1,
+ decref/1,
+
monitor/1,
monitored_by/1,
is_idle/1,
@@ -34,7 +37,9 @@
get_db_info/1,
get_doc_count/1,
get_epochs/1,
+ get_instance_start_time/1,
get_last_purged/1,
+ get_pid/1,
get_revs_limit/1,
get_security/1,
get_update_seq/1,
@@ -46,6 +51,7 @@
increment_update_seq/1,
set_revs_limit/2,
set_security/2,
+ set_user_ctx/2,
ensure_full_commit/1,
ensure_full_commit/2,
@@ -181,6 +187,14 @@ reopen(#db{main_pid = Pid, fd = Fd, fd_monitor = OldRef, user_ctx = UserCtx}) ->
{ok, NewDb#db{user_ctx = UserCtx, fd_monitor = NewRef}}
end.
+incref(#db{fd = Fd} = Db) ->
+ Ref = erlang:monitor(process, Fd),
+ {ok, Db#db{fd_monitor = Ref}}.
+
+decref(#db{fd_monitor = Monitor}) ->
+ erlang:demonitor(Monitor, [flush]),
+ ok.
+
is_system_db(#db{options = Options}) ->
lists:member(sys_db, Options).
@@ -381,6 +395,9 @@ get_last_purged(#db{}=Db) ->
couch_file:pread_term(Db#db.fd, Pointer)
end.
+get_pid(#db{main_pid = Pid}) ->
+ Pid.
+
get_doc_count(Db) ->
{ok, {Count, _, _}} = couch_btree:full_reduce(Db#db.id_tree),
{ok, Count}.
@@ -393,6 +410,9 @@ get_epochs(#db{}=Db) ->
validate_epochs(Epochs),
Epochs.
+get_instance_start_time(#db{instance_start_time = IST}) ->
+ IST.
+
get_compacted_seq(#db{}=Db) ->
couch_db_header:compacted_seq(Db#db.header).
@@ -585,6 +605,9 @@ set_security(#db{main_pid=Pid}=Db, {NewSecProps}) when is_list(NewSecProps) ->
set_security(_, _) ->
throw(bad_request).
+set_user_ctx(#db{} = Db, UserCtx) ->
+ {ok, Db#db{user_ctx = UserCtx}}.
+
validate_security_object(SecProps) ->
Admins = couch_util:get_value(<<"admins">>, SecProps, {[]}),
% we fallback to readers here for backwards compatibility
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_lru.erl
----------------------------------------------------------------------
diff --git a/src/couch_lru.erl b/src/couch_lru.erl
index b79286e..4bf0409 100644
--- a/src/couch_lru.erl
+++ b/src/couch_lru.erl
@@ -13,7 +13,7 @@
-module(couch_lru).
-export([new/0, insert/2, update/2, close/1]).
--include_lib("couch/include/couch_db.hrl").
+-include("couch_server_int.hrl").
new() ->
Updates = ets:new(couch_lru_updates, [ordered_set]),
@@ -49,18 +49,20 @@ close({Count, Updates, Dbs}) ->
close_int('$end_of_table', _Updates, _Dbs) ->
false;
close_int({_Count, DbName} = Key, Updates, Dbs) ->
- case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
+ case ets:update_element(couch_dbs, DbName, {#entry.lock, locked}) of
true ->
- [#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
+ [#entry{db = Db}] = ets:lookup(couch_dbs, DbName),
case couch_db:is_idle(Db) of true ->
+ DbPid = couch_db:get_pid(Db),
true = ets:delete(couch_dbs, DbName),
- true = ets:delete(couch_dbs_pid_to_name, Pid),
- exit(Pid, kill),
+ true = ets:delete(couch_dbs_pid_to_name, DbPid),
+ exit(DbPid, kill),
true = ets:delete(Updates, Key),
true = ets:delete(Dbs, DbName),
true;
false ->
- true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
+ ElemSpec = {#entry.lock, unlocked},
+ true = ets:update_element(couch_dbs, DbName, ElemSpec),
couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
close_int(ets:next(Updates, Key), Updates, Dbs)
end;
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_server.erl b/src/couch_server.erl
index 1152300..0513f63 100644
--- a/src/couch_server.erl
+++ b/src/couch_server.erl
@@ -26,6 +26,7 @@
-export([handle_config_change/5, handle_config_terminate/3]).
-include_lib("couch/include/couch_db.hrl").
+-include("couch_server_int.hrl").
-define(MAX_DBS_OPEN, 100).
-define(RELISTEN_DELAY, 5000).
@@ -73,16 +74,18 @@ sup_start_link() ->
open(DbName, Options0) ->
Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}),
case ets:lookup(couch_dbs, DbName) of
- [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked ->
+ [#entry{db = Db0, lock = Lock}] when Lock =/= locked ->
update_lru(DbName),
- {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
+ {ok, Db1} = couch_db:incref(Db0),
+ couch_db:set_user_ctx(Db1, Ctx);
_ ->
Options = maybe_add_sys_db_callbacks(DbName, Options0),
Timeout = couch_util:get_value(timeout, Options, infinity),
Create = couch_util:get_value(create_if_missing, Options, false),
case gen_server:call(couch_server, {open, DbName, Options}, Timeout) of
- {ok, #db{fd=Fd} = Db} ->
- {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
+ {ok, Db0} ->
+ {ok, Db1} = couch_db:incref(Db0),
+ couch_db:set_user_ctx(Db1, Ctx);
{not_found, no_db_file} when Create ->
couch_log:warning("creating missing database: ~s", [DbName]),
couch_server:create(DbName, Options);
@@ -100,9 +103,10 @@ close_lru() ->
create(DbName, Options0) ->
Options = maybe_add_sys_db_callbacks(DbName, Options0),
case gen_server:call(couch_server, {create, DbName, Options}, infinity) of
- {ok, #db{fd=Fd} = Db} ->
+ {ok, Db0} ->
Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}),
- {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}};
+ {ok, Db1} = couch_db:incref(Db0),
+ couch_db:set_user_ctx(Db1, Ctx);
Error ->
Error
end.
@@ -184,7 +188,7 @@ init([]) ->
ok = config:listen_for_changes(?MODULE, nil),
ok = couch_file:init_delete_dir(RootDir),
hash_admin_passwords(),
- ets:new(couch_dbs, [set, protected, named_table, {keypos, #db.name}]),
+ ets:new(couch_dbs, [set, protected, named_table, {keypos, #entry.name}]),
ets:new(couch_dbs_pid_to_name, [set, protected, named_table]),
process_flag(trap_exit, true),
{ok, #server{root_dir=RootDir,
@@ -196,8 +200,9 @@ terminate(Reason, Srv) ->
couch_log:error("couch_server terminating with ~p, state ~2048p",
[Reason,
Srv#server{lru = redacted}]),
- ets:foldl(fun(#db{main_pid=Pid}, _) -> couch_util:shutdown_sync(Pid) end,
- nil, couch_dbs),
+ ets:foldl(fun(Db, _) ->
+ couch_util:shutdown_sync(couch_db:get_pid(Db))
+ end, nil, couch_dbs),
ok.
handle_config_change("couchdb", "database_dir", _, _, _) ->
@@ -297,15 +302,13 @@ open_async(Server, From, DbName, Filepath, Options) ->
true -> create;
false -> open
end,
- % icky hack of field values - compactor_pid used to store clients
- % and fd used for opening request info
- true = ets:insert(couch_dbs, #db{
+ true = ets:insert(couch_dbs, #entry{
name = DbName,
- fd = ReqType,
- main_pid = Opener,
- compactor_pid = [From],
- fd_monitor = locked,
- options = Options
+ pid = Opener,
+ lock = locked,
+ waiters = [From],
+ req_type = ReqType,
+ db_options = Options
}),
true = ets:insert(couch_dbs_pid_to_name, {Opener, DbName}),
db_opened(Server).
@@ -329,16 +332,15 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) ->
true = ets:delete(couch_dbs_pid_to_name, FromPid),
OpenTime = timer:now_diff(os:timestamp(), T0) / 1000,
couch_stats:update_histogram([couchdb, db_open_time], OpenTime),
- % icky hack of field values - compactor_pid used to store clients
- % and fd used to possibly store a creation request
+ DbPid = couch_db:get_pid(Db),
case ets:lookup(couch_dbs, DbName) of
[] ->
% db was deleted during async open
- exit(Db#db.main_pid, kill),
+ exit(DbPid, kill),
{reply, ok, Server};
- [#db{fd=ReqType, compactor_pid=Froms}] ->
- link(Db#db.main_pid),
- [gen_server:reply(From, {ok, Db}) || From <- Froms],
+ [#entry{req_type = ReqType, waiters = Waiters} = Entry] ->
+ link(DbPid),
+ [gen_server:reply(Waiter, {ok, Db}) || Waiter <- Waiters],
% Cancel the creation request if it exists.
case ReqType of
{create, DbName, _Filepath, _Options, CrFrom} ->
@@ -346,21 +348,27 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) ->
_ ->
ok
end,
- true = ets:insert(couch_dbs, Db),
- true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}),
+ true = ets:insert(couch_dbs, #entry{
+ name = DbName,
+ db = Db,
+ pid = DbPid,
+ lock = unlocked,
+ db_options = Entry#entry.db_options,
+ start_time = couch_db:get_instance_start_time(Db)
+ }),
+ true = ets:insert(couch_dbs_pid_to_name, {DbPid, DbName}),
Lru = couch_lru:insert(DbName, Server#server.lru),
{reply, ok, Server#server{lru = Lru}}
end;
handle_call({open_result, T0, DbName, {error, eexist}}, From, Server) ->
handle_call({open_result, T0, DbName, file_exists}, From, Server);
handle_call({open_result, _T0, DbName, Error}, {FromPid, _Tag}, Server) ->
- % icky hack of field values - compactor_pid used to store clients
case ets:lookup(couch_dbs, DbName) of
[] ->
% db was deleted during async open
{reply, ok, Server};
- [#db{fd=ReqType, compactor_pid=Froms}] ->
- [gen_server:reply(From, Error) || From <- Froms],
+ [#entry{req_type = ReqType, waiters = Waiters} = Entry] ->
+ [gen_server:reply(Waiter, Error) || Waiter <- Waiters],
couch_log:info("open_result error ~p for ~s", [Error, DbName]),
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, FromPid),
@@ -384,15 +392,14 @@ handle_call({open, DbName, Options}, From, Server) ->
Error ->
{reply, Error, Server}
end;
- [#db{compactor_pid = Froms} = Db] when is_list(Froms) ->
- % icky hack of field values - compactor_pid used to store clients
- true = ets:insert(couch_dbs, Db#db{compactor_pid = [From|Froms]}),
- if length(Froms) =< 10 -> ok; true ->
+ [#entry{waiters = Waiters} = Entry] when is_list(Waiters) ->
+ true = ets:insert(couch_dbs, Entry#entry{waiters = [From | Waiters]}),
+ if length(Waiters) =< 10 -> ok; true ->
Fmt = "~b clients waiting to open db ~s",
- couch_log:info(Fmt, [length(Froms), DbName])
+ couch_log:info(Fmt, [length(Waiters), DbName])
end,
{noreply, Server};
- [#db{} = Db] ->
+ [#entry{db = Db}] ->
{reply, {ok, Db}, Server}
end;
handle_call({create, DbName, Options}, From, Server) ->
@@ -405,14 +412,13 @@ handle_call({create, DbName, Options}, From, Server) ->
{ok, Server2} = maybe_close_lru_db(Server),
{noreply, open_async(Server2, From, DbName, Filepath,
[create | Options])};
- [#db{fd=open}=Db] ->
+ [#entry{req_type = open} = Entry] ->
% We're trying to create a database while someone is in
% the middle of trying to open it. We allow one creator
% to wait while we figure out if it'll succeed.
- % icky hack of field values - fd used to store create request
CrOptions = [create | Options],
- NewDb = Db#db{fd={create, DbName, Filepath, CrOptions, From}},
- true = ets:insert(couch_dbs, NewDb),
+ Req = {create, DbName, Filepath, CrOptions, From},
+ true = ets:insert(couch_dbs, Entry#entry{req_type = Req}),
{noreply, Server};
[_AlreadyRunningDb] ->
{reply, file_exists, Server}
@@ -428,18 +434,17 @@ handle_call({delete, DbName, Options}, _From, Server) ->
Server2 =
case ets:lookup(couch_dbs, DbName) of
[] -> Server;
- [#db{main_pid=Pid, compactor_pid=Froms}] when is_list(Froms) ->
- % icky hack of field values - compactor_pid used to store clients
+ [#entry{pid = Pid, waiters = Waiters} = Entry] when is_list(Waiters) ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- [gen_server:reply(F, not_found) || F <- Froms],
+ [gen_server:reply(Waiter, not_found) || Waiter <- Waiters],
db_closed(Server);
- [#db{main_pid=Pid}] ->
+ [#entry{pid = Pid} = Entry] ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- db_closed(Server)
+ db_closed(Server);
end,
%% Delete any leftover compaction files. If we don't do this a
@@ -465,11 +470,12 @@ handle_call({delete, DbName, Options}, _From, Server) ->
Error ->
{reply, Error, Server}
end;
-handle_call({db_updated, #db{}=Db}, _From, Server0) ->
- #db{name = DbName, instance_start_time = StartTime} = Db,
- Server = try ets:lookup_element(couch_dbs, DbName, #db.instance_start_time) of
+handle_call({db_updated, Db}, _From, Server0) ->
+ DbName = couch_db:name(Db),
+ StartTime = couch_db:get_instance_start_time(Db),
+ Server = try ets:lookup_element(couch_dbs, DbName, #entry.start_time) of
StartTime ->
- true = ets:insert(couch_dbs, Db),
+ true = ets:update_element(couch_dbs, DbName, {#entry.db, Db}),
Lru = couch_lru:update(DbName, Server0#server.lru),
Server0#server{lru = Lru};
_ ->
@@ -494,18 +500,15 @@ handle_info({'EXIT', _Pid, config_change}, Server) ->
handle_info({'EXIT', Pid, Reason}, Server) ->
case ets:lookup(couch_dbs_pid_to_name, Pid) of
[{Pid, DbName}] ->
- [#db{compactor_pid=Froms}] = ets:lookup(couch_dbs, DbName),
+ [#entry{waiters = Waiters} = Entry] = ets:lookup(couch_dbs, DbName),
if Reason /= snappy_nif_not_loaded -> ok; true ->
Msg = io_lib:format("To open the database `~s`, Apache CouchDB "
"must be built with Erlang OTP R13B04 or higher.", [DbName]),
couch_log:error(Msg, [])
end,
couch_log:info("db ~s died with reason ~p", [DbName, Reason]),
- % icky hack of field values - compactor_pid used to store clients
- if is_list(Froms) ->
- [gen_server:reply(From, Reason) || From <- Froms];
- true ->
- ok
+ if not is_list(Waiters) -> ok; true ->
+ [gen_server:reply(Waiter, Reason) || Waiter <- Waiters]
end,
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_server_int.hrl
----------------------------------------------------------------------
diff --git a/src/couch_server_int.hrl b/src/couch_server_int.hrl
new file mode 100644
index 0000000..537a6ab
--- /dev/null
+++ b/src/couch_server_int.hrl
@@ -0,0 +1,23 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-record(entry, {
+ name,
+ db,
+ pid,
+ lock,
+ waiters,
+ req_type,
+ db_options,
+ start_time
+}).
[04/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge remote branch 'cloudant:add_sysdb_callback_once_per_db'
This closes #230
Signed-off-by: ILYA Khlopotov <ii...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/38d51803
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/38d51803
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/38d51803
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 38d51803aa3e51be3b15a42b3cbc1f922e386ed5
Parents: 66292db 8fc85c8
Author: ILYA Khlopotov <ii...@apache.org>
Authored: Tue Feb 28 13:51:29 2017 -0800
Committer: ILYA Khlopotov <ii...@apache.org>
Committed: Tue Feb 28 13:51:29 2017 -0800
----------------------------------------------------------------------
src/couch_server.erl | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[23/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Reorganize exports from couch_db.erl
Since we're getting ready to add API functions to couch_db.erl now is a
good time to clean up the exports list so that changes are more easily
tracked.
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/314ee06a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/314ee06a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/314ee06a
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 314ee06a5a1588a419c23d1f9c7205b612f4c4f0
Parents: 21c8d37
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 1 12:21:36 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:23:42 2017 -0500
----------------------------------------------------------------------
src/couch_db.erl | 114 +++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 90 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/314ee06a/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index d01a3e0..de9dd9f 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -12,30 +12,96 @@
-module(couch_db).
--export([open/2,open_int/2,close/1,create/2,get_db_info/1,get_design_docs/1]).
--export([start_compact/1, cancel_compact/1]).
--export([wait_for_compaction/1, wait_for_compaction/2]).
--export([is_idle/1,monitor/1,count_changes_since/2]).
--export([update_doc/3,update_doc/4,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]).
--export([get_doc_info/2,get_full_doc_info/2,get_full_doc_infos/2]).
--export([open_doc/2,open_doc/3,open_doc_revs/4]).
--export([set_revs_limit/2,get_revs_limit/1]).
--export([get_missing_revs/2,name/1,get_update_seq/1,get_committed_update_seq/1]).
--export([get_uuid/1, get_epochs/1, get_compacted_seq/1]).
--export([enum_docs/4,enum_docs_since/5]).
--export([enum_docs_since_reduce_to_count/1,enum_docs_reduce_to_count/1]).
--export([increment_update_seq/1,get_purge_seq/1,purge_docs/2,get_last_purged/1]).
--export([start_link/3,open_doc_int/3,ensure_full_commit/1,ensure_full_commit/2]).
--export([set_security/2,get_security/1]).
--export([changes_since/4,changes_since/5,read_doc/2,new_revid/1]).
--export([check_is_admin/1, is_admin/1, check_is_member/1, get_doc_count/1]).
--export([reopen/1, is_system_db/1, compression/1, make_doc/5]).
--export([load_validation_funs/1]).
--export([check_md5/2, with_stream/3]).
--export([monitored_by/1]).
--export([normalize_dbname/1]).
--export([validate_dbname/1]).
--export([dbname_suffix/1]).
+-export([
+ create/2,
+ open/2,
+ open_int/2,
+ reopen/1,
+ close/1,
+
+ monitor/1,
+ monitored_by/1,
+ is_idle/1,
+
+ is_admin/1,
+ check_is_admin/1,
+ check_is_member/1,
+
+ name/1,
+ compression/1,
+ get_committed_update_seq/1,
+ get_compacted_seq/1,
+ get_db_info/1,
+ get_doc_count/1,
+ get_epochs/1,
+ get_last_purged/1,
+ get_revs_limit/1,
+ get_security/1,
+ get_update_seq/1,
+ get_uuid/1,
+ get_purge_seq/1,
+
+ is_system_db/1,
+
+ increment_update_seq/1,
+ set_revs_limit/2,
+ set_security/2,
+
+ ensure_full_commit/1,
+ ensure_full_commit/2,
+
+ load_validation_funs/1,
+
+ open_doc/2,
+ open_doc/3,
+ open_doc_revs/4,
+ open_doc_int/3,
+ read_doc/2,
+ get_doc_info/2,
+ get_full_doc_info/2,
+ get_full_doc_infos/2,
+ get_missing_revs/2,
+ get_design_docs/1,
+
+ update_doc/3,
+ update_doc/4,
+ update_docs/4,
+ update_docs/2,
+ update_docs/3,
+ delete_doc/3,
+
+ purge_docs/2,
+
+ with_stream/3,
+
+ enum_docs/4,
+ enum_docs_reduce_to_count/1,
+
+ enum_docs_since/5,
+ enum_docs_since_reduce_to_count/1,
+ changes_since/4,
+ changes_since/5,
+ count_changes_since/2,
+
+ start_compact/1,
+ cancel_compact/1,
+ wait_for_compaction/1,
+ wait_for_compaction/2,
+
+ dbname_suffix/1,
+ normalize_dbname/1,
+ validate_dbname/1,
+
+ check_md5/2,
+ make_doc/5,
+ new_revid/1
+]).
+
+
+-export([
+ start_link/3
+]).
+
-include_lib("couch/include/couch_db.hrl").
[10/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Merge branch '64229-add-new-request-parameter'
Closes #233
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/9d8be06e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/9d8be06e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/9d8be06e
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 9d8be06e4ba823165f385667a360e7c050427918
Parents: f99e30c 0420f40
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 20:29:33 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 20:29:33 2017 -0500
----------------------------------------------------------------------
src/couch_httpd.erl | 5 ++---
src/couch_httpd_external.erl | 4 ++--
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[19/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Implement an ETS-basd couch_lru
Use a monotonicaly incrementing counter instead of `erlang:now()`. We don't
technically need to time-based functionality and just want to know relative
insertion order.
Instead of gb_tree, use an ordered_set ETS. This keep items sorted by their
update order, with most recent ones at the bottom.
An set ETS replaces the dictionary which maintains a mapping from database
names to their entry in updates table.
Interface is the same as the old couch_lru, so it is a direct swap in.
Thanks to Eric Avdey for intial version of test module.
COUCHDB-3326
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/016e1aa0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/016e1aa0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/016e1aa0
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: 016e1aa0ef4db0bbf47a28a2cce48b85200702d6
Parents: c50a8cd
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Mar 15 14:06:25 2017 -0400
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Fri Mar 17 12:16:40 2017 -0700
----------------------------------------------------------------------
src/couch_lru.erl | 60 +++++++++++++----------
test/couch_lru_tests.erl | 109 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 143 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/016e1aa0/src/couch_lru.erl
----------------------------------------------------------------------
diff --git a/src/couch_lru.erl b/src/couch_lru.erl
index 84d7bee..b79286e 100644
--- a/src/couch_lru.erl
+++ b/src/couch_lru.erl
@@ -16,33 +16,39 @@
-include_lib("couch/include/couch_db.hrl").
new() ->
- {gb_trees:empty(), dict:new()}.
-
-insert(DbName, {Tree0, Dict0}) ->
- Lru = erlang:now(),
- {gb_trees:insert(Lru, DbName, Tree0), dict:store(DbName, Lru, Dict0)}.
-
-update(DbName, {Tree0, Dict0}) ->
- case dict:find(DbName, Dict0) of
- {ok, Old} ->
- New = erlang:now(),
- Tree = gb_trees:insert(New, DbName, gb_trees:delete(Old, Tree0)),
- Dict = dict:store(DbName, New, Dict0),
- {Tree, Dict};
- error ->
- % We closed this database before processing the update. Ignore
- {Tree0, Dict0}
+ Updates = ets:new(couch_lru_updates, [ordered_set]),
+ Dbs = ets:new(couch_lru_dbs, [set]),
+ {0, Updates, Dbs}.
+
+insert(DbName, {Count, Updates, Dbs}) ->
+ update(DbName, {Count, Updates, Dbs}).
+
+update(DbName, {Count, Updates, Dbs}) ->
+ case ets:lookup(Dbs, DbName) of
+ [] ->
+ true = ets:insert(Dbs, {DbName, Count});
+ [{DbName, OldCount}] ->
+ true = ets:update_element(Dbs, DbName, {2, Count}),
+ true = ets:delete(Updates, {OldCount, DbName})
+ end,
+ true = ets:insert(Updates, {{Count, DbName}}),
+ {Count + 1, Updates, Dbs}.
+
+
+close({Count, Updates, Dbs}) ->
+ case close_int(ets:next(Updates, {-1, <<>>}), Updates, Dbs) of
+ true ->
+ {true, {Count, Updates, Dbs}};
+ false ->
+ false
end.
-%% Attempt to close the oldest idle database.
-close({Tree, _} = Cache) ->
- close_int(gb_trees:next(gb_trees:iterator(Tree)), Cache).
%% internals
-close_int(none, _) ->
+close_int('$end_of_table', _Updates, _Dbs) ->
false;
-close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
+close_int({_Count, DbName} = Key, Updates, Dbs) ->
case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
true ->
[#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
@@ -50,14 +56,16 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}};
+ true = ets:delete(Updates, Key),
+ true = ets:delete(Dbs, DbName),
+ true;
false ->
true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
- close_int(gb_trees:next(Iter), update(DbName, Cache))
+ close_int(ets:next(Updates, Key), Updates, Dbs)
end;
false ->
- NewTree = gb_trees:delete(Lru, Tree),
- NewIter = gb_trees:iterator(NewTree),
- close_int(gb_trees:next(NewIter), {NewTree, dict:erase(DbName, Dict)})
+ true = ets:delete(Updates, Key),
+ true = ets:delete(Dbs, DbName),
+ close_int(ets:next(Updates, Key), Updates, Dbs)
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/016e1aa0/test/couch_lru_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_lru_tests.erl b/test/couch_lru_tests.erl
new file mode 100644
index 0000000..1559835
--- /dev/null
+++ b/test/couch_lru_tests.erl
@@ -0,0 +1,109 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_lru_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+setup() ->
+ ok = meck:new(couch_db, [passthrough]),
+ ets:new(couch_dbs, [set, public, named_table, {keypos, #db.name}]),
+ ets:new(couch_dbs_pid_to_name, [set, public, named_table]),
+ couch_lru:new().
+
+teardown(_) ->
+ ets:delete(couch_dbs),
+ ets:delete(couch_dbs_pid_to_name),
+ (catch meck:unload(couch_db)).
+
+new_test_() ->
+ {setup,
+ fun() -> couch_lru:new() end,
+ fun(Lru) ->
+ ?_assertMatch({0, _, _}, Lru)
+ end
+ }.
+
+insert_test_() ->
+ {setup,
+ fun() -> couch_lru:new() end,
+ fun(Lru) ->
+ Key = <<"test">>,
+ {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
+ [
+ ?_assertEqual(1, ets_size(Dbs)),
+ ?_assert(ets:member(Dbs, Key)),
+ ?_assertEqual(1, ets_size(Updates)),
+ ?_assert(ets:member(Updates, {0, Key}))
+ ]
+ end
+ }.
+
+insert_same_test_() ->
+ {setup,
+ fun() -> couch_lru:new() end,
+ fun(Lru) ->
+ Key = <<"test">>,
+ Lru1 = {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
+ {2, Updates, Dbs} = couch_lru:insert(Key, Lru1),
+ [
+ ?_assertEqual(1, ets_size(Dbs)),
+ ?_assert(ets:member(Dbs, Key)),
+ ?_assertEqual(1, ets_size(Updates)),
+ ?_assert(ets:member(Updates, {1, Key}))
+ ]
+ end
+ }.
+
+update_test_() ->
+ {setup,
+ fun() -> couch_lru:new() end,
+ fun(Lru) ->
+ Key = <<"test">>,
+ Lru1 = {1, Updates, Dbs} = couch_lru:update(Key, Lru),
+ {2, Updates, Dbs} = couch_lru:update(Key, Lru1),
+ [
+ ?_assertEqual(1, ets_size(Dbs)),
+ ?_assert(ets:member(Dbs, Key)),
+ ?_assertEqual(1, ets_size(Updates)),
+ ?_assert(ets:member(Updates, {1, Key}))
+ ]
+ end
+ }.
+
+close_test_() ->
+ {setup,
+ fun setup/0,
+ fun teardown/1,
+ fun(Lru) ->
+ ok = meck:expect(couch_db, is_idle, 1, true),
+ {ok, Lru1} = add_record(Lru, <<"test1">>, c:pid(0, 1001, 0)),
+ {ok, Lru2} = add_record(Lru1, <<"test2">>, c:pid(0, 2001, 0)),
+ {true, {2, Updates, Dbs}} = couch_lru:close(Lru2),
+ [
+ ?_assertEqual(1, ets_size(Dbs)),
+ ?_assert(ets:member(Dbs, <<"test2">>)),
+ ?_assertEqual(1, ets_size(Updates)),
+ ?_assert(ets:member(Updates, {1, <<"test2">>}))
+ ]
+ end
+ }.
+
+add_record(Lru, Key, Pid) ->
+ true = ets:insert(couch_dbs, #db{name = Key, main_pid = Pid}),
+ true = ets:insert(couch_dbs_pid_to_name, {Pid, Key}),
+ {ok, couch_lru:insert(Key, Lru)}.
+
+ets_size(Ets) ->
+ proplists:get_value(size, ets:info(Ets)).
[22/26] couch commit: updated
refs/heads/COUCHDB-3288-remove-public-db-record to c515bca
Posted by da...@apache.org.
Move calculate_start_seq and owner_of
These functions were originally implemented in fabric_rpc.erl where they
really didn't belong. Moving them to couch_db.erl allows us to keep the
unit tests intact rather than just removing them now that the #db record
is being made private.
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/e1491f1c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/e1491f1c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/e1491f1c
Branch: refs/heads/COUCHDB-3288-remove-public-db-record
Commit: e1491f1cca38c5cd78e306d999731047f744680e
Parents: 314ee06
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Feb 3 09:59:23 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:23:42 2017 -0500
----------------------------------------------------------------------
src/couch_db.erl | 103 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 102 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/e1491f1c/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index de9dd9f..1f68200 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -83,6 +83,9 @@
changes_since/5,
count_changes_since/2,
+ calculate_start_seq/3,
+ owner_of/2,
+
start_compact/1,
cancel_compact/1,
wait_for_compaction/1,
@@ -386,7 +389,9 @@ get_uuid(#db{}=Db) ->
couch_db_header:uuid(Db#db.header).
get_epochs(#db{}=Db) ->
- couch_db_header:epochs(Db#db.header).
+ Epochs = couch_db_header:epochs(Db#db.header),
+ validate_epochs(Epochs),
+ Epochs.
get_compacted_seq(#db{}=Db) ->
couch_db_header:compacted_seq(Db#db.header).
@@ -1360,6 +1365,78 @@ enum_docs(Db, NS, InFun, InAcc, Options0) ->
Db#db.id_tree, FoldFun, InAcc, Options),
{ok, enum_docs_reduce_to_count(LastReduce), OutAcc}.
+
+calculate_start_seq(_Db, _Node, Seq) when is_integer(Seq) ->
+ Seq;
+calculate_start_seq(Db, Node, {Seq, Uuid}) ->
+ % Treat the current node as the epoch node
+ calculate_start_seq(Db, Node, {Seq, Uuid, Node});
+calculate_start_seq(Db, _Node, {Seq, Uuid, EpochNode}) ->
+ case is_prefix(Uuid, get_uuid(Db)) of
+ true ->
+ case is_owner(EpochNode, Seq, get_epochs(Db)) of
+ true -> Seq;
+ false -> 0
+ end;
+ false ->
+ %% The file was rebuilt, most likely in a different
+ %% order, so rewind.
+ 0
+ end;
+calculate_start_seq(Db, _Node, {replace, OriginalNode, Uuid, Seq}) ->
+ case is_prefix(Uuid, couch_db:get_uuid(Db)) of
+ true ->
+ start_seq(get_epochs(Db), OriginalNode, Seq);
+ false ->
+ {replace, OriginalNode, Uuid, Seq}
+ end.
+
+
+validate_epochs(Epochs) ->
+ %% Assert uniqueness.
+ case length(Epochs) == length(lists:ukeysort(2, Epochs)) of
+ true -> ok;
+ false -> erlang:error(duplicate_epoch)
+ end,
+ %% Assert order.
+ case Epochs == lists:sort(fun({_, A}, {_, B}) -> B =< A end, Epochs) of
+ true -> ok;
+ false -> erlang:error(epoch_order)
+ end.
+
+
+is_prefix(Pattern, Subject) ->
+ binary:longest_common_prefix([Pattern, Subject]) == size(Pattern).
+
+
+is_owner(Node, Seq, Epochs) ->
+ Node =:= owner_of(Epochs, Seq).
+
+
+owner_of(Db, Seq) when not is_list(Db) ->
+ owner_of(get_epochs(Db), Seq);
+owner_of([], _Seq) ->
+ undefined;
+owner_of([{EpochNode, EpochSeq} | _Rest], Seq) when Seq > EpochSeq ->
+ EpochNode;
+owner_of([_ | Rest], Seq) ->
+ owner_of(Rest, Seq).
+
+
+start_seq([{OrigNode, EpochSeq} | _], OrigNode, Seq) when Seq > EpochSeq ->
+ %% OrigNode is the owner of the Seq so we can safely stream from there
+ Seq;
+start_seq([{_, NewSeq}, {OrigNode, _} | _], OrigNode, Seq) when Seq > NewSeq ->
+ %% We transferred this file before Seq was written on OrigNode, so we need
+ %% to stream from the beginning of the next epoch. Note that it is _not_
+ %% necessary for the current node to own the epoch beginning at NewSeq
+ NewSeq;
+start_seq([_ | Rest], OrigNode, Seq) ->
+ start_seq(Rest, OrigNode, Seq);
+start_seq([], OrigNode, Seq) ->
+ erlang:error({epoch_mismatch, OrigNode, Seq}).
+
+
extract_namespace(Options0) ->
case proplists:split(Options0, [namespace]) of
{[[{namespace, NS}]], Options} ->
@@ -1698,6 +1775,30 @@ should_fail_validate_dbname(DbName) ->
ok
end)}.
+calculate_start_seq_test() ->
+ %% uuid mismatch is always a rewind.
+ Hdr1 = couch_db_header:new(),
+ Hdr2 = couch_db_header:set(Hdr1, [{epochs, [{node1, 1}]}, {uuid, <<"uuid1">>}]),
+ ?assertEqual(0, calculate_start_seq(#db{header=Hdr2}, node1, {1, <<"uuid2">>})),
+ %% uuid matches and seq is owned by node.
+ Hdr3 = couch_db_header:set(Hdr2, [{epochs, [{node1, 1}]}]),
+ ?assertEqual(2, calculate_start_seq(#db{header=Hdr3}, node1, {2, <<"uuid1">>})),
+ %% uuids match but seq is not owned by node.
+ Hdr4 = couch_db_header:set(Hdr2, [{epochs, [{node2, 2}, {node1, 1}]}]),
+ ?assertEqual(0, calculate_start_seq(#db{header=Hdr4}, node1, {3, <<"uuid1">>})),
+ %% return integer if we didn't get a vector.
+ ?assertEqual(4, calculate_start_seq(#db{}, foo, 4)).
+
+is_owner_test() ->
+ ?assertNot(is_owner(foo, 1, [])),
+ ?assertNot(is_owner(foo, 1, [{foo, 1}])),
+ ?assert(is_owner(foo, 2, [{foo, 1}])),
+ ?assert(is_owner(foo, 50, [{bar, 100}, {foo, 1}])),
+ ?assert(is_owner(foo, 50, [{baz, 200}, {bar, 100}, {foo, 1}])),
+ ?assert(is_owner(bar, 150, [{baz, 200}, {bar, 100}, {foo, 1}])),
+ ?assertError(duplicate_epoch, validate_epochs([{foo, 1}, {bar, 1}])),
+ ?assertError(epoch_order, validate_epochs([{foo, 100}, {bar, 200}])).
+
to_binary(DbName) when is_list(DbName) ->
?l2b(DbName);
to_binary(DbName) when is_binary(DbName) ->