You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by bb...@apache.org on 2017/04/19 22:18:35 UTC

[couchdb] branch COUCHDB-3377 updated (8be70d1 -> 9deccf8)

This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a change to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git.

     omits  8be70d1   squash
     omits  444ad5c   Update couch_mrview to use new index LRU
     omits  25b22c7   Use LRU for view indexes
     omits  9b5e2a2   Update couch view tests to use new index LRU
     omits  f9a0d86   Generalize couch_lru
      adds  cb823dd   chmod +x couchspawnkillable.sh
      adds  75cec86   Reorganize snap files into ./snap/ directory
      adds  8af1faf   Remove snap packaging, it lives in https://github.com/apache/couchdb-pkg.git now
      adds  7547e7f   Merge pull request #473 from mhall119/snap-from-source
      adds  96d10a9   Clarify & correct require_valid_user setting
      adds  a9dd4c4   Merge pull request #478 from apache/COUCHDB-3100
      adds  c0fda54   Fix mem3_sync_event_listener unit test
      adds  e367575   Merge pull request #482 from apache/COUCHDB-3380-fix-mem3-sync-event-listener-unit-tests
      adds  d754c8c   Fix couch_auth_cache reinitialization logic
      adds  3f0f806   Merge pull request #481 from apache/COUCHDB-3379-fix-couch-auth-cache-reinit
      adds  db2f19b   Fix mango full text detection
      adds  778738b   Merge pull request #480 from apache/COUCHDB-3378-fix-mango-full-text-detection
      adds  80963ca   Don't crash compactor when compacting process fails
      adds  1c94522   Add function to clean up after failed compaction
      adds  81166c5   Expose couch_index_compactor:get_compacting_pid/1
      adds  f7767a3   Update should_remove test
      adds  c3ff408   Merge pull request #474 from cloudant/3364-fix-view-compactor-unknown_info
       new  06f901b   Generalize couch_lru
       new  257c269   Update couch view tests to use new index LRU
       new  f0a8912   Use LRU for view indexes
       new  c7a9004   Update couch_mrview to use new index LRU
       new  9e06e3d   squash
       new  f1c3bb7   Remove unreachable clause
       new  9deccf8   squash

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (8be70d1)
            \
             N -- N -- N   refs/heads/COUCHDB-3377 (9deccf8)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omits" are not gone; other references still
refer to them.  Any revisions marked "discards" are gone forever.

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "adds" were already present in the repository and have only
been added to this reference.


Summary of changes:
 rel/overlay/etc/default.ini                        |  4 ++
 snapcfg/snap.ini                                   | 10 ---
 snapcfg/snap_run                                   |  9 ---
 snapcraft.yaml                                     | 81 ----------------------
 src/couch/priv/spawnkillable/couchspawnkillable.sh |  0
 src/couch/src/couch_auth_cache.erl                 | 17 ++---
 src/couch_index/src/couch_index.erl                | 12 +++-
 src/couch_index/src/couch_index_compactor.erl      | 17 ++++-
 src/couch_mrview/src/couch_mrview_compactor.erl    |  9 ++-
 src/couch_mrview/src/couch_mrview_index.erl        |  6 +-
 .../test/couch_mrview_compact_tests.erl            | 31 ++++++++-
 src/mango/rebar.config.script                      | 23 +++---
 src/mango/src/mango_cursor.erl                     | 21 ++++--
 ...or_text.erl.nocompile => mango_cursor_text.erl} |  4 ++
 src/mem3/src/mem3_sync_event_listener.erl          |  2 +-
 15 files changed, 105 insertions(+), 141 deletions(-)
 delete mode 100644 snapcfg/snap.ini
 delete mode 100755 snapcfg/snap_run
 delete mode 100644 snapcraft.yaml
 mode change 100644 => 100755 src/couch/priv/spawnkillable/couchspawnkillable.sh
 rename src/mango/src/{mango_cursor_text.erl.nocompile => mango_cursor_text.erl} (99%)

-- 
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].

[couchdb] 05/07: squash

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9e06e3dfc9ea771ba21e75d865e7b07f5ef93553
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Fri Apr 14 16:19:24 2017 -0700

    squash
---
 src/couch/test/couchdb_compaction_daemon_tests.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/couch/test/couchdb_compaction_daemon_tests.erl b/src/couch/test/couchdb_compaction_daemon_tests.erl
index 25d9b13..5af6a62 100644
--- a/src/couch/test/couchdb_compaction_daemon_tests.erl
+++ b/src/couch/test/couchdb_compaction_daemon_tests.erl
@@ -222,8 +222,9 @@ spawn_compaction_monitor(DbName) ->
         DbPid = couch_util:with_db(DbName, fun(Db) ->
             Db#db.main_pid
         end),
-        {ok, ViewPid} = couch_index_server:get_index(couch_mrview_index,
+        {ok, ViewPid, Mon} = couch_index_server:get_index(couch_mrview_index,
                 DbName, <<"_design/foo">>),
+        couch_index_server:close(Mon),
         TestPid ! {self(), started},
         receive
             {TestPid, go} -> ok

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 07/07: squash

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9deccf8ba9400adcca974067670ed2fccef394a7
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Wed Apr 19 15:17:43 2017 -0700

    squash
---
 src/couch_index/src/couch_index_compactor.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/couch_index/src/couch_index_compactor.erl b/src/couch_index/src/couch_index_compactor.erl
index ed83899..d224747 100644
--- a/src/couch_index/src/couch_index_compactor.erl
+++ b/src/couch_index/src/couch_index_compactor.erl
@@ -95,6 +95,7 @@ handle_info({'EXIT', Pid, Reason}, #st{pid = Pid} = State) ->
     IdxName = Mod:get(idx_name, IdxState),
     Args = [DbName, IdxName, Reason],
     couch_log:error("Compaction failed for db: ~s idx: ~s reason: ~p", Args),
+    ok = couch_index_server:set_compacting(State#st.idx, false),
     {noreply, State#st{pid = undefined}};
 handle_info({'EXIT', _Pid, normal}, State) ->
     {noreply, State};

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 04/07: Update couch_mrview to use new index LRU

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit c7a9004bca6ad89938378bab625c2c52a4145526
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Thu Apr 13 16:44:28 2017 -0700

    Update couch_mrview to use new index LRU
---
 src/couch_mrview/src/couch_mrview.erl      | 32 ++++++++++++++++++++----------
 src/couch_mrview/src/couch_mrview_util.erl |  9 +++++++--
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 088327c..1e49579 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -325,16 +325,19 @@ count_view_changes_since(Db, DDoc, VName, SinceSeq, Options) ->
 
 
 get_info(Db, DDoc) ->
-    {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
-    couch_index:get_info(Pid).
+    {ok, Pid, Mon} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
+    Info = couch_index:get_info(Pid),
+    ok = couch_index_server:close(Mon),
+    Info.
 
 
 trigger_update(Db, DDoc) ->
     trigger_update(Db, DDoc, couch_db:get_update_seq(Db)).
 
 trigger_update(Db, DDoc, UpdateSeq) ->
-    {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
-    couch_index:trigger_update(Pid, UpdateSeq).
+    {ok, Pid, Mon} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
+    couch_index:trigger_update(Pid, UpdateSeq),
+    couch_index_server:close(Mon).
 
 %% get informations on a view
 get_view_info(Db, DDoc, VName) ->
@@ -369,10 +372,14 @@ refresh(Db, DDoc) ->
             end),
 
     case couch_index_server:get_index(couch_mrview_index, Db, DDoc) of
-        {ok, Pid} ->
-            case catch couch_index:get_state(Pid, UpdateSeq) of
-                {ok, _} -> ok;
-                Error -> {error, Error}
+        {ok, Pid, Mon} ->
+            try
+                case catch couch_index:get_state(Pid, UpdateSeq) of
+                    {ok, _} -> ok;
+                    Error -> {error, Error}
+                end
+            after
+                couch_index_server:close(Mon)
             end;
         Error ->
             {error, Error}
@@ -383,17 +390,20 @@ compact(Db, DDoc) ->
 
 
 compact(Db, DDoc, Opts) ->
-    {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
-    couch_index:compact(Pid, Opts).
+    {ok, Pid, Mon} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
+    Ret = couch_index:compact(Pid, Opts),
+    couch_index_server:close(Mon),
+    Ret.
 
 
 cancel_compaction(Db, DDoc) ->
-    {ok, IPid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
+    {ok, IPid, Mon} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
     {ok, CPid} = couch_index:get_compactor_pid(IPid),
     ok = couch_index_compactor:cancel(CPid),
 
     % Cleanup the compaction file if it exists
     {ok, #mrst{sig=Sig, db_name=DbName}} = couch_index:get_state(IPid, 0),
+    couch_index_server:close(Mon),
     couch_mrview_util:delete_compaction_file(DbName, Sig),
     ok.
 
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 3830b96..be29cb3 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -40,7 +40,7 @@
 
 
 get_view(Db, DDoc, ViewName, Args0) ->
-    {ok, Pid, Args2} = get_view_index_pid(Db, DDoc, ViewName, Args0),
+    {ok, Pid, Mon, Args2} = get_view_index_pid(Db, DDoc, ViewName, Args0),
     DbUpdateSeq = couch_util:with_db(Db, fun(WDb) ->
         couch_db:get_update_seq(WDb)
     end),
@@ -51,9 +51,14 @@ get_view(Db, DDoc, ViewName, Args0) ->
         {ok, _} = Resp -> Resp;
         Error -> throw(Error)
     end,
+    ok = couch_index_server:close(Mon),
     Ref = erlang:monitor(process, State#mrst.fd),
     if Args2#mrargs.update == lazy ->
-        spawn(fun() -> catch couch_index:get_state(Pid, DbUpdateSeq) end);
+        spawn(fun() ->
+            {ok, Pid2, Mon2} = couch_index_server:get_index(?MOD, Db, DDoc),
+            catch couch_index:get_state(Pid2, DbUpdateSeq),
+            ok = couch_index_server:close(Mon2)
+        end);
         true -> ok
     end,
     #mrst{language=Lang, views=Views} = State,

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 01/07: Generalize couch_lru

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 06f901b94980a9fd46d2da0844299a22ab643c7d
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Fri Mar 17 11:47:29 2017 -0700

    Generalize couch_lru
    
    This commit makes couch_lru take a function which closes the database,
    thereby making couch_lru no longer be tightly coupled to couch_server.
    This change is necessary to be able to use couch_lru in
    couch_index_server for making an LRU for view indices.
---
 src/couch/include/couch_db.hrl     |  6 ++++
 src/couch/src/couch_lru.erl        | 67 ++++++++++++++++++--------------------
 src/couch/src/couch_server.erl     | 24 ++++++++++++--
 src/couch/test/couch_lru_tests.erl | 24 +++++++-------
 4 files changed, 71 insertions(+), 50 deletions(-)

diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl
index e7cd85d..135a2ea 100644
--- a/src/couch/include/couch_db.hrl
+++ b/src/couch/include/couch_db.hrl
@@ -231,6 +231,12 @@
     atts = []
 }).
 
+-record(couch_lru, {
+    count=0,
+    updates,
+    counts,
+    close_fun
+}).
 
 -type doc() :: #doc{}.
 -type ddoc() :: #doc{}.
diff --git a/src/couch/src/couch_lru.erl b/src/couch/src/couch_lru.erl
index b79286e..91ffdcc 100644
--- a/src/couch/src/couch_lru.erl
+++ b/src/couch/src/couch_lru.erl
@@ -11,34 +11,36 @@
 % the License.
 
 -module(couch_lru).
--export([new/0, insert/2, update/2, close/1]).
+-export([new/1, insert/2, update/2, close/1]).
 
 -include_lib("couch/include/couch_db.hrl").
 
-new() ->
+new(CloseFun) ->
     Updates = ets:new(couch_lru_updates, [ordered_set]),
-    Dbs = ets:new(couch_lru_dbs, [set]),
-    {0, Updates, Dbs}.
+    Counts = ets:new(couch_lru_counts, [set]),
+    #couch_lru{updates=Updates, counts=Counts, close_fun=CloseFun}.
 
-insert(DbName, {Count, Updates, Dbs}) ->
-    update(DbName, {Count, Updates, Dbs}).
+insert(Name, Lru) ->
+    update(Name, Lru).
 
-update(DbName, {Count, Updates, Dbs}) ->
-    case ets:lookup(Dbs, DbName) of
+update(Name, Lru) ->
+    #couch_lru{counts=Counts, updates=Updates, count=Count} = Lru,
+    case ets:lookup(Counts, Name) of
         [] ->
-            true = ets:insert(Dbs, {DbName, Count});
-        [{DbName, OldCount}] ->
-            true = ets:update_element(Dbs, DbName, {2, Count}),
-            true = ets:delete(Updates, {OldCount, DbName})
+            true = ets:insert(Counts, {Name, Count});
+        [{Name, OldCount}] ->
+            true = ets:update_element(Counts, Name, {2, Count}),
+            true = ets:delete(Updates, {OldCount, Name})
     end,
-    true = ets:insert(Updates, {{Count, DbName}}),
-    {Count + 1, Updates, Dbs}.
+    true = ets:insert(Updates, {{Count, Name}}),
+    Lru#couch_lru{count=Count+1}.
 
 
-close({Count, Updates, Dbs}) ->
-    case close_int(ets:next(Updates, {-1, <<>>}), Updates, Dbs) of
+close(Lru) ->
+    #couch_lru{updates=Updates} = Lru,
+    case close_int(ets:next(Updates, {-1, <<>>}), Lru) of
         true ->
-            {true, {Count, Updates, Dbs}};
+            {true, Lru};
         false ->
             false
     end.
@@ -46,26 +48,19 @@ close({Count, Updates, Dbs}) ->
 
 %% internals
 
-close_int('$end_of_table', _Updates, _Dbs) ->
+close_int('$end_of_table', _Lru) ->
     false;
-close_int({_Count, DbName} = Key, Updates, Dbs) ->
-    case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
-    true ->
-        [#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
-        case couch_db:is_idle(Db) of true ->
-            true = ets:delete(couch_dbs, DbName),
-            true = ets:delete(couch_dbs_pid_to_name, Pid),
-            exit(Pid, kill),
+close_int({_Count, Name} = Key, Lru) ->
+    #couch_lru{updates=Updates, counts=Counts, close_fun=CloseFun} = Lru,
+    {Stop, Remove} = CloseFun(Name),
+    case Remove of
+        true ->
             true = ets:delete(Updates, Key),
-            true = ets:delete(Dbs, DbName),
-            true;
+            true = ets:delete(Counts, Name);
         false ->
-            true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
-            couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
-            close_int(ets:next(Updates, Key), Updates, Dbs)
-        end;
-    false ->
-        true = ets:delete(Updates, Key),
-        true = ets:delete(Dbs, DbName),
-        close_int(ets:next(Updates, Key), Updates, Dbs)
+            ok
+    end,
+    case Stop of
+        false -> close_int(ets:next(Updates, Key), Lru);
+        true -> true
     end.
diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl
index 1152300..53dc9e1 100644
--- a/src/couch/src/couch_server.erl
+++ b/src/couch/src/couch_server.erl
@@ -20,7 +20,7 @@
 -export([init/1, handle_call/3,sup_start_link/0]).
 -export([handle_cast/2,code_change/3,handle_info/2,terminate/2]).
 -export([dev_start/0,is_admin/2,has_admins/0,get_stats/0]).
--export([close_lru/0]).
+-export([close_lru/0, maybe_close_db/1]).
 
 % config_listener api
 -export([handle_config_change/5, handle_config_terminate/3]).
@@ -36,7 +36,7 @@
     dbs_open=0,
     start_time="",
     update_lru_on_read=true,
-    lru = couch_lru:new()
+    lru = couch_lru:new(fun maybe_close_db/1)
     }).
 
 dev_start() ->
@@ -97,6 +97,26 @@ update_lru(DbName) ->
 close_lru() ->
     gen_server:call(couch_server, close_lru).
 
+maybe_close_db(DbName) ->
+    % First element of return indicates if we should stop closing DBs from the
+    % LRU and the second element indicates if we closed a DB.
+    case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
+    true ->
+        [#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
+        case couch_db:is_idle(Db) of true ->
+            true = ets:delete(couch_dbs, DbName),
+            true = ets:delete(couch_dbs_pid_to_name, Pid),
+            exit(Pid, kill),
+            {true, true};
+        false ->
+            true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
+            couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
+            {false, false}
+        end;
+    false ->
+        {false, true}
+    end.
+
 create(DbName, Options0) ->
     Options = maybe_add_sys_db_callbacks(DbName, Options0),
     case gen_server:call(couch_server, {create, DbName, Options}, infinity) of
diff --git a/src/couch/test/couch_lru_tests.erl b/src/couch/test/couch_lru_tests.erl
index 1559835..02a131c 100644
--- a/src/couch/test/couch_lru_tests.erl
+++ b/src/couch/test/couch_lru_tests.erl
@@ -20,7 +20,7 @@ setup() ->
     ok = meck:new(couch_db, [passthrough]),
     ets:new(couch_dbs, [set, public, named_table, {keypos, #db.name}]),
     ets:new(couch_dbs_pid_to_name, [set, public, named_table]),
-    couch_lru:new().
+    couch_lru:new(fun couch_server:maybe_close_db/1).
 
 teardown(_) ->
     ets:delete(couch_dbs),
@@ -29,18 +29,18 @@ teardown(_) ->
 
 new_test_() ->
     {setup,
-        fun() -> couch_lru:new() end,
+        fun() -> couch_lru:new(fun couch_server:maybe_close_db/1) end,
         fun(Lru) ->
-            ?_assertMatch({0, _, _}, Lru)
+            ?_assertEqual(0, Lru#couch_lru.count)
         end
     }.
 
 insert_test_() ->
     {setup,
-        fun() -> couch_lru:new() end,
+        fun() -> couch_lru:new(fun couch_server:maybe_close_db/1) end,
         fun(Lru) ->
             Key = <<"test">>,
-            {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
+            #couch_lru{count=1, updates=Updates, counts=Dbs} = couch_lru:insert(Key, Lru),
             [
                 ?_assertEqual(1, ets_size(Dbs)),
                 ?_assert(ets:member(Dbs, Key)),
@@ -52,11 +52,11 @@ insert_test_() ->
 
 insert_same_test_() ->
     {setup,
-        fun() -> couch_lru:new() end,
+        fun() -> couch_lru:new(fun couch_server:maybe_close_db/1) end,
         fun(Lru) ->
             Key = <<"test">>,
-            Lru1 = {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
-            {2, Updates, Dbs} = couch_lru:insert(Key, Lru1),
+            Lru1 = #couch_lru{count=1} = couch_lru:insert(Key, Lru),
+            #couch_lru{count=2, updates=Updates, counts=Dbs} = couch_lru:insert(Key, Lru1),
             [
                 ?_assertEqual(1, ets_size(Dbs)),
                 ?_assert(ets:member(Dbs, Key)),
@@ -68,11 +68,11 @@ insert_same_test_() ->
 
 update_test_() ->
     {setup,
-        fun() -> couch_lru:new() end,
+        fun() -> couch_lru:new(fun couch_server:maybe_close_db/1) end,
         fun(Lru) ->
             Key = <<"test">>,
-            Lru1 = {1, Updates, Dbs} = couch_lru:update(Key, Lru),
-            {2, Updates, Dbs} = couch_lru:update(Key, Lru1),
+            Lru1 = #couch_lru{count=1} = couch_lru:update(Key, Lru),
+            #couch_lru{count=2, updates=Updates, counts=Dbs} = couch_lru:update(Key, Lru1),
             [
                 ?_assertEqual(1, ets_size(Dbs)),
                 ?_assert(ets:member(Dbs, Key)),
@@ -90,7 +90,7 @@ close_test_() ->
             ok = meck:expect(couch_db, is_idle, 1, true),
             {ok, Lru1} = add_record(Lru, <<"test1">>, c:pid(0, 1001, 0)),
             {ok, Lru2} = add_record(Lru1, <<"test2">>, c:pid(0, 2001, 0)),
-            {true, {2, Updates, Dbs}} = couch_lru:close(Lru2),
+            {true, #couch_lru{count=2, updates=Updates, counts=Dbs}} = couch_lru:close(Lru2),
             [
                 ?_assertEqual(1, ets_size(Dbs)),
                 ?_assert(ets:member(Dbs, <<"test2">>)),

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 02/07: Update couch view tests to use new index LRU

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 257c26959350431479ae5d75bbd3825ee87ed315
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Thu Apr 13 15:40:49 2017 -0700

    Update couch view tests to use new index LRU
---
 src/couch/test/couchdb_views_tests.erl | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/couch/test/couchdb_views_tests.erl b/src/couch/test/couchdb_views_tests.erl
index f1fddfc..130be01 100644
--- a/src/couch/test/couchdb_views_tests.erl
+++ b/src/couch/test/couchdb_views_tests.erl
@@ -226,7 +226,7 @@ should_cleanup_all_index_files({DbName, {FooRev, BooRev}})->
 
 couchdb_1138(DbName) ->
     ?_test(begin
-        {ok, IndexerPid} = couch_index_server:get_index(
+        {ok, IndexerPid, Mon} = couch_index_server:get_index(
             couch_mrview_index, DbName, <<"_design/foo">>),
         ?assert(is_pid(IndexerPid)),
         ?assert(is_process_alive(IndexerPid)),
@@ -259,12 +259,14 @@ couchdb_1138(DbName) ->
         ?assertEqual(5, length(Rows2)),
         ?assertEqual(2, count_users(DbName)),
 
-        ?assert(is_process_alive(IndexerPid))
+        ?assert(is_process_alive(IndexerPid)),
+
+        couch_index_server:close(Mon)
     end).
 
 couchdb_1309(DbName) ->
     ?_test(begin
-        {ok, IndexerPid} = couch_index_server:get_index(
+        {ok, IndexerPid, Mon} = couch_index_server:get_index(
             couch_mrview_index, DbName, <<"_design/foo">>),
         ?assert(is_pid(IndexerPid)),
         ?assert(is_process_alive(IndexerPid)),
@@ -281,7 +283,7 @@ couchdb_1309(DbName) ->
         ?assert(is_process_alive(IndexerPid)),
 
         update_design_doc(DbName,  <<"_design/foo">>, <<"bar">>),
-        {ok, NewIndexerPid} = couch_index_server:get_index(
+        {ok, NewIndexerPid, NewMon} = couch_index_server:get_index(
             couch_mrview_index, DbName, <<"_design/foo">>),
         ?assert(is_pid(NewIndexerPid)),
         ?assert(is_process_alive(NewIndexerPid)),
@@ -301,12 +303,15 @@ couchdb_1309(DbName) ->
         ?assertEqual(4, length(Rows2)),
 
         ok = stop_indexer( %% FIXME we need to grab monitor earlier
-               fun() -> ok end,
+               fun() -> couch_index_server:close(Mon) end,
                IndexerPid, ?LINE,
                "old view group is not dead after ddoc update"),
 
         ok = stop_indexer(
-               fun() -> couch_server:delete(DbName, [?ADMIN_USER]) end,
+               fun() ->
+                   couch_index_server:close(NewMon),
+                   couch_server:delete(DbName, [?ADMIN_USER])
+               end,
                NewIndexerPid, ?LINE,
                "new view group did not die after DB deletion")
     end).
@@ -363,9 +368,10 @@ couchdb_1283() ->
         %% because we need have access to compaction Pid, not a Ref.
         %% {ok, MonRef} = couch_mrview:compact(MDb1#db.name, <<"_design/foo">>,
         %%                                     [monitor]),
-        {ok, Pid} = couch_index_server:get_index(
+        {ok, Pid, Mon} = couch_index_server:get_index(
             couch_mrview_index, MDb1#db.name, <<"_design/foo">>),
         {ok, CPid} = gen_server:call(Pid, compact),
+        couch_index_server:close(Mon),
         %% By suspending compaction process we ensure that compaction won't get
         %% finished too early to make get_writer_status assertion fail.
         erlang:suspend_process(CPid),

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 06/07: Remove unreachable clause

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f1c3bb7bfe1118b816404d6baa0a7821726b2eaf
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Wed Apr 19 15:15:20 2017 -0700

    Remove unreachable clause
---
 src/couch_index/src/couch_index_compactor.erl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/couch_index/src/couch_index_compactor.erl b/src/couch_index/src/couch_index_compactor.erl
index 2271c35..ed83899 100644
--- a/src/couch_index/src/couch_index_compactor.erl
+++ b/src/couch_index/src/couch_index_compactor.erl
@@ -98,8 +98,6 @@ handle_info({'EXIT', Pid, Reason}, #st{pid = Pid} = State) ->
     {noreply, State#st{pid = undefined}};
 handle_info({'EXIT', _Pid, normal}, State) ->
     {noreply, State};
-handle_info({'EXIT', Pid, _Reason}, #st{idx=Pid}=State) ->
-    {stop, normal, State};
 handle_info(_Mesg, State) ->
     {stop, unknown_info, State}.
 

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.

[couchdb] 03/07: Use LRU for view indexes

Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bbastian pushed a commit to branch COUCHDB-3377
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f0a891285ca393308093a24e3bf7f5c3e45812e1
Author: Benjamin Bastian <be...@gmail.com>
AuthorDate: Wed Mar 29 14:44:23 2017 -0700

    Use LRU for view indexes
---
 src/couch_index/priv/stats_descriptions.cfg        |  20 ++
 src/couch_index/src/couch_index.erl                |  16 +-
 src/couch_index/src/couch_index_compactor.erl      |   3 +
 src/couch_index/src/couch_index_server.erl         | 180 ++++++++++++----
 .../test/couch_index_compaction_tests.erl          |   7 +-
 src/couch_index/test/couch_index_lru_tests.erl     | 226 +++++++++++++++++++++
 6 files changed, 408 insertions(+), 44 deletions(-)

diff --git a/src/couch_index/priv/stats_descriptions.cfg b/src/couch_index/priv/stats_descriptions.cfg
new file mode 100644
index 0000000..aac317f
--- /dev/null
+++ b/src/couch_index/priv/stats_descriptions.cfg
@@ -0,0 +1,20 @@
+%% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+%% use this file except in compliance with the License. You may obtain a copy of
+%% the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+%% License for the specific language governing permissions and limitations under
+%% the License.
+
+% Style guide for descriptions: Start with a lowercase letter & do not add
+% a trailing full-stop / period
+% Please keep this in alphabetical order
+
+{[couchdb, couch_index_server, lru_skip], [
+    {type, counter},
+    {desc, <<"number of couch_index_server LRU operations skipped">>}
+]}.
diff --git a/src/couch_index/src/couch_index.erl b/src/couch_index/src/couch_index.erl
index c86f5e1..99b5e8a 100644
--- a/src/couch_index/src/couch_index.erl
+++ b/src/couch_index/src/couch_index.erl
@@ -240,8 +240,11 @@ handle_cast({new_state, NewIdxState}, State) ->
     couch_log:debug("Updated index for db: ~s idx: ~s seq: ~B", Args),
     Rest = send_replies(State#st.waiters, CurrSeq, NewIdxState),
     case State#st.committed of
-        true -> erlang:send_after(commit_delay(), self(), commit);
-        false -> ok
+        true ->
+            ok = couch_index_server:set_committing(self(), true),
+            erlang:send_after(commit_delay(), self(), commit);
+        false ->
+            ok
     end,
     {noreply, State#st{
         idx_state=NewIdxState,
@@ -297,6 +300,7 @@ handle_info(commit, State) ->
             % Commit the updates
             ok = Mod:commit(IdxState),
             couch_event:notify(DbName, {index_commit, IdxName}),
+            ok = couch_index_server:set_committing(self(), false),
             {noreply, State#st{committed=true}};
         _ ->
             % We can't commit the header because the database seq that's
@@ -305,6 +309,7 @@ handle_info(commit, State) ->
             % forever out of sync with the database. But a crash before we
             % commit these changes, no big deal, we only lose incremental
             % changes since last committal.
+            ok = couch_index_server:set_committing(self(), true),
             erlang:send_after(commit_delay(), self(), commit),
             {noreply, State}
     end;
@@ -385,8 +390,11 @@ commit_compacted(NewIdxState, State) ->
         false -> ok
     end,
     case State#st.committed of
-        true -> erlang:send_after(commit_delay(), self(), commit);
-        false -> ok
+        true ->
+            ok = couch_index_server:set_committing(self(), true),
+            erlang:send_after(commit_delay(), self(), commit);
+        false ->
+            ok
     end,
     State#st{
         idx_state=NewIdxState1,
diff --git a/src/couch_index/src/couch_index_compactor.erl b/src/couch_index/src/couch_index_compactor.erl
index 61f406c..2271c35 100644
--- a/src/couch_index/src/couch_index_compactor.erl
+++ b/src/couch_index/src/couch_index_compactor.erl
@@ -64,12 +64,14 @@ handle_call({compact, _}, _From, #st{pid=Pid}=State) when is_pid(Pid) ->
     {reply, {ok, Pid}, State};
 handle_call({compact, IdxState}, _From, #st{idx=Idx}=State) ->
     Pid = spawn_link(fun() -> compact(Idx, State#st.mod, IdxState) end),
+    ok = couch_index_server:set_compacting(Idx, true),
     {reply, {ok, Pid}, State#st{pid=Pid}};
 handle_call(cancel, _From, #st{pid=undefined}=State) ->
     {reply, ok, State};
 handle_call(cancel, _From, #st{pid=Pid}=State) ->
     unlink(Pid),
     exit(Pid, kill),
+    ok = couch_index_server:set_compacting(State#st.idx, false),
     {reply, ok, State#st{pid=undefined}};
 handle_call(get_compacting_pid, _From, #st{pid=Pid}=State) ->
     {reply, {ok, Pid}, State};
@@ -84,6 +86,7 @@ handle_cast(_Mesg, State) ->
 
 
 handle_info({'EXIT', Pid, normal}, #st{pid=Pid}=State) ->
+    ok = couch_index_server:set_compacting(State#st.idx, false),
     {noreply, State#st{pid=undefined}};
 handle_info({'EXIT', Pid, Reason}, #st{pid = Pid} = State) ->
     #st{idx = Idx, mod = Mod} = State,
diff --git a/src/couch_index/src/couch_index_server.erl b/src/couch_index/src/couch_index_server.erl
index 4e86f5e..7ec91c3 100644
--- a/src/couch_index/src/couch_index_server.erl
+++ b/src/couch_index/src/couch_index_server.erl
@@ -16,7 +16,8 @@
 
 -vsn(2).
 
--export([start_link/0, validate/2, get_index/4, get_index/3, get_index/2]).
+-export([start_link/0, validate/2, get_index/4, get_index/3, get_index/2, close/1]).
+-export([set_committing/2, set_compacting/2]).
 
 -export([init/1, terminate/2, code_change/3]).
 -export([handle_call/3, handle_cast/2, handle_info/2]).
@@ -34,8 +35,76 @@
 -define(BY_PID, couchdb_indexes_by_pid).
 -define(BY_DB, couchdb_indexes_by_db).
 -define(RELISTEN_DELAY, 5000).
+-define(MAX_INDICES_OPEN, 500).
+
+-record(st, {
+    lru=couch_lru:new(fun maybe_close_index/1),
+    open=0,
+    max_open=?MAX_INDICES_OPEN,
+    root_dir
+}).
+
+-record(entry, {
+    name,
+    pid,
+    locked=false,
+    committing=false,
+    compacting=false,
+    waiters=undefined
+}).
+
+close(Mon) ->
+    erlang:demonitor(Mon, [flush]),
+    ok.
+
+maybe_close_lru_view(#st{open=Open, max_open=Max}=State) when Open =< Max ->
+    {ok, State};
+maybe_close_lru_view(State) ->
+    #st{lru=Lru, open=Open} = State,
+    case couch_lru:close(Lru) of
+        false ->
+            {ok, State};
+        {true, NewLru} ->
+            maybe_close_lru_view(State#st{lru=NewLru, open=Open-1})
+    end.
+
+is_idle(Pid) ->
+    case erlang:process_info(Pid, monitored_by) of
+        undefined ->
+            true;
+        {monitored_by, Pids} ->
+            [] =:= Pids -- [whereis(couch_stats_process_tracker)]
+    end.
 
--record(st, {root_dir}).
+set_compacting(Idx, IsCompacting) ->
+    gen_server:call(?MODULE, {compacting, Idx, IsCompacting}, infinity).
+
+set_committing(Pid, IsCommitting) ->
+    gen_server:call(?MODULE, {committing, Pid, IsCommitting}, infinity).
+
+maybe_close_index({DbName, DDocId, Sig}) ->
+    case ets:update_element(?BY_SIG, {DbName, Sig}, {#entry.locked, true}) of
+        true ->
+            case ets:lookup(?BY_SIG, {DbName, Sig}) of
+                [#entry{pid=Pid, committing=false, compacting=false}] ->
+                    case is_idle(Pid) of
+                        true ->
+                            rem_from_ets(DbName, Sig, DDocId, Pid),
+                            couch_index:stop(Pid),
+                            {true, true};
+                        false ->
+                            ets:update_element(?BY_SIG, {DbName, Sig}, {#entry.locked, false}),
+                            couch_stats:increment_counter([couchdb, couch_index_server, lru_skip]),
+                            {false, false}
+                    end;
+                _ ->
+                    ets:update_element(?BY_SIG, {DbName, Sig}, {#entry.locked, false}),
+                    couch_stats:increment_counter([couchdb, couch_index_server, lru_skip]),
+                    {false, false}
+            end;
+        false ->
+            {false, true}
+    end.
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
@@ -94,8 +163,8 @@ get_index(Module, Db, DDoc, Fun) when is_binary(DDoc) ->
 get_index(Module, Db, DDoc, Fun) when is_function(Fun, 1) ->
     {ok, InitState} = Module:init(Db, DDoc),
     {ok, FunResp} = Fun(InitState),
-    {ok, Pid} = get_index(Module, InitState),
-    {ok, Pid, FunResp};
+    {ok, Pid, Monitor} = get_index(Module, InitState),
+    {ok, Pid, Monitor, FunResp};
 get_index(Module, Db, DDoc, _Fun) ->
     {ok, InitState} = Module:init(Db, DDoc),
     get_index(Module, InitState).
@@ -105,24 +174,31 @@ get_index(Module, IdxState) ->
     DbName = Module:get(db_name, IdxState),
     Sig = Module:get(signature, IdxState),
     case ets:lookup(?BY_SIG, {DbName, Sig}) of
-        [{_, Pid}] when is_pid(Pid) ->
-            {ok, Pid};
+        [#entry{pid=Pid, locked=false}] when is_pid(Pid) ->
+            Monitor = erlang:monitor(process, Pid),
+            {ok, Pid, Monitor};
         _ ->
             Args = {Module, IdxState, DbName, Sig},
-            gen_server:call(?MODULE, {get_index, Args}, infinity)
+            case gen_server:call(?MODULE, {get_index, Args}, infinity) of
+                {ok, Pid} ->
+                    Monitor = erlang:monitor(process, Pid),
+                    {ok, Pid, Monitor};
+                {error, Reason} ->
+                    {error, Reason}
+            end
     end.
 
-
 init([]) ->
     process_flag(trap_exit, true),
     ok = config:listen_for_changes(?MODULE, couch_index_util:root_dir()),
-    ets:new(?BY_SIG, [protected, set, named_table]),
-    ets:new(?BY_PID, [private, set, named_table]),
+    ets:new(?BY_SIG, [protected, set, named_table, {keypos, #entry.name}]),
+    ets:new(?BY_PID, [protected, set, named_table]),
     ets:new(?BY_DB, [protected, bag, named_table]),
     couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
     RootDir = couch_index_util:root_dir(),
+    MaxIndicesOpen = config:get_integer("couchdb", "max_indices_open", ?MAX_INDICES_OPEN),
     couch_file:init_delete_dir(RootDir),
-    {ok, #st{root_dir=RootDir}}.
+    {ok, #st{root_dir=RootDir, max_open=MaxIndicesOpen}}.
 
 
 terminate(_Reason, _State) ->
@@ -134,47 +210,69 @@ terminate(_Reason, _State) ->
 handle_call({get_index, {_Mod, _IdxState, DbName, Sig}=Args}, From, State) ->
     case ets:lookup(?BY_SIG, {DbName, Sig}) of
         [] ->
+            {ok, NewState} = maybe_close_lru_view(State#st{open=(State#st.open)+1}),
             spawn_link(fun() -> new_index(Args) end),
-            ets:insert(?BY_SIG, {{DbName, Sig}, [From]}),
+            ets:insert(?BY_SIG, #entry{name={DbName, Sig}, waiters=[From]}),
+            {noreply, NewState};
+        [#entry{waiters=Waiters}=Entry] when is_list(Waiters) ->
+            ets:insert(?BY_SIG, Entry#entry{waiters=[From | Waiters]}),
             {noreply, State};
-        [{_, Waiters}] when is_list(Waiters) ->
-            ets:insert(?BY_SIG, {{DbName, Sig}, [From | Waiters]}),
-            {noreply, State};
-        [{_, Pid}] when is_pid(Pid) ->
+        [#entry{pid=Pid}] when is_pid(Pid) ->
             {reply, {ok, Pid}, State}
     end;
 handle_call({async_open, {DbName, DDocId, Sig}, {ok, Pid}}, _From, State) ->
-    [{_, Waiters}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+    [#entry{waiters=Waiters}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+    NewLru = couch_lru:insert({DbName, DDocId, Sig}, State#st.lru),
     [gen_server:reply(From, {ok, Pid}) || From <- Waiters],
     link(Pid),
     add_to_ets(DbName, Sig, DDocId, Pid),
-    {reply, ok, State};
+    {reply, ok, State#st{lru=NewLru}};
 handle_call({async_error, {DbName, _DDocId, Sig}, Error}, _From, State) ->
-    [{_, Waiters}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+    [#entry{waiters=Waiters}] = ets:lookup(?BY_SIG, {DbName, Sig}),
     [gen_server:reply(From, Error) || From <- Waiters],
     ets:delete(?BY_SIG, {DbName, Sig}),
+    {reply, ok, State#st{open=(State#st.open)-1}};
+handle_call({compacting, Pid, IsCompacting}, _From, State) ->
+    case ets:lookup(?BY_PID, Pid) of
+        [{Pid, {DbName, Sig}}] ->
+            ets:update_element(?BY_SIG, {DbName, Sig}, {#entry.compacting, IsCompacting});
+        [] ->
+            ok
+    end,
+    {reply, ok, State};
+handle_call({committing, Pid, IsCommitting}, _From, State) ->
+    case ets:lookup(?BY_PID, Pid) of
+        [{Pid, {DbName, Sig}}] ->
+            ets:update_element(?BY_SIG, {DbName, Sig}, {#entry.committing, IsCommitting});
+        [] ->
+            ok
+    end,
     {reply, ok, State};
 handle_call({reset_indexes, DbName}, _From, State) ->
     reset_indexes(DbName, State#st.root_dir),
-    {reply, ok, State}.
+    {reply, ok, State};
+handle_call(get_open_count, _From, State) ->
+    {reply, State#st.open, State}.
 
 
 handle_cast({reset_indexes, DbName}, State) ->
     reset_indexes(DbName, State#st.root_dir),
-    {noreply, State}.
+    {noreply, State};
+handle_cast(close_indexes, State) ->
+    {ok, NewState} = maybe_close_lru_view(State),
+    {noreply, NewState}.
 
-handle_info({'EXIT', Pid, Reason}, Server) ->
-    case ets:lookup(?BY_PID, Pid) of
+handle_info({'EXIT', Pid, Reason}, State) ->
+    NewState = case ets:lookup(?BY_PID, Pid) of
         [{Pid, {DbName, Sig}}] ->
-            [{DbName, {DDocId, Sig}}] =
-                ets:match_object(?BY_DB, {DbName, {'$1', Sig}}),
-            rem_from_ets(DbName, Sig, DDocId, Pid);
+            rem_from_ets(DbName, Sig, Pid),
+            State#st{open=(State#st.open)-1};
         [] when Reason /= normal ->
             exit(Reason);
         _Else ->
-            ok
+            State
     end,
-    {noreply, Server};
+    {noreply, NewState};
 handle_info(restart_config_listener, State) ->
     ok = config:listen_for_changes(?MODULE, couch_index_util:root_dir()),
     {noreply, State};
@@ -187,18 +285,20 @@ code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
 
-handle_config_change("couchdb", "index_dir", RootDir, _, RootDir) ->
-    {ok, RootDir};
-handle_config_change("couchdb", "view_index_dir", RootDir, _, RootDir) ->
-    {ok, RootDir};
+handle_config_change("couchdb", "index_dir", RootDir, _, #st{root_dir=RootDir}=State) ->
+    {ok, State};
+handle_config_change("couchdb", "view_index_dir", RootDir, _, #st{root_dir=RootDir}=State) ->
+    {ok, State};
 handle_config_change("couchdb", "index_dir", _, _, _) ->
     exit(whereis(couch_index_server), config_change),
     remove_handler;
 handle_config_change("couchdb", "view_index_dir", _, _, _) ->
     exit(whereis(couch_index_server), config_change),
     remove_handler;
-handle_config_change(_, _, _, _, RootDir) ->
-    {ok, RootDir}.
+handle_config_change("couchdb", "max_indices_open", Max, _, State) ->
+    {ok, State#st{max_open=list_to_integer(Max)}};
+handle_config_change(_, _, _, _, State) ->
+    {ok, State}.
 
 handle_config_terminate(_, stop, _) ->
     ok;
@@ -222,7 +322,7 @@ new_index({Mod, IdxState, DbName, Sig}) ->
 reset_indexes(DbName, Root) ->
     % shutdown all the updaters and clear the files, the db got changed
     Fun = fun({_, {DDocId, Sig}}) ->
-        [{_, Pid}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+        [#entry{pid=Pid}] = ets:lookup(?BY_SIG, {DbName, Sig}),
         MRef = erlang:monitor(process, Pid),
         gen_server:cast(Pid, delete),
         receive {'DOWN', MRef, _, _, _} -> ok end,
@@ -234,11 +334,17 @@ reset_indexes(DbName, Root) ->
 
 
 add_to_ets(DbName, Sig, DDocId, Pid) ->
-    ets:insert(?BY_SIG, {{DbName, Sig}, Pid}),
+    ets:insert(?BY_SIG, #entry{name={DbName, Sig}, pid=Pid}),
     ets:insert(?BY_PID, {Pid, {DbName, Sig}}),
     ets:insert(?BY_DB, {DbName, {DDocId, Sig}}).
 
 
+rem_from_ets(DbName, Sig, Pid) ->
+    [{DbName, {DDocId, Sig}}] =
+        ets:match_object(?BY_DB, {DbName, {'$1', Sig}}),
+    rem_from_ets(DbName, Sig, DDocId, Pid).
+
+
 rem_from_ets(DbName, Sig, DDocId, Pid) ->
     ets:delete(?BY_SIG, {DbName, Sig}),
     ets:delete(?BY_PID, Pid),
@@ -254,7 +360,7 @@ handle_db_event(DbName, deleted, St) ->
 handle_db_event(DbName, {ddoc_updated, DDocId}, St) ->
     lists:foreach(fun({_DbName, {_DDocId, Sig}}) ->
         case ets:lookup(?BY_SIG, {DbName, Sig}) of
-            [{_, IndexPid}] ->
+            [#entry{pid=IndexPid}] ->
                 (catch gen_server:cast(IndexPid, ddoc_updated));
             [] ->
                 ok
diff --git a/src/couch_index/test/couch_index_compaction_tests.erl b/src/couch_index/test/couch_index_compaction_tests.erl
index 0787151..6cbffc8 100644
--- a/src/couch_index/test/couch_index_compaction_tests.erl
+++ b/src/couch_index/test/couch_index_compaction_tests.erl
@@ -19,9 +19,9 @@ setup() ->
     DbName = ?tempdb(),
     {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
     couch_db:close(Db),
-    {ok, IndexerPid} = fake_index(Db),
+    {ok, IndexerPid, Mon} = fake_index(Db),
     ?assertNot(is_opened(Db)),
-    {Db, IndexerPid}.
+    {Db, IndexerPid, Mon}.
 
 fake_index(#db{name = DbName} = Db) ->
     ok = meck:new([test_index], [non_strict]),
@@ -67,7 +67,7 @@ compaction_test_() ->
     }.
 
 
-hold_db_for_recompaction({Db, Idx}) ->
+hold_db_for_recompaction({Db, Idx, Mon}) ->
     ?_test(begin
         ?assertNot(is_opened(Db)),
         ok = meck:reset(test_index),
@@ -87,6 +87,7 @@ hold_db_for_recompaction({Db, Idx}) ->
         end,
 
         ?assertNot(is_opened(Db)),
+        couch_index_server:close(Mon),
         ok
     end).
 
diff --git a/src/couch_index/test/couch_index_lru_tests.erl b/src/couch_index/test/couch_index_lru_tests.erl
new file mode 100644
index 0000000..52f4244
--- /dev/null
+++ b/src/couch_index/test/couch_index_lru_tests.erl
@@ -0,0 +1,226 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_index_lru_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(MAX_INDICES_OPEN, 10).
+
+-record(test_idx, {
+    db_name,
+    idx_name,
+    signature
+}).
+
+
+setup() ->
+    test_util:start_couch([]),
+    DbName = ?tempdb(),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
+    config:set("couchdb", "max_indices_open", integer_to_list(?MAX_INDICES_OPEN)),
+    Db.
+
+
+teardown(Db) ->
+    ok = couch_server:delete(Db#db.name, [?ADMIN_CTX]),
+    config:delete("couchdb", "max_indices_open"),
+    (catch couch_db:close(Db)),
+    ok.
+
+
+lru_test_() ->
+    {
+        "Test the view index LRU",
+        {
+            setup,
+            fun() -> test_util:start_couch([]) end, fun test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun test_close_while_compacting/1,
+                    fun test_soft_max/1
+                ]
+            }
+        }
+    }.
+
+
+test_close_while_compacting(Db) ->
+    ?_test(begin
+        ok = meck:new([couch_index_server], [passthrough]),
+        Self = self(),
+        ok = meck:expect(couch_index_server, set_compacting, fun(Idx, IsCompacting) ->
+            meck:passthrough([Idx, IsCompacting]),
+            Self ! {compact, IsCompacting, self()},
+            receive finish ->
+                ok
+            end,
+            ok
+        end),
+
+        ok = meck:expect(couch_index_server, set_committing, fun(Idx, IsCommitting) ->
+            meck:passthrough([Idx, IsCommitting]),
+            Self ! {commit, IsCommitting, self()},
+            ok
+        end),
+
+        % create ddocs
+        DDocIds = lists:map(fun(I) ->
+            BI = integer_to_binary(I),
+            <<"_design/ddoc_", BI/binary>>
+        end, lists:seq(1,?MAX_INDICES_OPEN+10)),
+        ok = create_ddocs(Db, DDocIds),
+
+        % open and compact indexes
+        Openers = lists:map(fun(DDocId) ->
+            spawn_link(fun() ->
+                {ok, Pid, Mon} = couch_index_server:get_index(couch_mrview_index, Db#db.name, DDocId),
+                couch_index:compact(Pid),
+                receive close ->
+                    ok
+                end,
+                couch_index_server:close(Mon)
+            end)
+        end, DDocIds),
+
+        % check that all indexes are open
+        ToClose = wait_all_compacting(true, [], ?MAX_INDICES_OPEN+10),
+        ?assertEqual(?MAX_INDICES_OPEN+10, gen_server:call(couch_index_server, get_open_count)),
+        % close compactor pids, but still block flag from being set in BY_SIG table
+        lists:foreach(fun(Opener) -> Opener ! close end, Openers),
+        % check that compaction flag block pids from closing
+        gen_server:cast(couch_index_server, close_indexes),
+        ?assertEqual(?MAX_INDICES_OPEN+10, gen_server:call(couch_index_server, get_open_count)),
+        % allow compaction flag to be unset
+        lists:foreach(fun(CPid) -> CPid ! finish end, ToClose),
+        % wait until all compaction flags are unset
+        Finished = wait_all_compacting(false, [], ?MAX_INDICES_OPEN+10),
+        lists:foreach(fun(CPid) -> CPid ! finish end, Finished),
+        gen_server:cast(couch_index_server, close_indexes),
+        ?assertEqual(?MAX_INDICES_OPEN+10, gen_server:call(couch_index_server, get_open_count)),
+        % wait for all commits to start
+        Indexers = wait_all_committing(dict:new(), true, ?MAX_INDICES_OPEN+10),
+        ?assertEqual(?MAX_INDICES_OPEN+10, gen_server:call(couch_index_server, get_open_count)),
+        % force premature commit
+        [Indexer ! commit || Indexer <- Indexers],
+        % wait until commits happen
+        wait_all_committing(dict:new(), false, ?MAX_INDICES_OPEN+10),
+        gen_server:cast(couch_index_server, close_indexes),
+        % since all commits and all compacts are done, make sure indexes are closed
+        ?assertEqual(?MAX_INDICES_OPEN, gen_server:call(couch_index_server, get_open_count)),
+        % clean up
+        (catch meck:unload(couch_index_server)),
+        ok
+    end).
+
+
+test_soft_max(Db) ->
+    ?_test(begin
+        ok = meck:new([test_index], [non_strict]),
+        ok = meck:expect(test_index, init, fun(Db0, DDoc) ->
+            Sig = couch_crypto:hash(md5, term_to_binary({Db0#db.name, DDoc})),
+            {ok, #test_idx{db_name=Db0#db.name, idx_name=DDoc, signature=Sig}}
+        end),
+        ok = meck:expect(test_index, close, ['_'], {true, true}),
+        ok = meck:expect(test_index, open, fun(_Db, State) ->
+            {ok, State}
+        end),
+        ok = meck:expect(test_index, compact, ['_', '_', '_'],
+            meck:seq([{ok, 9}, {ok, 10}])), %% to trigger recompaction
+        ok = meck:expect(test_index, commit, ['_'], ok),
+        ok = meck:expect(test_index, get, fun
+            (db_name, State) ->
+                State#test_idx.db_name;
+            (idx_name, State) ->
+                State#test_idx.idx_name;
+            (signature, State) ->
+                State#test_idx.signature;
+            (update_seq, Seq) ->
+                Seq
+        end),
+
+        ok = meck:reset(test_index),
+
+        IdxOpens = lists:map(fun(I) ->
+            BI = integer_to_binary(I),
+            % hack: use tuple as index name so couch_index_server won't try to open
+            % it as a design document.
+            IndexName = {<<"_design/i", BI/binary>>},
+            ?assertEqual(I-1, gen_server:call(couch_index_server, get_open_count)),
+            couch_index_server:get_index(test_index, Db, IndexName)
+        end, lists:seq(1, 500)),
+
+        lists:foldl(fun(IdxOpen, Acc) ->
+            ?assertMatch({ok, _, _}, IdxOpen),
+            {ok, Pid, Mon} = IdxOpen,
+            ?assert(is_pid(Pid)),
+            ?assert(is_reference(Mon)),
+            ?assertNotEqual(undefined, process_info(Pid)),
+            gen_server:cast(couch_index_server, close_indexes),
+            OpenCount = gen_server:call(couch_index_server, get_open_count),
+            ?assertEqual(max(?MAX_INDICES_OPEN, Acc), OpenCount),
+            couch_index_server:close(Mon),
+            Acc-1
+        end, 500, IdxOpens),
+
+        config:delete("couchdb", "max_indices_open"),
+        (catch meck:unload(test_index)),
+        (catch meck:unload(couch_util)),
+        ok
+    end).
+
+
+wait_all_compacting(_IsCompacting, Acc, 0) ->
+    Acc;
+wait_all_compacting(IsCompacting, Acc, Remaining) ->
+    receive {compact, IsCompacting, From} ->
+        wait_all_compacting(IsCompacting, [From | Acc], Remaining-1)
+    end.
+
+
+wait_all_committing(Pids, ShouldBe, Count) ->
+    receive {commit, IsCommitting, From} ->
+        Pids0 = dict:store(From, IsCommitting, Pids),
+        CommitCount = dict:fold(fun(_K, V, Acc) ->
+            case V of
+                ShouldBe -> Acc+1;
+                _ -> Acc
+            end
+        end, 0, Pids0),
+        case Count =:= CommitCount of
+            true ->
+                [Pid || {Pid, _} <- dict:to_list(Pids0)];
+            false ->
+                wait_all_committing(Pids0, ShouldBe, Count)
+        end
+    end.
+
+
+create_ddocs(Db, DDocIds) ->
+    Docs = lists:map(fun(DDocId) ->
+        MapFun = <<"function(doc) {emit(\"", DDocId/binary, "\", 1);}">>,
+        Json = {[
+            {<<"_id">>, DDocId},
+            {<<"language">>, <<"javascript">>},
+            {<<"views">>, {[
+                {<<"v">>, {[
+                    {<<"map">>, MapFun}
+                ]}}
+            ]}}
+        ]},
+        couch_doc:from_json_obj(Json)
+    end, DDocIds),
+    {ok, _} = couch_db:update_docs(Db, Docs, [?ADMIN_CTX]),
+    ok.

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.