You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2020/03/05 17:05:36 UTC
[couchdb] branch ken-2 updated (b16c4a1 -> 9d95348)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch ken-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard b16c4a1 Add fabric2_index
discard f0b0c95 wip 1 - separate ken2 app
add c9175af Fix couch_views type in wait_for_job
add ee150f4 standardize couch_views_indexer_tests
add 4bd68d1 Add Key/Value size limit for map indexes
add 735b3f6 Enable fdb feature
new 9d95348 wip 4
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b16c4a1)
\
N -- N -- N refs/heads/ken-2 (9d95348)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rebar.config.script | 1 -
rel/overlay/etc/default.ini | 4 +
rel/reltool.config | 2 -
src/couch_jobs/src/couch_jobs_fdb.erl | 4 +
src/couch_views/src/couch_views.erl | 22 +-
.../src/couch_views_epi.erl} | 6 +-
.../src/couch_views_fabric2_plugin.erl} | 19 +-
src/couch_views/src/couch_views_fdb.erl | 104 ++++++---
src/couch_views/src/couch_views_indexer.erl | 4 +-
.../src/couch_views_indices.erl} | 31 ++-
src/couch_views/src/couch_views_jobs.erl | 2 +-
src/couch_views/src/couch_views_sup.erl | 12 +-
src/couch_views/src/couch_views_updater.erl | 190 ++++++++++++++++
src/couch_views/test/couch_views_indexer_test.erl | 163 ++++++++++++--
src/fabric/src/fabric2_db_plugin.erl | 5 +
src/fabric/src/fabric2_fdb.erl | 2 +
src/fabric/src/fabric2_index.erl | 63 ------
src/fabric/src/fabric2_sup.erl | 1 +
src/ken2/README.md | 16 --
src/ken2/rebar.config | 14 --
src/ken2/src/ken2.app.src | 30 ---
src/ken2/src/ken2.hrl | 24 --
src/ken2/src/ken2_app.erl | 26 ---
src/ken2/src/ken2_fabric2_plugin.erl | 34 ---
src/ken2/src/ken2_fdb.erl | 165 --------------
src/ken2/src/ken2_notifier.erl | 244 ---------------------
src/ken2/src/ken2_server.erl | 193 ----------------
src/ken2/src/ken2_sup.erl | 46 ----
src/ken2/test/config.ini | 2 -
src/ken2/test/ken_server_test.erl | 97 --------
30 files changed, 489 insertions(+), 1037 deletions(-)
rename src/{ken2/src/ken2_epi.erl => couch_views/src/couch_views_epi.erl} (90%)
copy src/{couch_js/src/couch_js_app.erl => couch_views/src/couch_views_fabric2_plugin.erl} (76%)
rename src/{ken2/src/ken2.erl => couch_views/src/couch_views_indices.erl} (53%)
create mode 100644 src/couch_views/src/couch_views_updater.erl
delete mode 100644 src/fabric/src/fabric2_index.erl
delete mode 100644 src/ken2/README.md
delete mode 100644 src/ken2/rebar.config
delete mode 100644 src/ken2/src/ken2.app.src
delete mode 100644 src/ken2/src/ken2.hrl
delete mode 100644 src/ken2/src/ken2_app.erl
delete mode 100644 src/ken2/src/ken2_fabric2_plugin.erl
delete mode 100644 src/ken2/src/ken2_fdb.erl
delete mode 100644 src/ken2/src/ken2_notifier.erl
delete mode 100644 src/ken2/src/ken2_server.erl
delete mode 100644 src/ken2/src/ken2_sup.erl
delete mode 100644 src/ken2/test/config.ini
delete mode 100644 src/ken2/test/ken_server_test.erl
[couchdb] 01/01: wip 4
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch ken-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9d95348f000ddd059a8e4855340e85b857c50b71
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu Mar 5 11:50:24 2020 -0500
wip 4
---
src/couch_jobs/src/couch_jobs_fdb.erl | 4 +
src/couch_views/src/couch_views.erl | 22 ++-
src/couch_views/src/couch_views_epi.erl | 57 +++++++
src/couch_views/src/couch_views_fabric2_plugin.erl | 24 +++
src/couch_views/src/couch_views_indices.erl | 37 ++++
src/couch_views/src/couch_views_sup.erl | 12 +-
src/couch_views/src/couch_views_updater.erl | 190 +++++++++++++++++++++
src/fabric/src/fabric2_db_plugin.erl | 5 +
src/fabric/src/fabric2_fdb.erl | 2 +
9 files changed, 351 insertions(+), 2 deletions(-)
diff --git a/src/couch_jobs/src/couch_jobs_fdb.erl b/src/couch_jobs/src/couch_jobs_fdb.erl
index e59387e..26d01e9 100644
--- a/src/couch_jobs/src/couch_jobs_fdb.erl
+++ b/src/couch_jobs/src/couch_jobs_fdb.erl
@@ -225,6 +225,10 @@ resubmit(#{jtx := true} = JTx0, #{job := true} = Job, NewSTime) ->
data => Data
},
{ok, Job1};
+ pending when STime == OldSTime ->
+ % If pending and we don't actually update the scheduled
+ % time avoid generating useless db writes.
+ {ok, Job};
pending ->
JV1 = JV#jv{seq = ?PENDING_SEQ, stime = STime},
set_job_val(Tx, Key, JV1),
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 322415b..0145ca1 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -12,8 +12,14 @@
-module(couch_views).
+
+-behavior(couch_views_indices).
+
+
-export([
- query/6
+ query/6,
+ register_index/0,
+ build_indices/2
]).
@@ -55,6 +61,20 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
end.
+register_index() ->
+ % Called from supervisor so return as we don't spawn a process
+ couch_view_indices:register_index(?MODULE),
+ ignore.
+
+
+build_indices(#{} = Db, DDocs) when is_list(DDocs) ->
+ lists:map(fun(DDoc) ->
+ DbName = fabric2_db:name(Db),
+ {ok, #mrst{} = Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+ couch_view_jobs:build_view_async(Db, Mrst)
+ end).
+
+
read_view(Db, Mrst, ViewName, Callback, Acc0, Args) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
try
diff --git a/src/couch_views/src/couch_views_epi.erl b/src/couch_views/src/couch_views_epi.erl
new file mode 100644
index 0000000..e3a961d
--- /dev/null
+++ b/src/couch_views/src/couch_views_epi.erl
@@ -0,0 +1,57 @@
+%
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_epi).
+
+
+-behaviour(couch_epi_plugin).
+
+
+-export([
+ app/0,
+ providers/0,
+ services/0,
+ data_subscriptions/0,
+ data_providers/0,
+ processes/0,
+ notify/3
+]).
+
+
+app() ->
+ couch_views.
+
+
+providers() ->
+ [
+ {fabric2_db, couch_views_fabric2_plugin}
+ ].
+
+services() ->
+ [].
+
+
+data_subscriptions() ->
+ [].
+
+
+data_providers() ->
+ [].
+
+
+processes() ->
+ [].
+
+
+notify(_Key, _Old, _New) ->
+ ok.
diff --git a/src/couch_views/src/couch_views_fabric2_plugin.erl b/src/couch_views/src/couch_views_fabric2_plugin.erl
new file mode 100644
index 0000000..28b5228
--- /dev/null
+++ b/src/couch_views/src/couch_views_fabric2_plugin.erl
@@ -0,0 +1,24 @@
+%
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_fabric2_plugin).
+
+
+-export([
+ after_doc_write/4
+]).
+
+
+after_doc_write(#{} = Db, _Winner, _OldWinner, _UpdateType) ->
+ ok = couch_views_updater:db_updated(Db),
+ Db.
diff --git a/src/couch_views/src/couch_views_indices.erl b/src/couch_views/src/couch_views_indices.erl
new file mode 100644
index 0000000..7b06a5f
--- /dev/null
+++ b/src/couch_views/src/couch_views_indices.erl
@@ -0,0 +1,37 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_indices).
+
+
+-callback build_indices(Db :: map(), DDocs :: list()) -> [ok | {error, term()}].
+
+
+-export([
+ register_index/1,
+ build_indices/2
+]).
+
+
+register_index(Mod) when is_atom(Mod) ->
+ Indices = lists:usort([Mod | registrations()]),
+ application:set_env(couch_views, indices, Indices).
+
+
+build_indices(Db, DDocs) ->
+ lists:flatmap(fun(Mod) ->
+ Mod:build_indices(Db, DDocs)
+ end, registrations()).
+
+
+registrations() ->
+ application:get_env(couch_views, indices, []).
diff --git a/src/couch_views/src/couch_views_sup.erl b/src/couch_views/src/couch_views_sup.erl
index 7a72a1f..6038427 100644
--- a/src/couch_views/src/couch_views_sup.erl
+++ b/src/couch_views/src/couch_views_sup.erl
@@ -40,9 +40,19 @@ init(normal) ->
#{
id => couch_views_server,
start => {couch_views_server, start_link, []}
+ },
+ #{
+ id => couch_views,
+ start => {couch_views, register_index, []}
+
+ },
+ #{
+ id => couch_views_updater,
+ start => {couch_views_updater, start_link, []}
}
],
- {ok, {flags(), Children}};
+ {ok, {flags(), couch_epi:register(couch_view_epi, Children)}};
+
init(builds_disabled) ->
couch_log:notice("~p : view_indexing disabled", [?MODULE]),
diff --git a/src/couch_views/src/couch_views_updater.erl b/src/couch_views/src/couch_views_updater.erl
new file mode 100644
index 0000000..d9937d6
--- /dev/null
+++ b/src/couch_views/src/couch_views_updater.erl
@@ -0,0 +1,190 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_updater).
+
+
+-behaviour(gen_server).
+
+
+-export([
+ db_updated/1,
+ start_link/0,
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+-define(SHARDS, 32).
+-define(DEFAULT_DELAY_MSEC, 60000).
+-define(DEFAULT_RESOLUTION_MSEC, 5000).
+
+
+db_updated(#{} = Db) ->
+ DbName = fabric2_db:name(Db),
+ Table = table(erlang:phash2(Db) rem ?SHARDS),
+ NowMsec = now_msec(),
+ ets:insert_new(Table, {DbName, NowMsec}).
+
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+ lists:foreach(fun(T) ->
+ spawn_link(fun() -> process_loop(T) end)
+ end, create_tables()),
+ {ok, nil}.
+
+
+terminate(_, _St) ->
+ ok.
+
+
+handle_call(Msg, _From, St) ->
+ {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+ {stop, {bad_cast, Msg}, St}.
+
+
+handle_info(Msg, St) ->
+ {stop, {bad_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+now_msec() ->
+ erlang:monotonic_time(millisecond).
+
+
+process_loop(Table) ->
+ Now = now_msec(),
+ Delay = delay_msec(),
+ Since = Now - Delay,
+ process_updates(Table, Since),
+ clean_stale(Table, Since),
+ Resolution = resolution_msec(),
+ Jitter = rand:uniform(1 + trunc(Resolution * 0.25)),
+ timer:sleep(Resolution + Jitter),
+ process_loop(Table).
+
+
+clean_stale(Table, Since) ->
+ Head = {'_', '$1'},
+ Guard = {'<', '$1', Since},
+ % Monotonic is not strictly monotonic, process items using `=<` but clean
+ % with `<` in case there was an update with the same time as NowMsec after
+ % we started processing processed update in that cycle.
+ ets:select_delete(Table, [{Head, [Guard], [true]}]).
+
+
+process_updates(Table, Since) ->
+ Head = {'$1', '$2'},
+ Guard = {'=<', '$2', Since},
+ case ets:select(Table, [{Head, [Guard], ['$1']}], 25) of
+ '$end_of_table' -> ok;
+ {Match, Cont} -> process_updates_iter(Match, Cont)
+ end.
+
+
+process_updates_iter([], Cont) ->
+ case ets:select(Cont) of
+ '$end_of_table' -> ok;
+ {Match, Cont1} -> process_updates_iter(Match, Cont1)
+ end;
+
+
+process_updates_iter([Db | Rest], Cont) ->
+ try
+ process_db(Db)
+ catch
+ error:database_does_not_exist ->
+ ok;
+ Tag:Reason ->
+ LogMsg = "~p failed to build indices for `~s` ~p:~p",
+ couch_log:error(LogMsg, [?MODULE, Db, Tag, Reason])
+ end,
+ process_updates_iter(Rest, Cont).
+
+
+delay_msec() ->
+ config:get_integer("couch_views", "updater_delay_msec",
+ ?DEFAULT_DELAY_MSEC).
+
+
+resolution_msec() ->
+ config:get_integer("couch_views", "updater_resolution_msec",
+ ?DEFAULT_RESOLUTION_MSEC).
+
+
+create_tables() ->
+ Opts = [
+ named_table,
+ public,
+ {write_concurency, true},
+ {read_concurrency, true}
+ ],
+ Tables = [table(N) || N <- lists:seq(0, ?SHARDS - 1)],
+ [ets:new(T, Opts) || T <- Tables].
+
+
+table(Id) when is_integer(Id), Id >= 0 andalso Id < ?SHARDS ->
+ list_to_atom("couch_views_updater_" ++ integer_to_list(Id)).
+
+
+process_db(DbName) when is_binary(DbName) ->
+ {ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
+ fabric2_db:transactional(Db, fun(TxDb) ->
+ case fabric2_db:get_doc_count(TxDb, <<"_design">>) of
+ 0 ->
+ % Quick optimization here to avoid a range read since we already
+ % maintain an explicit ddoc count
+ ok;
+ N when is_integer(N), N > 0 ->
+ DDocs = get_design_docs(TxDb),
+ couch_views_indices:build_indices(TxDb, shuffle(DDocs))
+ end
+ end).
+
+get_design_docs(Db) ->
+ Callback = fun
+ ({meta, _}, Acc) -> {ok, Acc};
+ (complete, Acc) -> {ok, Acc};
+ ({row, Row}, Acc) -> {ok, acc_doc(Db, Row, Acc)}
+ end,
+ {ok, DDocs} = fabric2_db:fold_design_docs(Db, Callback, [], []),
+ DDocs.
+
+
+acc_doc(Db, Row, Acc) ->
+ {_, DocId} = lists:keyfind(id, 1, Row),
+ case fabric2_db:open_doc(Db, DocId, []) of
+ {not_found, missing} -> Acc;
+ {ok, #doc{deleted = true}} -> Acc;
+ {ok, #doc{} = Doc} -> [couch_doc:to_json_obj(Doc) | Acc]
+ end.
+
+
+shuffle(DDocs) ->
+ [D || {_, D} <- lists:sort([{random:uniform(), D} || D <- DDocs])].
diff --git a/src/fabric/src/fabric2_db_plugin.erl b/src/fabric/src/fabric2_db_plugin.erl
index fb83ed4..80851a6 100644
--- a/src/fabric/src/fabric2_db_plugin.erl
+++ b/src/fabric/src/fabric2_db_plugin.erl
@@ -15,6 +15,7 @@
-export([
validate_dbname/3,
before_doc_update/3,
+ after_doc_write/4,
after_doc_read/2,
validate_docid/1,
check_is_admin/1,
@@ -49,6 +50,10 @@ before_doc_update(Db, Doc0, UpdateType) ->
end.
+after_doc_write(Db, Winner, OldWinner, UpdateStatus)->
+ with_pipe(after_doc_write, [Db, Winner, OldWinner, UpdateStatus]).
+
+
after_doc_read(Db, Doc0) ->
Fun = fabric2_db:get_after_doc_read_fun(Db),
case with_pipe(after_doc_read, [Doc0, Db]) of
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index c34b33c..2610198 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -782,6 +782,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ fabric2_db_plugin:after_doc_write(Db, NewWinner, OldWinner, UpdateStatus),
+
% Update database size
AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
RemSize = sum_rem_rev_sizes(ToRemove),