You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/19 08:22:24 UTC
[couchdb] 12/23: more work on background indexer
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5294ab841045302efede84f7820a183f44067dc9
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Feb 6 15:58:19 2020 +0200
more work on background indexer
---
src/couch/src/couch_proc_manager.erl | 2 +-
src/couch_js/src/couch_js_proc_manager.erl | 2 +-
src/mango/src/mango_fdb.erl | 2 +-
src/mango/src/mango_httpd.erl | 3 ++-
src/mango/src/mango_idx.erl | 31 ++++++++++++++++--------
src/mango/src/mango_idx_view.hrl | 3 +--
src/mango/src/mango_indexer_server.erl | 2 +-
src/mango/src/mango_jobs.erl | 5 ++--
src/mango/src/mango_jobs_indexer.erl | 1 -
src/mango/src/mango_native_proc.erl | 3 ++-
src/mango/test/13-users-db-find-test.py | 4 ++-
src/mango/test/eunit/mango_jobs_indexer_test.erl | 9 ++++---
src/mango/test/mango.py | 9 ++-----
13 files changed, 44 insertions(+), 32 deletions(-)
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 3366b2b..0f686ef 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl
index 0964696..0f2916b 100644
--- a/src/couch_js/src/couch_js_proc_manager.erl
+++ b/src/couch_js/src/couch_js_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index a54d658..3e0c48e 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -224,7 +224,7 @@ fold_cb({Key, _}, Acc) ->
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
JSONDoc = couch_doc:to_json_obj(Doc, []),
-%% io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
+ io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
case Callback({doc, JSONDoc}, Cursor) of
{ok, Cursor1} ->
Acc#{
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index d5e9cfa..b8525dc 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -62,7 +62,8 @@ handle_req_int(_, _) ->
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
- Idxs = lists:sort(mango_idx:list(Db)),
+ Idxs0 = mango_idx:add_build_status(Db, mango_idx:list(Db)),
+ Idxs = lists:sort(Idxs0),
JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
TotalRows = length(JsonIdxs0),
Limit = case couch_util:get_value(limit, Params, TotalRows) of
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 3aadd49..c1deaa9 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -26,6 +26,7 @@
add/2,
remove/2,
from_ddoc/2,
+ add_build_status/2,
special/1,
dbname/1,
@@ -58,11 +59,9 @@ list(Db) ->
rows => []
},
{ok, Indexes} = fabric2_db:fold_design_docs(Db, fun ddoc_fold_cb/2, Acc0, []),
-%% io:format("INDEXES ~p ~n", [Indexes]),
Indexes ++ special(Db).
-% Todo this should all be in fabric2_db
ddoc_fold_cb({meta, _}, Acc) ->
{ok, Acc};
@@ -237,14 +236,25 @@ from_ddoc(Db, {Props}) ->
%% [mango_idx_view]
%% end,
Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods),
+ lists:map(fun(Idx) ->
+ Idx#idx{
+ dbname = DbName,
+ ddoc = DDoc,
+ partitioned = get_idx_partitioned(Db, Props)
+ }
+ end, Idxs).
+
+
+add_build_status(Db, Idxs) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
- lists:map(fun(Idx) ->
- Idx#idx{
- dbname = DbName,
- ddoc = DDoc,
- partitioned = get_idx_partitioned(Db, Props),
- build_status = mango_fdb:get_build_status(TxDb, DDoc)
- }
+ lists:map(fun
+ (#idx{type = <<"special">>} = Idx) ->
+ Idx;
+ (Idx) ->
+ DDoc = mango_idx:ddoc(Idx),
+ Idx#idx{
+ build_status = mango_fdb:get_build_status(TxDb, DDoc)
+ }
end, Idxs)
end).
@@ -255,7 +265,8 @@ special(Db) ->
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?MANGO_INDEX_READY
},
% Add one for _update_seq
[AllDocs].
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
index 6ebe68e..14ce87c 100644
--- a/src/mango/src/mango_idx_view.hrl
+++ b/src/mango/src/mango_idx_view.hrl
@@ -11,5 +11,4 @@
% the License.
%%-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
--define(MAX_JSON_OBJ, <<255>>).
-%%-define(MAX_JSON_OBJ, {[{<<"ZZZ">>, <<"ZZZ">>}]}).
+-define(MAX_JSON_OBJ, {[{<<"\ufff0">>, <<"\ufff0">>}]}).
diff --git a/src/mango/src/mango_indexer_server.erl b/src/mango/src/mango_indexer_server.erl
index 29530bb..6942c9f 100644
--- a/src/mango/src/mango_indexer_server.erl
+++ b/src/mango/src/mango_indexer_server.erl
@@ -31,7 +31,7 @@
]).
--define(MAX_WORKERS, 1).
+-define(MAX_WORKERS, 100).
start_link() ->
diff --git a/src/mango/src/mango_jobs.erl b/src/mango/src/mango_jobs.erl
index 6739d62..c5a70ff 100644
--- a/src/mango/src/mango_jobs.erl
+++ b/src/mango/src/mango_jobs.erl
@@ -39,8 +39,9 @@ build_index(TxDb, #idx{} = Idx) ->
{ok, JobId}.
-job_id(#{name := DbName}, #idx{ddoc = DDoc}) ->
- <<DbName/binary, "-",DDoc/binary>>.
+job_id(#{name := DbName}, #idx{ddoc = DDoc} = Idx) ->
+ Cols = iolist_to_binary(mango_idx:columns(Idx)),
+ <<DbName/binary, "_",DDoc/binary, Cols/binary>>.
job_data(Db, Idx) ->
diff --git a/src/mango/src/mango_jobs_indexer.erl b/src/mango/src/mango_jobs_indexer.erl
index ce6b850..d68c80b 100644
--- a/src/mango/src/mango_jobs_indexer.erl
+++ b/src/mango/src/mango_jobs_indexer.erl
@@ -94,7 +94,6 @@ init() ->
exit:normal ->
ok;
Error:Reason ->
- io:format("ERROR in index worker ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
NewRetry = Retries + 1,
RetryLimit = retry_limit(),
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
index cbf3622..5a05083 100644
--- a/src/mango/src/mango_native_proc.erl
+++ b/src/mango/src/mango_native_proc.erl
@@ -47,7 +47,8 @@
start_link() ->
- gen_server:start_link(?MODULE, [], []).
+ throw({error, mango_native_proc_is_no_longer_needed}).
+%% gen_server:start_link(?MODULE, [], []).
set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py
index 73d15ea..32d919a 100644
--- a/src/mango/test/13-users-db-find-test.py
+++ b/src/mango/test/13-users-db-find-test.py
@@ -12,9 +12,10 @@
# the License.
-import mango, requests
+import mango, requests, unittest
+@unittest.skip("this FDB doesn't support this")
class UsersDbFindTests(mango.UsersDbTests):
def test_simple_find(self):
docs = self.db.find({"name": {"$eq": "demo02"}})
@@ -57,6 +58,7 @@ class UsersDbFindTests(mango.UsersDbTests):
assert len(docs) == 3
+@unittest.skip("this FDB doesn't support this")
class UsersDbIndexFindTests(UsersDbFindTests):
def setUp(self):
self.db.create_index(["name"])
diff --git a/src/mango/test/eunit/mango_jobs_indexer_test.erl b/src/mango/test/eunit/mango_jobs_indexer_test.erl
index 7a8cb24..9641163 100644
--- a/src/mango/test/eunit/mango_jobs_indexer_test.erl
+++ b/src/mango/test/eunit/mango_jobs_indexer_test.erl
@@ -89,7 +89,6 @@ index_lots_of_docs(Db) ->
index_can_recover_from_crash(Db) ->
meck:new(mango_indexer, [passthrough]),
meck:expect(mango_indexer, write_doc, fun (Db, Doc, Idxs) ->
- ?debugFmt("doc ~p ~p ~n", [Doc, Idxs]),
Id = Doc#doc.id,
case Id == <<"2">> of
true ->
@@ -112,8 +111,12 @@ index_can_recover_from_crash(Db) ->
wait_while_ddoc_builds(Db) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
Idxs = mango_idx:list(TxDb),
- [Idx] = lists:filter(fun (Idx) -> Idx#idx.type == <<"json">> end, Idxs),
- if Idx#idx.build_status == ?MANGO_INDEX_READY -> ok; true ->
+
+ Ready = lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, mango_idx:add_build_status(TxDb, Idxs)),
+
+ if length(Ready) > 1 -> ok; true ->
timer:sleep(100),
wait_while_ddoc_builds(Db)
end
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 92cf211..5b1c7a7 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -110,11 +110,7 @@ class Database(object):
def save_docs(self, docs, **kwargs):
body = json.dumps({"docs": docs})
r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs)
- print("DOC")
- print(docs)
r.raise_for_status()
- print("RES")
- print(r.json())
for doc, result in zip(docs, r.json()):
doc["_id"] = result["id"]
doc["_rev"] = result["rev"]
@@ -142,7 +138,7 @@ class Database(object):
name=None,
ddoc=None,
partial_filter_selector=None,
- selector=None,
+ selector=None
):
body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
if name is not None:
@@ -167,7 +163,7 @@ class Database(object):
for i in self.get_index(r.json()["id"], r.json()["name"])
if i["build_status"] == "ready"
]) < 1:
- delay(t=0.1)
+ delay(t=0.2)
return created
@@ -285,7 +281,6 @@ class Database(object):
else:
path = self.path("_find")
r = self.sess.post(path, data=body)
- print(r.json())
r.raise_for_status()
if explain or return_raw:
return r.json()