You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/06 17:10:04 UTC
[couchdb] branch fdb-mango-indexes updated (2e1de9c -> f48742b)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 2e1de9c Revert "Simplify modify_int"
omit b4a45ca Simplify modify_int
new f48742b more work on background indexer
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2e1de9c)
\
N -- N -- N refs/heads/fdb-mango-indexes (f48742b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/src/couch_proc_manager.erl | 2 +-
src/couch_js/src/couch_js_proc_manager.erl | 2 +-
src/mango/src/mango_fdb.erl | 2 +-
src/mango/src/mango_httpd.erl | 3 ++-
src/mango/src/mango_idx.erl | 31 ++++++++++++++++--------
src/mango/src/mango_idx_view.hrl | 3 +--
src/mango/src/mango_indexer_server.erl | 2 +-
src/mango/src/mango_jobs.erl | 5 ++--
src/mango/src/mango_jobs_indexer.erl | 1 -
src/mango/src/mango_native_proc.erl | 3 ++-
src/mango/test/13-users-db-find-test.py | 4 ++-
src/mango/test/eunit/mango_jobs_indexer_test.erl | 9 ++++---
src/mango/test/mango.py | 9 ++-----
13 files changed, 44 insertions(+), 32 deletions(-)
[couchdb] 01/01: more work on background indexer
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f48742b8c93f9b05e2d72ea510a480041a1fe1c0
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Feb 6 15:58:19 2020 +0200
more work on background indexer
---
src/couch/src/couch_proc_manager.erl | 2 +-
src/couch_js/src/couch_js_proc_manager.erl | 2 +-
src/mango/src/mango_fdb.erl | 2 +-
src/mango/src/mango_httpd.erl | 3 ++-
src/mango/src/mango_idx.erl | 31 ++++++++++++++++--------
src/mango/src/mango_idx_view.hrl | 3 +--
src/mango/src/mango_indexer_server.erl | 2 +-
src/mango/src/mango_jobs.erl | 5 ++--
src/mango/src/mango_jobs_indexer.erl | 1 -
src/mango/src/mango_native_proc.erl | 3 ++-
src/mango/test/13-users-db-find-test.py | 4 ++-
src/mango/test/eunit/mango_jobs_indexer_test.erl | 9 ++++---
src/mango/test/mango.py | 9 ++-----
13 files changed, 44 insertions(+), 32 deletions(-)
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 3366b2b..0f686ef 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl
index 0964696..0f2916b 100644
--- a/src/couch_js/src/couch_js_proc_manager.erl
+++ b/src/couch_js/src/couch_js_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index a54d658..3e0c48e 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -224,7 +224,7 @@ fold_cb({Key, _}, Acc) ->
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
JSONDoc = couch_doc:to_json_obj(Doc, []),
-%% io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
+ io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
case Callback({doc, JSONDoc}, Cursor) of
{ok, Cursor1} ->
Acc#{
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index d5e9cfa..b8525dc 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -62,7 +62,8 @@ handle_req_int(_, _) ->
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
- Idxs = lists:sort(mango_idx:list(Db)),
+ Idxs0 = mango_idx:add_build_status(Db, mango_idx:list(Db)),
+ Idxs = lists:sort(Idxs0),
JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
TotalRows = length(JsonIdxs0),
Limit = case couch_util:get_value(limit, Params, TotalRows) of
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 3aadd49..c1deaa9 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -26,6 +26,7 @@
add/2,
remove/2,
from_ddoc/2,
+ add_build_status/2,
special/1,
dbname/1,
@@ -58,11 +59,9 @@ list(Db) ->
rows => []
},
{ok, Indexes} = fabric2_db:fold_design_docs(Db, fun ddoc_fold_cb/2, Acc0, []),
-%% io:format("INDEXES ~p ~n", [Indexes]),
Indexes ++ special(Db).
-% Todo this should all be in fabric2_db
ddoc_fold_cb({meta, _}, Acc) ->
{ok, Acc};
@@ -237,14 +236,25 @@ from_ddoc(Db, {Props}) ->
%% [mango_idx_view]
%% end,
Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods),
+ lists:map(fun(Idx) ->
+ Idx#idx{
+ dbname = DbName,
+ ddoc = DDoc,
+ partitioned = get_idx_partitioned(Db, Props)
+ }
+ end, Idxs).
+
+
+add_build_status(Db, Idxs) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
- lists:map(fun(Idx) ->
- Idx#idx{
- dbname = DbName,
- ddoc = DDoc,
- partitioned = get_idx_partitioned(Db, Props),
- build_status = mango_fdb:get_build_status(TxDb, DDoc)
- }
+ lists:map(fun
+ (#idx{type = <<"special">>} = Idx) ->
+ Idx;
+ (Idx) ->
+ DDoc = mango_idx:ddoc(Idx),
+ Idx#idx{
+ build_status = mango_fdb:get_build_status(TxDb, DDoc)
+ }
end, Idxs)
end).
@@ -255,7 +265,8 @@ special(Db) ->
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?MANGO_INDEX_READY
},
% Add one for _update_seq
[AllDocs].
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
index 6ebe68e..14ce87c 100644
--- a/src/mango/src/mango_idx_view.hrl
+++ b/src/mango/src/mango_idx_view.hrl
@@ -11,5 +11,4 @@
% the License.
%%-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
--define(MAX_JSON_OBJ, <<255>>).
-%%-define(MAX_JSON_OBJ, {[{<<"ZZZ">>, <<"ZZZ">>}]}).
+-define(MAX_JSON_OBJ, {[{<<"\ufff0">>, <<"\ufff0">>}]}).
diff --git a/src/mango/src/mango_indexer_server.erl b/src/mango/src/mango_indexer_server.erl
index 29530bb..6942c9f 100644
--- a/src/mango/src/mango_indexer_server.erl
+++ b/src/mango/src/mango_indexer_server.erl
@@ -31,7 +31,7 @@
]).
--define(MAX_WORKERS, 1).
+-define(MAX_WORKERS, 100).
start_link() ->
diff --git a/src/mango/src/mango_jobs.erl b/src/mango/src/mango_jobs.erl
index 6739d62..c5a70ff 100644
--- a/src/mango/src/mango_jobs.erl
+++ b/src/mango/src/mango_jobs.erl
@@ -39,8 +39,9 @@ build_index(TxDb, #idx{} = Idx) ->
{ok, JobId}.
-job_id(#{name := DbName}, #idx{ddoc = DDoc}) ->
- <<DbName/binary, "-",DDoc/binary>>.
+job_id(#{name := DbName}, #idx{ddoc = DDoc} = Idx) ->
+ Cols = iolist_to_binary(mango_idx:columns(Idx)),
+ <<DbName/binary, "_",DDoc/binary, Cols/binary>>.
job_data(Db, Idx) ->
diff --git a/src/mango/src/mango_jobs_indexer.erl b/src/mango/src/mango_jobs_indexer.erl
index ce6b850..d68c80b 100644
--- a/src/mango/src/mango_jobs_indexer.erl
+++ b/src/mango/src/mango_jobs_indexer.erl
@@ -94,7 +94,6 @@ init() ->
exit:normal ->
ok;
Error:Reason ->
- io:format("ERROR in index worker ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
NewRetry = Retries + 1,
RetryLimit = retry_limit(),
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
index cbf3622..5a05083 100644
--- a/src/mango/src/mango_native_proc.erl
+++ b/src/mango/src/mango_native_proc.erl
@@ -47,7 +47,8 @@
start_link() ->
- gen_server:start_link(?MODULE, [], []).
+ throw({error, mango_native_proc_is_no_longer_needed}).
+%% gen_server:start_link(?MODULE, [], []).
set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py
index 73d15ea..32d919a 100644
--- a/src/mango/test/13-users-db-find-test.py
+++ b/src/mango/test/13-users-db-find-test.py
@@ -12,9 +12,10 @@
# the License.
-import mango, requests
+import mango, requests, unittest
+@unittest.skip("this FDB doesn't support this")
class UsersDbFindTests(mango.UsersDbTests):
def test_simple_find(self):
docs = self.db.find({"name": {"$eq": "demo02"}})
@@ -57,6 +58,7 @@ class UsersDbFindTests(mango.UsersDbTests):
assert len(docs) == 3
+@unittest.skip("this FDB doesn't support this")
class UsersDbIndexFindTests(UsersDbFindTests):
def setUp(self):
self.db.create_index(["name"])
diff --git a/src/mango/test/eunit/mango_jobs_indexer_test.erl b/src/mango/test/eunit/mango_jobs_indexer_test.erl
index 7a8cb24..9641163 100644
--- a/src/mango/test/eunit/mango_jobs_indexer_test.erl
+++ b/src/mango/test/eunit/mango_jobs_indexer_test.erl
@@ -89,7 +89,6 @@ index_lots_of_docs(Db) ->
index_can_recover_from_crash(Db) ->
meck:new(mango_indexer, [passthrough]),
meck:expect(mango_indexer, write_doc, fun (Db, Doc, Idxs) ->
- ?debugFmt("doc ~p ~p ~n", [Doc, Idxs]),
Id = Doc#doc.id,
case Id == <<"2">> of
true ->
@@ -112,8 +111,12 @@ index_can_recover_from_crash(Db) ->
wait_while_ddoc_builds(Db) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
Idxs = mango_idx:list(TxDb),
- [Idx] = lists:filter(fun (Idx) -> Idx#idx.type == <<"json">> end, Idxs),
- if Idx#idx.build_status == ?MANGO_INDEX_READY -> ok; true ->
+
+ Ready = lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, mango_idx:add_build_status(TxDb, Idxs)),
+
+ if length(Ready) > 1 -> ok; true ->
timer:sleep(100),
wait_while_ddoc_builds(Db)
end
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 92cf211..5b1c7a7 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -110,11 +110,7 @@ class Database(object):
def save_docs(self, docs, **kwargs):
body = json.dumps({"docs": docs})
r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs)
- print("DOC")
- print(docs)
r.raise_for_status()
- print("RES")
- print(r.json())
for doc, result in zip(docs, r.json()):
doc["_id"] = result["id"]
doc["_rev"] = result["rev"]
@@ -142,7 +138,7 @@ class Database(object):
name=None,
ddoc=None,
partial_filter_selector=None,
- selector=None,
+ selector=None
):
body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
if name is not None:
@@ -167,7 +163,7 @@ class Database(object):
for i in self.get_index(r.json()["id"], r.json()["name"])
if i["build_status"] == "ready"
]) < 1:
- delay(t=0.1)
+ delay(t=0.2)
return created
@@ -285,7 +281,6 @@ class Database(object):
else:
path = self.path("_find")
r = self.sess.post(path, data=body)
- print(r.json())
r.raise_for_status()
if explain or return_raw:
return r.json()