You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/01/21 11:47:11 UTC
[couchdb] branch fdb-mango-indexes created (now 1223bb9)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 1223bb9 mango crud index definitions
This branch includes the following new commits:
new 332c983 add crude mango hook and indexer setup
new e88fc7f change mango test auth to match elixir
new 1223bb9 mango crud index definitions
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/03: add crude mango hook and indexer setup
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 332c9835d4b0b9b0f2808913a16fa6f1d461db6c
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Jan 21 13:39:56 2020 +0200
add crude mango hook and indexer setup
---
src/fabric/src/fabric2_fdb.erl | 13 ++--
src/mango/src/mango_fdb.erl | 35 ++++++++++
src/mango/src/mango_indexer.erl | 101 +++++++++++++++++++++++++++
src/mango/test/exunit/mango_indexer_test.exs | 68 ++++++++++++++++++
src/mango/test/exunit/test_helper.exs | 2 +
5 files changed, 215 insertions(+), 4 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 6abe1f6..723f14d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -697,13 +697,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
- incr_stat(Db, <<"doc_count">>, 1);
+ incr_stat(Db, <<"doc_count">>, 1),
+ mango_indexer:update(Db, created, Doc, not_found);
recreated ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
incr_stat(Db, <<"doc_count">>, 1),
- incr_stat(Db, <<"doc_del_count">>, -1);
+ incr_stat(Db, <<"doc_del_count">>, -1),
+ mango_indexer:update(Db, created, Doc, not_found);
replicate_deleted ->
incr_stat(Db, <<"doc_del_count">>, 1);
ignore ->
@@ -713,9 +715,12 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
incr_stat(Db, <<"doc_design_count">>, -1)
end,
incr_stat(Db, <<"doc_count">>, -1),
- incr_stat(Db, <<"doc_del_count">>, 1);
+ incr_stat(Db, <<"doc_del_count">>, 1),
+ OldDoc = get_doc_body(Db, DocId, OldWinner),
+ mango_indexer:update(Db, deleted, not_found, OldDoc);
updated ->
- ok
+ OldDoc = get_doc_body(Db, DocId, OldWinner),
+ mango_indexer:update(Db, updated, Doc, OldDoc)
end,
ok.
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
new file mode 100644
index 0000000..c29ae8f
--- /dev/null
+++ b/src/mango/src/mango_fdb.erl
@@ -0,0 +1,35 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_fdb).
+
+
+-export([
+ write_doc/4
+]).
+
+
+write_doc(Db, Doc, Indexes, Results) ->
+ lists:foreach(fun (Index) ->
+ MangoIdxPrefix = mango_idx_prefix(Db, Index),
+ ok
+ end, Indexes).
+
+
+mango_idx_prefix(Db, Index) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ io:format("INDEX ~p ~n", [Index]),
+ ok.
+
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
new file mode 100644
index 0000000..b217ce1
--- /dev/null
+++ b/src/mango/src/mango_indexer.erl
@@ -0,0 +1,101 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_indexer).
+
+
+-export([
+ update/4
+]).
+
+
+update(Db, deleted, _, OldDoc) ->
+ ok;
+update(Db, updated, Doc, OldDoc) ->
+ ok;
+update(Db, created, Doc, _) ->
+%% Indexes = mango_idx:list(Db),
+%% Fun = fun (DDoc, Acc) ->
+%% io:format("DESIGN DOC ~p ~n", [DDoc]),
+%% Acc
+%% end,
+%% fabric2_db:fold_design_docs(Db, Fun, [], []),
+%% % maybe validate indexes here
+%% JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+%% io:format("Update ~p ~n, ~p ~n", [Doc, JSONDoc]),
+%% Results = index_doc(Indexes, JSONDoc),
+ ok.
+
+
+index_doc(Indexes, Doc) ->
+ lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes).
+
+
+get_index_entries({IdxProps}, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
+ Selector = get_index_partial_filter_selector(IdxProps),
+ case should_index(Selector, Doc) of
+ false ->
+ [];
+ true ->
+ Values = get_index_values(Fields, Doc),
+ case lists:member(not_found, Values) of
+ true -> [];
+ false -> [[Values, null]]
+ end
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxProps) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxProps, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the partial_filter_selector
+ % set as selector
+ couch_util:get_value(<<"selector">>, IdxProps, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
+
+
+validate_index_info(IndexInfo) ->
+ IdxTypes = [mango_idx_view, mango_idx_text],
+ Results = lists:foldl(fun(IdxType, Results0) ->
+ try
+ IdxType:validate_index_def(IndexInfo),
+ [valid_index | Results0]
+ catch _:_ ->
+ [invalid_index | Results0]
+ end
+ end, [], IdxTypes),
+ lists:member(valid_index, Results).
+
diff --git a/src/mango/test/exunit/mango_indexer_test.exs b/src/mango/test/exunit/mango_indexer_test.exs
new file mode 100644
index 0000000..3a86ae4
--- /dev/null
+++ b/src/mango/test/exunit/mango_indexer_test.exs
@@ -0,0 +1,68 @@
+defmodule MangoIndexerTest do
+ use Couch.Test.ExUnit.Case
+
+ alias Couch.Test.Utils
+ alias Couch.Test.Setup
+ alias Couch.Test.Setup.Step
+
+ setup_all do
+ test_ctx =
+ :test_util.start_couch([:couch_log, :fabric, :couch_js, :couch_jobs])
+
+ on_exit(fn ->
+ :test_util.stop_couch(test_ctx)
+ end)
+ end
+
+ setup do
+ db_name = Utils.random_name("db")
+
+ admin_ctx =
+ {:user_ctx,
+ Utils.erlang_record(:user_ctx, "couch/include/couch_db.hrl", roles: ["_admin"])}
+
+ {:ok, db} = :fabric2_db.create(db_name, [admin_ctx])
+
+ docs = create_docs()
+ ddoc = create_ddoc()
+
+ {ok, _} = :fabric2_db.update_docs(db, [ddoc | docs])
+
+ on_exit(fn ->
+ :fabric2_db.delete(db_name, [admin_ctx])
+ end)
+
+ %{
+ :db_name => db_name,
+ :db => db,
+ :ddoc => ddoc
+ }
+ end
+
+ test "create design doc through _index", context do
+ db = context[:db]
+ end
+
+# Create 1 design doc that should be filtered out and ignored
+ defp create_ddocs() do
+ views = %{
+ "_id" => "_design/bar",
+ "views" => %{
+ "dates_sum" => %{
+ "map" => """
+ function(doc) {
+ if (doc.date) {
+ emit(doc.date, doc.date_val);
+ }
+ }
+ """
+ }
+ }
+ }
+ :couch_doc.from_json_obj(:jiffy.decode(:jiffy.encode(views)))
+ end
+
+ defp create_docs() do
+ []
+ end
+end
\ No newline at end of file
diff --git a/src/mango/test/exunit/test_helper.exs b/src/mango/test/exunit/test_helper.exs
new file mode 100644
index 0000000..f4ab64f
--- /dev/null
+++ b/src/mango/test/exunit/test_helper.exs
@@ -0,0 +1,2 @@
+ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
+ExUnit.start()
\ No newline at end of file
[couchdb] 02/03: change mango test auth to match elixir
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e88fc7fab9e39ae9285fe0d056e1ae6a2dbc5b16
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Jan 21 13:41:41 2020 +0200
change mango test auth to match elixir
---
src/mango/test/README.md | 4 ++--
src/mango/test/mango.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/mango/test/README.md b/src/mango/test/README.md
index 509e32e..08693a2 100644
--- a/src/mango/test/README.md
+++ b/src/mango/test/README.md
@@ -23,7 +23,7 @@ Test configuration
The following environment variables can be used to configure the test fixtures:
* `COUCH_HOST` - root url (including port) of the CouchDB instance to run the tests against. Default is `"http://127.0.0.1:15984"`.
- * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"testuser"`.
- * `COUCH_PASSWORD` - CouchDB password. Default is `"testpass"`.
+ * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"adm"`.
+ * `COUCH_PASSWORD` - CouchDB password. Default is `"pass"`.
* `COUCH_AUTH_HEADER` - Optional Authorization header value. If specified, this is used instead of basic authentication with the username/password variables above.
* `MANGO_TEXT_INDEXES` - Set to `"1"` to run the tests only applicable to text indexes.
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index de8a638..e8ce2c5 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -48,8 +48,8 @@ class Database(object):
dbname,
host="127.0.0.1",
port="15984",
- user="testuser",
- password="testpass",
+ user="adm",
+ password="pass",
):
root_url = get_from_environment("COUCH_HOST", "http://{}:{}".format(host, port))
auth_header = get_from_environment("COUCH_AUTH_HEADER", None)
[couchdb] 03/03: mango crud index definitions
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1223bb98df13a4e3b286d0f033ff3c63c2872245
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Jan 21 13:45:21 2020 +0200
mango crud index definitions
---
src/mango/src/mango_crud.erl | 6 +++--
src/mango/src/mango_httpd.erl | 8 +------
src/mango/src/mango_idx.erl | 44 +++++++++++++++++++++++++++++++-----
src/mango/src/mango_util.erl | 20 ++++++++--------
src/mango/test/01-index-crud-test.py | 15 ++++++++++++
5 files changed, 69 insertions(+), 24 deletions(-)
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 41a4d14..735531d 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -35,8 +35,9 @@ insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, Docs, Opts0) when is_list(Docs) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
+ % Todo: I dont think we need to support w = 3?
Opts2 = maybe_int_to_str(w, Opts1),
- case fabric:update_docs(Db, Docs, Opts2) of
+ case fabric2_db:update_docs(Db, Docs, Opts2) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -111,7 +112,8 @@ maybe_add_user_ctx(Db, Opts) ->
{user_ctx, _} ->
Opts;
false ->
- [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
+ UserCtx = maps:get(user_ctx, Db),
+ [{user_ctx, UserCtx} | Opts]
end.
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 379d2e1..b046229 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,9 +32,8 @@
threshold = 1490
}).
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
try
- Db = set_user_ctx(Req, Db0),
handle_req_int(Req, Db)
catch
throw:{mango_error, Module, Reason} ->
@@ -198,11 +197,6 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
- {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
- NewDb.
-
-
get_idx_w_opts(Opts) ->
case lists:keyfind(w, 1, Opts) of
{w, N} when is_integer(N), N > 0 ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5d06a8f..7997057 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -52,10 +52,40 @@
-include("mango.hrl").
-include("mango_idx.hrl").
-
list(Db) ->
- {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
- Indexes.
+ Acc0 = #{
+ db => Db,
+ rows => []
+ },
+ {ok, Indexes} = fabric2_db:fold_design_docs(Db, fun ddoc_fold_cb/2, Acc0, []),
+ io:format("INDEXES ~p ~n", [Indexes]),
+ Indexes ++ special(Db).
+
+
+% Todo this should all be in fabric2_db
+ddoc_fold_cb({meta, _}, Acc) ->
+ {ok, Acc};
+
+ddoc_fold_cb(complete, Acc) ->
+ #{rows := Rows} = Acc,
+ {ok, Rows};
+
+ddoc_fold_cb({row, Row}, Acc) ->
+ #{
+ db := Db,
+ rows := Rows
+ } = Acc,
+ {_, Id} = lists:keyfind(id, 1, Row),
+ {ok, Doc} = fabric2_db:open_doc(Db, Id),
+ JSONDoc = couch_doc:to_json_obj(Doc, []),
+ try
+ Idx = from_ddoc(Db, JSONDoc),
+ {ok, Acc#{rows:= Rows ++ Idx}}
+ catch
+ throw:{mango_error, _, invalid_query_ddoc_language} ->
+ io:format("ERROR ~p ~n", [JSONDoc]),
+ {ok, Acc}
+ end.
get_usable_indexes(Db, Selector, Opts) ->
@@ -294,7 +324,7 @@ db_to_name(Name) when is_binary(Name) ->
db_to_name(Name) when is_list(Name) ->
iolist_to_binary(Name);
db_to_name(Db) ->
- couch_db:name(Db).
+ maps:get(name, Db).
get_idx_def(Opts) ->
@@ -407,8 +437,10 @@ set_ddoc_partitioned_option(DDoc, Partitioned) ->
DDoc#doc{body = {NewProps}}.
-get_idx_partitioned(Db, DDocProps) ->
- Default = fabric_util:is_partitioned(Db),
+get_idx_partitioned(_Db, DDocProps) ->
+ % TODO: Add in partition support
+%% Default = fabric_util:is_partitioned(Db),
+ Default = false,
case couch_util:get_value(<<"options">>, DDocProps) of
{DesignOpts} ->
case couch_util:get_value(<<"partitioned">>, DesignOpts) of
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index a734717..50fa79a 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,16 @@ open_doc(Db, DocId) ->
open_doc(Db, DocId, Options) ->
- case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
- {ok, Doc} ->
- {ok, Doc};
- {not_found, _} ->
- not_found;
- _ ->
- ?MANGO_ERROR({error_loading_doc, DocId})
- end.
+ fabric2_db:open_doc(Db, DocId, Options).
+ % TODO: is this defer still required?
+%% case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
+%% {ok, Doc} ->
+%% {ok, Doc};
+%% {not_found, _} ->
+%% not_found;
+%% _ ->
+%% ?MANGO_ERROR({error_loading_doc, DocId})
+%% end.
open_ddocs(Db) ->
@@ -111,7 +113,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
case open_doc(Db, DDocId, DbOpts) of
{ok, Doc} ->
{ok, check_lang(Doc)};
- not_found ->
+ {not_found, missing} ->
Body = {[
{<<"language">>, <<"query">>}
]},
diff --git a/src/mango/test/01-index-crud-test.py b/src/mango/test/01-index-crud-test.py
index b602399..dd9ab1a 100644
--- a/src/mango/test/01-index-crud-test.py
+++ b/src/mango/test/01-index-crud-test.py
@@ -113,6 +113,21 @@ class IndexCrudTests(mango.DbPerClass):
return
raise AssertionError("index not created")
+ def test_ignore_design_docs(self):
+ fields = ["baz", "foo"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+ self.db.save_doc({
+ "_id": "_design/ignore",
+ "views": {
+ "view1": {
+ "map": "function (doc) { emit(doc._id, 1)}"
+ }
+ }
+ })
+ Indexes = self.db.list_indexes()
+ self.assertEqual(len(Indexes), 2)
+
def test_read_idx_doc(self):
self.db.create_index(["foo", "bar"], name="idx_01")
self.db.create_index(["hello", "bar"])