You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/01/28 06:18:59 UTC
[couchdb] 01/05: add crude mango hook and indexer setup
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a03c704a213432ba973e5dd0f02d5cc86b7fc298
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Jan 21 13:39:56 2020 +0200
add crude mango hook and indexer setup
---
src/fabric/src/fabric2_fdb.erl | 13 ++--
src/mango/src/mango_fdb.erl | 35 ++++++++++
src/mango/src/mango_indexer.erl | 101 +++++++++++++++++++++++++++
src/mango/test/exunit/mango_indexer_test.exs | 68 ++++++++++++++++++
src/mango/test/exunit/test_helper.exs | 2 +
5 files changed, 215 insertions(+), 4 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 6abe1f6..723f14d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -697,13 +697,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
- incr_stat(Db, <<"doc_count">>, 1);
+ incr_stat(Db, <<"doc_count">>, 1),
+ mango_indexer:update(Db, created, Doc, not_found);
recreated ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
incr_stat(Db, <<"doc_count">>, 1),
- incr_stat(Db, <<"doc_del_count">>, -1);
+ incr_stat(Db, <<"doc_del_count">>, -1),
+ mango_indexer:update(Db, created, Doc, not_found);
replicate_deleted ->
incr_stat(Db, <<"doc_del_count">>, 1);
ignore ->
@@ -713,9 +715,12 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
incr_stat(Db, <<"doc_design_count">>, -1)
end,
incr_stat(Db, <<"doc_count">>, -1),
- incr_stat(Db, <<"doc_del_count">>, 1);
+ incr_stat(Db, <<"doc_del_count">>, 1),
+ OldDoc = get_doc_body(Db, DocId, OldWinner),
+ mango_indexer:update(Db, deleted, not_found, OldDoc);
updated ->
- ok
+ OldDoc = get_doc_body(Db, DocId, OldWinner),
+ mango_indexer:update(Db, updated, Doc, OldDoc)
end,
ok.
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
new file mode 100644
index 0000000..c29ae8f
--- /dev/null
+++ b/src/mango/src/mango_fdb.erl
@@ -0,0 +1,35 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_fdb).
+
+
+-export([
+ write_doc/4
+]).
+
+
+write_doc(Db, Doc, Indexes, Results) ->
+ lists:foreach(fun (Index) ->
+ MangoIdxPrefix = mango_idx_prefix(Db, Index),
+ ok
+ end, Indexes).
+
+
+mango_idx_prefix(Db, Index) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ io:format("INDEX ~p ~n", [Index]),
+ ok.
+
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
new file mode 100644
index 0000000..b217ce1
--- /dev/null
+++ b/src/mango/src/mango_indexer.erl
@@ -0,0 +1,101 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_indexer).
+
+
+-export([
+ update/4
+]).
+
+
+update(Db, deleted, _, OldDoc) ->
+ ok;
+update(Db, updated, Doc, OldDoc) ->
+ ok;
+update(Db, created, Doc, _) ->
+%% Indexes = mango_idx:list(Db),
+%% Fun = fun (DDoc, Acc) ->
+%% io:format("DESIGN DOC ~p ~n", [DDoc]),
+%% Acc
+%% end,
+%% fabric2_db:fold_design_docs(Db, Fun, [], []),
+%% % maybe validate indexes here
+%% JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+%% io:format("Update ~p ~n, ~p ~n", [Doc, JSONDoc]),
+%% Results = index_doc(Indexes, JSONDoc),
+ ok.
+
+
+index_doc(Indexes, Doc) ->
+ lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes).
+
+
+get_index_entries({IdxProps}, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
+ Selector = get_index_partial_filter_selector(IdxProps),
+ case should_index(Selector, Doc) of
+ false ->
+ [];
+ true ->
+ Values = get_index_values(Fields, Doc),
+ case lists:member(not_found, Values) of
+ true -> [];
+ false -> [[Values, null]]
+ end
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxProps) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxProps, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the partial_filter_selector
+ % set as selector
+ couch_util:get_value(<<"selector">>, IdxProps, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
+
+
+validate_index_info(IndexInfo) ->
+ IdxTypes = [mango_idx_view, mango_idx_text],
+ Results = lists:foldl(fun(IdxType, Results0) ->
+ try
+ IdxType:validate_index_def(IndexInfo),
+ [valid_index | Results0]
+ catch _:_ ->
+ [invalid_index | Results0]
+ end
+ end, [], IdxTypes),
+ lists:member(valid_index, Results).
+
diff --git a/src/mango/test/exunit/mango_indexer_test.exs b/src/mango/test/exunit/mango_indexer_test.exs
new file mode 100644
index 0000000..3a86ae4
--- /dev/null
+++ b/src/mango/test/exunit/mango_indexer_test.exs
@@ -0,0 +1,68 @@
+defmodule MangoIndexerTest do
+ use Couch.Test.ExUnit.Case
+
+ alias Couch.Test.Utils
+ alias Couch.Test.Setup
+ alias Couch.Test.Setup.Step
+
+ setup_all do
+ test_ctx =
+ :test_util.start_couch([:couch_log, :fabric, :couch_js, :couch_jobs])
+
+ on_exit(fn ->
+ :test_util.stop_couch(test_ctx)
+ end)
+ end
+
+ setup do
+ db_name = Utils.random_name("db")
+
+ admin_ctx =
+ {:user_ctx,
+ Utils.erlang_record(:user_ctx, "couch/include/couch_db.hrl", roles: ["_admin"])}
+
+ {:ok, db} = :fabric2_db.create(db_name, [admin_ctx])
+
+ docs = create_docs()
+ ddoc = create_ddoc()
+
+ {ok, _} = :fabric2_db.update_docs(db, [ddoc | docs])
+
+ on_exit(fn ->
+ :fabric2_db.delete(db_name, [admin_ctx])
+ end)
+
+ %{
+ :db_name => db_name,
+ :db => db,
+ :ddoc => ddoc
+ }
+ end
+
+ test "create design doc through _index", context do
+ db = context[:db]
+ end
+
+# Create 1 design doc that should be filtered out and ignored
+ defp create_ddocs() do
+ views = %{
+ "_id" => "_design/bar",
+ "views" => %{
+ "dates_sum" => %{
+ "map" => """
+ function(doc) {
+ if (doc.date) {
+ emit(doc.date, doc.date_val);
+ }
+ }
+ """
+ }
+ }
+ }
+ :couch_doc.from_json_obj(:jiffy.decode(:jiffy.encode(views)))
+ end
+
+ defp create_docs() do
+ []
+ end
+end
\ No newline at end of file
diff --git a/src/mango/test/exunit/test_helper.exs b/src/mango/test/exunit/test_helper.exs
new file mode 100644
index 0000000..f4ab64f
--- /dev/null
+++ b/src/mango/test/exunit/test_helper.exs
@@ -0,0 +1,2 @@
+ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
+ExUnit.start()
\ No newline at end of file