You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/01/30 14:07:41 UTC
[couchdb] 01/01: able to add/delete/update mango fdb indexes
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8c97ee78f8fd145f5ce5feb5df7a9c000ecbccf6
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Jan 30 15:48:30 2020 +0200
able to add/delete/update mango fdb indexes
---
src/fabric/src/fabric2_fdb.erl | 11 +-
src/mango/src/mango_fdb.erl | 28 ++++-
src/mango/src/mango_indexer.erl | 78 +++++++++----
src/mango/test/21-fdb-indexing.py | 48 --------
src/mango/test/eunit/mango_indexer_test.erl | 165 +++++++++++++++++++++++++++
src/mango/test/exunit/mango_indexer_test.exs | 86 --------------
src/mango/test/exunit/test_helper.exs | 2 -
7 files changed, 252 insertions(+), 166 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 723f14d..9771acb 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -588,7 +588,10 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
} = Doc,
% Doc body
-
+ % Fetch the old doc body for the mango hooks later
+ PrevDoc = if OldWinner == not_found -> not_found; true ->
+ get_doc_body(Db, DocId, OldWinner)
+ end,
ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -716,11 +719,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
end,
incr_stat(Db, <<"doc_count">>, -1),
incr_stat(Db, <<"doc_del_count">>, 1),
- OldDoc = get_doc_body(Db, DocId, OldWinner),
- mango_indexer:update(Db, deleted, not_found, OldDoc);
+ mango_indexer:update(Db, deleted, not_found, PrevDoc);
updated ->
- OldDoc = get_doc_body(Db, DocId, OldWinner),
- mango_indexer:update(Db, updated, Doc, OldDoc)
+ mango_indexer:update(Db, updated, Doc, PrevDoc)
end,
ok.
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index dbd22fa..def942f 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -23,6 +23,7 @@
-export([
query_all_docs/4,
+ remove_doc/3,
write_doc/3,
query/4
]).
@@ -143,6 +144,17 @@ fold_cb({Key, _}, Acc) ->
end.
+remove_doc(TxDb, DocId, IdxResults) ->
+ lists:foreach(fun (IdxResult) ->
+ #{
+ ddoc_id := DDocId,
+ results := Results
+ } = IdxResult,
+ MangoIdxPrefix = mango_idx_prefix(TxDb, DDocId),
+ clear_key(TxDb, MangoIdxPrefix, Results, DocId)
+ end, IdxResults).
+
+
write_doc(TxDb, DocId, IdxResults) ->
lists:foreach(fun (IdxResult) ->
#{
@@ -162,11 +174,23 @@ mango_idx_prefix(TxDb, Id) ->
erlfdb_tuple:pack(Key, DbPrefix).
+create_key(MangoIdxPrefix, Results, DocId) ->
+ EncodedResults = couch_views_encoding:encode(Results, key),
+ erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix).
+
+
+clear_key(TxDb, MangoIdxPrefix, Results, DocId) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = create_key(MangoIdxPrefix, Results, DocId),
+ erlfdb:clear(Tx, Key).
+
+
add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
#{
tx := Tx
} = TxDb,
- EncodedResults = couch_views_encoding:encode(Results, key),
- Key = erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix),
+ Key = create_key(MangoIdxPrefix, Results, DocId),
erlfdb:set(Tx, Key, <<0>>).
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index 20af5bd..0cb15f7 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -22,37 +22,69 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango_idx.hrl").
+
+update(Db, State, Doc, PrevDoc) ->
+ try
+ update_int(Db, State, Doc, PrevDoc)
+ catch
+ Error:Reason ->
+ io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
+ #{
+ name := DbName
+ } = Db,
+
+ Id = case Doc of
+ not_found when is_record(PrevDoc, doc) ->
+ #doc{id = DocId} = PrevDoc,
+ DocId;
+ not_found ->
+ <<"unknown_doc_id">>;
+ #doc{} ->
+ #doc{id = DocId} = Doc,
+ DocId
+ end,
+
+ couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+ [DbName, Id, Error, Reason])
+ end,
+ ok.
+
% Design doc
% Todo: Check if design doc is mango index and kick off background worker
% to build new index
-update(Db, Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, OldDoc) ->
+update_int(Db, State, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, PrevDoc) ->
io:format("DESIGN DOC SAVED ~p ~n", [Doc]),
ok;
-update(Db, deleted, _, OldDoc) ->
- ok;
+update_int(Db, deleted, _, PrevDoc) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ remove_doc(Db, PrevDoc, Indexes1);
-update(Db, updated, Doc, OldDoc) ->
- ok;
+update_int(Db, updated, Doc, PrevDoc) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ remove_doc(Db, PrevDoc, Indexes1),
+ write_doc(Db, Doc, Indexes1);
-update(Db, created, Doc, _) ->
- try
- io:format("CREATED ~p ~n", [Doc]),
- #doc{id = DocId} = Doc,
- Indexes = mango_idx:list(Db),
- Indexes1 = filter_json_indexes(Indexes),
- io:format("UPDATE INDEXES ~p ~n filtered ~p ~n", [Indexes, Indexes1]),
- JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
- io:format("DOC ~p ~n", [Doc]),
- Results = index_doc(Indexes1, JSONDoc),
- io:format("Update ~p ~n, ~p ~n Results ~p ~n", [Doc, JSONDoc, Results]),
- mango_fdb:write_doc(Db, DocId, Results)
- catch
- Error:Reason ->
- io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
- ok
- end,
- ok.
+update_int(Db, created, Doc, _) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ write_doc(Db, Doc, Indexes1).
+
+
+remove_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ PrevJSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ PrevResults = index_doc(Indexes, PrevJSONDoc),
+ mango_fdb:remove_doc(Db, DocId, PrevResults).
+
+
+write_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ Results = index_doc(Indexes, JSONDoc),
+ mango_fdb:write_doc(Db, DocId, Results).
filter_json_indexes(Indexes) ->
diff --git a/src/mango/test/21-fdb-indexing.py b/src/mango/test/21-fdb-indexing.py
deleted file mode 100644
index e1cfd90..0000000
--- a/src/mango/test/21-fdb-indexing.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- coding: latin-1 -*-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import mango
-import copy
-
-DOCS = [
- {"_id": "100", "name": "Jimi", "location": "AUS", "user_id": 1, "same": "value"},
- {"_id": "200", "name": "Eddie", "location": "BRA", "user_id": 2, "same": "value"},
- {"_id": "300", "name": "Harry", "location": "CAN", "user_id": 3, "same": "value"},
- {"_id": "400", "name": "Eddie", "location": "DEN", "user_id": 4, "same": "value"},
- {"_id": "500", "name": "Jones", "location": "ETH", "user_id": 5, "same": "value"}
-]
-
-class FdbIndexingTests(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- self.db.create_index(["name"], name="name")
- self.db.save_docs(copy.deepcopy(DOCS))
-
- def test_doc_update(self):
- docs = self.db.find({"name": "Eddie"})
- self.assertEqual(len(docs), 2)
- self.assertEqual(docs[0]["_id"], "200")
- self.assertEqual(docs[1]["_id"], "400")
-
- doc = self.db.open_doc("400")
- doc["name"] = "NotEddie"
- self.db.save_doc(doc)
-
- docs = self.db.find({"name": "Eddie"})
- print("DD")
- print(docs)
- self.assertEqual(len(docs), 1)
- self.assertEqual(docs[0]["_id"], "200")
-
-
-
diff --git a/src/mango/test/eunit/mango_indexer_test.erl b/src/mango/test/eunit/mango_indexer_test.erl
new file mode 100644
index 0000000..778caea
--- /dev/null
+++ b/src/mango/test/eunit/mango_indexer_test.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_indexer_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("mango/src/mango_cursor.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [with([
+ ?TDEF(index_docs),
+ ?TDEF(update_doc),
+ ?TDEF(delete_doc)
+ ])]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_idx_ddoc(Db),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Docs = make_docs(3),
+ fabric2_db:update_docs(Db, Docs),
+ {Db, couch_doc:to_json_obj(DDoc, [])}.
+
+
+foreach_teardown({Db, _}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+update_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"2">>}, {value, 4}]
+ ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+run_query(Db, DDoc) ->
+ Args = #{
+ start_key => [],
+ start_key_docid => <<>>,
+ end_key => [],
+ end_key_docid => <<255>>,
+ dir => fwd,
+ skip => 0
+ },
+ [Idx] = mango_idx:from_ddoc(Db, DDoc),
+ Cursor = #cursor{
+ db = Db,
+ index = Idx,
+ user_acc = []
+ },
+ {ok, Cursor1} = mango_fdb:query(Db, fun query_cb/2, Cursor, Args),
+ Acc = Cursor1#cursor.user_acc,
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props)}
+ ]
+
+ end, Acc).
+
+
+create_idx_ddoc(Db) ->
+ Opts = [
+ {def, {[{<<"fields">>,{[{<<"value">>,<<"asc">>}]}}]}},
+ {type, <<"json">>},
+ {name, <<"idx_01">>},
+ {ddoc, auto_name},
+ {w, 3},
+ {partitioned, db_default}
+ ],
+
+ {ok, Idx} = mango_idx:new(Db, Opts),
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), []),
+ {ok, NewDDoc} = mango_idx:add(DDoc, Idx),
+ NewDDoc.
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({doc, Doc}, #cursor{user_acc = Acc} = Cursor) ->
+ {ok, Cursor#cursor{
+ user_acc = Acc ++ [Doc]
+ }}.
+
diff --git a/src/mango/test/exunit/mango_indexer_test.exs b/src/mango/test/exunit/mango_indexer_test.exs
deleted file mode 100644
index f62f47e..0000000
--- a/src/mango/test/exunit/mango_indexer_test.exs
+++ /dev/null
@@ -1,86 +0,0 @@
-defmodule MangoIndexerTest do
- use Couch.Test.ExUnit.Case
-
- alias Couch.Test.Utils
- alias Couch.Test.Setup
- alias Couch.Test.Setup.Step
-
- setup_all do
- test_ctx = :test_util.start_couch([:couch_log, :fabric, :couch_js, :couch_jobs])
-
- on_exit(fn ->
- :test_util.stop_couch(test_ctx)
- end)
- end
-
- setup do
- db_name = Utils.random_name("db")
-
- admin_ctx =
- {:user_ctx,
- Utils.erlang_record(:user_ctx, "couch/include/couch_db.hrl", roles: ["_admin"])}
-
- {:ok, db} = :fabric2_db.create(db_name, [admin_ctx])
-
- ddocs = create_ddocs()
- idx_ddocs = create_indexes(db)
- docs = create_docs()
-
- IO.inspect(idx_ddocs)
- {ok, _} = :fabric2_db.update_docs(db, ddocs ++ idx_ddocs)
- {ok, _} = :fabric2_db.update_docs(db, docs)
-
- on_exit(fn ->
- :fabric2_db.delete(db_name, [admin_ctx])
- end)
-
- %{
- db_name: db_name,
- db: db,
- ddoc: ddocs,
- idx: idx_ddocs
- }
- end
-
- test "update doc", context do
- db = context[:db]
- end
-
- defp create_indexes(db) do
- opts = [
- {:def, {[{"fields", ["group", "value"]}]}},
- {:type, "json"},
- {:name, "idx_01"},
- {:ddoc, :auto_name},
- {:w, 3},
- {:partitioned, :db_default}
- ]
-
- {:ok, idx} = :mango_idx.new(db, opts)
- db_opts = [{:user_ctx, db["user_ctx"]}, :deleted, :ejson_body]
- {:ok, ddoc} = :mango_util.load_ddoc(db, :mango_idx.ddoc(idx), db_opts)
- {:ok, new_ddoc} = :mango_idx.add(ddoc, idx)
- [new_ddoc]
- end
-
- defp create_docs() do
- for i <- 1..1 do
- group =
- if rem(i, 3) == 0 do
- "first"
- else
- "second"
- end
-
- :couch_doc.from_json_obj(
- {[
- {"_id", "doc-id-#{i}"},
- {"value", i},
- {"val_str", Integer.to_string(i, 8)},
- {"some", "field"},
- {"group", group}
- ]}
- )
- end
- end
-end
diff --git a/src/mango/test/exunit/test_helper.exs b/src/mango/test/exunit/test_helper.exs
deleted file mode 100644
index 3140500..0000000
--- a/src/mango/test/exunit/test_helper.exs
+++ /dev/null
@@ -1,2 +0,0 @@
-ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
-ExUnit.start()