You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/01/30 14:07:40 UTC
[couchdb] branch fdb-mango-indexes updated (90bbba8 -> 8c97ee7)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 90bbba8 able to add/delete/update mango fdb indexes
new 8c97ee7 able to add/delete/update mango fdb indexes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (90bbba8)
\
N -- N -- N refs/heads/fdb-mango-indexes (8c97ee7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/mango/test/eunit/mango_indexer_test.erl | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
[couchdb] 01/01: able to add/delete/update mango fdb indexes
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8c97ee78f8fd145f5ce5feb5df7a9c000ecbccf6
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Jan 30 15:48:30 2020 +0200
able to add/delete/update mango fdb indexes
---
src/fabric/src/fabric2_fdb.erl | 11 +-
src/mango/src/mango_fdb.erl | 28 ++++-
src/mango/src/mango_indexer.erl | 78 +++++++++----
src/mango/test/21-fdb-indexing.py | 48 --------
src/mango/test/eunit/mango_indexer_test.erl | 165 +++++++++++++++++++++++++++
src/mango/test/exunit/mango_indexer_test.exs | 86 --------------
src/mango/test/exunit/test_helper.exs | 2 -
7 files changed, 252 insertions(+), 166 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 723f14d..9771acb 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -588,7 +588,10 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
} = Doc,
% Doc body
-
+ % Fetch the old doc body for the mango hooks later
+ PrevDoc = if OldWinner == not_found -> not_found; true ->
+ get_doc_body(Db, DocId, OldWinner)
+ end,
ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -716,11 +719,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
end,
incr_stat(Db, <<"doc_count">>, -1),
incr_stat(Db, <<"doc_del_count">>, 1),
- OldDoc = get_doc_body(Db, DocId, OldWinner),
- mango_indexer:update(Db, deleted, not_found, OldDoc);
+ mango_indexer:update(Db, deleted, not_found, PrevDoc);
updated ->
- OldDoc = get_doc_body(Db, DocId, OldWinner),
- mango_indexer:update(Db, updated, Doc, OldDoc)
+ mango_indexer:update(Db, updated, Doc, PrevDoc)
end,
ok.
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index dbd22fa..def942f 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -23,6 +23,7 @@
-export([
query_all_docs/4,
+ remove_doc/3,
write_doc/3,
query/4
]).
@@ -143,6 +144,17 @@ fold_cb({Key, _}, Acc) ->
end.
+remove_doc(TxDb, DocId, IdxResults) ->
+ lists:foreach(fun (IdxResult) ->
+ #{
+ ddoc_id := DDocId,
+ results := Results
+ } = IdxResult,
+ MangoIdxPrefix = mango_idx_prefix(TxDb, DDocId),
+ clear_key(TxDb, MangoIdxPrefix, Results, DocId)
+ end, IdxResults).
+
+
write_doc(TxDb, DocId, IdxResults) ->
lists:foreach(fun (IdxResult) ->
#{
@@ -162,11 +174,23 @@ mango_idx_prefix(TxDb, Id) ->
erlfdb_tuple:pack(Key, DbPrefix).
+create_key(MangoIdxPrefix, Results, DocId) ->
+ EncodedResults = couch_views_encoding:encode(Results, key),
+ erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix).
+
+
+clear_key(TxDb, MangoIdxPrefix, Results, DocId) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = create_key(MangoIdxPrefix, Results, DocId),
+ erlfdb:clear(Tx, Key).
+
+
add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
#{
tx := Tx
} = TxDb,
- EncodedResults = couch_views_encoding:encode(Results, key),
- Key = erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix),
+ Key = create_key(MangoIdxPrefix, Results, DocId),
erlfdb:set(Tx, Key, <<0>>).
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index 20af5bd..0cb15f7 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -22,37 +22,69 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango_idx.hrl").
+
+update(Db, State, Doc, PrevDoc) ->
+ try
+ update_int(Db, State, Doc, PrevDoc)
+ catch
+ Error:Reason ->
+ io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
+ #{
+ name := DbName
+ } = Db,
+
+ Id = case Doc of
+ not_found when is_record(PrevDoc, doc) ->
+ #doc{id = DocId} = PrevDoc,
+ DocId;
+ not_found ->
+ <<"unknown_doc_id">>;
+ #doc{} ->
+ #doc{id = DocId} = Doc,
+ DocId
+ end,
+
+ couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+ [DbName, Id, Error, Reason])
+ end,
+ ok.
+
% Design doc
% Todo: Check if design doc is mango index and kick off background worker
% to build new index
-update(Db, Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, OldDoc) ->
+update_int(Db, State, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, PrevDoc) ->
io:format("DESIGN DOC SAVED ~p ~n", [Doc]),
ok;
-update(Db, deleted, _, OldDoc) ->
- ok;
+update_int(Db, deleted, _, PrevDoc) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ remove_doc(Db, PrevDoc, Indexes1);
-update(Db, updated, Doc, OldDoc) ->
- ok;
+update_int(Db, updated, Doc, PrevDoc) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ remove_doc(Db, PrevDoc, Indexes1),
+ write_doc(Db, Doc, Indexes1);
-update(Db, created, Doc, _) ->
- try
- io:format("CREATED ~p ~n", [Doc]),
- #doc{id = DocId} = Doc,
- Indexes = mango_idx:list(Db),
- Indexes1 = filter_json_indexes(Indexes),
- io:format("UPDATE INDEXES ~p ~n filtered ~p ~n", [Indexes, Indexes1]),
- JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
- io:format("DOC ~p ~n", [Doc]),
- Results = index_doc(Indexes1, JSONDoc),
- io:format("Update ~p ~n, ~p ~n Results ~p ~n", [Doc, JSONDoc, Results]),
- mango_fdb:write_doc(Db, DocId, Results)
- catch
- Error:Reason ->
- io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
- ok
- end,
- ok.
+update_int(Db, created, Doc, _) ->
+ Indexes = mango_idx:list(Db),
+ Indexes1 = filter_json_indexes(Indexes),
+ write_doc(Db, Doc, Indexes1).
+
+
+remove_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ PrevJSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ PrevResults = index_doc(Indexes, PrevJSONDoc),
+ mango_fdb:remove_doc(Db, DocId, PrevResults).
+
+
+write_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ Results = index_doc(Indexes, JSONDoc),
+ mango_fdb:write_doc(Db, DocId, Results).
filter_json_indexes(Indexes) ->
diff --git a/src/mango/test/21-fdb-indexing.py b/src/mango/test/21-fdb-indexing.py
deleted file mode 100644
index e1cfd90..0000000
--- a/src/mango/test/21-fdb-indexing.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- coding: latin-1 -*-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import mango
-import copy
-
-DOCS = [
- {"_id": "100", "name": "Jimi", "location": "AUS", "user_id": 1, "same": "value"},
- {"_id": "200", "name": "Eddie", "location": "BRA", "user_id": 2, "same": "value"},
- {"_id": "300", "name": "Harry", "location": "CAN", "user_id": 3, "same": "value"},
- {"_id": "400", "name": "Eddie", "location": "DEN", "user_id": 4, "same": "value"},
- {"_id": "500", "name": "Jones", "location": "ETH", "user_id": 5, "same": "value"}
-]
-
-class FdbIndexingTests(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- self.db.create_index(["name"], name="name")
- self.db.save_docs(copy.deepcopy(DOCS))
-
- def test_doc_update(self):
- docs = self.db.find({"name": "Eddie"})
- self.assertEqual(len(docs), 2)
- self.assertEqual(docs[0]["_id"], "200")
- self.assertEqual(docs[1]["_id"], "400")
-
- doc = self.db.open_doc("400")
- doc["name"] = "NotEddie"
- self.db.save_doc(doc)
-
- docs = self.db.find({"name": "Eddie"})
- print("DD")
- print(docs)
- self.assertEqual(len(docs), 1)
- self.assertEqual(docs[0]["_id"], "200")
-
-
-
diff --git a/src/mango/test/eunit/mango_indexer_test.erl b/src/mango/test/eunit/mango_indexer_test.erl
new file mode 100644
index 0000000..778caea
--- /dev/null
+++ b/src/mango/test/eunit/mango_indexer_test.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_indexer_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("mango/src/mango_cursor.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [with([
+ ?TDEF(index_docs),
+ ?TDEF(update_doc),
+ ?TDEF(delete_doc)
+ ])]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_idx_ddoc(Db),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Docs = make_docs(3),
+ fabric2_db:update_docs(Db, Docs),
+ {Db, couch_doc:to_json_obj(DDoc, [])}.
+
+
+foreach_teardown({Db, _}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+update_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"2">>}, {value, 4}]
+ ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+run_query(Db, DDoc) ->
+ Args = #{
+ start_key => [],
+ start_key_docid => <<>>,
+ end_key => [],
+ end_key_docid => <<255>>,
+ dir => fwd,
+ skip => 0
+ },
+ [Idx] = mango_idx:from_ddoc(Db, DDoc),
+ Cursor = #cursor{
+ db = Db,
+ index = Idx,
+ user_acc = []
+ },
+ {ok, Cursor1} = mango_fdb:query(Db, fun query_cb/2, Cursor, Args),
+ Acc = Cursor1#cursor.user_acc,
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props)}
+ ]
+
+ end, Acc).
+
+
+create_idx_ddoc(Db) ->
+ Opts = [
+ {def, {[{<<"fields">>,{[{<<"value">>,<<"asc">>}]}}]}},
+ {type, <<"json">>},
+ {name, <<"idx_01">>},
+ {ddoc, auto_name},
+ {w, 3},
+ {partitioned, db_default}
+ ],
+
+ {ok, Idx} = mango_idx:new(Db, Opts),
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), []),
+ {ok, NewDDoc} = mango_idx:add(DDoc, Idx),
+ NewDDoc.
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({doc, Doc}, #cursor{user_acc = Acc} = Cursor) ->
+ {ok, Cursor#cursor{
+ user_acc = Acc ++ [Doc]
+ }}.
+
diff --git a/src/mango/test/exunit/mango_indexer_test.exs b/src/mango/test/exunit/mango_indexer_test.exs
deleted file mode 100644
index f62f47e..0000000
--- a/src/mango/test/exunit/mango_indexer_test.exs
+++ /dev/null
@@ -1,86 +0,0 @@
-defmodule MangoIndexerTest do
- use Couch.Test.ExUnit.Case
-
- alias Couch.Test.Utils
- alias Couch.Test.Setup
- alias Couch.Test.Setup.Step
-
- setup_all do
- test_ctx = :test_util.start_couch([:couch_log, :fabric, :couch_js, :couch_jobs])
-
- on_exit(fn ->
- :test_util.stop_couch(test_ctx)
- end)
- end
-
- setup do
- db_name = Utils.random_name("db")
-
- admin_ctx =
- {:user_ctx,
- Utils.erlang_record(:user_ctx, "couch/include/couch_db.hrl", roles: ["_admin"])}
-
- {:ok, db} = :fabric2_db.create(db_name, [admin_ctx])
-
- ddocs = create_ddocs()
- idx_ddocs = create_indexes(db)
- docs = create_docs()
-
- IO.inspect(idx_ddocs)
- {ok, _} = :fabric2_db.update_docs(db, ddocs ++ idx_ddocs)
- {ok, _} = :fabric2_db.update_docs(db, docs)
-
- on_exit(fn ->
- :fabric2_db.delete(db_name, [admin_ctx])
- end)
-
- %{
- db_name: db_name,
- db: db,
- ddoc: ddocs,
- idx: idx_ddocs
- }
- end
-
- test "update doc", context do
- db = context[:db]
- end
-
- defp create_indexes(db) do
- opts = [
- {:def, {[{"fields", ["group", "value"]}]}},
- {:type, "json"},
- {:name, "idx_01"},
- {:ddoc, :auto_name},
- {:w, 3},
- {:partitioned, :db_default}
- ]
-
- {:ok, idx} = :mango_idx.new(db, opts)
- db_opts = [{:user_ctx, db["user_ctx"]}, :deleted, :ejson_body]
- {:ok, ddoc} = :mango_util.load_ddoc(db, :mango_idx.ddoc(idx), db_opts)
- {:ok, new_ddoc} = :mango_idx.add(ddoc, idx)
- [new_ddoc]
- end
-
- defp create_docs() do
- for i <- 1..1 do
- group =
- if rem(i, 3) == 0 do
- "first"
- else
- "second"
- end
-
- :couch_doc.from_json_obj(
- {[
- {"_id", "doc-id-#{i}"},
- {"value", i},
- {"val_str", Integer.to_string(i, 8)},
- {"some", "field"},
- {"group", group}
- ]}
- )
- end
- end
-end
diff --git a/src/mango/test/exunit/test_helper.exs b/src/mango/test/exunit/test_helper.exs
deleted file mode 100644
index 3140500..0000000
--- a/src/mango/test/exunit/test_helper.exs
+++ /dev/null
@@ -1,2 +0,0 @@
-ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
-ExUnit.start()