You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2019/09/04 17:59:22 UTC
[couchdb] branch prototype/fdb-layer updated: Implement
_design_docs and _local_docs
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/prototype/fdb-layer by this push:
new 247bbef Implement _design_docs and _local_docs
247bbef is described below
commit 247bbef844445c4aca39662dbfcc19fcc915a015
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Fri Aug 30 20:17:42 2019 -0400
Implement _design_docs and _local_docs
`_design_docs` reuses `_all_docs` logic and adjusts `start_key` and
`end_key` to be within the `_design/` prefix range.
Namespace setting was simplified to never have an `undefined`
value. This way it doesn't need extra case statements to handle it
further down in the FDB code.
---
src/chttpd/src/chttpd_db.erl | 38 +++++++------
src/fabric/src/fabric2_db.erl | 89 +++++++++++++++++++++++++++--
src/fabric/src/fabric2_fdb.erl | 6 ++
test/elixir/test/basics_test.exs | 119 +++++++++++++++++++++++++++++++++++++++
4 files changed, 231 insertions(+), 21 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 1837585..04da99c 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -859,22 +859,21 @@ all_docs_view(Req, Db, Keys, OP) ->
send_all_docs(Db, #mrargs{keys = undefined} = Args, UserCtx, VAcc0) ->
Opts = all_docs_view_opts(Args, UserCtx),
- Acc = {iter, Db, Args, VAcc0},
+ NS = couch_util:get_value(namespace, Opts),
+ FoldFun = case NS of
+ <<"_all_docs">> -> fold_docs;
+ <<"_design">> -> fold_design_docs;
+ <<"_local">> -> fold_local_docs
+ end,
ViewCb = fun view_cb/2,
- {ok, {iter, _, _, VAcc1}} = fabric2_db:fold_docs(Db, ViewCb, Acc, Opts),
+ Acc = {iter, Db, Args, VAcc0},
+ {ok, {iter, _, _, VAcc1}} = fabric2_db:FoldFun(Db, ViewCb, Acc, Opts),
VAcc1.
send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
Keys = apply_args_to_keylist(Args, Args#mrargs.keys),
- %% namespace can be _set_ to `undefined`, so we
- %% want simulate enum here
- NS = case couch_util:get_value(namespace, Args#mrargs.extra) of
- <<"_all_docs">> -> <<"_all_docs">>;
- <<"_design">> -> <<"_design">>;
- <<"_local">> -> <<"_local">>;
- _ -> <<"_all_docs">>
- end,
+ NS = couch_util:get_value(namespace, Args#mrargs.extra),
TotalRows = fabric2_db:get_doc_count(Db, NS),
Meta = case Args#mrargs.update_seq of
true ->
@@ -889,7 +888,7 @@ send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
_ -> Args#mrargs.doc_options
end ++ [{user_ctx, UserCtx}],
IncludeDocs = Args#mrargs.include_docs,
- VAcc2 = lists:foldl(fun(DocId, Acc) ->
+ lists:foldl(fun(DocId, Acc) ->
OpenOpts = [deleted | DocOpts],
Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
{not_found, missing} ->
@@ -931,6 +930,7 @@ send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
all_docs_view_opts(Args, UserCtx) ->
+ NS = couch_util:get_value(namespace, Args#mrargs.extra),
StartKey = case Args#mrargs.start_key of
undefined -> Args#mrargs.start_key_docid;
SKey -> SKey
@@ -939,18 +939,23 @@ all_docs_view_opts(Args, UserCtx) ->
undefined -> Args#mrargs.end_key_docid;
EKey -> EKey
end,
+ StartKeyOpts = case StartKey of
+ <<_/binary>> -> [{start_key, StartKey}];
+ undefined -> []
+ end,
EndKeyOpts = case {EndKey, Args#mrargs.inclusive_end} of
{<<_/binary>>, false} -> [{end_key_gt, EndKey}];
- {_, _} -> [{end_key, EndKey}]
+ {<<_/binary>>, true} -> [{end_key, EndKey}];
+ {undefined, _} -> []
end,
[
{user_ctx, UserCtx},
{dir, Args#mrargs.direction},
- {start_key, StartKey},
{limit, Args#mrargs.limit},
{skip, Args#mrargs.skip},
- {update_seq, Args#mrargs.update_seq}
- ] ++ EndKeyOpts.
+ {update_seq, Args#mrargs.update_seq},
+ {namespace, NS}
+ ] ++ StartKeyOpts ++ EndKeyOpts.
apply_args_to_keylist(Args, Keys0) ->
@@ -2015,8 +2020,7 @@ monitor_attachments(Att) ->
demonitor_refs(Refs) when is_list(Refs) ->
[demonitor(Ref) || Ref <- Refs].
-set_namespace(<<"_all_docs">>, Args) ->
- set_namespace(undefined, Args);
+
set_namespace(<<"_local_docs">>, Args) ->
set_namespace(<<"_local">>, Args);
set_namespace(<<"_design_docs">>, Args) ->
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 2afb780..b2945b6 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -102,8 +102,8 @@
fold_docs/3,
fold_docs/4,
- %% fold_local_docs/4,
- %% fold_design_docs/4,
+ fold_design_docs/4,
+ fold_local_docs/4,
fold_changes/4,
fold_changes/5,
%% count_changes_since/2,
@@ -136,6 +136,8 @@
"(\\.[0-9]{10,})?$" % but allow an optional shard timestamp at the end
).
+-define(FIRST_DDOC_KEY, <<"_design/">>).
+-define(LAST_DDOC_KEY, <<"_design0">>).
-define(RETURN(Term), throw({?MODULE, Term})).
@@ -314,6 +316,9 @@ get_doc_count(Db) ->
get_doc_count(Db, <<"doc_count">>).
+get_doc_count(Db, undefined) ->
+ get_doc_count(Db, <<"doc_count">>);
+
get_doc_count(Db, <<"_all_docs">>) ->
get_doc_count(Db, <<"doc_count">>);
@@ -729,8 +734,8 @@ fold_docs(Db, UserFun, UserAcc0, Options) ->
} = TxDb,
Prefix = erlfdb_tuple:pack({?DB_ALL_DOCS}, DbPrefix),
- DocCount = get_doc_count(TxDb),
-
+ NS = couch_util:get_value(namespace, Options),
+ DocCount = get_doc_count(TxDb, NS),
Meta = case lists:keyfind(update_seq, 1, Options) of
{_, true} ->
UpdateSeq = fabric2_db:get_update_seq(TxDb),
@@ -758,6 +763,42 @@ fold_docs(Db, UserFun, UserAcc0, Options) ->
end).
+fold_design_docs(Db, UserFun, UserAcc0, Options1) ->
+ Options2 = set_design_doc_keys(Options1),
+ fold_docs(Db, UserFun, UserAcc0, Options2).
+
+
+fold_local_docs(Db, UserFun, UserAcc0, Options) ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ try
+ #{
+ db_prefix := DbPrefix
+ } = TxDb,
+
+ Prefix = erlfdb_tuple:pack({?DB_LOCAL_DOCS}, DbPrefix),
+ DocCount = get_doc_count(TxDb, <<"doc_local_count">>),
+ Meta = [{total, DocCount}, {offset, null}],
+
+ UserAcc1 = maybe_stop(UserFun({meta, Meta}, UserAcc0)),
+
+ UserAcc2 = fabric2_fdb:fold_range(TxDb, Prefix, fun({K, V}, Acc) ->
+ {DocId} = erlfdb_tuple:unpack(K, Prefix),
+ LDoc = fabric2_fdb:get_local_doc(TxDb, DocId, V),
+ #doc{revs = {Pos, [Rev]}} = LDoc,
+ maybe_stop(UserFun({row, [
+ {id, DocId},
+ {key, DocId},
+ {value, {[{rev, couch_doc:rev_to_str({Pos, Rev})}]}}
+ ]}, Acc))
+ end, UserAcc1, Options),
+
+ {ok, maybe_stop(UserFun(complete, UserAcc2))}
+ catch throw:{stop, FinalUserAcc} ->
+ {ok, FinalUserAcc}
+ end
+ end).
+
+
fold_changes(Db, SinceSeq, UserFun, UserAcc) ->
fold_changes(Db, SinceSeq, UserFun, UserAcc, []).
@@ -1615,3 +1656,43 @@ maybe_stop({ok, Acc}) ->
Acc;
maybe_stop({stop, Acc}) ->
throw({stop, Acc}).
+
+
+set_design_doc_keys(Options1) ->
+ Dir = couch_util:get_value(dir, Options1, fwd),
+ Options2 = set_design_doc_start_key(Options1, Dir),
+ set_design_doc_end_key(Options2, Dir).
+
+
+set_design_doc_start_key(Options, fwd) ->
+ Key1 = couch_util:get_value(start_key, Options, ?FIRST_DDOC_KEY),
+ Key2 = max(Key1, ?FIRST_DDOC_KEY),
+ lists:keystore(start_key, 1, Options, {start_key, Key2});
+
+set_design_doc_start_key(Options, rev) ->
+ Key1 = couch_util:get_value(start_key, Options, ?LAST_DDOC_KEY),
+ Key2 = min(Key1, ?LAST_DDOC_KEY),
+ lists:keystore(start_key, 1, Options, {start_key, Key2}).
+
+
+set_design_doc_end_key(Options, fwd) ->
+ case couch_util:get_value(end_key_gt, Options) of
+ undefined ->
+ Key1 = couch_util:get_value(end_key, Options, ?LAST_DDOC_KEY),
+ Key2 = min(Key1, ?LAST_DDOC_KEY),
+ lists:keystore(end_key, 1, Options, {end_key, Key2});
+ EKeyGT ->
+ Key2 = min(EKeyGT, ?LAST_DDOC_KEY),
+ lists:keystore(end_key_gt, 1, Options, {end_key_gt, Key2})
+ end;
+
+set_design_doc_end_key(Options, rev) ->
+ case couch_util:get_value(end_key_gt, Options) of
+ undefined ->
+ Key1 = couch_util:get_value(end_key, Options, ?FIRST_DDOC_KEY),
+ Key2 = max(Key1, ?FIRST_DDOC_KEY),
+ lists:keystore(end_key, 1, Options, {end_key, Key2});
+ EKeyGT ->
+ Key2 = max(EKeyGT, ?FIRST_DDOC_KEY),
+ lists:keystore(end_key_gt, 1, Options, {end_key_gt, Key2})
+ end.
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index df37096..391122e 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -43,6 +43,7 @@
get_doc_body_future/3,
get_doc_body_wait/4,
get_local_doc/2,
+ get_local_doc/3,
write_doc/6,
write_local_doc/2,
@@ -498,6 +499,11 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
fdb_to_local_doc(Db, DocId, Val).
+get_local_doc(#{} = Db, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId, Val)
+ when is_binary(Val) orelse Val =:= not_found ->
+ fdb_to_local_doc(ensure_current(Db), DocId, Val).
+
+
write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
#{
tx := Tx,
diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs
index 70dd6e8..736cdbb 100644
--- a/test/elixir/test/basics_test.exs
+++ b/test/elixir/test/basics_test.exs
@@ -356,4 +356,123 @@ defmodule BasicsTest do
assert Map.has_key?(val, "rev")
end
+ @tag :with_db
+ test "_design_docs works", context do
+ db_name = context[:db_name]
+ body = %{:a => 1}
+
+ resp = Couch.get("/#{db_name}/_design_docs")
+ assert resp.status_code == 200
+ assert resp.body == %{"offset" => :null, "rows" => [], "total_rows" => 0}
+
+ assert Couch.put("/#{db_name}/doc1", body: body).body["ok"]
+
+ # Make sure regular documents didn't get picked up
+ resp = Couch.get("/#{db_name}/_design_docs")
+ assert resp.status_code == 200
+ assert resp.body == %{"offset" => :null, "rows" => [], "total_rows" => 0}
+
+ # Add _design/doc1
+ assert Couch.put("/#{db_name}/_design/doc1", body: body).body["ok"]
+ resp = Couch.get("/#{db_name}/_design_docs")
+ assert resp.status_code == 200
+ assert resp.body["total_rows"] == 1
+ [row] = resp.body["rows"]
+
+ assert row["id"] == "_design/doc1"
+ assert row["key"] == "_design/doc1"
+
+ val = row["value"]
+ assert Map.has_key?(val, "rev")
+
+ # Add _design/doc5
+ assert Couch.put("/#{db_name}/_design/doc5", body: body).body["ok"]
+ resp = Couch.get("/#{db_name}/_design_docs")
+ assert resp.status_code == 200
+ [row1, row2] = resp.body["rows"]
+ assert row1["id"] == "_design/doc1"
+ assert row2["id"] == "_design/doc5"
+
+ # descending=true
+ resp = Couch.get("/#{db_name}/_design_docs?descending=true")
+ assert resp.status_code == 200
+ [row1, row2] = resp.body["rows"]
+ assert row1["id"] == "_design/doc5"
+ assert row2["id"] == "_design/doc1"
+
+ # start_key=doc2
+ resp = Couch.get("/#{db_name}/_design_docs?start_key=\"_design/doc2\"")
+ assert resp.status_code == 200
+ [row] = resp.body["rows"]
+ assert row["id"] == "_design/doc5"
+
+ # end_key=doc2
+ resp = Couch.get("/#{db_name}/_design_docs?end_key=\"_design/doc2\"")
+ assert resp.status_code == 200
+ [row] = resp.body["rows"]
+ assert row["id"] == "_design/doc1"
+
+ # inclusive_end=false
+ qstr = "start_key=\"_design/doc2\"&end_key=\"_design/doc5\"&inclusive_end=false"
+ resp = Couch.get("/#{db_name}/_design_docs?" <> qstr)
+ assert resp.status_code == 200
+ assert resp.body == %{"offset" => :null, "rows" => [], "total_rows" => 2}
+ end
+
+ @tag :with_db
+ test "_local_docs works", context do
+ db_name = context[:db_name]
+ body = %{:a => 1}
+
+ resp = Couch.get("/#{db_name}/_local_docs")
+ assert resp.status_code == 200
+ assert resp.body == %{"offset" => :null, "rows" => [], "total_rows" => 0}
+
+ # Add _local/doc1
+ assert Couch.put("/#{db_name}/_local/doc1", body: body).body["ok"]
+ resp = Couch.get("/#{db_name}/_local_docs")
+ assert resp.status_code == 200
+ assert resp.body["total_rows"] == 1
+ [row] = resp.body["rows"]
+
+ assert row["id"] == "_local/doc1"
+ assert row["key"] == "_local/doc1"
+
+ val = row["value"]
+ assert Map.has_key?(val, "rev")
+
+ # Add _local/doc5
+ assert Couch.put("/#{db_name}/_local/doc5", body: body).body["ok"]
+ resp = Couch.get("/#{db_name}/_local_docs")
+ assert resp.status_code == 200
+ [row1, row2] = resp.body["rows"]
+ assert row1["id"] == "_local/doc1"
+ assert row2["id"] == "_local/doc5"
+
+ # descending=true
+ resp = Couch.get("/#{db_name}/_local_docs?descending=true")
+ assert resp.status_code == 200
+ [row1, row2] = resp.body["rows"]
+ assert row1["id"] == "_local/doc5"
+ assert row2["id"] == "_local/doc1"
+
+ # start_key=doc2
+ resp = Couch.get("/#{db_name}/_local_docs?start_key=\"_local/doc2\"")
+ assert resp.status_code == 200
+ [row] = resp.body["rows"]
+ assert row["id"] == "_local/doc5"
+
+ # end_key=doc2
+ resp = Couch.get("/#{db_name}/_local_docs?end_key=\"_local/doc2\"")
+ assert resp.status_code == 200
+ [row] = resp.body["rows"]
+ assert row["id"] == "_local/doc1"
+
+ # inclusive_end=false
+ qstr = "start_key=\"_local/doc2\"&end_key=\"_local/doc5\"&inclusive_end=false"
+ resp = Couch.get("/#{db_name}/_local_docs?" <> qstr)
+ assert resp.status_code == 200
+ assert resp.body == %{"offset" => :null, "rows" => [], "total_rows" => 2}
+ end
+
end