You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/06/10 19:52:18 UTC
[couchdb] branch prototype/fdb-layer updated (4fbabcc -> 21af513)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 4fbabcc Fix fabric2_txids:terminate/2
new e3f24aa Fix revision generation on attachment upload
new 09f4489 Convert attachment info to disk terms correctly
new 4933c08 Allow for previously configured filters
new 5331966 Fix validate_doc_update when recreating a document
new 7073798 Database config changes should bump the db version
new 21af513 Implement `_users` db authentication
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd_auth_cache.erl | 40 ++++--
src/chttpd/src/chttpd_changes.erl | 3 +
src/chttpd/src/chttpd_db.erl | 5 +-
src/fabric/src/fabric2_db.erl | 144 ++++++++++++++-------
src/fabric/src/fabric2_fdb.erl | 32 +++--
.../src/fabric2_users_db.erl} | 61 +++++----
src/fabric/src/fabric2_util.erl | 7 +
7 files changed, 191 insertions(+), 101 deletions(-)
copy src/{couch/src/couch_users_db.erl => fabric/src/fabric2_users_db.erl} (79%)
[couchdb] 05/06: Database config changes should bump the db version
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7073798325475fd9056b8cd221a691a28848f1dd
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Jun 10 14:36:55 2019 -0500
Database config changes should bump the db version
This was a remnant before we used a version per database.
---
src/fabric/src/fabric2_fdb.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 4f08d97..d179387 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -338,7 +338,7 @@ set_config(#{} = Db, ConfigKey, ConfigVal) ->
Key = erlfdb_tuple:pack({?DB_CONFIG, ConfigKey}, DbPrefix),
erlfdb:set(Tx, Key, ConfigVal),
- bump_metadata_version(Tx).
+ bump_db_version(Db).
get_stat(#{} = Db, StatKey) ->
[couchdb] 04/06: Fix validate_doc_update when recreating a document
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5331966c20c516ed2fac9497b68d49047d6b72e9
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Jun 10 14:35:11 2019 -0500
Fix validate_doc_update when recreating a document
This fixes the behavior when validating a document update that is
recreating a previously deleted document. Before this fix we were
sending a document body with `"_deleted":true` as the existing document.
However, CouchDB behavior expects the previous document passed to VDU's
to be `null` in this case.
---
src/fabric/src/fabric2_db.erl | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index acd473f..48e50f1 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1196,8 +1196,13 @@ prep_and_validate(Db, NewDoc, PrevRevInfo) ->
_ -> false
end,
+ WasDeleted = case PrevRevInfo of
+ not_found -> false;
+ #{deleted := D} -> D
+ end,
+
PrevDoc = case HasStubs orelse (HasVDUs and not IsDDoc) of
- true when PrevRevInfo /= not_found ->
+ true when PrevRevInfo /= not_found, not WasDeleted ->
case fabric2_fdb:get_doc_body(Db, NewDoc#doc.id, PrevRevInfo) of
#doc{} = PDoc -> PDoc;
{not_found, _} -> nil
[couchdb] 06/06: Implement `_users` db authentication
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 21af513ce4c67caa77f12e650a3d3462fbf42948
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Jun 7 12:46:06 2019 -0500
Implement `_users` db authentication
This changes `chttpd_auth_cache` to use FoundationDB to back the
`_users` database including the `before_doc_update` and `after_doc_read`
features.
---
src/chttpd/src/chttpd_auth_cache.erl | 40 ++++++----
src/chttpd/src/chttpd_db.erl | 5 +-
src/fabric/src/fabric2_db.erl | 34 ++++++---
src/fabric/src/fabric2_fdb.erl | 8 +-
src/fabric/src/fabric2_users_db.erl | 144 +++++++++++++++++++++++++++++++++++
src/fabric/src/fabric2_util.erl | 7 ++
6 files changed, 212 insertions(+), 26 deletions(-)
diff --git a/src/chttpd/src/chttpd_auth_cache.erl b/src/chttpd/src/chttpd_auth_cache.erl
index 638d8c7..d947fe6 100644
--- a/src/chttpd/src/chttpd_auth_cache.erl
+++ b/src/chttpd/src/chttpd_auth_cache.erl
@@ -52,7 +52,8 @@ get_user_creds(_Req, UserName) when is_binary(UserName) ->
update_user_creds(_Req, UserDoc, _Ctx) ->
{_, Ref} = spawn_monitor(fun() ->
- case fabric:update_doc(dbname(), UserDoc, []) of
+ {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+ case fabric2_db:update_doc(Db, UserDoc) of
{ok, _} ->
exit(ok);
Else ->
@@ -100,6 +101,14 @@ maybe_increment_auth_cache_miss(UserName) ->
%% gen_server callbacks
init([]) ->
+ try
+ fabric2_db:open(dbname(), [?ADMIN_CTX])
+ catch error:database_does_not_exist ->
+ case fabric2_db:create(dbname(), [?ADMIN_CTX]) of
+ {ok, _} -> ok;
+ {error, file_exists} -> ok
+ end
+ end,
self() ! {start_listener, 0},
{ok, #state{}}.
@@ -139,7 +148,8 @@ spawn_changes(Since) ->
Pid.
listen_for_changes(Since) ->
- ensure_auth_ddoc_exists(dbname(), <<"_design/_auth">>),
+ {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+ ensure_auth_ddoc_exists(Db, <<"_design/_auth">>),
CBFun = fun ?MODULE:changes_callback/2,
Args = #changes_args{
feed = "continuous",
@@ -147,7 +157,8 @@ listen_for_changes(Since) ->
heartbeat = true,
filter = {default, main_only}
},
- fabric:changes(dbname(), CBFun, Since, Args).
+ ChangesFun = chttpd_changes:handle_db_changes(Args, nil, Db),
+ ChangesFun({CBFun, Since}).
changes_callback(waiting_for_updates, Acc) ->
{ok, Acc};
@@ -156,7 +167,7 @@ changes_callback(start, Since) ->
changes_callback({stop, EndSeq, _Pending}, _) ->
exit({seq, EndSeq});
changes_callback({change, {Change}}, _) ->
- case couch_util:get_value(id, Change) of
+ case couch_util:get_value(<<"id">>, Change) of
<<"_design/", _/binary>> ->
ok;
DocId ->
@@ -171,7 +182,8 @@ changes_callback({error, _}, EndSeq) ->
exit({seq, EndSeq}).
load_user_from_db(UserName) ->
- try fabric:open_doc(dbname(), docid(UserName), [?ADMIN_CTX, ejson_body, conflicts]) of
+ {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+ try fabric2_db:open_doc(Db, docid(UserName), [conflicts]) of
{ok, Doc} ->
{Props} = couch_doc:to_json_obj(Doc, []),
Props;
@@ -183,7 +195,8 @@ load_user_from_db(UserName) ->
end.
dbname() ->
- config:get("chttpd_auth", "authentication_db", "_users").
+ DbNameStr = config:get("chttpd_auth", "authentication_db", "_users"),
+ iolist_to_binary(DbNameStr).
docid(UserName) ->
<<"org.couchdb.user:", UserName/binary>>.
@@ -191,11 +204,11 @@ docid(UserName) ->
username(<<"org.couchdb.user:", UserName/binary>>) ->
UserName.
-ensure_auth_ddoc_exists(DbName, DDocId) ->
- case fabric:open_doc(DbName, DDocId, [?ADMIN_CTX, ejson_body]) of
+ensure_auth_ddoc_exists(Db, DDocId) ->
+ case fabric2_db:open_doc(Db, DDocId) of
{not_found, _Reason} ->
{ok, AuthDesign} = couch_auth_cache:auth_design_doc(DDocId),
- update_doc_ignoring_conflict(DbName, AuthDesign, [?ADMIN_CTX]);
+ update_doc_ignoring_conflict(Db, AuthDesign);
{ok, Doc} ->
{Props} = couch_doc:to_json_obj(Doc, []),
case couch_util:get_value(<<"validate_doc_update">>, Props, []) of
@@ -205,17 +218,18 @@ ensure_auth_ddoc_exists(DbName, DDocId) ->
Props1 = lists:keyreplace(<<"validate_doc_update">>, 1, Props,
{<<"validate_doc_update">>,
?AUTH_DB_DOC_VALIDATE_FUNCTION}),
- update_doc_ignoring_conflict(DbName, couch_doc:from_json_obj({Props1}), [?ADMIN_CTX])
+ NewDoc = couch_doc:from_json_obj({Props1}),
+ update_doc_ignoring_conflict(Db, NewDoc)
end;
{error, Reason} ->
- couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [DbName, DDocId, Reason]),
+ couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [dbname(), DDocId, Reason]),
ok
end,
ok.
-update_doc_ignoring_conflict(DbName, Doc, Options) ->
+update_doc_ignoring_conflict(DbName, Doc) ->
try
- fabric:update_doc(DbName, Doc, Options)
+ fabric2_db:update_doc(DbName, Doc)
catch
throw:conflict ->
ok
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 40c1a1e..4337041 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -724,10 +724,9 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_revs_diff">>]}=Req, Db) ->
db_req(#httpd{path_parts=[_,<<"_revs_diff">>]}=Req, _Db) ->
send_method_not_allowed(Req, "POST");
-db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>],user_ctx=Ctx}=Req,
- Db) ->
+db_req(#httpd{method = 'PUT',path_parts = [_, <<"_security">>]} = Req, Db) ->
SecObj = chttpd:json_body(Req),
- case fabric:set_security(Db, SecObj, [{user_ctx, Ctx}]) of
+ case fabric2_db:set_security(Db, SecObj) of
ok ->
send_json(Req, {[{<<"ok">>, true}]});
Else ->
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 48e50f1..80028a6 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -149,9 +149,10 @@ create(DbName, Options) ->
% We cache outside of the transaction so that we're sure
% that the transaction was committed.
case Result of
- #{} = Db ->
- ok = fabric2_server:store(Db),
- {ok, Db#{tx := undefined}};
+ #{} = Db0 ->
+ Db1 = maybe_add_sys_db_callbacks(Db0),
+ ok = fabric2_server:store(Db1),
+ {ok, Db1#{tx := undefined}};
Error ->
Error
end.
@@ -167,9 +168,10 @@ open(DbName, Options) ->
end),
% Cache outside the transaction retry loop
case Result of
- #{} = Db ->
- ok = fabric2_server:store(Db),
- {ok, Db#{tx := undefined}};
+ #{} = Db0 ->
+ Db1 = maybe_add_sys_db_callbacks(Db0),
+ ok = fabric2_server:store(Db1),
+ {ok, Db1#{tx := undefined}};
Error ->
Error
end
@@ -552,18 +554,19 @@ update_docs(Db, Docs) ->
update_docs(Db, Docs, []).
-update_docs(Db, Docs, Options) ->
+update_docs(Db, Docs0, Options) ->
+ Docs1 = apply_before_doc_update(Db, Docs0, Options),
Resps0 = case lists:member(replicated_changes, Options) of
false ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
- update_docs_interactive(TxDb, Docs, Options)
+ update_docs_interactive(TxDb, Docs1, Options)
end);
true ->
lists:map(fun(Doc) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
update_doc_int(TxDb, Doc, Options)
end)
- end, Docs)
+ end, Docs1)
end,
% Convert errors
Resps1 = lists:map(fun(Resp) ->
@@ -882,6 +885,19 @@ find_possible_ancestors(RevInfos, MissingRevs) ->
end, RevInfos).
+apply_before_doc_update(Db, Docs, Options) ->
+ #{before_doc_update := BDU} = Db,
+ UpdateType = case lists:member(replicated_changes, Options) of
+ true -> replicated_changes;
+ false -> interactive_edit
+ end,
+ if BDU == undefined -> Docs; true ->
+ lists:map(fun(Doc) ->
+ BDU(Doc, Db, UpdateType)
+ end, Docs)
+ end.
+
+
update_doc_int(#{} = Db, #doc{} = Doc, Options) ->
IsLocal = case Doc#doc.id of
<<?LOCAL_DOC_PREFIX, _/binary>> -> true;
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index d179387..4b01826 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -944,7 +944,13 @@ fdb_to_doc(Db, DocId, Pos, Path, Bin) when is_binary(Bin) ->
body = Body,
atts = Atts,
deleted = Deleted
- };
+ },
+
+ case Db of
+ #{after_doc_read := undefined} -> Doc0;
+ #{after_doc_read := ADR} -> ADR(Doc0, Db)
+ end;
+
fdb_to_doc(_Db, _DocId, _Pos, _Path, not_found) ->
{not_found, missing}.
diff --git a/src/fabric/src/fabric2_users_db.erl b/src/fabric/src/fabric2_users_db.erl
new file mode 100644
index 0000000..9a8a462
--- /dev/null
+++ b/src/fabric/src/fabric2_users_db.erl
@@ -0,0 +1,144 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_users_db).
+
+-export([
+ before_doc_update/3,
+ after_doc_read/2,
+ strip_non_public_fields/1
+]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+-define(NAME, <<"name">>).
+-define(PASSWORD, <<"password">>).
+-define(DERIVED_KEY, <<"derived_key">>).
+-define(PASSWORD_SCHEME, <<"password_scheme">>).
+-define(SIMPLE, <<"simple">>).
+-define(PASSWORD_SHA, <<"password_sha">>).
+-define(PBKDF2, <<"pbkdf2">>).
+-define(ITERATIONS, <<"iterations">>).
+-define(SALT, <<"salt">>).
+-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
+
+-define(
+ DDOCS_ADMIN_ONLY,
+ <<"Only administrators can view design docs in the users database.">>
+).
+
+% If the request's userCtx identifies an admin
+% -> save_doc (see below)
+%
+% If the request's userCtx.name is null:
+% -> save_doc
+% // this is an anonymous user registering a new document
+% // in case a user doc with the same id already exists, the anonymous
+% // user will get a regular doc update conflict.
+% If the request's userCtx.name doesn't match the doc's name
+% -> 404 // Not Found
+% Else
+% -> save_doc
+before_doc_update(Doc, Db, _UpdateType) ->
+ #user_ctx{name = Name} = fabric2_db:get_user_ctx(Db),
+ DocName = get_doc_name(Doc),
+ case (catch fabric2_db:check_is_admin(Db)) of
+ ok ->
+ save_doc(Doc);
+ _ when Name =:= DocName orelse Name =:= null ->
+ save_doc(Doc);
+ _ ->
+ throw(not_found)
+ end.
+
+% If newDoc.password == null || newDoc.password == undefined:
+% ->
+% noop
+% Else -> // calculate password hash server side
+% newDoc.password_sha = hash_pw(newDoc.password + salt)
+% newDoc.salt = salt
+% newDoc.password = null
+save_doc(#doc{body={Body}} = Doc) ->
+ %% Support both schemes to smooth migration from legacy scheme
+ Scheme = config:get("couch_httpd_auth", "password_scheme", "pbkdf2"),
+ case {fabric2_util:get_value(?PASSWORD, Body), Scheme} of
+ {null, _} -> % server admins don't have a user-db password entry
+ Doc;
+ {undefined, _} ->
+ Doc;
+ {ClearPassword, "simple"} -> % deprecated
+ Salt = couch_uuids:random(),
+ PasswordSha = couch_passwords:simple(ClearPassword, Salt),
+ Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?SIMPLE),
+ Body1 = ?replace(Body0, ?SALT, Salt),
+ Body2 = ?replace(Body1, ?PASSWORD_SHA, PasswordSha),
+ Body3 = proplists:delete(?PASSWORD, Body2),
+ Doc#doc{body={Body3}};
+ {ClearPassword, "pbkdf2"} ->
+ Iterations = list_to_integer(config:get("couch_httpd_auth", "iterations", "1000")),
+ Salt = couch_uuids:random(),
+ DerivedKey = couch_passwords:pbkdf2(ClearPassword, Salt, Iterations),
+ Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?PBKDF2),
+ Body1 = ?replace(Body0, ?ITERATIONS, Iterations),
+ Body2 = ?replace(Body1, ?DERIVED_KEY, DerivedKey),
+ Body3 = ?replace(Body2, ?SALT, Salt),
+ Body4 = proplists:delete(?PASSWORD, Body3),
+ Doc#doc{body={Body4}};
+ {_ClearPassword, Scheme} ->
+ couch_log:error("[couch_httpd_auth] password_scheme value of '~p' is invalid.", [Scheme]),
+ throw({forbidden, "Server cannot hash passwords at this time."})
+ end.
+
+
+% If the doc is a design doc
+% If the request's userCtx identifies an admin
+% -> return doc
+% Else
+% -> 403 // Forbidden
+% If the request's userCtx identifies an admin
+% -> return doc
+% If the request's userCtx.name doesn't match the doc's name
+% -> 404 // Not Found
+% Else
+% -> return doc
+after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, Db) ->
+ case (catch fabric2_db:check_is_admin(Db)) of
+ ok -> Doc;
+ _ -> throw({forbidden, ?DDOCS_ADMIN_ONLY})
+ end;
+after_doc_read(Doc, Db) ->
+ #user_ctx{name = Name} = fabric2_db:get_user_ctx(Db),
+ DocName = get_doc_name(Doc),
+ case (catch fabric2_db:check_is_admin(Db)) of
+ ok ->
+ Doc;
+ _ when Name =:= DocName ->
+ Doc;
+ _ ->
+ Doc1 = strip_non_public_fields(Doc),
+ case Doc1 of
+ #doc{body={[]}} -> throw(not_found);
+ _ -> Doc1
+ end
+ end.
+
+
+get_doc_name(#doc{id= <<"org.couchdb.user:", Name/binary>>}) ->
+ Name;
+get_doc_name(_) ->
+ undefined.
+
+
+strip_non_public_fields(#doc{body={Props}}=Doc) ->
+ PublicFields = config:get("couch_httpd_auth", "public_fields", ""),
+ Public = re:split(PublicFields, "\\s*,\\s*", [{return, binary}]),
+ Doc#doc{body={[{K, V} || {K, V} <- Props, lists:member(K, Public)]}}.
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 6e2df67..fb59d59 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -24,6 +24,8 @@
validate_security_object/1,
+ dbname_ends_with/2,
+
get_value/2,
get_value/3,
to_hex/1,
@@ -113,6 +115,11 @@ validate_json_list_of_strings(Member, Props) ->
end.
+dbname_ends_with(#{} = Db, Suffix) when is_binary(Suffix) ->
+ DbName = fabric2_db:name(Db),
+ Suffix == filename:basename(DbName).
+
+
get_value(Key, List) ->
get_value(Key, List, undefined).
[couchdb] 01/06: Fix revision generation on attachment upload
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e3f24aa1aa6eb389a0512670db0090cf03014506
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Jun 7 15:02:19 2019 -0500
Fix revision generation on attachment upload
When uploading an attachment we hadn't yet flushed data to FoundationDB
which caused the md5 to be empty. The `new_revid` algorithm then
declared that was because it was an old style attachment and thus our
new revision would be a random number.
This fix just flushes our attachments earlier in the process of updating
a document.
---
src/fabric/src/fabric2_db.erl | 103 +++++++++++++++++++++++++++--------------
src/fabric/src/fabric2_fdb.erl | 9 +---
2 files changed, 70 insertions(+), 42 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 02a18fa..acd473f 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -120,7 +120,7 @@
%% validate_dbname/1,
%% make_doc/5,
- new_revid/1
+ new_revid/2
]).
@@ -604,9 +604,7 @@ read_attachment(Db, DocId, AttId) ->
write_attachment(Db, DocId, Att) ->
Data = couch_att:fetch(data, Att),
- {ok, AttId} = fabric2_fdb:transactional(Db, fun(TxDb) ->
- fabric2_fdb:write_attachment(TxDb, DocId, Data)
- end),
+ {ok, AttId} = fabric2_fdb:write_attachment(Db, DocId, Data),
couch_att:store(data, {loc, Db, DocId, AttId}, Att).
@@ -630,33 +628,69 @@ fold_changes(Db, SinceSeq, UserFun, UserAcc, Options) ->
end).
-new_revid(Doc) ->
+maybe_add_sys_db_callbacks(Db) ->
+ IsReplicatorDb = fabric2_util:dbname_ends_with(Db, <<"_replicator">>),
+
+ CfgUsersSuffix = config:get("couchdb", "users_db_suffix", "_users"),
+ IsCfgUsersDb = fabric2_util:dbname_ends_with(Db, ?l2b(CfgUsersSuffix)),
+ IsGlobalUsersDb = fabric2_util:dbname_ends_with(Db, <<"_users">>),
+ IsUsersDb = IsCfgUsersDb orelse IsGlobalUsersDb,
+
+ {BDU, ADR} = if
+ IsReplicatorDb ->
+ {
+ fun couch_replicator_docs:before_doc_update/3,
+ fun couch_replicator_docs:after_doc_read/2
+ };
+ IsUsersDb ->
+ {
+ fun fabric2_users_db:before_doc_update/3,
+ fun fabric2_users_db:after_doc_read/2
+ };
+ true ->
+ {undefined, undefined}
+ end,
+
+ Db#{
+ before_doc_update := BDU,
+ after_doc_read := ADR
+ }.
+
+
+new_revid(Db, Doc) ->
#doc{
+ id = DocId,
body = Body,
revs = {OldStart, OldRevs},
atts = Atts,
deleted = Deleted
} = Doc,
- DigestedAtts = lists:foldl(fun(Att, Acc) ->
- [N, T, M] = couch_att:fetch([name, type, md5], Att),
- case M == <<>> of
- true -> Acc;
- false -> [{N, T, M} | Acc]
+ {NewAtts, AttSigInfo} = lists:mapfoldl(fun(Att, Acc) ->
+ [Name, Type, Data, Md5] = couch_att:fetch([name, type, data, md5], Att),
+ case Data of
+ {loc, _, _, _} ->
+ {Att, [{Name, Type, Md5} | Acc]};
+ _ ->
+ Att1 = couch_att:flush(Db, DocId, Att),
+ Att2 = couch_att:store(revpos, OldStart + 1, Att1),
+ {Att2, [{Name, Type, couch_att:fetch(md5, Att2)} | Acc]}
end
end, [], Atts),
- Rev = case DigestedAtts of
- Atts2 when length(Atts) =/= length(Atts2) ->
- % We must have old style non-md5 attachments
- list_to_binary(integer_to_list(couch_util:rand32()));
- Atts2 ->
+ Rev = case length(Atts) == length(AttSigInfo) of
+ true ->
OldRev = case OldRevs of [] -> 0; [OldRev0 | _] -> OldRev0 end,
- SigTerm = [Deleted, OldStart, OldRev, Body, Atts2],
- couch_hash:md5_hash(term_to_binary(SigTerm, [{minor_version, 1}]))
+ SigTerm = [Deleted, OldStart, OldRev, Body, AttSigInfo],
+ couch_hash:md5_hash(term_to_binary(SigTerm, [{minor_version, 1}]));
+ false ->
+ erlang:error(missing_att_info)
end,
- Doc#doc{revs = {OldStart + 1, [Rev | OldRevs]}}.
+ Doc#doc{
+ revs = {OldStart + 1, [Rev | OldRevs]},
+ atts = NewAtts
+ }.
maybe_set_user_ctx(Db, Options) ->
@@ -970,12 +1004,11 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
% Validate the doc update and create the
% new revinfo map
Doc2 = prep_and_validate(Db, Doc1, Target),
+
#doc{
deleted = NewDeleted,
revs = {NewRevPos, [NewRev | NewRevPath]}
- } = Doc3 = new_revid(Doc2),
-
- Doc4 = update_attachment_revpos(Doc3),
+ } = Doc3 = new_revid(Db, Doc2),
NewRevInfo = #{
winner => undefined,
@@ -988,9 +1021,9 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
% Gather the list of possible winnig revisions
Possible = case Target == Winner of
- true when not Doc4#doc.deleted ->
+ true when not Doc3#doc.deleted ->
[NewRevInfo];
- true when Doc4#doc.deleted ->
+ true when Doc3#doc.deleted ->
case SecondPlace of
#{} -> [NewRevInfo, SecondPlace];
not_found -> [NewRevInfo]
@@ -1015,7 +1048,7 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
ok = fabric2_fdb:write_doc(
Db,
- Doc4,
+ Doc3,
NewWinner,
Winner,
ToUpdate,
@@ -1076,6 +1109,7 @@ update_doc_replicated(Db, Doc0, _Options) ->
LeafPath = get_leaf_path(RevPos, Rev, AllLeafsFull),
PrevRevInfo = find_prev_revinfo(RevPos, LeafPath),
Doc2 = prep_and_validate(Db, Doc1, PrevRevInfo),
+ Doc3 = flush_doc_atts(Db, Doc2),
% Possible winners are the previous winner and
% the new DocRevInfo
@@ -1097,7 +1131,7 @@ update_doc_replicated(Db, Doc0, _Options) ->
ok = fabric2_fdb:write_doc(
Db,
- Doc2,
+ Doc3,
NewWinner,
Winner,
ToUpdate,
@@ -1119,19 +1153,20 @@ update_local_doc(Db, Doc0, _Options) ->
{ok, {0, integer_to_binary(Rev)}}.
-update_attachment_revpos(#doc{revs = {RevPos, _Revs}, atts = Atts0} = Doc) ->
- Atts = lists:map(fun(Att) ->
+flush_doc_atts(Db, Doc) ->
+ #doc{
+ id = DocId,
+ atts = Atts
+ } = Doc,
+ NewAtts = lists:map(fun(Att) ->
case couch_att:fetch(data, Att) of
- {loc, _Db, _DocId, _AttId} ->
- % Attachment was already on disk
+ {loc, _, _, _} ->
Att;
_ ->
- % We will write this attachment with this update
- % so mark it with the RevPos that will be written
- couch_att:store(revpos, RevPos, Att)
+ couch_att:flush(Db, DocId, Att)
end
- end, Atts0),
- Doc#doc{atts = Atts}.
+ end, Atts),
+ Doc#doc{atts = NewAtts}.
get_winning_rev_futures(Db, Docs) ->
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 0a4f298..788bbc6 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -924,7 +924,7 @@ doc_to_fdb(Db, #doc{} = Doc) ->
body = Body,
atts = Atts,
deleted = Deleted
- } = doc_flush_atts(Db, Doc),
+ } = Doc,
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev}, DbPrefix),
Val = {Body, Atts, Deleted},
@@ -977,13 +977,6 @@ fdb_to_local_doc(_Db, _DocId, not_found) ->
{not_found, missing}.
-doc_flush_atts(Db, Doc) ->
- Atts = lists:map(fun(Att) ->
- couch_att:flush(Db, Doc#doc.id, Att)
- end, Doc#doc.atts),
- Doc#doc{atts = Atts}.
-
-
chunkify_attachment(Data) ->
case Data of
<<>> ->
[couchdb] 02/06: Convert attachment info to disk terms correctly
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 09f4489533ee1976c3c683301dc7e1332b02bafb
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Jun 7 16:12:28 2019 -0500
Convert attachment info to disk terms correctly
I was accidentally skipping this step around properly
serializing/deserializing attachments.
Note to self: If someon specifies attachment headers this will likely
break when we attempt to pack the value tuple here.
---
src/fabric/src/fabric2_fdb.erl | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 788bbc6..4f08d97 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -926,14 +926,19 @@ doc_to_fdb(Db, #doc{} = Doc) ->
deleted = Deleted
} = Doc,
+ DiskAtts = lists:map(fun couch_att:to_disk_term/1, Atts),
+
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev}, DbPrefix),
- Val = {Body, Atts, Deleted},
+ Val = {Body, DiskAtts, Deleted},
{Key, term_to_binary(Val, [{minor_version, 1}])}.
-fdb_to_doc(_Db, DocId, Pos, Path, Bin) when is_binary(Bin) ->
- {Body, Atts, Deleted} = binary_to_term(Bin, [safe]),
- #doc{
+fdb_to_doc(Db, DocId, Pos, Path, Bin) when is_binary(Bin) ->
+ {Body, DiskAtts, Deleted} = binary_to_term(Bin, [safe]),
+ Atts = lists:map(fun(Att) ->
+ couch_att:from_disk_term(Db, DocId, Att)
+ end, DiskAtts),
+ Doc0 = #doc{
id = DocId,
revs = {Pos, Path},
body = Body,
[couchdb] 03/06: Allow for previously configured filters
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4933c08efee96b894abfd463266e438f8e708703
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Jun 10 14:33:12 2019 -0500
Allow for previously configured filters
The older chttpd/fabric split configured filters as one step in the
coordinator instead of within each RPC worker.
---
src/chttpd/src/chttpd_changes.erl | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/chttpd/src/chttpd_changes.erl b/src/chttpd/src/chttpd_changes.erl
index 620f68d..d27bbad 100644
--- a/src/chttpd/src/chttpd_changes.erl
+++ b/src/chttpd/src/chttpd_changes.erl
@@ -197,6 +197,9 @@ get_callback_acc(Callback) when is_function(Callback, 1) ->
{fun(Ev, _) -> Callback(Ev) end, ok}.
+configure_filter(Filter, _Style, _Req, _Db) when is_tuple(Filter) ->
+ % Filter has already been configured
+ Filter;
configure_filter("_doc_ids", Style, Req, _Db) ->
{doc_ids, Style, get_doc_ids(Req)};
configure_filter("_selector", Style, Req, _Db) ->