You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2019/08/22 19:35:22 UTC
[couchdb] 01/01: Implement fabric2_db EPI plugin
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch prototype/fdb-layer-epi
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4814664d533c32908e9b6e7c905028e5e6effe64
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu Aug 22 15:23:53 2019 -0400
Implement fabric2_db EPI plugin
This mostly equivalent to the `couch_db` EPI plugin, but using fabric2 calls
and without some of the functions that are not relevent to FDB such as
on_compact/1 and others..
Unfortunately had to revert fabric2_sup to tuples temporarily until we rebase
from master and pick up commit 1db0294eb1093066773760b75a200d99aa453be8
---
rel/apps/couch_epi.config | 1 +
src/chttpd/src/chttpd_db.erl | 8 +-
src/fabric/src/fabric.app.src | 1 +
src/fabric/src/fabric2_db.erl | 136 ++++++++++++++++-----
src/fabric/src/fabric2_db_plugin.erl | 92 ++++++++++++++
.../fabric/src/fabric2_epi.erl | 48 ++++++--
src/fabric/src/fabric2_sup.erl | 30 +++--
7 files changed, 259 insertions(+), 57 deletions(-)
diff --git a/rel/apps/couch_epi.config b/rel/apps/couch_epi.config
index a53721a..0f3d2da 100644
--- a/rel/apps/couch_epi.config
+++ b/rel/apps/couch_epi.config
@@ -12,6 +12,7 @@
{plugins, [
couch_db_epi,
+ fabric2_epi,
chttpd_epi,
couch_index_epi,
dreyfus_epi,
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 785ca3f..dbb92fa 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -1047,7 +1047,7 @@ db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) ->
db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
couch_httpd:validate_referer(Req),
- couch_doc:validate_docid(DocId, fabric2_db:name(Db)),
+ fabric2_db:validate_docid(DocId),
chttpd:validate_ctype(Req, "multipart/form-data"),
Options = [{user_ctx,Ctx}],
@@ -1107,7 +1107,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
update_type = UpdateType
} = parse_doc_query(Req),
DbName = fabric2_db:name(Db),
- couch_doc:validate_docid(DocId, fabric2_db:name(Db)),
+ fabric2_db:validate_docid(DocId),
Options = [{user_ctx, Ctx}],
@@ -1668,7 +1668,7 @@ db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts)
% check for the existence of the doc to handle the 404 case.
couch_doc_open(Db, DocId, nil, [])
end,
- couch_doc:validate_docid(DocId, fabric2_db:name(Db)),
+ fabric2_db:validate_docid(DocId),
#doc{id=DocId};
Rev ->
case fabric2_db:open_doc_revs(Db, DocId, [Rev], [{user_ctx,Ctx}]) of
@@ -2031,7 +2031,7 @@ bulk_get_open_doc_revs1(Db, Props, Options, {}) ->
{null, {error, Error}, Options};
DocId ->
try
- couch_doc:validate_docid(DocId, fabric2_db:name(Db)),
+ fabric2_db:validate_docid(DocId),
bulk_get_open_doc_revs1(Db, Props, Options, {DocId})
catch throw:{Error, Reason} ->
{DocId, {error, {null, Error, Reason}}, Options}
diff --git a/src/fabric/src/fabric.app.src b/src/fabric/src/fabric.app.src
index 20fbb1e..77260f9 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/fabric/src/fabric.app.src
@@ -21,6 +21,7 @@
kernel,
stdlib,
config,
+ couch_epi,
couch,
rexi,
mem3,
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index c926da9..2afb780 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -85,7 +85,7 @@
%% get_minimum_purge_seq/1,
%% purge_client_exists/3,
- %% validate_docid/2,
+ validate_docid/1,
%% doc_from_json_obj_validate/2,
update_doc/2,
@@ -118,9 +118,9 @@
%% wait_for_compaction/1,
%% wait_for_compaction/2,
- %% dbname_suffix/1,
- %% normalize_dbname/1,
- %% validate_dbname/1,
+ dbname_suffix/1,
+ normalize_dbname/1,
+ validate_dbname/1,
%% make_doc/5,
new_revid/2
@@ -141,21 +141,26 @@
create(DbName, Options) ->
- Result = fabric2_fdb:transactional(DbName, Options, fun(TxDb) ->
- case fabric2_fdb:exists(TxDb) of
- true ->
- {error, file_exists};
- false ->
- fabric2_fdb:create(TxDb, Options)
- end
- end),
- % We cache outside of the transaction so that we're sure
- % that the transaction was committed.
- case Result of
- #{} = Db0 ->
- Db1 = maybe_add_sys_db_callbacks(Db0),
- ok = fabric2_server:store(Db1),
- {ok, Db1#{tx := undefined}};
+ case validate_dbname(DbName) of
+ ok ->
+ Result = fabric2_fdb:transactional(DbName, Options, fun(TxDb) ->
+ case fabric2_fdb:exists(TxDb) of
+ true ->
+ {error, file_exists};
+ false ->
+ fabric2_fdb:create(TxDb, Options)
+ end
+ end),
+ % We cache outside of the transaction so that we're sure
+ % that the transaction was committed.
+ case Result of
+ #{} = Db0 ->
+ Db1 = maybe_add_sys_db_callbacks(Db0),
+ ok = fabric2_server:store(Db1),
+ {ok, Db1#{tx := undefined}};
+ Error ->
+ Error
+ end;
Error ->
Error
end.
@@ -225,11 +230,15 @@ list_dbs(UserFun, UserAcc0, Options) ->
is_admin(Db) ->
- % TODO: Need to re-consider couch_db_plugin:check_is_admin/1
- {SecProps} = get_security(Db),
- UserCtx = get_user_ctx(Db),
- {Admins} = get_admins(SecProps),
- is_authorized(Admins, UserCtx).
+ case fabric2_db_plugin:check_is_admin(Db) of
+ true ->
+ true;
+ false ->
+ {SecProps} = get_security(Db),
+ UserCtx = get_user_ctx(Db),
+ {Admins} = get_admins(SecProps),
+ is_authorized(Admins, UserCtx)
+ end.
check_is_admin(Db) ->
@@ -582,6 +591,44 @@ get_missing_revs(Db, JsonIdRevs) ->
{ok, AllMissing}.
+validate_docid(<<"">>) ->
+ throw({illegal_docid, <<"Document id must not be empty">>});
+validate_docid(<<"_design/">>) ->
+ throw({illegal_docid, <<"Illegal document id `_design/`">>});
+validate_docid(<<"_local/">>) ->
+ throw({illegal_docid, <<"Illegal document id `_local/`">>});
+validate_docid(Id) when is_binary(Id) ->
+ MaxLen = case config:get("couchdb", "max_document_id_length", "infinity") of
+ "infinity" -> infinity;
+ IntegerVal -> list_to_integer(IntegerVal)
+ end,
+ case MaxLen > 0 andalso byte_size(Id) > MaxLen of
+ true -> throw({illegal_docid, <<"Document id is too long">>});
+ false -> ok
+ end,
+ case couch_util:validate_utf8(Id) of
+ false -> throw({illegal_docid, <<"Document id must be valid UTF-8">>});
+ true -> ok
+ end,
+ case Id of
+ <<?DESIGN_DOC_PREFIX, _/binary>> -> ok;
+ <<?LOCAL_DOC_PREFIX, _/binary>> -> ok;
+ <<"_", _/binary>> ->
+ case fabric2_db_plugin:validate_docid(Id) of
+ true ->
+ ok;
+ false ->
+ throw(
+ {illegal_docid,
+ <<"Only reserved document ids may start with underscore.">>})
+ end;
+ _Else -> ok
+ end;
+validate_docid(Id) ->
+ couch_log:debug("Document id is not a string: ~p", [Id]),
+ throw({illegal_docid, <<"Document id must be a string">>}).
+
+
update_doc(Db, Doc) ->
update_doc(Db, Doc, []).
@@ -758,6 +805,38 @@ fold_changes(Db, SinceSeq, UserFun, UserAcc, Options) ->
end).
+dbname_suffix(DbName) ->
+ filename:basename(normalize_dbname(DbName)).
+
+
+normalize_dbname(DbName) ->
+ % Remove in the final cleanup. We don't need to handle shards prefix or
+ % remove .couch suffixes anymore. Keep it for now to pass all the existing
+ % tests.
+ couch_db:normalize_dbname(DbName).
+
+
+validate_dbname(DbName) when is_list(DbName) ->
+ validate_dbname(?l2b(DbName));
+
+validate_dbname(DbName) when is_binary(DbName) ->
+ Normalized = normalize_dbname(DbName),
+ fabric2_db_plugin:validate_dbname(
+ DbName, Normalized, fun validate_dbname_int/2).
+
+validate_dbname_int(DbName, Normalized) when is_binary(DbName) ->
+ DbNoExt = couch_util:drop_dot_couch_ext(DbName),
+ case re:run(DbNoExt, ?DBNAME_REGEX, [{capture,none}, dollar_endonly]) of
+ match ->
+ ok;
+ nomatch ->
+ case is_system_db_name(Normalized) of
+ true -> ok;
+ false -> {error, {illegal_database_name, DbName}}
+ end
+ end.
+
+
maybe_add_sys_db_callbacks(Db) ->
IsReplicatorDb = is_replicator_db(Db),
IsUsersDb = is_users_db(Db),
@@ -1030,16 +1109,13 @@ find_possible_ancestors(RevInfos, MissingRevs) ->
apply_before_doc_update(Db, Docs, Options) ->
- #{before_doc_update := BDU} = Db,
UpdateType = case lists:member(replicated_changes, Options) of
true -> replicated_changes;
false -> interactive_edit
end,
- if BDU == undefined -> Docs; true ->
- lists:map(fun(Doc) ->
- BDU(Doc, Db, UpdateType)
- end, Docs)
- end.
+ lists:map(fun(Doc) ->
+ fabric2_db_plugin:before_doc_update(Db, Doc, UpdateType)
+ end, Docs).
update_doc_int(#{} = Db, #doc{} = Doc, Options) ->
diff --git a/src/fabric/src/fabric2_db_plugin.erl b/src/fabric/src/fabric2_db_plugin.erl
new file mode 100644
index 0000000..41f9e9d
--- /dev/null
+++ b/src/fabric/src/fabric2_db_plugin.erl
@@ -0,0 +1,92 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_db_plugin).
+
+-export([
+ validate_dbname/3,
+ before_doc_update/3,
+ after_doc_read/2,
+ validate_docid/1,
+ check_is_admin/1,
+ is_valid_purge_client/2
+]).
+
+-define(SERVICE_ID, fabric2_db).
+
+
+%% ------------------------------------------------------------------
+%% API Function Definitions
+%% ------------------------------------------------------------------
+
+validate_dbname(DbName, Normalized, Default) ->
+ maybe_handle(validate_dbname, [DbName, Normalized], Default).
+
+
+before_doc_update(Db, Doc0, UpdateType) ->
+ Fun = fabric2_db:get_before_doc_update_fun(Db),
+ case with_pipe(before_doc_update, [Doc0, Db, UpdateType]) of
+ [Doc1, _Db, UpdateType1] when is_function(Fun) ->
+ Fun(Doc1, Db, UpdateType1);
+ [Doc1, _Db, _UpdateType] ->
+ Doc1
+ end.
+
+
+after_doc_read(Db, Doc0) ->
+ Fun = fabric2_db:get_after_doc_read_fun(Db),
+ case with_pipe(after_doc_read, [Doc0, Db]) of
+ [Doc1, _Db] when is_function(Fun) -> Fun(Doc1, Db);
+ [Doc1, _Db] -> Doc1
+ end.
+
+
+validate_docid(Id) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ %% callbacks return true only if it specifically allow the given Id
+ couch_epi:any(Handle, ?SERVICE_ID, validate_docid, [Id], []).
+
+
+check_is_admin(Db) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ %% callbacks return true only if it specifically allow the given Id
+ R = couch_epi:any(Handle, ?SERVICE_ID, check_is_admin, [Db], []),
+ %io:format(standard_error, "~n FFFFFFF ~p check_is_admin Db:~p => ~p~n", [?MODULE, fabric2_db:name(Db), R]),
+ R.
+
+
+is_valid_purge_client(DbName, Props) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ %% callbacks return true only if it specifically allow the given Id
+ couch_epi:any(Handle, ?SERVICE_ID, is_valid_purge_client, [DbName, Props], []).
+
+%% ------------------------------------------------------------------
+%% Internal Function Definitions
+%% ------------------------------------------------------------------
+
+with_pipe(Func, Args) ->
+ do_apply(Func, Args, [pipe]).
+
+do_apply(Func, Args, Opts) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ couch_epi:apply(Handle, ?SERVICE_ID, Func, Args, Opts).
+
+maybe_handle(Func, Args, Default) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ case couch_epi:decide(Handle, ?SERVICE_ID, Func, Args, []) of
+ no_decision when is_function(Default) ->
+ apply(Default, Args);
+ no_decision ->
+ Default;
+ {decided, Result} ->
+ Result
+ end.
diff --git a/rel/apps/couch_epi.config b/src/fabric/src/fabric2_epi.erl
similarity index 51%
copy from rel/apps/couch_epi.config
copy to src/fabric/src/fabric2_epi.erl
index a53721a..f73eeb0 100644
--- a/rel/apps/couch_epi.config
+++ b/src/fabric/src/fabric2_epi.erl
@@ -2,7 +2,7 @@
% use this file except in compliance with the License. You may obtain a copy of
% the License at
%
-% http://www.apache.org/licenses/LICENSE-2.0
+% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@@ -10,13 +10,39 @@
% License for the specific language governing permissions and limitations under
% the License.
-{plugins, [
- couch_db_epi,
- chttpd_epi,
- couch_index_epi,
- dreyfus_epi,
- global_changes_epi,
- mango_epi,
- mem3_epi,
- setup_epi
-]}.
+-module(fabric2_epi).
+
+-behaviour(couch_epi_plugin).
+
+-export([
+ app/0,
+ providers/0,
+ services/0,
+ data_subscriptions/0,
+ data_providers/0,
+ processes/0,
+ notify/3
+]).
+
+app() ->
+ fabric.
+
+providers() ->
+ [].
+
+services() ->
+ [
+ {fabric2_db, fabric2_db_plugin}
+ ].
+
+data_subscriptions() ->
+ [].
+
+data_providers() ->
+ [].
+
+processes() ->
+ [].
+
+notify(_Key, _Old, _New) ->
+ ok.
diff --git a/src/fabric/src/fabric2_sup.erl b/src/fabric/src/fabric2_sup.erl
index 73c6c1f..b0d71c7 100644
--- a/src/fabric/src/fabric2_sup.erl
+++ b/src/fabric/src/fabric2_sup.erl
@@ -29,19 +29,25 @@ start_link(Args) ->
init([]) ->
- Flags = #{
- strategy => one_for_one,
- intensity => 1,
- period => 5
- },
+ Flags = {one_for_one, 1, 5},
Children = [
- #{
- id => fabric2_server,
- start => {fabric2_server, start_link, []}
+ {
+ fabric2_server,
+ {fabric2_server, start_link, []},
+ permanent,
+ 5000,
+ worker,
+ [fabric2_server]
},
- #{
- id => fabric2_txids,
- start => {fabric2_txids, start_link, []}
+ {
+ fabric2_txids,
+ {fabric2_txids, start_link, []},
+ permanent,
+ 5000,
+ worker,
+ [fabric2_server]
}
],
- {ok, {Flags, Children}}.
+ % Once 1db0294eb1093066773760b75a200d99aa453be8 is merged, switch to maps
+ ChildrenWithEpi = couch_epi:register_service(fabric2_epi, Children),
+ {ok, {Flags, ChildrenWithEpi}}.