You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2020/07/26 18:10:02 UTC

[couchdb] branch feat/access-master-clean created (now adc4bee)

This is an automated email from the ASF dual-hosted git repository.

jan pushed a change to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


      at adc4bee  feat(fabric): handle access requests

This branch includes the following new commits:

     new 45405fc  chore: rm .venv on make clean
     new 66ffdd2  chore(emilio): ignore ioq
     new 3914d3c  test: per doc access test suite
     new c499bb0  feat(couch): various records now have an access field
     new 4a64a8b  feat(access): introduce new access query server
     new 02d1918  feat(btree): handle access field in btree
     new 0701b1a  feat(utils): add ddoc validation fun
     new df3b76f  test: update existing tests to match
     new 92d1c43  feat(access): _users users now have a default _users role
     new 672a790  feat(access): main db update and validation logic
     new e34cad8  feat(access): use access by-seq/by-id for regular users
     new 5650b03  feat(chttpd): add access support to chttpd
     new 4de5f66  feat(replicator): add access support to replicator
     new 3434cea  feat(ddoc_cache): make access aware
     new 9922c62  feat(global_changes): make access aware
     new 9da1e4b  test: adjust peruser tests to access
     new adc4bee  feat(fabric): handle access requests

The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[couchdb] 08/17: test: update existing tests to match

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit df3b76f144de843ca71ba516c4edcfa563335bec
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:00:19 2020 +0200

    test: update existing tests to match
---
 src/couch/test/eunit/couchdb_mrview_cors_tests.erl      | 3 ++-
 src/couch/test/eunit/couchdb_update_conflicts_tests.erl | 2 +-
 test/elixir/test/bulk_docs_test.exs                     | 1 +
 test/elixir/test/cookie_auth_test.exs                   | 2 +-
 test/elixir/test/security_validation_test.exs           | 2 +-
 test/javascript/tests/security_validation.js            | 2 +-
 6 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/couch/test/eunit/couchdb_mrview_cors_tests.erl b/src/couch/test/eunit/couchdb_mrview_cors_tests.erl
index 0f69048..9d6f726 100644
--- a/src/couch/test/eunit/couchdb_mrview_cors_tests.erl
+++ b/src/couch/test/eunit/couchdb_mrview_cors_tests.erl
@@ -19,6 +19,7 @@
 
 -define(DDOC, {[
     {<<"_id">>, <<"_design/foo">>},
+    {<<"_access">>, [<<"user_a">>]},
     {<<"shows">>, {[
         {<<"bar">>, <<"function(doc, req) {return '<h1>wosh</h1>';}">>}
     ]}}
@@ -93,7 +94,7 @@ should_make_shows_request(_, {Host, DbName}) ->
     end).
 
 create_db(backdoor, DbName) ->
-    {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX, {access, true}]),
     couch_db:close(Db);
 create_db(clustered, DbName) ->
     {ok, Status, _, _} = test_request:put(db_url(DbName), [?AUTH], ""),
diff --git a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
index 1329aba..7f9d1db 100644
--- a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
+++ b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
@@ -19,7 +19,7 @@
 -define(DOC_ID, <<"foobar">>).
 -define(LOCAL_DOC_ID, <<"_local/foobar">>).
 -define(NUM_CLIENTS, [100, 500, 1000, 2000, 5000, 10000]).
--define(TIMEOUT, 20000).
+-define(TIMEOUT, 100000).
 
 start() ->
     test_util:start_couch().
diff --git a/test/elixir/test/bulk_docs_test.exs b/test/elixir/test/bulk_docs_test.exs
index 1a7c110..596454b 100644
--- a/test/elixir/test/bulk_docs_test.exs
+++ b/test/elixir/test/bulk_docs_test.exs
@@ -124,6 +124,7 @@ defmodule BulkDocsTest do
   test "bulk docs emits conflict error for duplicate doc `_id`s", ctx do
     docs = [%{_id: "0", a: 0}, %{_id: "1", a: 1}, %{_id: "1", a: 2}, %{_id: "3", a: 3}]
     rows = bulk_post(docs, ctx[:db_name]).body
+
     assert Enum.at(rows, 1)["id"] == "1"
     assert Enum.at(rows, 1)["ok"]
     assert Enum.at(rows, 2)["error"] == "conflict"
diff --git a/test/elixir/test/cookie_auth_test.exs b/test/elixir/test/cookie_auth_test.exs
index abc0fd7..b8027e4 100644
--- a/test/elixir/test/cookie_auth_test.exs
+++ b/test/elixir/test/cookie_auth_test.exs
@@ -318,7 +318,7 @@ defmodule CookieAuthTest do
     session = login("jchris", "funnybone")
     info = Couch.Session.info(session)
     assert info["userCtx"]["name"] == "jchris"
-    assert Enum.empty?(info["userCtx"]["roles"])
+    assert info["userCtx"]["roles"] == ["_users"]
 
     jason_user_doc =
       jason_user_doc
diff --git a/test/elixir/test/security_validation_test.exs b/test/elixir/test/security_validation_test.exs
index 0df3a78..9aaed76 100644
--- a/test/elixir/test/security_validation_test.exs
+++ b/test/elixir/test/security_validation_test.exs
@@ -145,7 +145,7 @@ defmodule SecurityValidationTest do
     headers = @auth_headers[:jerry]
     resp = Couch.get("/_session", headers: headers)
     assert resp.body["userCtx"]["name"] == "jerry"
-    assert resp.body["userCtx"]["roles"] == []
+    assert resp.body["userCtx"]["roles"] == ["_users"]
   end
 
   @tag :with_db
diff --git a/test/javascript/tests/security_validation.js b/test/javascript/tests/security_validation.js
index 6f0bd0f..1299d90 100644
--- a/test/javascript/tests/security_validation.js
+++ b/test/javascript/tests/security_validation.js
@@ -131,7 +131,7 @@ couchTests.security_validation = function(debug) {
       var user = JSON.parse(resp.responseText).userCtx;
       T(user.name == "jerry");
       // test that the roles are listed properly
-      TEquals(user.roles, []);
+      TEquals(["_users"], user.roles);
 
 
       // update the document


[couchdb] 02/17: chore(emilio): ignore ioq

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 66ffdd22747f587a14724d166972519d21c41ed1
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:52:58 2020 +0200

    chore(emilio): ignore ioq
---
 emilio.config | 1 +
 1 file changed, 1 insertion(+)

diff --git a/emilio.config b/emilio.config
index 0dad938..d9f7b6e 100644
--- a/emilio.config
+++ b/emilio.config
@@ -11,6 +11,7 @@
     "src[\/]snappy[\/]*",
     "src[\/]ssl_verify_fun[\/]*",
     "src[\/]ibrowse[\/]*",
+    "src[\/]ioq[\/]*",
     "src[\/]jiffy[\/]*",
     "src[\/]meck[\/]*",
     "src[\/]proper[\/]*",


[couchdb] 07/17: feat(utils): add ddoc validation fun

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 0701b1afaaaa4a58ee50edcea229df963d240689
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:59:27 2020 +0200

    feat(utils): add ddoc validation fun
---
 src/couch/src/couch_util.erl | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index dffb681..dbd7755 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -40,6 +40,7 @@
 -export([check_md5/2]).
 -export([set_mqd_off_heap/1]).
 -export([set_process_priority/2]).
+-export([validate_design_access/1, validate_design_access/2]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -763,3 +764,16 @@ check_config_blacklist(Section) ->
     _ ->
         ok
     end.
+
+validate_design_access(DDoc) ->
+    validate_design_access1(DDoc, true).
+
+validate_design_access(Db, DDoc) ->
+    validate_design_access1(DDoc, couch_db:has_access_enabled(Db)).
+
+validate_design_access1(_DDoc, false) -> ok;
+validate_design_access1(DDoc, true) ->
+    is_users_ddoc(DDoc).
+
+is_users_ddoc(#doc{access=[<<"_users">>]}) -> ok;
+is_users_ddoc(_) -> throw({forbidden, <<"per-user ddoc access">>}).


[couchdb] 03/17: test: per doc access test suite

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 3914d3c00d313ce05fe8e7a7f530fdfedec2c20e
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:54:54 2020 +0200

    test: per doc access test suite
---
 src/couch/test/eunit/couchdb_access_tests.erl | 1035 +++++++++++++++++++++++++
 1 file changed, 1035 insertions(+)

diff --git a/src/couch/test/eunit/couchdb_access_tests.erl b/src/couch/test/eunit/couchdb_access_tests.erl
new file mode 100644
index 0000000..45650ee
--- /dev/null
+++ b/src/couch/test/eunit/couchdb_access_tests.erl
@@ -0,0 +1,1035 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couchdb_access_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+-define(ADMIN_REQ_HEADERS, [?CONTENT_JSON, {basic_auth, {"a", "a"}}]).
+-define(USERX_REQ_HEADERS, [?CONTENT_JSON, {basic_auth, {"x", "x"}}]).
+-define(USERY_REQ_HEADERS, [?CONTENT_JSON, {basic_auth, {"y", "y"}}]).
+-define(SECURITY_OBJECT, {[
+ {<<"members">>,{[{<<"roles">>,[<<"_admin">>, <<"_users">>]}]}},
+ {<<"admins">>, {[{<<"roles">>,[<<"_admin">>]}]}}
+]}).
+
+url() ->
+    Addr = config:get("httpd", "bind_address", "127.0.0.1"),
+    lists:concat(["http://", Addr, ":", port()]).
+
+before_each(_) ->
+    {ok, 201, _, _} = test_request:put(url() ++ "/db?q=1&n=1&access=true", ?ADMIN_REQ_HEADERS, ""),
+    {ok, _, _, _} = test_request:put(url() ++ "/db/_security", ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+    url().
+
+after_each(_, Url) ->
+    {ok, 200, _, _} = test_request:delete(Url ++ "/db", ?ADMIN_REQ_HEADERS),
+    {_, _, _, _} = test_request:delete(Url ++ "/db2", ?ADMIN_REQ_HEADERS),
+    {_, _, _, _} = test_request:delete(Url ++ "/db3", ?ADMIN_REQ_HEADERS),
+    ok.
+
+before_all() ->
+    Couch = test_util:start_couch([chttpd, couch_replicator]),
+    Hashed = couch_passwords:hash_admin_password("a"),
+    ok = config:set("admins", "a", binary_to_list(Hashed), _Persist=false),
+    ok = config:set("couchdb", "uuid", "21ac467c1bc05e9d9e9d2d850bb1108f", _Persist=false),
+    ok = config:set("log", "level", "debug", _Persist=false),
+
+    % cleanup and setup
+    {ok, _, _, _} = test_request:delete(url() ++ "/db", ?ADMIN_REQ_HEADERS),
+    % {ok, _, _, _} = test_request:put(url() ++ "/db?q=1&n=1&access=true", ?ADMIN_REQ_HEADERS, ""),
+
+    % create users
+    UserDbUrl = url() ++ "/_users?q=1&n=1",
+    {ok, _, _, _} = test_request:delete(UserDbUrl, ?ADMIN_REQ_HEADERS, ""),
+    {ok, 201, _, _} = test_request:put(UserDbUrl, ?ADMIN_REQ_HEADERS, ""),
+
+    UserXDocUrl = url() ++ "/_users/org.couchdb.user:x",
+    UserXDocBody = "{ \"name\":\"x\", \"roles\": [], \"password\":\"x\", \"type\": \"user\" }",
+    {ok, 201, _, _} = test_request:put(UserXDocUrl, ?ADMIN_REQ_HEADERS, UserXDocBody),
+
+    UserYDocUrl = url() ++ "/_users/org.couchdb.user:y",
+    UserYDocBody = "{ \"name\":\"y\", \"roles\": [], \"password\":\"y\", \"type\": \"user\" }",
+    {ok, 201, _, _} = test_request:put(UserYDocUrl, ?ADMIN_REQ_HEADERS, UserYDocBody),
+    Couch.
+
+after_all(_) ->
+    ok = test_util:stop_couch(done).
+
+access_test_() ->
+    Tests = [
+        % Doc creation
+        fun should_not_let_anonymous_user_create_doc/2,
+        fun should_let_admin_create_doc_with_access/2,
+        fun should_let_admin_create_doc_without_access/2,
+        fun should_let_user_create_doc_for_themselves/2,
+        fun should_not_let_user_create_doc_for_someone_else/2,
+        fun should_let_user_create_access_ddoc/2,
+        fun access_ddoc_should_have_no_effects/2,
+
+        % Doc updates
+        fun users_with_access_can_update_doc/2,
+        fun users_without_access_can_not_update_doc/2,
+        fun users_with_access_can_not_change_access/2,
+        fun users_with_access_can_not_remove_access/2,
+
+        % Doc reads
+        fun should_let_admin_read_doc_with_access/2,
+        fun user_with_access_can_read_doc/2,
+        fun user_without_access_can_not_read_doc/2,
+        fun user_can_not_read_doc_without_access/2,
+        fun admin_with_access_can_read_conflicted_doc/2,
+        fun user_with_access_can_not_read_conflicted_doc/2,
+
+        % Doc deletes
+        fun should_let_admin_delete_doc_with_access/2,
+        fun should_let_user_delete_doc_for_themselves/2,
+        fun should_not_let_user_delete_doc_for_someone_else/2,
+
+        % _all_docs with include_docs
+        fun should_let_admin_fetch_all_docs/2,
+        fun should_let_user_fetch_their_own_all_docs/2,
+
+
+        % _changes
+        fun should_let_admin_fetch_changes/2,
+        fun should_let_user_fetch_their_own_changes/2,
+
+        % views
+        fun should_not_allow_admin_access_ddoc_view_request/2,
+        fun should_not_allow_user_access_ddoc_view_request/2,
+        fun should_allow_admin_users_access_ddoc_view_request/2,
+        fun should_allow_user_users_access_ddoc_view_request/2,
+
+        % replication
+        fun should_allow_admin_to_replicate_from_access_to_access/2,
+        fun should_allow_admin_to_replicate_from_no_access_to_access/2,
+        fun should_allow_admin_to_replicate_from_access_to_no_access/2,
+        fun should_allow_admin_to_replicate_from_no_access_to_no_access/2,
+        %
+        fun should_allow_user_to_replicate_from_access_to_access/2,
+        fun should_allow_user_to_replicate_from_access_to_no_access/2,
+        fun should_allow_user_to_replicate_from_no_access_to_access/2,
+        fun should_allow_user_to_replicate_from_no_access_to_no_access/2,
+
+        % _revs_diff for docs you don’t have access to
+        fun should_not_allow_user_to_revs_diff_other_docs/2
+
+
+        % TODO: create test db with role and not _users in _security.members
+        % and make sure a user in that group can access while a user not
+        % in that group cant
+        % % potential future feature
+        % % fun should_let_user_fetch_their_own_all_docs_plus_users_ddocs/2%,
+    ],
+    {
+        "Access tests",
+        {
+            setup,
+            fun before_all/0, fun after_all/1,
+            [
+                make_test_cases(clustered, Tests)
+            ]
+        }
+    }.
+
+make_test_cases(Mod, Funs) ->
+    {
+        lists:flatten(io_lib:format("~s", [Mod])),
+        {foreachx, fun before_each/1, fun after_each/2, [{Mod, Fun} || Fun <- Funs]}
+    }.
+
+% Doc creation
+ % http://127.0.0.1:64903/db/a?revs=true&open_revs=%5B%221-23202479633c2b380f79507a776743d5%22%5D&latest=true
+
+% should_do_the_thing(_PortType, Url) ->
+%   ?_test(begin
+%       {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+%           ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+%       {ok, Code, _, _} = test_request:get(Url ++ "/db/a?revs=true&open_revs=%5B%221-23202479633c2b380f79507a776743d5%22%5D&latest=true",
+%           ?USERX_REQ_HEADERS),
+%       ?assertEqual(200, Code)
+%   end).
+%
+
+should_not_let_anonymous_user_create_doc(_PortType, Url) ->
+    % TODO: debugging leftover
+    % BulkDocsBody = {[
+    %   {<<"docs">>, [
+    %       {[{<<"_id">>, <<"a">>}]},
+    %       {[{<<"_id">>, <<"a">>}]},
+    %       {[{<<"_id">>, <<"b">>}]},
+    %       {[{<<"_id">>, <<"c">>}]}
+    %   ]}
+    % ]},
+    % Resp = test_request:post(Url ++ "/db/_bulk_docs", ?ADMIN_REQ_HEADERS, jiffy:encode(BulkDocsBody)),
+    % ?debugFmt("~nResp: ~p~n", [Resp]),
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/a", "{\"a\":1,\"_access\":[\"x\"]}"),
+    ?_assertEqual(401, Code).
+
+should_let_admin_create_doc_with_access(_PortType, Url) ->
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    ?_assertEqual(201, Code).
+
+should_let_admin_create_doc_without_access(_PortType, Url) ->
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1}"),
+    ?_assertEqual(201, Code).
+
+should_let_user_create_doc_for_themselves(_PortType, Url) ->
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    ?_assertEqual(201, Code).
+
+should_not_let_user_create_doc_for_someone_else(_PortType, Url) ->
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/c",
+        ?USERY_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    ?_assertEqual(403, Code).
+
+should_let_user_create_access_ddoc(_PortType, Url) ->
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/dx",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    ?_assertEqual(201, Code).
+
+access_ddoc_should_have_no_effects(_PortType, Url) ->
+    ?_test(begin
+        Ddoc = "{ \"_access\":[\"x\"], \"validate_doc_update\": \"function(newDoc, oldDoc, userCtx) { throw({unauthorized: 'throw error'})}\",   \"views\": {     \"foo\": {       \"map\": \"function(doc) { emit(doc._id) }\"     }   },   \"shows\": {     \"boo\": \"function() {}\"   },   \"lists\": {     \"hoo\": \"function() {}\"   },   \"update\": {     \"goo\": \"function() {}\"   },   \"filters\": {     \"loo\": \"function() {}\"   }   }",
+        {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/dx",
+            ?USERX_REQ_HEADERS, Ddoc),
+        ?assertEqual(201, Code),
+        {ok, Code1, _, _} = test_request:put(Url ++ "/db/b",
+            ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        ?assertEqual(201, Code1),
+        {ok, Code2, _, _} = test_request:get(Url ++ "/db/_design/dx/_view/foo",
+            ?USERX_REQ_HEADERS),
+        ?assertEqual(404, Code2),
+        {ok, Code3, _, _} = test_request:get(Url ++ "/db/_design/dx/_show/boo/b",
+            ?USERX_REQ_HEADERS),
+        ?assertEqual(404, Code3),
+        {ok, Code4, _, _} = test_request:get(Url ++ "/db/_design/dx/_list/hoo/foo",
+            ?USERX_REQ_HEADERS),
+        ?assertEqual(404, Code4),
+        {ok, Code5, _, _} = test_request:post(Url ++ "/db/_design/dx/_update/goo",
+            ?USERX_REQ_HEADERS, ""),
+        ?assertEqual(404, Code5),
+        {ok, Code6, _, _} = test_request:get(Url ++ "/db/_changes?filter=dx/loo",
+            ?USERX_REQ_HEADERS),
+        ?assertEqual(404, Code6),
+        {ok, Code7, _, _} = test_request:get(Url ++ "/db/_changes?filter=_view&view=dx/foo",
+            ?USERX_REQ_HEADERS),
+        ?assertEqual(404, Code7)
+    end).
+
+% Doc updates
+
+users_with_access_can_update_doc(_PortType, Url) ->
+    {ok, _, _, Body} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {Json} = jiffy:decode(Body),
+    Rev = couch_util:get_value(<<"rev">>, Json),
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS,
+        "{\"a\":2,\"_access\":[\"x\"],\"_rev\":\"" ++ binary_to_list(Rev) ++ "\"}"),
+    ?_assertEqual(201, Code).
+
+users_without_access_can_not_update_doc(_PortType, Url) ->
+    {ok, _, _, Body} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {Json} = jiffy:decode(Body),
+    Rev = couch_util:get_value(<<"rev">>, Json),
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/b",
+        ?USERY_REQ_HEADERS,
+        "{\"a\":2,\"_access\":[\"y\"],\"_rev\":\"" ++ binary_to_list(Rev) ++ "\"}"),
+    ?_assertEqual(403, Code).
+
+users_with_access_can_not_change_access(_PortType, Url) ->
+    {ok, _, _, Body} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {Json} = jiffy:decode(Body),
+    Rev = couch_util:get_value(<<"rev">>, Json),
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS,
+        "{\"a\":2,\"_access\":[\"y\"],\"_rev\":\"" ++ binary_to_list(Rev) ++ "\"}"),
+    ?_assertEqual(403, Code).
+
+users_with_access_can_not_remove_access(_PortType, Url) ->
+    {ok, _, _, Body} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {Json} = jiffy:decode(Body),
+    Rev = couch_util:get_value(<<"rev">>, Json),
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/b",
+        ?USERX_REQ_HEADERS,
+        "{\"a\":2,\"_rev\":\"" ++ binary_to_list(Rev) ++ "\"}"),
+    ?_assertEqual(403, Code).
+
+% Doc reads
+
+should_let_admin_read_doc_with_access(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS),
+    ?_assertEqual(200, Code).
+
+user_with_access_can_read_doc(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(200, Code).
+
+user_with_access_can_not_read_conflicted_doc(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"_id\":\"f1\",\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a?new_edits=false",
+        ?ADMIN_REQ_HEADERS, "{\"_id\":\"f1\",\"_rev\":\"7-XYZ\",\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(403, Code).
+
+admin_with_access_can_read_conflicted_doc(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"_id\":\"a\",\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a?new_edits=false",
+        ?ADMIN_REQ_HEADERS, "{\"_id\":\"a\",\"_rev\":\"7-XYZ\",\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS),
+    ?_assertEqual(200, Code).
+
+user_without_access_can_not_read_doc(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?USERY_REQ_HEADERS),
+    ?_assertEqual(403, Code).
+
+user_can_not_read_doc_without_access(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1}"),
+    {ok, Code, _, _} = test_request:get(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(403, Code).
+
+% Doc deletes
+
+should_let_admin_delete_doc_with_access(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:delete(Url ++ "/db/a?rev=1-23202479633c2b380f79507a776743d5",
+        ?ADMIN_REQ_HEADERS),
+    ?_assertEqual(200, Code).
+
+should_let_user_delete_doc_for_themselves(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, _, _, _} = test_request:get(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS),
+    {ok, Code, _, _} = test_request:delete(Url ++ "/db/a?rev=1-23202479633c2b380f79507a776743d5",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(200, Code).
+
+should_not_let_user_delete_doc_for_someone_else(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?USERX_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, Code, _, _} = test_request:delete(Url ++ "/db/a?rev=1-23202479633c2b380f79507a776743d5",
+        ?USERY_REQ_HEADERS),
+    ?_assertEqual(403, Code).
+
+% _all_docs with include_docs
+
+should_let_admin_fetch_all_docs(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/b",
+        ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/c",
+        ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/d",
+        ?ADMIN_REQ_HEADERS, "{\"d\":4,\"_access\":[\"y\"]}"),
+    {ok, 200, _, Body} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+        ?ADMIN_REQ_HEADERS),
+    {Json} = jiffy:decode(Body),
+    ?_assertEqual(4, proplists:get_value(<<"total_rows">>, Json)).
+
+should_let_user_fetch_their_own_all_docs(_PortType, Url) ->
+    ?_test(begin
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/b",
+            ?USERX_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/d",
+            ?USERY_REQ_HEADERS, "{\"d\":4,\"_access\":[\"y\"]}"),
+        {ok, 200, _, Body} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+            ?USERX_REQ_HEADERS),
+        {Json} = jiffy:decode(Body),
+        Rows = proplists:get_value(<<"rows">>, Json),
+        ?assertEqual([{[{<<"id">>,<<"a">>},
+               {<<"key">>,<<"a">>},
+               {<<"value">>,<<"1-23202479633c2b380f79507a776743d5">>},
+               {<<"doc">>,
+                {[{<<"_id">>,<<"a">>},
+                  {<<"_rev">>,<<"1-23202479633c2b380f79507a776743d5">>},
+                  {<<"a">>,1},
+                  {<<"_access">>,[<<"x">>]}]}}]},
+             {[{<<"id">>,<<"b">>},
+               {<<"key">>,<<"b">>},
+               {<<"value">>,<<"1-d33fb05384fa65a8081da2046595de0f">>},
+               {<<"doc">>,
+                {[{<<"_id">>,<<"b">>},
+                  {<<"_rev">>,<<"1-d33fb05384fa65a8081da2046595de0f">>},
+                  {<<"b">>,2},
+                  {<<"_access">>,[<<"x">>]}]}}]}], Rows),
+        ?assertEqual(2, length(Rows)),
+        ?assertEqual(4, proplists:get_value(<<"total_rows">>, Json)),
+
+        {ok, 200, _, Body1} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+            ?USERY_REQ_HEADERS),
+        {Json1} = jiffy:decode(Body1),
+        ?assertEqual( [{<<"total_rows">>,4},
+            {<<"offset">>,2},
+            {<<"rows">>,
+                [{[{<<"id">>,<<"c">>},
+                 {<<"key">>,<<"c">>},
+                 {<<"value">>,<<"1-92aef5b0e4a3f4db0aba1320869bc95d">>},
+                 {<<"doc">>,
+                  {[{<<"_id">>,<<"c">>},
+                    {<<"_rev">>,<<"1-92aef5b0e4a3f4db0aba1320869bc95d">>},
+                    {<<"c">>,3},
+                    {<<"_access">>,[<<"y">>]}]}}]},
+                {[{<<"id">>,<<"d">>},
+                 {<<"key">>,<<"d">>},
+                 {<<"value">>,<<"1-ae984f6550038b1ed1565ac4b6cd8c5d">>},
+                 {<<"doc">>,
+                  {[{<<"_id">>,<<"d">>},
+                    {<<"_rev">>,<<"1-ae984f6550038b1ed1565ac4b6cd8c5d">>},
+                    {<<"d">>,4},
+                    {<<"_access">>,[<<"y">>]}]}}]}]}], Json1)
+    end).
+
+
+% _changes
+
+should_let_admin_fetch_changes(_PortType, Url) ->
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+        ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/b",
+        ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/c",
+        ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+    {ok, 201, _, _} = test_request:put(Url ++ "/db/d",
+        ?ADMIN_REQ_HEADERS, "{\"d\":4,\"_access\":[\"y\"]}"),
+    {ok, 200, _, Body} = test_request:get(Url ++ "/db/_changes",
+        ?ADMIN_REQ_HEADERS),
+    {Json} = jiffy:decode(Body),
+    AmountOfDocs = length(proplists:get_value(<<"results">>, Json)),
+    ?_assertEqual(4, AmountOfDocs).
+
+should_let_user_fetch_their_own_changes(_PortType, Url) ->
+    ?_test(begin
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+        {ok, 201, _, _} = test_request:put(Url ++ "/db/d",
+            ?ADMIN_REQ_HEADERS, "{\"d\":4,\"_access\":[\"y\"]}"),
+        {ok, 200, _, Body} = test_request:get(Url ++ "/db/_changes",
+            ?USERX_REQ_HEADERS),
+        {Json} = jiffy:decode(Body),
+        ?assertMatch([{<<"results">>,
+               [{[{<<"seq">>,
+                   <<"2-", _/binary>>},
+                  {<<"id">>,<<"a">>},
+                  {<<"changes">>,
+                   [{[{<<"rev">>,<<"1-23202479633c2b380f79507a776743d5">>}]}]}]},
+                {[{<<"seq">>,
+                   <<"3-", _/binary>>},
+                  {<<"id">>,<<"b">>},
+                  {<<"changes">>,
+                   [{[{<<"rev">>,<<"1-d33fb05384fa65a8081da2046595de0f">>}]}]}]}]},
+              {<<"last_seq">>,
+               <<"3-", _/binary>>},
+              {<<"pending">>,2}], Json),
+        AmountOfDocs = length(proplists:get_value(<<"results">>, Json)),
+        ?assertEqual(2, AmountOfDocs)
+    end).
+
+% views
+
+should_not_allow_admin_access_ddoc_view_request(_PortType, Url) ->
+    DDoc = "{\"a\":1,\"_access\":[\"x\"],\"views\":{\"foo\":{\"map\":\"function() {}\"}}}",
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/a",
+        ?ADMIN_REQ_HEADERS, DDoc),
+    ?assertEqual(201, Code),
+    {ok, Code1, _, _} = test_request:get(Url ++ "/db/_design/a/_view/foo",
+        ?ADMIN_REQ_HEADERS),
+    ?_assertEqual(404, Code1).
+
+should_not_allow_user_access_ddoc_view_request(_PortType, Url) ->
+    DDoc = "{\"a\":1,\"_access\":[\"x\"],\"views\":{\"foo\":{\"map\":\"function() {}\"}}}",
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/a",
+        ?ADMIN_REQ_HEADERS, DDoc),
+    ?assertEqual(201, Code),
+    {ok, Code1, _, _} = test_request:get(Url ++ "/db/_design/a/_view/foo",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(404, Code1).
+
+should_allow_admin_users_access_ddoc_view_request(_PortType, Url) ->
+    DDoc = "{\"a\":1,\"_access\":[\"_users\"],\"views\":{\"foo\":{\"map\":\"function() {}\"}}}",
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/a",
+        ?ADMIN_REQ_HEADERS, DDoc),
+    ?assertEqual(201, Code),
+    {ok, Code1, _, _} = test_request:get(Url ++ "/db/_design/a/_view/foo",
+        ?ADMIN_REQ_HEADERS),
+    ?_assertEqual(200, Code1).
+
+should_allow_user_users_access_ddoc_view_request(_PortType, Url) ->
+    DDoc = "{\"a\":1,\"_access\":[\"_users\"],\"views\":{\"foo\":{\"map\":\"function() {}\"}}}",
+    {ok, Code, _, _} = test_request:put(Url ++ "/db/_design/a",
+        ?ADMIN_REQ_HEADERS, DDoc),
+    ?assertEqual(201, Code),
+    {ok, Code1, _, _} = test_request:get(Url ++ "/db/_design/a/_view/foo",
+        ?USERX_REQ_HEADERS),
+    ?_assertEqual(200, Code1).
+
+% replication
+
+should_allow_admin_to_replicate_from_access_to_access(_PortType, Url) ->
+    ?_test(begin
+        % create target db
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1&access=true",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"x\"]}"),
+
+        % replicate
+        AdminUrl = string:replace(Url, "http://", "http://a:a@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(AdminUrl ++ "/db")},
+          {<<"target">>, list_to_binary(AdminUrl ++ "/db2")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?ADMIN_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(3, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db2/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(3, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_admin_to_replicate_from_no_access_to_access(_PortType, Url) ->
+    ?_test(begin
+        % create target db
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"x\"]}"),
+
+        % replicate
+        AdminUrl = string:replace(Url, "http://", "http://a:a@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(AdminUrl ++ "/db2")},
+          {<<"target">>, list_to_binary(AdminUrl ++ "/db")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?ADMIN_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(3, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(3, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_admin_to_replicate_from_access_to_no_access(_PortType, Url) ->
+    ?_test(begin
+        % create target db
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"x\"]}"),
+
+        % replicate
+        AdminUrl = string:replace(Url, "http://", "http://a:a@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(AdminUrl ++ "/db")},
+          {<<"target">>, list_to_binary(AdminUrl ++ "/db2")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?ADMIN_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(3, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db2/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(3, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_admin_to_replicate_from_no_access_to_no_access(_PortType, Url) ->
+    ?_test(begin
+        % create source and target dbs
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        {ok, 201, _, _} = test_request:put(url() ++ "/db3?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db3/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"x\"]}"),
+
+        % replicate
+        AdminUrl = string:replace(Url, "http://", "http://a:a@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(AdminUrl ++ "/db2")},
+          {<<"target">>, list_to_binary(AdminUrl ++ "/db3")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?ADMIN_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(3, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db3/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(3, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_user_to_replicate_from_access_to_access(_PortType, Url) ->
+    ?_test(begin
+        % create source and target dbs
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1&access=true",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+
+        % replicate
+        UserXUrl = string:replace(Url, "http://", "http://x:x@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(UserXUrl ++ "/db")},
+          {<<"target">>, list_to_binary(UserXUrl ++ "/db2")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?USERX_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+        % ?debugFmt("~nResponseBody: ~p~n", [ResponseBody]),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(2, MissingChecked),
+        ?assertEqual(2, MissingFound),
+        ?assertEqual(2, DocsReard),
+        ?assertEqual(2, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert access in local doc
+        ReplicationId = couch_util:get_value(<<"replication_id">>, EJResponseBody),
+        {ok, 200, _, CheckPoint} = test_request:get(Url ++ "/db/_local/" ++ ReplicationId,
+            ?USERX_REQ_HEADERS),
+        {EJCheckPoint} = jiffy:decode(CheckPoint),
+        Access = couch_util:get_value(<<"_access">>, EJCheckPoint),
+        ?assertEqual([<<"x">>], Access),
+
+        % make sure others can’t read our local docs
+        {ok, 403, _, _} = test_request:get(Url ++ "/db/_local/" ++ ReplicationId,
+            ?USERY_REQ_HEADERS),
+
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db2/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(2, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_user_to_replicate_from_access_to_no_access(_PortType, Url) ->
+    ?_test(begin
+        % create source and target dbs
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+
+        % replicate
+        UserXUrl = string:replace(Url, "http://", "http://x:x@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(UserXUrl ++ "/db")},
+          {<<"target">>, list_to_binary(UserXUrl ++ "/db2")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?USERX_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(2, MissingChecked),
+        ?assertEqual(2, MissingFound),
+        ?assertEqual(2, DocsReard),
+        ?assertEqual(2, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db2/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(2, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_user_to_replicate_from_no_access_to_access(_PortType, Url) ->
+    ?_test(begin
+        % create source and target dbs
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        % leave for easier debugging
+        % VduFun = <<"function(newdoc, olddoc, userctx) {if(newdoc._id == \"b\") throw({'forbidden':'fail'})}">>,
+        % DDoc = {[
+        %    {<<"_id">>, <<"_design/vdu">>},
+        %    {<<"validate_doc_update">>, VduFun}
+        % ]},
+        % {ok, _, _, _} = test_request:put(Url ++ "/db/_design/vdu",
+        %     ?ADMIN_REQ_HEADERS, jiffy:encode(DDoc)),
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+
+
+        % replicate
+        UserXUrl = string:replace(Url, "http://", "http://x:x@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(UserXUrl ++ "/db2")},
+          {<<"target">>, list_to_binary(UserXUrl ++ "/db")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?USERX_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(2, DocsWritten),
+        ?assertEqual(1, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(2, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+should_allow_user_to_replicate_from_no_access_to_no_access(_PortType, Url) ->
+    ?_test(begin
+        % create source and target dbs
+        {ok, 201, _, _} = test_request:put(url() ++ "/db2?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db2/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+
+        {ok, 201, _, _} = test_request:put(url() ++ "/db3?q=1&n=1",
+          ?ADMIN_REQ_HEADERS, ""),
+        % set target db security
+        {ok, _, _, _} = test_request:put(url() ++ "/db3/_security",
+          ?ADMIN_REQ_HEADERS, jiffy:encode(?SECURITY_OBJECT)),
+        % create source docs
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/a",
+            ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/b",
+            ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+        {ok, _, _, _} = test_request:put(Url ++ "/db2/c",
+            ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+
+        % replicate
+        UserXUrl = string:replace(Url, "http://", "http://x:x@"),
+        EJRequestBody = {[
+          {<<"source">>, list_to_binary(UserXUrl ++ "/db2")},
+          {<<"target">>, list_to_binary(UserXUrl ++ "/db3")}
+        ]},
+        {ok, ResponseCode, _, ResponseBody} = test_request:post(Url ++ "/_replicate",
+            ?USERX_REQ_HEADERS, jiffy:encode(EJRequestBody)),
+
+        % assert replication status
+        {EJResponseBody} = jiffy:decode(ResponseBody),
+        ?assertEqual(ResponseCode, 200),
+        ?assertEqual(true, couch_util:get_value(<<"ok">>, EJResponseBody)),
+        [{History}] = couch_util:get_value(<<"history">>, EJResponseBody),
+
+        MissingChecked = couch_util:get_value(<<"missing_checked">>, History),
+        MissingFound = couch_util:get_value(<<"missing_found">>, History),
+        DocsReard = couch_util:get_value(<<"docs_read">>, History),
+        DocsWritten = couch_util:get_value(<<"docs_written">>, History),
+        DocWriteFailures = couch_util:get_value(<<"doc_write_failures">>, History),
+     
+        ?assertEqual(3, MissingChecked),
+        ?assertEqual(3, MissingFound),
+        ?assertEqual(3, DocsReard),
+        ?assertEqual(3, DocsWritten),
+        ?assertEqual(0, DocWriteFailures),
+      
+        % assert docs in target db
+        {ok, 200, _, ADBody} = test_request:get(Url ++ "/db3/_all_docs?include_docs=true",
+            ?ADMIN_REQ_HEADERS),
+        {Json} = jiffy:decode(ADBody),
+        ?assertEqual(3, proplists:get_value(<<"total_rows">>, Json))
+    end).
+
+% revs_diff
+should_not_allow_user_to_revs_diff_other_docs(_PortType, Url) ->
+  ?_test(begin
+      % create test docs
+      {ok, _, _, _} = test_request:put(Url ++ "/db/a",
+          ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+      {ok, _, _, _} = test_request:put(Url ++ "/db/b",
+          ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+      {ok, _, _, V} = test_request:put(Url ++ "/db/c",
+          ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+
+      % nothing missing
+      RevsDiff = {[
+          {<<"a">>, [
+              <<"1-23202479633c2b380f79507a776743d5">>
+          ]}
+      ]},
+      {ok, GoodCode, _, GoodBody} = test_request:post(Url ++ "/db/_revs_diff",
+          ?USERX_REQ_HEADERS, jiffy:encode(RevsDiff)),
+      EJGoodBody = jiffy:decode(GoodBody),
+      ?assertEqual(200, GoodCode),
+      ?assertEqual({[]}, EJGoodBody),
+
+      % something missing
+      MissingRevsDiff = {[
+          {<<"a">>, [
+              <<"1-missing">>
+          ]}
+      ]},
+      {ok, MissingCode, _, MissingBody} = test_request:post(Url ++ "/db/_revs_diff",
+          ?USERX_REQ_HEADERS, jiffy:encode(MissingRevsDiff)),
+      EJMissingBody = jiffy:decode(MissingBody),
+      ?assertEqual(200, MissingCode),
+      MissingExpect = {[
+          {<<"a">>, {[
+              {<<"missing">>, [<<"1-missing">>]}
+          ]}}
+      ]},
+      ?assertEqual(MissingExpect, EJMissingBody),
+
+      % other doc
+      OtherRevsDiff = {[
+          {<<"c">>, [
+              <<"1-92aef5b0e4a3f4db0aba1320869bc95d">>
+          ]}
+      ]},
+      {ok, OtherCode, _, OtherBody} = test_request:post(Url ++ "/db/_revs_diff",
+          ?USERX_REQ_HEADERS, jiffy:encode(OtherRevsDiff)),
+      EJOtherBody = jiffy:decode(OtherBody),
+      ?assertEqual(200, OtherCode),
+      ?assertEqual({[]}, EJOtherBody)
+  end).
+%% ------------------------------------------------------------------
+%% Internal Function Definitions
+%% ------------------------------------------------------------------
+
+port() ->
+    integer_to_list(mochiweb_socket_server:get(chttpd, port)).
+
+% Potential future feature:%
+% should_let_user_fetch_their_own_all_docs_plus_users_ddocs(_PortType, Url) ->
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/a",
+%         ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"x\"]}"),
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/_design/foo",
+%         ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"_users\"]}"),
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/_design/bar",
+%         ?ADMIN_REQ_HEADERS, "{\"a\":1,\"_access\":[\"houdini\"]}"),
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/b",
+%         ?USERX_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+%
+%     % % TODO: add allowing non-admin users adding non-admin ddocs
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/_design/x",
+%         ?ADMIN_REQ_HEADERS, "{\"b\":2,\"_access\":[\"x\"]}"),
+%
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/c",
+%         ?ADMIN_REQ_HEADERS, "{\"c\":3,\"_access\":[\"y\"]}"),
+%     {ok, 201, _, _} = test_request:put(Url ++ "/db/d",
+%         ?USERY_REQ_HEADERS, "{\"d\":4,\"_access\":[\"y\"]}"),
+%     {ok, 200, _, Body} = test_request:get(Url ++ "/db/_all_docs?include_docs=true",
+%         ?USERX_REQ_HEADERS),
+%     {Json} = jiffy:decode(Body),
+%     ?debugFmt("~nHSOIN: ~p~n", [Json]),
+%     ?_assertEqual(3, length(proplists:get_value(<<"rows">>, Json))).


[couchdb] 14/17: feat(ddoc_cache): make access aware

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 3434cea45ad968ee1d75ff954d292e4875c30e3c
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:07:13 2020 +0200

    feat(ddoc_cache): make access aware
---
 src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl          | 2 +-
 src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl      | 2 +-
 src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
index 5248469..03a4b14 100644
--- a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
@@ -33,7 +33,7 @@ ddocid({_, DDocId}) ->
 
 
 recover({DbName, DDocId}) ->
-    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]).
+    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX, ddoc_cache]).
 
 
 insert({DbName, DDocId}, {ok, #doc{revs = Revs} = DDoc}) ->
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
index 868fa77..fa7187f 100644
--- a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
@@ -33,7 +33,7 @@ ddocid({_, DDocId, _}) ->
 
 
 recover({DbName, DDocId, Rev}) ->
-    Opts = [ejson_body, ?ADMIN_CTX],
+    Opts = [ejson_body, ?ADMIN_CTX, ddoc_cache],
     {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], Opts),
     Resp.
 
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
index 2182dea..dc22b01 100644
--- a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
@@ -30,7 +30,8 @@ ddocid(_) ->
 
 
 recover(DbName) ->
-    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
+    {ok, DDocs0} = fabric:design_docs(mem3:dbname(DbName)),
+    DDocs = lists:filter(fun couch_doc:has_no_access/1, DDocs0),
     Funs = lists:flatmap(fun(DDoc) ->
         case couch_doc:get_validate_doc_fun(DDoc) of
             nil -> [];


[couchdb] 15/17: feat(global_changes): make access aware

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9922c62bd29c3d713888a4fd440402a2266753c0
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:07:27 2020 +0200

    feat(global_changes): make access aware
---
 src/global_changes/src/global_changes_server.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/global_changes/src/global_changes_server.erl b/src/global_changes/src/global_changes_server.erl
index 7e30625..d0783be 100644
--- a/src/global_changes/src/global_changes_server.erl
+++ b/src/global_changes/src/global_changes_server.erl
@@ -160,7 +160,7 @@ flush_updates(State) ->
         end, [], GroupedIds),
 
         spawn(fun() ->
-            fabric:update_docs(State#state.dbname, Docs, [])
+            fabric:update_docs(State#state.dbname, Docs, [?ADMIN_CTX])
         end),
 
         Count = State#state.pending_update_count,


[couchdb] 11/17: feat(access): use access by-seq/by-id for regular users

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit e34cad849981d7247f1f5eff4862d2f785804ce1
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:04:33 2020 +0200

    feat(access): use access by-seq/by-id for regular users
---
 src/couch/src/couch_btree.erl                 |  12 +++
 src/couch_index/src/couch_index_updater.erl   |  35 +++++---
 src/couch_mrview/include/couch_mrview.hrl     |   3 +-
 src/couch_mrview/src/couch_mrview.erl         | 119 +++++++++++++++++++++++++-
 src/couch_mrview/src/couch_mrview_updater.erl |  45 ++++++++--
 src/couch_mrview/src/couch_mrview_util.erl    |   8 +-
 6 files changed, 203 insertions(+), 19 deletions(-)

diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index ea0cf69..75e801e 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -16,6 +16,7 @@
 -export([fold/4, full_reduce/1, final_reduce/2, size/1, foldl/3, foldl/4]).
 -export([fold_reduce/4, lookup/2, get_state/1, set_options/2]).
 -export([extract/2, assemble/3, less/3]).
+-export([full_reduce_with_options/2]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -92,6 +93,17 @@ fold_reduce(#btree{root=Root}=Bt, Fun, Acc, Options) ->
         throw:{stop, AccDone} -> {ok, AccDone}
     end.
 
+full_reduce_with_options(Bt, Options0) ->
+    CountFun = fun(_SeqStart, PartialReds, 0) ->
+        {ok, couch_btree:final_reduce(Bt, PartialReds)}
+    end,
+    [UserName] = proplists:get_value(start_key, Options0, <<"">>),
+    EndKey = {[UserName, {[]}]},
+    Options = Options0 ++ [
+        {end_key, EndKey}
+    ],
+    fold_reduce(Bt, CountFun, 0, Options).
+
 full_reduce(#btree{root=nil,reduce=Reduce}) ->
     {ok, Reduce(reduce, [])};
 full_reduce(#btree{root=Root}) ->
diff --git a/src/couch_index/src/couch_index_updater.erl b/src/couch_index/src/couch_index_updater.erl
index fb15db0..2f65c1c 100644
--- a/src/couch_index/src/couch_index_updater.erl
+++ b/src/couch_index/src/couch_index_updater.erl
@@ -135,8 +135,8 @@ update(Idx, Mod, IdxState) ->
     CommittedOnly = lists:member(committed_only, UpdateOpts),
     IncludeDesign = lists:member(include_design, UpdateOpts),
     DocOpts = case lists:member(local_seq, UpdateOpts) of
-        true -> [conflicts, deleted_conflicts, local_seq];
-        _ -> [conflicts, deleted_conflicts]
+        true -> [conflicts, deleted_conflicts, local_seq, deleted];
+        _ -> [conflicts, deleted_conflicts, local_seq, deleted]
     end,
 
     couch_util:with_db(DbName, fun(Db) ->
@@ -154,23 +154,36 @@ update(Idx, Mod, IdxState) ->
         end,
 
         GetInfo = fun
-            (#full_doc_info{id=Id, update_seq=Seq, deleted=Del}=FDI) ->
-                {Id, Seq, Del, couch_doc:to_doc_info(FDI)};
-            (#doc_info{id=Id, high_seq=Seq, revs=[RI|_]}=DI) ->
-                {Id, Seq, RI#rev_info.deleted, DI}
+            (#full_doc_info{id=Id, update_seq=Seq, deleted=Del,access=Access}=FDI) ->
+                {Id, Seq, Del, couch_doc:to_doc_info(FDI), Access};
+            (#doc_info{id=Id, high_seq=Seq, revs=[RI|_],access=Access}=DI) ->
+                {Id, Seq, RI#rev_info.deleted, DI, Access}
         end,
 
         LoadDoc = fun(DI) ->
-            {DocId, Seq, Deleted, DocInfo} = GetInfo(DI),
+            {DocId, Seq, Deleted, DocInfo, Access} = GetInfo(DI),
 
             case {IncludeDesign, DocId} of
                 {false, <<"_design/", _/binary>>} ->
                     {nil, Seq};
-                _ when Deleted ->
-                    {#doc{id=DocId, deleted=true}, Seq};
                 _ ->
-                    {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
-                    {Doc, Seq}
+                    case IndexName of % TODO: move into outer case statement
+                        <<"_design/_access">> ->
+                            {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+                            % TODO: hande conflicted docs in _access index
+                            % probably remove
+                            [RevInfo|_] = DocInfo#doc_info.revs,
+                            Doc1 = Doc#doc{
+                                meta = [{body_sp, RevInfo#rev_info.body_sp}],
+                                access = Access
+                            },
+                            {Doc1, Seq};
+                        _ when Deleted ->
+                            {#doc{id=DocId, deleted=true}, Seq};
+                        _ ->
+                            {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+                            {Doc, Seq}
+                    end
             end
         end,
 
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index bb0ab0b..d9d7a8b 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -81,7 +81,8 @@
     conflicts,
     callback,
     sorted = true,
-    extra = []
+    extra = [],
+    deleted = false
 }).
 
 -record(vacc, {
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 1cdc918..298576d 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -13,7 +13,7 @@
 -module(couch_mrview).
 
 -export([validate/2]).
--export([query_all_docs/2, query_all_docs/4]).
+-export([query_all_docs/2, query_all_docs/4, query_changes_access/5]).
 -export([query_view/3, query_view/4, query_view/6, get_view_index_pid/4]).
 -export([get_info/2]).
 -export([trigger_update/2, trigger_update/3]).
@@ -233,6 +233,123 @@ query_all_docs(Db, Args) ->
 query_all_docs(Db, Args, Callback, Acc) when is_list(Args) ->
     query_all_docs(Db, to_mrargs(Args), Callback, Acc);
 query_all_docs(Db, Args0, Callback, Acc) ->
+    case couch_db:has_access_enabled(Db) and not couch_db:is_admin(Db) of
+        true -> query_all_docs_access(Db, Args0, Callback, Acc);
+        false -> query_all_docs_admin(Db, Args0, Callback, Acc)
+    end.
+
+access_ddoc() ->
+    #doc{
+        id = <<"_design/_access">>,
+        body = {[
+            {<<"language">>,<<"_access">>},
+            {<<"options">>, {[
+                {<<"include_design">>, true}
+            ]}},
+            {<<"views">>, {[
+                {<<"_access_by_id">>, {[
+                    {<<"map">>, <<"_access/by-id-map">>},
+                    {<<"reduce">>, <<"_count">>}
+                ]}},
+                {<<"_access_by_seq">>, {[
+                    {<<"map">>, <<"_access/by-seq-map">>},
+                    {<<"reduce">>, <<"_count">>}
+                ]}}
+            ]}}
+        ]}
+    }.
+
+query_changes_access(Db, StartSeq, Fun, Options, Acc) ->
+    DDoc = access_ddoc(),
+    UserCtx = couch_db:get_user_ctx(Db),
+    UserName = UserCtx#user_ctx.name,
+    %% % TODO: add roles
+    Args1 = prefix_startkey_endkey(UserName, #mrargs{}, fwd),
+    Args2 = Args1#mrargs{deleted=true},
+    Args = Args2#mrargs{reduce=false},
+    %% % filter out the user-prefix from the key, so _all_docs looks normal
+    %% % this isn’t a separate function because I’m binding Callback0 and I don’t
+    %% % know the Erlang equivalent of JS’s fun.bind(this, newarg)
+    Callback = fun
+         ({meta, _}, Acc0) ->
+            {ok, Acc0}; % ignore for now
+         ({row, Props}, Acc0) ->
+            % turn row into FDI
+            Value = couch_util:get_value(value, Props),
+            [Owner, Seq] = couch_util:get_value(key, Props),
+
+            Rev = couch_util:get_value(rev, Value),
+            Deleted = couch_util:get_value(deleted, Value, false),
+            BodySp = couch_util:get_value(body_sp, Value),
+
+            [Pos, RevId] = string:split(?b2l(Rev), "-"),
+            FDI = #full_doc_info{
+                id = proplists:get_value(id, Props),
+                rev_tree = [{list_to_integer(Pos), {?l2b(RevId), #leaf{deleted=Deleted, ptr=BodySp, seq=Seq, sizes=#size_info{}}, []}}],
+                deleted = Deleted,
+                update_seq = 0,
+                sizes = #size_info{},
+                access = [Owner]
+            },
+            Fun(FDI, Acc0);
+        (_Else, Acc0) ->
+            {ok, Acc0} % ignore for now
+        end,
+    VName = <<"_access_by_seq">>,
+    query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+query_all_docs_access(Db, Args0, Callback0, Acc) ->
+    % query our not yest existing, home-grown _access view.
+    % use query_view for this.
+    DDoc = access_ddoc(),
+    UserCtx = couch_db:get_user_ctx(Db),
+    UserName = UserCtx#user_ctx.name,
+    Args1 = prefix_startkey_endkey(UserName, Args0, Args0#mrargs.direction),
+    Args = Args1#mrargs{reduce=false, extra=Args1#mrargs.extra ++ [{all_docs_access, true}]},
+
+    Callback = fun
+        ({row, Props}, Acc0) ->
+
+            % filter out the user-prefix from the key, so _all_docs looks normal
+            % this isn’t a separate function because I’m binding Callback0 and I
+            % don’t know the Erlang equivalent of JS’s fun.bind(this, newarg)
+            [_User, Key] = proplists:get_value(key, Props),
+            Row0 = proplists:delete(key, Props),
+            Row = [{key, Key} | Row0],
+
+            Callback0({row, Row}, Acc0);
+        (Row, Acc0) ->
+            Callback0(Row, Acc0)
+        end,
+    VName = <<"_access_by_id">>,
+    query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+prefix_startkey_endkey(UserName, Args, fwd) ->
+    #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+    Args#mrargs {
+        start_key = case StartKey of
+            undefined -> [UserName];
+            StartKey -> [UserName, StartKey]
+        end,
+        end_key = case EndKey of
+            undefined -> [UserName, {}];
+            EndKey -> [UserName, EndKey, {}]
+        end
+    };
+prefix_startkey_endkey(UserName, Args, rev) ->
+    #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+    Args#mrargs {
+        end_key = case StartKey of
+            undefined -> [UserName];
+            StartKey -> [UserName, StartKey]
+        end,
+        start_key = case EndKey of
+            undefined -> [UserName, {}];
+            EndKey -> [UserName, EndKey, {}]
+        end
+    }.
+
+query_all_docs_admin(Db, Args0, Callback, Acc) ->
     Sig = couch_util:with_db(Db, fun(WDb) ->
         {ok, Info} = couch_db:get_db_info(WDb),
         couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info)))
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 522367c..0dc9f85 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -116,8 +116,9 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
     process_doc(Doc, Seq, State#mrst{doc_acc=[]});
 process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
     {ok, State#mrst{doc_acc=[{nil, Seq, nil} | Acc]}};
-process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
-    {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
+% TODO: re-evaluate why this is commented out
+% process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
+%     {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
 process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
     {ok, State#mrst{doc_acc=[{Id, Seq, Doc} | Acc]}}.
 
@@ -140,6 +141,13 @@ finish_update(#mrst{doc_acc=Acc}=State) ->
             }}
     end.
 
+make_deleted_body({Props}, Meta, Seq) ->
+    BodySp = couch_util:get_value(body_sp, Meta),
+    Result = [{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}],
+    case couch_util:get_value(<<"_access">>, Props) of
+    undefined -> Result;
+    Access -> [{<<"_access">>, Access} | Result]
+    end.
 
 map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
     erlang:put(io_priority, {view_update, DbName, IdxName}),
@@ -158,11 +166,38 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
             DocFun = fun
                 ({nil, Seq, _}, {SeqAcc, Results}) ->
                     {erlang:max(Seq, SeqAcc), Results};
-                ({Id, Seq, deleted}, {SeqAcc, Results}) ->
-                    {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
+                ({Id, Seq, Rev, #doc{deleted=true, body=Body, meta=Meta}}, {SeqAcc, Results}) ->
+                    % _access needs deleted docs
+                    case IdxName of
+                        <<"_design/_access">> ->
+                            % splice in seq
+                            {Start, Rev1} = Rev,
+                            Doc = #doc{
+                                id = Id,
+                                revs = {Start, [Rev1]},
+                                body = {make_deleted_body(Body, Meta, Seq)}, %% todo: only keep _access and add _seq
+                                deleted = true
+                            },
+                            {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+                            {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, Res} | Results]};
+                        _Else ->
+                            {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, []} | Results]}
+                        end;
                 ({Id, Seq, Doc}, {SeqAcc, Results}) ->
                     couch_stats:increment_counter([couchdb, mrview, map_doc]),
-                    {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+                    % IdxName: ~p, Doc: ~p~n~n", [IdxName, Doc]),
+                    Doc0 = case IdxName of
+                        <<"_design/_access">> ->
+                            % splice in seq
+                            {Props} = Doc#doc.body,
+                            BodySp = couch_util:get_value(body_sp, Doc#doc.meta),
+                            Doc#doc{
+                                body = {Props++[{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}]}
+                            };
+                        _Else ->
+                            Doc
+                        end,
+                    {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc0),
                     {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
             end,
             FoldFun = fun(Docs, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index e971720..698adb6 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -20,6 +20,7 @@
 -export([index_file/2, compaction_file/2, open_file/1]).
 -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
 -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_access_row_count/2]).
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
@@ -340,6 +341,10 @@ temp_view_to_ddoc({Props}) ->
     ]},
     couch_doc:from_json_obj(DDoc).
 
+get_access_row_count(#mrview{btree=Bt}, UserName) ->
+    couch_btree:full_reduce_with_options(Bt, [
+        {start_key, UserName}
+    ]).
 
 get_row_count(#mrview{btree=Bt}) ->
     Count = case couch_btree:full_reduce(Bt) of
@@ -407,8 +412,9 @@ validate_args(#mrst{} = State, Args0) ->
 
     ViewPartitioned = State#mrst.partitioned,
     Partition = get_extra(Args, partition),
+    AllDocsAccess = get_extra(Args, all_docs_access, false),
 
-    case {ViewPartitioned, Partition} of
+    case {ViewPartitioned and not AllDocsAccess, Partition} of
         {true, undefined} ->
             Msg1 = <<"`partition` parameter is mandatory "
                     "for queries to this view.">>,


[couchdb] 13/17: feat(replicator): add access support to replicator

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 4de5f6616864e5812b3046987c34ef45d2ada554
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:06:38 2020 +0200

    feat(replicator): add access support to replicator
---
 src/couch_replicator/src/couch_replicator.erl      |  8 +++++-
 .../src/couch_replicator_scheduler_job.erl         | 30 ++++++++++++++++++----
 .../couch_replicator_error_reporting_tests.erl     |  6 ++---
 3 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/src/couch_replicator/src/couch_replicator.erl b/src/couch_replicator/src/couch_replicator.erl
index b38f31b..a509674 100644
--- a/src/couch_replicator/src/couch_replicator.erl
+++ b/src/couch_replicator/src/couch_replicator.erl
@@ -72,7 +72,13 @@ replicate(PostBody, Ctx) ->
     false ->
         check_authorization(RepId, UserCtx),
         {ok, Listener} = rep_result_listener(RepId),
-        Result = do_replication_loop(Rep),
+        Result = case do_replication_loop(Rep) of
+            {ok, {ResultJson}} ->
+                {PublicRepId, _} = couch_replicator_ids:replication_id(Rep), % TODO: check with options
+                {ok, {[{<<"replication_id">>, ?l2b(PublicRepId)} | ResultJson]}};
+            Else ->
+                Else
+        end,
         couch_replicator_notifier:stop(Listener),
         Result
     end.
diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
index 0b33419..c18fe20 100644
--- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl
+++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
@@ -66,6 +66,8 @@
     rep_starttime,
     src_starttime,
     tgt_starttime,
+    src_access,
+    tgt_access,
     timer, % checkpoint timer
     changes_queue,
     changes_manager,
@@ -610,6 +612,8 @@ init_state(Rep) ->
         rep_starttime = StartTime,
         src_starttime = get_value(<<"instance_start_time">>, SourceInfo),
         tgt_starttime = get_value(<<"instance_start_time">>, TargetInfo),
+        src_access = get_value(<<"access">>, SourceInfo),
+        tgt_access = get_value(<<"access">>, TargetInfo),
         session_id = couch_uuids:random(),
         source_seq = SourceSeq,
         use_checkpoints = get_value(use_checkpoints, Options, true),
@@ -713,8 +717,10 @@ do_checkpoint(State) ->
         rep_starttime = ReplicationStartTime,
         src_starttime = SrcInstanceStartTime,
         tgt_starttime = TgtInstanceStartTime,
+        src_access = SrcAccess,
+        tgt_access = TgtAccess,
         stats = Stats,
-        rep_details = #rep{options = Options},
+        rep_details = #rep{options = Options, user_ctx = UserCtx},
         session_id = SessionId
     } = State,
     case commit_to_both(Source, Target) of
@@ -770,9 +776,9 @@ do_checkpoint(State) ->
 
         try
             {SrcRevPos, SrcRevId} = update_checkpoint(
-                Source, SourceLog#doc{body = NewRepHistory}, source),
+                Source, SourceLog#doc{body = NewRepHistory}, SrcAccess, UserCtx, source),
             {TgtRevPos, TgtRevId} = update_checkpoint(
-                Target, TargetLog#doc{body = NewRepHistory}, target),
+                Target, TargetLog#doc{body = NewRepHistory}, TgtAccess, UserCtx, target),
             NewState = State#rep_state{
                 checkpoint_history = NewRepHistory,
                 committed_seq = NewTsSeq,
@@ -797,16 +803,28 @@ do_checkpoint(State) ->
 
 
 update_checkpoint(Db, Doc, DbType) ->
+    update_checkpoint(Db, Doc, false, #user_ctx{}, DbType).
+
+update_checkpoint(Db, Doc) ->
+    update_checkpoint(Db, Doc, false, #user_ctx{}).
+
+update_checkpoint(Db, Doc, Access, UserCtx, DbType) ->
     try
-        update_checkpoint(Db, Doc)
+        update_checkpoint(Db, Doc, Access, UserCtx)
     catch throw:{checkpoint_commit_failure, Reason} ->
         throw({checkpoint_commit_failure,
             <<"Error updating the ", (to_binary(DbType))/binary,
                 " checkpoint document: ", (to_binary(Reason))/binary>>})
     end.
 
+update_checkpoint(Db, #doc{id = LogId} = Doc0, Access, UserCtx) ->
+    % if db has _access, then:
+    %    get userCtx from replication and splice into doc _access
+    Doc = case Access of
+        true -> Doc0#doc{access = [UserCtx#user_ctx.name]};
+        _False -> Doc0
+    end,
 
-update_checkpoint(Db, #doc{id = LogId, body = LogBody} = Doc) ->
     try
         case couch_replicator_api_wrap:update_doc(Db, Doc, [delay_commit]) of
         {ok, PosRevId} ->
@@ -815,6 +833,8 @@ update_checkpoint(Db, #doc{id = LogId, body = LogBody} = Doc) ->
             throw({checkpoint_commit_failure, Reason})
         end
     catch throw:conflict ->
+        % TODO: An admin could have changed the access on the checkpoint doc.
+        %       However unlikely, we can handle this gracefully here.
         case (catch couch_replicator_api_wrap:open_doc(Db, LogId, [ejson_body])) of
         {ok, #doc{body = LogBody, revs = {Pos, [RevId | _]}}} ->
             % This means that we were able to update successfully the
diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
index 6b4f95c..be15bd3 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
@@ -119,7 +119,7 @@ t_fail_changes_queue({Source, Target}) ->
 
         RepPid = couch_replicator_test_helper:get_pid(RepId),
         State = sys:get_state(RepPid),
-        ChangesQueue = element(20, State),
+        ChangesQueue = element(22, State),
         ?assert(is_process_alive(ChangesQueue)),
 
         {ok, Listener} = rep_result_listener(RepId),
@@ -139,7 +139,7 @@ t_fail_changes_manager({Source, Target}) ->
 
         RepPid = couch_replicator_test_helper:get_pid(RepId),
         State = sys:get_state(RepPid),
-        ChangesManager = element(21, State),
+        ChangesManager = element(23, State),
         ?assert(is_process_alive(ChangesManager)),
 
         {ok, Listener} = rep_result_listener(RepId),
@@ -159,7 +159,7 @@ t_fail_changes_reader_proc({Source, Target}) ->
 
         RepPid = couch_replicator_test_helper:get_pid(RepId),
         State = sys:get_state(RepPid),
-        ChangesReader = element(22, State),
+        ChangesReader = element(24, State),
         ?assert(is_process_alive(ChangesReader)),
 
         {ok, Listener} = rep_result_listener(RepId),


[couchdb] 06/17: feat(btree): handle access field in btree

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 02d191820d38d9ec4cd3a8c875b11aee8921768b
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:58:38 2020 +0200

    feat(btree): handle access field in btree
---
 src/couch/src/couch_bt_engine.erl | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index 48e751a..71594ef 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -675,22 +675,25 @@ id_tree_split(#full_doc_info{}=Info) ->
         update_seq = Seq,
         deleted = Deleted,
         sizes = SizeInfo,
-        rev_tree = Tree
+        rev_tree = Tree,
+        access = Access
     } = Info,
-    {Id, {Seq, ?b2i(Deleted), split_sizes(SizeInfo), disk_tree(Tree)}}.
+    {Id, {Seq, ?b2i(Deleted), split_sizes(SizeInfo), disk_tree(Tree), split_access(Access)}}.
 
 
 id_tree_join(Id, {HighSeq, Deleted, DiskTree}) ->
     % Handle old formats before data_size was added
     id_tree_join(Id, {HighSeq, Deleted, #size_info{}, DiskTree});
-
 id_tree_join(Id, {HighSeq, Deleted, Sizes, DiskTree}) ->
+    id_tree_join(Id, {HighSeq, Deleted, Sizes, DiskTree, []});
+id_tree_join(Id, {HighSeq, Deleted, Sizes, DiskTree, Access}) ->
     #full_doc_info{
         id = Id,
         update_seq = HighSeq,
         deleted = ?i2b(Deleted),
         sizes = couch_db_updater:upgrade_sizes(Sizes),
-        rev_tree = rev_tree(DiskTree)
+        rev_tree = rev_tree(DiskTree),
+        access = join_access(Access)
     }.
 
 
@@ -721,21 +724,24 @@ seq_tree_split(#full_doc_info{}=Info) ->
         update_seq = Seq,
         deleted = Del,
         sizes = SizeInfo,
-        rev_tree = Tree
+        rev_tree = Tree,
+        access = Access
     } = Info,
-    {Seq, {Id, ?b2i(Del), split_sizes(SizeInfo), disk_tree(Tree)}}.
+    {Seq, {Id, ?b2i(Del), split_sizes(SizeInfo), disk_tree(Tree), split_access(Access)}}.
 
 
 seq_tree_join(Seq, {Id, Del, DiskTree}) when is_integer(Del) ->
     seq_tree_join(Seq, {Id, Del, {0, 0}, DiskTree});
-
 seq_tree_join(Seq, {Id, Del, Sizes, DiskTree}) when is_integer(Del) ->
+    seq_tree_join(Seq, {Id, Del, Sizes, DiskTree, []});
+seq_tree_join(Seq, {Id, Del, Sizes, DiskTree, Access}) when is_integer(Del) ->
     #full_doc_info{
         id = Id,
         update_seq = Seq,
         deleted = ?i2b(Del),
         sizes = join_sizes(Sizes),
-        rev_tree = rev_tree(DiskTree)
+        rev_tree = rev_tree(DiskTree),
+        access = join_access(Access)
     };
 
 seq_tree_join(KeySeq, {Id, RevInfos, DeletedRevInfos}) ->
@@ -760,6 +766,8 @@ seq_tree_reduce(reduce, DocInfos) ->
 seq_tree_reduce(rereduce, Reds) ->
     lists:sum(Reds).
 
+join_access(Access) -> Access.
+split_access(Access) -> Access.
 
 local_tree_split(#doc{revs = {0, [Rev]}} = Doc) when is_binary(Rev) ->
     #doc{


[couchdb] 12/17: feat(chttpd): add access support to chttpd

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 5650b03693c43015825ccd831652a58add83f0a3
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:06:12 2020 +0200

    feat(chttpd): add access support to chttpd
---
 src/chttpd/src/chttpd.erl      |  2 ++
 src/chttpd/src/chttpd_db.erl   | 47 +++++++++++++++++++++++++++++++-----------
 src/chttpd/src/chttpd_view.erl | 14 ++++++++++---
 3 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index adde073..10f8ce3 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -860,6 +860,8 @@ start_delayed_response(#delayed_resp{}=DelayedResp) ->
 
 error_info({Error, Reason}) when is_list(Reason) ->
     error_info({Error, couch_util:to_binary(Reason)});
+error_info(access) ->
+    {403, <<"forbidden">>, <<"access">>};
 error_info(bad_request) ->
     {400, <<"bad_request">>, <<>>};
 error_info({bad_request, Reason}) ->
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 6a3df6d..04b6cda 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -106,7 +106,7 @@ handle_changes_req1(#httpd{}=Req, Db) ->
         Etag = chttpd:make_etag({Info, Suffix}),
         DeltaT = timer:now_diff(os:timestamp(), T0) / 1000,
         couch_stats:update_histogram([couchdb, dbinfo], DeltaT),
-        chttpd:etag_respond(Req, Etag, fun() ->
+        case chttpd:etag_respond(Req, Etag, fun() ->
             Acc0 = #cacc{
                 feed = normal,
                 etag = Etag,
@@ -114,7 +114,14 @@ handle_changes_req1(#httpd{}=Req, Db) ->
                 threshold = Max
             },
             fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs)
-        end);
+        end) of
+            % TODO: This may be a debugging leftover, undo by just returning
+            %       chttpd:etag_respond()
+            {error, {forbidden, Message, _Stacktrace}} ->
+                throw({forbidden, Message});
+            Response ->
+                Response
+        end;
     Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource"  ->
         couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]),
         Acc0 = #cacc{
@@ -123,7 +130,14 @@ handle_changes_req1(#httpd{}=Req, Db) ->
             threshold = Max
         },
         try
-            fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs)
+            % TODO: This may be a debugging leftover, undo by just returning
+            %       fabric:changes()
+            case fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) of
+                {error, {forbidden, Message, _Stacktrace}} ->
+                    throw({forbidden, Message});
+                Response ->
+                    Response
+            end
         after
             couch_stats:decrement_counter([couchdb, httpd, clients_requesting_changes])
         end;
@@ -386,6 +400,7 @@ create_db_req(#httpd{}=Req, DbName) ->
     N = chttpd:qs_value(Req, "n", config:get("cluster", "n", "3")),
     Q = chttpd:qs_value(Req, "q", config:get("cluster", "q", "8")),
     P = chttpd:qs_value(Req, "placement", config:get("cluster", "placement")),
+    Access = chttpd:qs_value(Req, "access", false),
     EngineOpt = parse_engine_opt(Req),
     DbProps = parse_partitioned_opt(Req),
     Options = [
@@ -394,8 +409,12 @@ create_db_req(#httpd{}=Req, DbName) ->
         {placement, P},
         {props, DbProps}
     ] ++ EngineOpt,
+    Options1 = case Access of
+        "true" -> [{access, true} | Options];
+        _ -> Options
+    end,
     DocUrl = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName)),
-    case fabric:create_db(DbName, Options) of
+    case fabric:create_db(DbName, Options1) of
     ok ->
         send_json(Req, 201, [{"Location", DocUrl}], {[{ok, true}]});
     accepted ->
@@ -906,16 +925,18 @@ view_cb(Msg, Acc) ->
     couch_mrview_http:view_cb(Msg, Acc).
 
 db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) ->
-    % check for the existence of the doc to handle the 404 case.
-    couch_doc_open(Db, DocId, nil, []),
-    case chttpd:qs_value(Req, "rev") of
+    % fetch the old doc revision, so we can compare access control
+    % in send_update_doc() later.
+    Doc0 = couch_doc_open(Db, DocId, nil, [{user_ctx, Req#httpd.user_ctx}]),
+    Revs = chttpd:qs_value(Req, "rev"),
+    case Revs of
     undefined ->
         Body = {[{<<"_deleted">>,true}]};
     Rev ->
         Body = {[{<<"_rev">>, ?l2b(Rev)},{<<"_deleted">>,true}]}
     end,
-    Doc = couch_doc_from_req(Req, Db, DocId, Body),
-    send_updated_doc(Req, Db, DocId, Doc);
+    Doc = Doc0#doc{revs=Revs,body=Body,deleted=true},
+    send_updated_doc(Req, Db, DocId, couch_doc_from_req(Req, Db, DocId, Doc));
 
 db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) ->
     #doc_query_args{
@@ -1255,10 +1276,12 @@ receive_request_data(Req, LenLeft) when LenLeft > 0 ->
 receive_request_data(_Req, _) ->
     throw(<<"expected more data">>).
 
+update_doc_result_to_json({#doc{id=Id,revs=Rev}, access}) ->
+    update_doc_result_to_json({{Id, Rev}, access});
 update_doc_result_to_json({{Id, Rev}, Error}) ->
-        {_Code, Err, Msg} = chttpd:error_info(Error),
-        {[{id, Id}, {rev, couch_doc:rev_to_str(Rev)},
-            {error, Err}, {reason, Msg}]}.
+    {_Code, Err, Msg} = chttpd:error_info(Error),
+    {[{id, Id}, {rev, couch_doc:rev_to_str(Rev)},
+        {error, Err}, {reason, Msg}]}.
 
 update_doc_result_to_json(#doc{id=DocId}, Result) ->
     update_doc_result_to_json(DocId, Result);
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index f73a8b7..31f59ec 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -51,9 +51,17 @@ fabric_query_view(Db, Req, DDoc, ViewName, Args) ->
     Max = chttpd:chunked_response_buffer_size(),
     VAcc = #vacc{db=Db, req=Req, threshold=Max},
     Options = [{user_ctx, Req#httpd.user_ctx}],
-    {ok, Resp} = fabric:query_view(Db, Options, DDoc, ViewName,
-            fun view_cb/2, VAcc, Args),
-    {ok, Resp#vacc.resp}.
+    % TODO: This might just be a debugging leftover, we might be able
+    %       to undo this by just returning {ok, Resp#vacc.resp}
+    %       However, this *might* be here because we need to handle
+    %       errors here now, because access might tell us to.
+    case fabric:query_view(Db, Options, DDoc, ViewName,
+            fun view_cb/2, VAcc, Args) of
+        {ok, Resp} ->
+            {ok, Resp#vacc.resp};
+        {error, Error} ->
+            throw(Error)
+    end.
 
 
 view_cb({row, Row} = Msg, Acc) ->


[couchdb] 16/17: test: adjust peruser tests to access

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9da1e4b8507e160ef124a7d635e093acdb596afb
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:07:51 2020 +0200

    test: adjust peruser tests to access
---
 src/couch_peruser/test/eunit/couch_peruser_test.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/couch_peruser/test/eunit/couch_peruser_test.erl b/src/couch_peruser/test/eunit/couch_peruser_test.erl
index 5ddbe7a..151c493 100644
--- a/src/couch_peruser/test/eunit/couch_peruser_test.erl
+++ b/src/couch_peruser/test/eunit/couch_peruser_test.erl
@@ -53,8 +53,8 @@ teardown(TestAuthDb) ->
     do_request(delete, get_cluster_base_url() ++ "/" ++ ?b2l(TestAuthDb)),
     do_request(delete, get_base_url() ++ "/" ++ ?b2l(TestAuthDb)),
     lists:foreach(fun(DbName) ->
-        case binary:part(DbName, 0, 7) of
-            <<"userdb-">> -> delete_db(DbName);
+        case binary:part(DbName, 0, 6) of
+            <<"userdb">> -> delete_db(DbName);
             _ -> ok
         end
     end, all_dbs()).


[couchdb] 10/17: feat(access): main db update and validation logic

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 672a790139febef2c0924cd1875d167b7ae95031
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:02:35 2020 +0200

    feat(access): main db update and validation logic
---
 src/couch/src/couch_db.erl         | 230 +++++++++++++++++++++++++++++++------
 src/couch/src/couch_db_updater.erl | 147 ++++++++++++++++++++----
 2 files changed, 319 insertions(+), 58 deletions(-)

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index e1d726d..cac768d 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -30,6 +30,9 @@
     is_admin/1,
     check_is_admin/1,
     check_is_member/1,
+    validate_access/2,
+    check_access/2,
+    has_access_enabled/1,
 
     name/1,
     get_after_doc_read_fun/1,
@@ -135,6 +138,8 @@
 
 
 -include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
 -include("couch_db_int.hrl").
 
 -define(DBNAME_REGEX,
@@ -276,6 +281,12 @@ wait_for_compaction(#db{main_pid=Pid}=Db, Timeout) ->
             ok
     end.
 
+has_access_enabled(#db{access=true}) -> true;
+has_access_enabled(_) -> false.
+
+is_read_from_ddoc_cache(Options) ->
+    lists:member(ddoc_cache, Options).
+
 delete_doc(Db, Id, Revisions) ->
     DeletedDocs = [#doc{id=Id, revs=[Rev], deleted=true} || Rev <- Revisions],
     {ok, [Result]} = update_docs(Db, DeletedDocs, []),
@@ -284,26 +295,36 @@ delete_doc(Db, Id, Revisions) ->
 open_doc(Db, IdOrDocInfo) ->
     open_doc(Db, IdOrDocInfo, []).
 
-open_doc(Db, Id, Options) ->
+open_doc(Db, Id, Options0) ->
     increment_stat(Db, [couchdb, database_reads]),
+    Options = case has_access_enabled(Db) of
+        true -> Options0 ++ [conflicts];
+        _Else -> Options0
+    end,
     case open_doc_int(Db, Id, Options) of
     {ok, #doc{deleted=true}=Doc} ->
         case lists:member(deleted, Options) of
         true ->
-            apply_open_options({ok, Doc},Options);
+            {ok, Doc};
         false ->
             {not_found, deleted}
         end;
     Else ->
-        apply_open_options(Else,Options)
+        Else
     end.
 
-apply_open_options({ok, Doc},Options) ->
-    apply_open_options2(Doc,Options);
-apply_open_options(Else,_Options) ->
+apply_open_options(Db, {ok, Doc}, Options) ->
+    ok = validate_access(Db, Doc, Options),
+    apply_open_options1({ok, Doc}, Options);
+apply_open_options(_Db, Else, _Options) ->
+    Else.
+
+apply_open_options1({ok, Doc}, Options) ->
+    apply_open_options2(Doc, Options);
+apply_open_options1(Else, _Options) ->
     Else.
 
-apply_open_options2(Doc,[]) ->
+apply_open_options2(Doc, []) ->
     {ok, Doc};
 apply_open_options2(#doc{atts=Atts0,revs=Revs}=Doc,
         [{atts_since, PossibleAncestors}|Rest]) ->
@@ -317,8 +338,8 @@ apply_open_options2(#doc{atts=Atts0,revs=Revs}=Doc,
     apply_open_options2(Doc#doc{atts=Atts}, Rest);
 apply_open_options2(Doc, [ejson_body | Rest]) ->
     apply_open_options2(couch_doc:with_ejson_body(Doc), Rest);
-apply_open_options2(Doc,[_|Rest]) ->
-    apply_open_options2(Doc,Rest).
+apply_open_options2(Doc, [_|Rest]) ->
+    apply_open_options2(Doc, Rest).
 
 
 find_ancestor_rev_pos({_, []}, _AttsSinceRevs) ->
@@ -336,7 +357,7 @@ find_ancestor_rev_pos({RevPos, [RevId|Rest]}, AttsSinceRevs) ->
 open_doc_revs(Db, Id, Revs, Options) ->
     increment_stat(Db, [couchdb, database_reads]),
     [{ok, Results}] = open_doc_revs_int(Db, [{Id, Revs}], Options),
-    {ok, [apply_open_options(Result, Options) || Result <- Results]}.
+    {ok, [apply_open_options(Db, Result, Options) || Result <- Results]}.
 
 % Each returned result is a list of tuples:
 % {Id, MissingRevs, PossibleAncestors}
@@ -577,7 +598,8 @@ get_db_info(Db) ->
         name = Name,
         compactor_pid = Compactor,
         instance_start_time = StartTime,
-        committed_update_seq = CommittedUpdateSeq
+        committed_update_seq = CommittedUpdateSeq,
+        access = Access
     } = Db,
     {ok, DocCount} = get_doc_count(Db),
     {ok, DelDocCount} = get_del_doc_count(Db),
@@ -609,7 +631,8 @@ get_db_info(Db) ->
         {committed_update_seq, CommittedUpdateSeq},
         {compacted_seq, CompactedSeq},
         {props, Props},
-        {uuid, Uuid}
+        {uuid, Uuid},
+        {access, Access}
     ],
     {ok, InfoList}.
 
@@ -724,6 +747,73 @@ security_error_type(#user_ctx{name=null}) ->
 security_error_type(#user_ctx{name=_}) ->
     forbidden.
 
+is_per_user_ddoc(#doc{access=[]}) -> false;
+is_per_user_ddoc(#doc{access=[<<"_users">>]}) -> false;
+is_per_user_ddoc(_) -> true.
+    
+
+validate_access(Db, Doc) ->
+    validate_access(Db, Doc, []).
+
+validate_access(Db, Doc, Options) ->
+    validate_access1(has_access_enabled(Db), Db, Doc, Options).
+
+validate_access1(false, _Db, _Doc, _Options) -> ok;
+validate_access1(true, Db, #doc{meta=Meta}=Doc, Options) ->
+    case proplists:get_value(conflicts, Meta) of
+        undefined -> % no conflicts
+            case is_read_from_ddoc_cache(Options) andalso is_per_user_ddoc(Doc) of
+                true -> throw({not_found, missing});
+                _False -> validate_access2(Db, Doc)
+            end;
+        _Else -> % only admins can read conflicted docs in _access dbs
+            case is_admin(Db) of
+                true -> ok;
+                _Else2 -> throw({forbidden, <<"document is in conflict">>})
+            end
+    end.
+validate_access2(Db, Doc) ->
+    validate_access3(check_access(Db, Doc)).
+
+validate_access3(true) -> ok;
+validate_access3(_) -> throw({forbidden, <<"can't touch this">>}).
+
+check_access(Db, #doc{access=Access}=Doc) ->
+    check_access(Db, Access);
+check_access(Db, Access) ->
+    #user_ctx{
+        name=UserName,
+        roles=UserRoles
+    } = Db#db.user_ctx,
+    case Access of
+    [] ->
+        % if doc has no _access, userCtX must be admin
+        is_admin(Db);
+    Access ->
+        % if doc has _access, userCtx must be admin OR matching user or role
+        % _access = ["a", "b", ]
+        case is_admin(Db) of
+        true ->
+            true;
+        _ ->
+            case {check_name(UserName, Access), check_roles(UserRoles, Access)} of
+            {true, _} -> true;
+            {_, true} -> true;
+            _ -> false
+            end
+        end
+    end.
+
+check_name(null, _Access) -> true;
+check_name(UserName, Access) ->
+            lists:member(UserName, Access).
+% nicked from couch_db:check_security
+
+check_roles(Roles, Access) ->
+    UserRolesSet = ordsets:from_list(Roles),
+    RolesSet = ordsets:from_list(Access ++ ["_users"]),
+    not ordsets:is_disjoint(UserRolesSet, RolesSet).
+
 
 get_admins(#db{security=SecProps}) ->
     couch_util:get_value(<<"admins">>, SecProps, {[]}).
@@ -863,9 +953,14 @@ group_alike_docs([Doc|Rest], [Bucket|RestBuckets]) ->
     end.
 
 validate_doc_update(#db{}=Db, #doc{id= <<"_design/",_/binary>>}=Doc, _GetDiskDocFun) ->
-    case catch check_is_admin(Db) of
-        ok -> validate_ddoc(Db, Doc);
-        Error -> Error
+    case couch_doc:has_access(Doc) of
+        true ->
+            validate_ddoc(Db, Doc);
+        _Else ->
+            case catch check_is_admin(Db) of
+                ok -> validate_ddoc(Db, Doc);
+                Error -> Error
+            end
     end;
 validate_doc_update(#db{validate_doc_funs = undefined} = Db, Doc, Fun) ->
     ValidationFuns = load_validation_funs(Db),
@@ -1172,6 +1267,32 @@ doc_tag(#doc{meta=Meta}) ->
         Else -> throw({invalid_doc_tag, Else})
     end.
 
+validate_update(Db, Doc) ->
+    case catch validate_access(Db, Doc) of
+        ok -> Doc;
+        Error -> Error
+    end.
+
+
+validate_docs_access(Db, DocBuckets, DocErrors) ->
+            validate_docs_access1(Db, DocBuckets, {[], DocErrors}).
+
+validate_docs_access1(_Db, [], {DocBuckets0, DocErrors}) ->
+            DocBuckets1 = lists:reverse(lists:map(fun lists:reverse/1, DocBuckets0)),
+    DocBuckets = case DocBuckets1 of
+        [[]] -> [];
+        Else -> Else
+    end,
+    {ok, DocBuckets, lists:reverse(DocErrors)};
+validate_docs_access1(Db, [DocBucket|RestBuckets], {DocAcc, ErrorAcc}) ->
+    {NewBuckets, NewErrors} = lists:foldl(fun(Doc, {Acc, ErrAcc}) ->
+        case catch validate_access(Db, Doc) of
+            ok -> {[Doc|Acc], ErrAcc};
+            Error -> {Acc, [{doc_tag(Doc), Error}|ErrAcc]}
+        end
+    end, {[], ErrorAcc}, DocBucket),
+    validate_docs_access1(Db, RestBuckets, {[NewBuckets | DocAcc], NewErrors}).
+
 update_docs(Db, Docs0, Options, replicated_changes) ->
     Docs = tag_docs(Docs0),
 
@@ -1185,9 +1306,31 @@ update_docs(Db, Docs0, Options, replicated_changes) ->
 
     DocBuckets2 = [[doc_flush_atts(Db, check_dup_atts(Doc))
             || Doc <- Bucket] || Bucket <- DocBuckets],
-    {ok, _} = write_and_commit(Db, DocBuckets2,
+    {ok, Results} = write_and_commit(Db, DocBuckets2,
         NonRepDocs, [merge_conflicts | Options]),
-    {ok, DocErrors};
+
+    case couch_db:has_access_enabled(Db) of
+        false ->
+            % we’re done here
+            {ok, DocErrors};
+        _ ->
+            AccessViolations = lists:filter(fun({_Ref, Tag}) -> Tag =:= access end, Results),
+            case length(AccessViolations) of
+                0 ->
+                    % we’re done here
+                    {ok, DocErrors};
+                _ ->
+                    % dig out FDIs from Docs matching our tags/refs
+                    DocsDict = lists:foldl(fun(Doc, Dict) ->
+                        Tag = doc_tag(Doc),
+                        dict:store(Tag, Doc, Dict)
+                    end, dict:new(), Docs),
+                    AccessResults = lists:map(fun({Ref, Access}) ->
+                        { dict:fetch(Ref, DocsDict), Access }
+                    end, AccessViolations),
+                    {ok, AccessResults}
+            end
+    end;
 
 update_docs(Db, Docs0, Options, interactive_edit) ->
     Docs = tag_docs(Docs0),
@@ -1272,7 +1415,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1,
     MergeConflicts = lists:member(merge_conflicts, Options),
     MRef = erlang:monitor(process, Pid),
     try
-        Pid ! {update_docs, self(), DocBuckets, NonRepDocs, MergeConflicts},
+        Pid ! {update_docs, self(), DocBuckets, NonRepDocs, MergeConflicts, Ctx},
         case collect_results_with_metrics(Pid, MRef, []) of
         {ok, Results} -> {ok, Results};
         retry ->
@@ -1286,7 +1429,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1,
             % We only retry once
             DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2),
             close(Db2),
-            Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts},
+            Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts, Ctx},
             case collect_results_with_metrics(Pid, MRef, []) of
             {ok, Results} -> {ok, Results};
             retry -> throw({update_error, compaction_retry})
@@ -1475,6 +1618,11 @@ open_read_stream(Db, AttState) ->
 is_active_stream(Db, StreamEngine) ->
     couch_db_engine:is_active_stream(Db, StreamEngine).
 
+changes_since(Db, StartSeq, Fun, Options, Acc) when is_record(Db, db) ->
+    case couch_db:has_access_enabled(Db) and not couch_db:is_admin(Db) of
+        true -> couch_mrview:query_changes_access(Db, StartSeq, Fun, Options, Acc);
+        false -> couch_db_engine:fold_changes(Db, StartSeq, Fun, Options, Acc)
+    end.
 
 calculate_start_seq(_Db, _Node, Seq) when is_integer(Seq) ->
     Seq;
@@ -1594,8 +1742,10 @@ fold_changes(Db, StartSeq, UserFun, UserAcc) ->
 
 
 fold_changes(Db, StartSeq, UserFun, UserAcc, Opts) ->
-    couch_db_engine:fold_changes(Db, StartSeq, UserFun, UserAcc, Opts).
-
+    case couch_db:has_access_enabled(Db) and not couch_db:is_admin(Db) of
+        true -> couch_mrview:query_changes_access(Db, StartSeq, UserFun, Opts, UserAcc);
+        false -> couch_db_engine:fold_changes(Db, StartSeq, UserFun, UserAcc, Opts)
+    end.
 
 fold_purge_infos(Db, StartPurgeSeq, Fun, Acc) ->
     fold_purge_infos(Db, StartPurgeSeq, Fun, Acc, []).
@@ -1616,7 +1766,7 @@ open_doc_revs_int(Db, IdRevs, Options) ->
     lists:zipwith(
         fun({Id, Revs}, Lookup) ->
             case Lookup of
-            #full_doc_info{rev_tree=RevTree} ->
+            #full_doc_info{rev_tree=RevTree, access=Access} ->
                 {FoundRevs, MissingRevs} =
                 case Revs of
                 all ->
@@ -1636,7 +1786,7 @@ open_doc_revs_int(Db, IdRevs, Options) ->
                         % we have the rev in our list but know nothing about it
                         {{not_found, missing}, {Pos, Rev}};
                     #leaf{deleted=IsDeleted, ptr=SummaryPtr} ->
-                        {ok, make_doc(Db, Id, IsDeleted, SummaryPtr, FoundRevPath)}
+                        {ok, make_doc(Db, Id, IsDeleted, SummaryPtr, FoundRevPath, Access)}
                     end
                 end, FoundRevs),
                 Results = FoundResults ++ [{{not_found, missing}, MissingRev} || MissingRev <- MissingRevs],
@@ -1652,21 +1802,27 @@ open_doc_revs_int(Db, IdRevs, Options) ->
 open_doc_int(Db, <<?LOCAL_DOC_PREFIX, _/binary>> = Id, Options) ->
     case couch_db_engine:open_local_docs(Db, [Id]) of
     [#doc{} = Doc] ->
-        apply_open_options({ok, Doc}, Options);
+        case Doc#doc.body of
+            { Body } ->
+                Access = couch_util:get_value(<<"_access">>, Body),
+                apply_open_options(Db, {ok, Doc#doc{access = Access}}, Options);
+            _Else ->
+                apply_open_options(Db, {ok, Doc}, Options)
+        end;
     [not_found] ->
         {not_found, missing}
     end;
-open_doc_int(Db, #doc_info{id=Id,revs=[RevInfo|_]}=DocInfo, Options) ->
+open_doc_int(Db, #doc_info{id=Id,revs=[RevInfo|_],access=Access}=DocInfo, Options) ->
     #rev_info{deleted=IsDeleted,rev={Pos,RevId},body_sp=Bp} = RevInfo,
-    Doc = make_doc(Db, Id, IsDeleted, Bp, {Pos,[RevId]}),
-    apply_open_options(
+    Doc = make_doc(Db, Id, IsDeleted, Bp, {Pos,[RevId]}, Access),
+    apply_open_options(Db,
        {ok, Doc#doc{meta=doc_meta_info(DocInfo, [], Options)}}, Options);
-open_doc_int(Db, #full_doc_info{id=Id,rev_tree=RevTree}=FullDocInfo, Options) ->
+open_doc_int(Db, #full_doc_info{id=Id,rev_tree=RevTree,access=Access}=FullDocInfo, Options) ->
     #doc_info{revs=[#rev_info{deleted=IsDeleted,rev=Rev,body_sp=Bp}|_]} =
         DocInfo = couch_doc:to_doc_info(FullDocInfo),
     {[{_, RevPath}], []} = couch_key_tree:get(RevTree, [Rev]),
-    Doc = make_doc(Db, Id, IsDeleted, Bp, RevPath),
-    apply_open_options(
+    Doc = make_doc(Db, Id, IsDeleted, Bp, RevPath, Access),
+    apply_open_options(Db,
         {ok, Doc#doc{meta=doc_meta_info(DocInfo, RevTree, Options)}}, Options);
 open_doc_int(Db, Id, Options) ->
     case get_full_doc_info(Db, Id) of
@@ -1716,22 +1872,28 @@ doc_meta_info(#doc_info{high_seq=Seq,revs=[#rev_info{rev=Rev}|RestInfo]}, RevTre
     true -> [{local_seq, Seq}]
     end.
 
+make_doc(Db, Id, Deleted, Bp, RevisionPath) ->
+    make_doc(Db, Id, Deleted, Bp, RevisionPath, []);
+make_doc(Db, Id, Deleted, Bp, {Pos, Revs}) ->
+    make_doc(Db, Id, Deleted, Bp, {Pos, Revs}, []).
 
-make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath) ->
+make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath, Access) ->
     #doc{
         id = Id,
         revs = RevisionPath,
         body = [],
         atts = [],
-        deleted = Deleted
+        deleted = Deleted,
+        access = Access
     };
-make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}) ->
+make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}, Access) ->
     RevsLimit = get_revs_limit(Db),
     Doc0 = couch_db_engine:read_doc_body(Db, #doc{
         id = Id,
         revs = {Pos, lists:sublist(Revs, 1, RevsLimit)},
         body = Bp,
-        deleted = Deleted
+        deleted = Deleted,
+        access = Access
     }),
     Doc1 = case Doc0#doc.atts of
         BinAtts when is_binary(BinAtts) ->
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 1ca804c..41d3661 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -22,7 +22,10 @@
 
 -define(IDLE_LIMIT_DEFAULT, 61000).
 -define(DEFAULT_MAX_PARTITION_SIZE, 16#280000000). % 10 GiB
-
+-define(DEFAULT_SECURITY_OBJECT, [
+ {<<"members">>,{[{<<"roles">>,[<<"_admin">>]}]}},
+ {<<"admins">>, {[{<<"roles">>,[<<"_admin">>]}]}}
+]).
 
 -record(merge_acc, {
     revs_limit,
@@ -37,7 +40,7 @@
 init({Engine, DbName, FilePath, Options0}) ->
     erlang:put(io_priority, {db_update, DbName}),
     update_idle_limit_from_config(),
-    DefaultSecObj = default_security_object(DbName),
+    DefaultSecObj = default_security_object(DbName, Options0),
     Options = [{default_security_object, DefaultSecObj} | Options0],
     try
         {ok, EngineState} = couch_db_engine:init(Engine, FilePath, Options),
@@ -162,7 +165,7 @@ handle_cast(Msg, #db{name = Name} = Db) ->
     {stop, Msg, Db}.
 
 
-handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts},
+handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts, UserCtx},
         Db) ->
     GroupedDocs2 = sort_and_tag_grouped_docs(Client, GroupedDocs),
     if NonRepDocs == [] ->
@@ -173,7 +176,7 @@ handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts},
         Clients = [Client]
     end,
     NonRepDocs2 = [{Client, NRDoc} || NRDoc <- NonRepDocs],
-    try update_docs_int(Db, GroupedDocs3, NonRepDocs2, MergeConflicts) of
+    try update_docs_int(Db, GroupedDocs3, NonRepDocs2, MergeConflicts, UserCtx) of
     {ok, Db2, UpdatedDDocIds} ->
         ok = gen_server:call(couch_server, {db_updated, Db2}, infinity),
         case {couch_db:get_update_seq(Db), couch_db:get_update_seq(Db2)} of
@@ -248,7 +251,11 @@ sort_and_tag_grouped_docs(Client, GroupedDocs) ->
     % The merge_updates function will fail and the database can end up with
     % duplicate documents if the incoming groups are not sorted, so as a sanity
     % check we sort them again here. See COUCHDB-2735.
-    Cmp = fun([#doc{id=A}|_], [#doc{id=B}|_]) -> A < B end,
+    Cmp = fun
+        ([], []) -> false; % TODO: re-evaluate this addition, might be a
+                           %       superflous now
+        ([#doc{id=A}|_], [#doc{id=B}|_]) -> A < B
+     end,
     lists:map(fun(DocGroup) ->
         [{Client, maybe_tag_doc(D)} || D <- DocGroup]
     end, lists:sort(Cmp, GroupedDocs)).
@@ -298,7 +305,7 @@ init_db(DbName, FilePath, EngineState, Options) ->
 
     BDU = couch_util:get_value(before_doc_update, Options, nil),
     ADR = couch_util:get_value(after_doc_read, Options, nil),
-
+    Access = couch_util:get_value(access, Options, false),
     NonCreateOpts = [Opt || Opt <- Options, Opt /= create],
 
     InitDb = #db{
@@ -308,7 +315,8 @@ init_db(DbName, FilePath, EngineState, Options) ->
         instance_start_time = StartTime,
         options = NonCreateOpts,
         before_doc_update = BDU,
-        after_doc_read = ADR
+        after_doc_read = ADR,
+        access = Access
     },
 
     DbProps = couch_db_engine:get_props(InitDb),
@@ -364,7 +372,8 @@ flush_trees(#db{} = Db,
                             active = WrittenSize,
                             external = ExternalSize
                         },
-                        atts = AttSizeInfo
+                        atts = AttSizeInfo,
+                        access = NewDoc#doc.access
                     },
                     {Leaf, add_sizes(Type, Leaf, SizesAcc)};
                 #leaf{} ->
@@ -439,6 +448,9 @@ doc_tag(#doc{meta=Meta}) ->
         Else -> throw({invalid_doc_tag, Else})
     end.
 
+merge_rev_trees([[]], [], Acc) ->
+    % validate_docs_access left us with no docs to merge
+    {ok, Acc};
 merge_rev_trees([], [], Acc) ->
     {ok, Acc#merge_acc{
         add_infos = lists:reverse(Acc#merge_acc.add_infos)
@@ -611,19 +623,26 @@ maybe_stem_full_doc_info(#full_doc_info{rev_tree = Tree} = Info, Limit) ->
             Info
     end.
 
-update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) ->
+update_docs_int(Db, DocsList, LocalDocs, MergeConflicts, UserCtx) ->
     UpdateSeq = couch_db_engine:get_update_seq(Db),
     RevsLimit = couch_db_engine:get_revs_limit(Db),
 
     Ids = [Id || [{_Client, #doc{id=Id}}|_] <- DocsList],
+    % TODO: maybe a perf hit, instead of zip3-ing existin Accesses into
+    %       our doc lists, maybe find 404 docs differently down in
+    %       validate_docs_access (revs is [], which we can then use
+    %       to skip validation as we know it is the first doc rev)
+    Accesses = [Access || [{_Client, #doc{access=Access}}|_] <- DocsList],
+
     % lookup up the old documents, if they exist.
     OldDocLookups = couch_db_engine:open_docs(Db, Ids),
-    OldDocInfos = lists:zipwith(fun
-        (_Id, #full_doc_info{} = FDI) ->
+    
+    OldDocInfos = lists:zipwith3(fun
+        (_Id, #full_doc_info{} = FDI, _Access) ->
             FDI;
-        (Id, not_found) ->
-            #full_doc_info{id=Id}
-    end, Ids, OldDocLookups),
+        (Id, not_found, Access) ->
+            #full_doc_info{id=Id,access=Access}
+    end, Ids, OldDocLookups, Accesses),
 
     %% Get the list of full partitions
     FullPartitions = case couch_db:is_partitioned(Db) of
@@ -653,7 +672,14 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) ->
         cur_seq = UpdateSeq,
         full_partitions = FullPartitions
     },
-    {ok, AccOut} = merge_rev_trees(DocsList, OldDocInfos, AccIn),
+    % Loop over DocsList, validate_access for each OldDocInfo on Db,
+    %.  if no OldDocInfo, then send to DocsListValidated, keep OldDocsInfo
+    %   if valid, then send to DocsListValidated, OldDocsInfo
+    %.  if invalid, then send_result tagged `access`(c.f. `conflict)
+    %.    and don’t add to DLV, nor ODI
+
+    { DocsListValidated, OldDocInfosValidated } = validate_docs_access(Db, UserCtx, DocsList, OldDocInfos),
+    {ok, AccOut} = merge_rev_trees(DocsListValidated, OldDocInfosValidated, AccIn),
     #merge_acc{
         add_infos = NewFullDocInfos,
         rem_seqs = RemSeqs
@@ -663,7 +689,8 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) ->
     % the trees, the attachments are already written to disk)
     {ok, IndexFDIs} = flush_trees(Db, NewFullDocInfos, []),
     Pairs = pair_write_info(OldDocLookups, IndexFDIs),
-    LocalDocs2 = update_local_doc_revs(LocalDocs),
+    LocalDocs1 = apply_local_docs_access(Db, LocalDocs),
+    LocalDocs2 = update_local_doc_revs(LocalDocs1),
 
     {ok, Db1} = couch_db_engine:write_doc_infos(Db, Pairs, LocalDocs2),
 
@@ -676,15 +703,83 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) ->
         length(LocalDocs2)
     ),
 
-    % Check if we just updated any design documents, and update the validation
-    % funs if we did.
+    % Check if we just updated any non-access design documents,
+    % and update the validation funs if we did.
+    NonAccessIds = [Id || [{_Client, #doc{id=Id,access=[]}}|_] <- DocsList],
     UpdatedDDocIds = lists:flatmap(fun
         (<<"_design/", _/binary>> = Id) -> [Id];
         (_) -> []
-    end, Ids),
+    end, NonAccessIds),
 
     {ok, commit_data(Db1), UpdatedDDocIds}.
 
+% check_access(Db, UserCtx, Access) ->
+%     check_access(Db, UserCtx, couch_db:has_access_enabled(Db), Access).
+%
+% check_access(_Db, UserCtx, false, _Access) ->
+%     true;
+
+% at this point, we already validated this Db is access enabled, so do the checks right away.
+check_access(Db, UserCtx, Access) -> couch_db:check_access(Db#db{user_ctx=UserCtx}, Access).
+
+% TODO: looks like we go into validation here unconditionally and only check in
+%       check_access() whether the Db has_access_enabled(), we should do this
+%       here on the outside. Might be our perf issue.
+%       However, if it is, that means we have to speed this up as it would still
+%       be too slow for when access is enabled.
+validate_docs_access(Db, UserCtx, DocsList, OldDocInfos) ->
+    case couch_db:has_access_enabled(Db) of
+        true -> validate_docs_access_int(Db, UserCtx, DocsList, OldDocInfos);
+        _Else -> { DocsList, OldDocInfos }
+    end.
+
+validate_docs_access_int(Db, UserCtx, DocsList, OldDocInfos) ->
+    validate_docs_access(Db, UserCtx, DocsList, OldDocInfos, [], []).
+
+validate_docs_access(_Db, UserCtx, [], [], DocsListValidated, OldDocInfosValidated) ->
+    { lists:reverse(DocsListValidated), lists:reverse(OldDocInfosValidated) };
+validate_docs_access(Db, UserCtx, [Docs | DocRest], [OldInfo | OldInfoRest], DocsListValidated, OldDocInfosValidated) ->
+    % loop over Docs as {Client,  NewDoc}
+    %   validate Doc
+    %   if valid, then put back in Docs
+    %   if not, then send_result and skip
+    NewDocs = lists:foldl(fun({ Client, Doc }, Acc) ->
+        % check if we are allowed to update the doc, skip when new doc
+        OldDocMatchesAccess = case OldInfo#full_doc_info.rev_tree of
+            [] -> true;
+            _ -> check_access(Db, UserCtx, OldInfo#full_doc_info.access)
+        end,
+        
+        NewDocMatchesAccess = check_access(Db, UserCtx, Doc#doc.access),
+        case OldDocMatchesAccess andalso NewDocMatchesAccess of
+            true -> % if valid, then send to DocsListValidated, OldDocsInfo
+                    % and store the access context on the new doc
+                [{Client, Doc} | Acc];
+            _Else2 -> % if invalid, then send_result tagged `access`(c.f. `conflict)
+                      % and don’t add to DLV, nor ODI
+                send_result(Client, Doc, access),
+                Acc
+        end
+    end, [], Docs),
+
+    { NewDocsListValidated, NewOldDocInfosValidated } = case length(NewDocs) of
+        0 -> % we sent out all docs as invalid access, drop the old doc info associated with it
+            { [NewDocs | DocsListValidated], OldDocInfosValidated };
+        _ ->
+            { [NewDocs | DocsListValidated], [OldInfo | OldDocInfosValidated] }
+    end,
+    validate_docs_access(Db, UserCtx, DocRest, OldInfoRest, NewDocsListValidated, NewOldDocInfosValidated).
+
+apply_local_docs_access(Db, Docs) ->
+    apply_local_docs_access1(couch_db:has_access_enabled(Db), Docs).
+
+apply_local_docs_access1(false, Docs) ->
+    Docs;
+apply_local_docs_access1(true, Docs) ->
+    lists:map(fun({Client, #doc{access = Access, body = {Body}} = Doc}) ->
+        Doc1 = Doc#doc{body = {[{<<"_access">>, Access} | Body]}},
+        {Client, Doc1}
+    end, Docs).
 
 update_local_doc_revs(Docs) ->
     lists:foldl(fun({Client, Doc}, Acc) ->
@@ -850,19 +945,23 @@ get_meta_body_size(Meta) ->
     ExternalSize.
 
 
+default_security_object(DbName, []) ->
+    default_security_object(DbName);
+default_security_object(DbName, Options) ->
+    case lists:member({access, true}, Options) of
+        false -> default_security_object(DbName);
+        true -> ?DEFAULT_SECURITY_OBJECT
+    end.
 default_security_object(<<"shards/", _/binary>>) ->
     case config:get("couchdb", "default_security", "admin_only") of
-        "admin_only" ->
-            [{<<"members">>,{[{<<"roles">>,[<<"_admin">>]}]}},
-             {<<"admins">>,{[{<<"roles">>,[<<"_admin">>]}]}}];
+        "admin_only" -> ?DEFAULT_SECURITY_OBJECT;
         Everyone when Everyone == "everyone"; Everyone == "admin_local" ->
             []
     end;
 default_security_object(_DbName) ->
     case config:get("couchdb", "default_security", "admin_only") of
         Admin when Admin == "admin_only"; Admin == "admin_local" ->
-            [{<<"members">>,{[{<<"roles">>,[<<"_admin">>]}]}},
-             {<<"admins">>,{[{<<"roles">>,[<<"_admin">>]}]}}];
+            ?DEFAULT_SECURITY_OBJECT;
         "everyone" ->
             []
     end.


[couchdb] 01/17: chore: rm .venv on make clean

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 45405fc41cbf2a31d118b506c009b11e34b3333c
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:52:47 2020 +0200

    chore: rm .venv on make clean
---
 Makefile     | 1 +
 Makefile.win | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index 53cea3b..3180605 100644
--- a/Makefile
+++ b/Makefile
@@ -465,6 +465,7 @@ clean:
 	@rm -f src/couch/priv/couchspawnkillable
 	@rm -f src/couch/priv/couch_js/config.h
 	@rm -f dev/boot_node.beam dev/pbkdf2.pyc log/crash.log
+	@rm -f .venv
 
 
 .PHONY: distclean
diff --git a/Makefile.win b/Makefile.win
index 7e14a53..6006be0 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -389,6 +389,7 @@ clean:
 	-@rmdir /s/q src\mango\.venv
 	-@del /f/q src\couch\priv\couch_js\config.h
 	-@del /f/q dev\boot_node.beam dev\pbkdf2.pyc log\crash.log
+	-@rmdir /s/q .venv
 
 
 .PHONY: distclean


[couchdb] 05/17: feat(access): introduce new access query server

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 4a64a8b8261ecb8bea4b3aea9d473949a032a8d6
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:57:32 2020 +0200

    feat(access): introduce new access query server
---
 src/couch/src/couch_access_native_proc.erl | 143 +++++++++++++++++++++++++++++
 src/couch/src/couch_proc_manager.erl       |   1 +
 2 files changed, 144 insertions(+)

diff --git a/src/couch/src/couch_access_native_proc.erl b/src/couch/src/couch_access_native_proc.erl
new file mode 100644
index 0000000..965b124
--- /dev/null
+++ b/src/couch/src/couch_access_native_proc.erl
@@ -0,0 +1,143 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_access_native_proc).
+-behavior(gen_server).
+
+
+-export([
+    start_link/0,
+    set_timeout/2,
+    prompt/2
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+
+-record(st, {
+    indexes = [],
+    timeout = 5000 % TODO: make configurable
+}).
+
+start_link() ->
+    gen_server:start_link(?MODULE, [], []).
+
+
+set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
+    gen_server:call(Pid, {set_timeout, TimeOut}).
+
+
+prompt(Pid, Data) ->
+    gen_server:call(Pid, {prompt, Data}).
+
+
+init(_) ->
+    {ok, #st{}}.
+
+
+terminate(_Reason, _St) ->
+    ok.
+
+
+handle_call({set_timeout, TimeOut}, _From, St) ->
+    {reply, ok, St#st{timeout=TimeOut}};
+
+handle_call({prompt, [<<"reset">>]}, _From, St) ->
+    {reply, true, St#st{indexes=[]}};
+
+handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) ->
+    {reply, true, St#st{indexes=[]}};
+
+handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
+    {reply, true, St};
+
+handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) ->
+    {reply, map_doc(St, mango_json:to_binary(Doc)), St};
+
+handle_call({prompt, [<<"reduce">>, _, _]}, _From, St) ->
+    {reply, null, St};
+
+handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) ->
+    {reply, null, St};
+
+handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
+    {reply, [[]], St};
+
+handle_call(Msg, _From, St) ->
+    {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
+
+handle_cast(garbage_collect, St) ->
+    erlang:garbage_collect(),
+    {noreply, St};
+
+handle_cast(Msg, St) ->
+    {stop, {invalid_cast, Msg}, St}.
+
+
+handle_info(Msg, St) ->
+    {stop, {invalid_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+    {ok, St}.
+
+% return value is an array of arrays, first dimension is the different indexes
+% [0] will be by-access-id // for this test, later we should make this by-access
+% -seq, since that one we will always need, and by-access-id can be opt-in.
+% the second dimension is the number of emit kv pairs:
+% [ // the return value
+%   [ // the first view
+%     ['k1', 'v1'], // the first k/v pair for the first view
+%     ['k2', 'v2']  // second, etc.
+%   ],
+%   [ // second view
+%     ['l1', 'w1'] // first k/v par in second view
+%   ]
+% ]
+% {"id":"account/bongel","key":"account/bongel","value":{"rev":"1-967a00dff5e02add41819138abb3284d"}},
+
+map_doc(_St, {Doc}) ->
+    case couch_util:get_value(<<"_access">>, Doc) of
+        undefined ->
+            [[],[]]; % do not index this doc
+        Access when is_list(Access) ->
+            Id = couch_util:get_value(<<"_id">>, Doc),
+            Rev = couch_util:get_value(<<"_rev">>, Doc),
+            Seq = couch_util:get_value(<<"_seq">>, Doc),
+            Deleted = couch_util:get_value(<<"_deleted">>, Doc, false),
+            BodySp = couch_util:get_value(<<"_body_sp">>, Doc),
+            % by-access-id
+            ById = case Deleted of
+                false ->
+                    lists:map(fun(UserOrRole) -> [
+                        [[UserOrRole, Id], Rev]
+                    ] end, Access);
+                _True -> [[]]
+            end,
+
+            % by-access-seq
+            BySeq = lists:map(fun(UserOrRole) -> [
+                [[UserOrRole, Seq], [{rev, Rev}, {deleted, Deleted}, {body_sp, BodySp}]]
+            ] end, Access),
+            ById ++ BySeq;
+        Else ->
+            % TODO: no comprende: should not be needed once we implement
+            % _access field validation
+            [[],[]]
+    end.
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 0daef3e..901495a 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -110,6 +110,7 @@ init([]) ->
     ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
     ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
     ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+    ets:insert(?SERVERS, [{"_ACCESS", {couch_access_native_proc, start_link, []}}]),
     maybe_configure_erlang_native_servers(),
 
     {ok, #state{


[couchdb] 17/17: feat(fabric): handle access requests

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit adc4bee93fa47681a4bfb03b3eb86f09579ddf87
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:08:23 2020 +0200

    feat(fabric): handle access requests
---
 src/fabric/src/fabric_db_info.erl      |  2 ++
 src/fabric/src/fabric_doc_update.erl   | 13 +++++++++----
 src/fabric/src/fabric_view_changes.erl | 14 ++++++++++----
 src/fabric/src/fabric_view_map.erl     |  4 ++++
 src/fabric/src/fabric_view_reduce.erl  |  4 ++++
 5 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/fabric/src/fabric_db_info.erl b/src/fabric/src/fabric_db_info.erl
index 40da678..a50ba9e 100644
--- a/src/fabric/src/fabric_db_info.erl
+++ b/src/fabric/src/fabric_db_info.erl
@@ -105,6 +105,8 @@ merge_results(Info) ->
             [{disk_format_version, lists:max(X)} | Acc];
         (cluster, [X], Acc) ->
             [{cluster, {X}} | Acc];
+        (access, [X], Acc) ->
+            [{access, X} | Acc];
         (props, Xs, Acc) ->
             [{props, {merge_object(Xs)}} | Acc];
         (_K, _V, Acc) ->
diff --git a/src/fabric/src/fabric_doc_update.erl b/src/fabric/src/fabric_doc_update.erl
index 69babc1..2607240 100644
--- a/src/fabric/src/fabric_doc_update.erl
+++ b/src/fabric/src/fabric_doc_update.erl
@@ -315,7 +315,9 @@ doc_update1() ->
     {ok, StW5_3} = handle_message({rexi_EXIT, nil}, SA2, StW5_2),
     {stop, ReplyW5} = handle_message({rexi_EXIT, nil}, SB2, StW5_3),
     ?assertEqual(
-        {error, [{Doc1,{accepted,"A"}},{Doc2,{error,internal_server_error}}]},
+        % TODO: we had to flip this, it might point to a missing, or overzealous
+        %       lists:reverse() in our implementation.
+        {error, [{Doc2,{error,internal_server_error}},{Doc1,{accepted,"A"}}]},
         ReplyW5
     ).
 
@@ -339,8 +341,9 @@ doc_update2() ->
 
     {stop, Reply} =
         handle_message({rexi_EXIT, 1},lists:nth(3,Shards),Acc2),
-
-    ?assertEqual({accepted, [{Doc1,{accepted,Doc2}}, {Doc2,{accepted,Doc1}}]},
+    % TODO: we had to flip this, it might point to a missing, or overzealous
+    %       lists:reverse() in our implementation.
+    ?assertEqual({accepted, [{Doc2,{accepted,Doc1}}, {Doc1,{accepted,Doc2}}]},
         Reply).
 
 doc_update3() ->
@@ -364,7 +367,9 @@ doc_update3() ->
     {stop, Reply} =
         handle_message({ok, [{ok, Doc1},{ok, Doc2}]},lists:nth(3,Shards),Acc2),
 
-    ?assertEqual({ok, [{Doc1, {ok, Doc2}},{Doc2, {ok,Doc1}}]},Reply).
+    % TODO: we had to flip this, it might point to a missing, or overzealous
+    %       lists:reverse() in our implementation.
+    ?assertEqual({ok, [{Doc2, {ok,Doc1}},{Doc1, {ok, Doc2}}]},Reply).
 
 % needed for testing to avoid having to start the mem3 application
 group_docs_by_shard_hack(_DbName, Shards, Docs) ->
diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl
index febbd31..5b9a866 100644
--- a/src/fabric/src/fabric_view_changes.erl
+++ b/src/fabric/src/fabric_view_changes.erl
@@ -63,16 +63,22 @@ go(DbName, "normal", Options, Callback, Acc0) ->
     case validate_start_seq(DbName, Since) of
     ok ->
         {ok, Acc} = Callback(start, Acc0),
-        {ok, Collector} = send_changes(
+        catch case send_changes(
             DbName,
             Args,
             Callback,
             Since,
             Acc,
             5000
-        ),
-        #collector{counters=Seqs, user_acc=AccOut, offset=Offset} = Collector,
-        Callback({stop, pack_seqs(Seqs), pending_count(Offset)}, AccOut);
+        ) of
+            % TODO: This may be a debugging leftover, undo by just returning
+            %       Callback({stop, pack_seqs…
+            {ok, Collector} ->
+                #collector{counters=Seqs, user_acc=AccOut, offset=Offset} = Collector,
+                Callback({stop, pack_seqs(Seqs), pending_count(Offset)}, AccOut);
+            {error, Error} ->
+                throw(Error)
+        end;       
     Error ->
         Callback(Error, Acc0)
     end.
diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl
index b8d0d39..801fa82 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -58,6 +58,10 @@ go(Db, Options, DDoc, View, Args, Callback, Acc, VInfo) ->
                     "map_view"
                 ),
                 Callback({error, timeout}, Acc);
+            % TODO: this might be a debugging leftover, revert by deleting the
+            %       next two lines
+            {error, {forbidden, Error, _Stacktrace}} ->
+                {error, {forbidden, Error}};
             {error, Error} ->
                 Callback({error, Error}, Acc)
         end
diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl
index a432b2c..831d2dd 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -57,6 +57,10 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
                     "reduce_view"
                 ),
                 Callback({error, timeout}, Acc);
+            % TODO: this might be a debugging leftover, revert by deleting the
+            %       next two lines
+            {error, {forbidden, Error, _Stacktrace}} ->
+                {error, {forbidden, Error}};
             {error, Error} ->
                 Callback({error, Error}, Acc)
         end


[couchdb] 04/17: feat(couch): various records now have an access field

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit c499bb09f36ea1ad378921cf49ef3006568bf2af
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 19:56:41 2020 +0200

    feat(couch): various records now have an access field
---
 src/couch/include/couch_db.hrl | 12 +++++++----
 src/couch/src/couch_db_int.hrl |  3 ++-
 src/couch/src/couch_doc.erl    | 45 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl
index 830b9bc..454ee36 100644
--- a/src/couch/include/couch_db.hrl
+++ b/src/couch/include/couch_db.hrl
@@ -61,7 +61,8 @@
 -record(doc_info, {
     id = <<"">>,
     high_seq = 0,
-    revs = [] % rev_info
+    revs = [], % rev_info
+    access = []
 }).
 
 -record(size_info, {
@@ -74,7 +75,8 @@
     update_seq = 0,
     deleted = false,
     rev_tree = [],
-    sizes = #size_info{}
+    sizes = #size_info{},
+    access = []
 }).
 
 -record(httpd, {
@@ -118,7 +120,8 @@
 
     % key/value tuple of meta information, provided when using special options:
     % couch_db:open_doc(Db, Id, Options).
-    meta = []
+    meta = [],
+    access = []
 }).
 
 
@@ -201,7 +204,8 @@
     ptr,
     seq,
     sizes = #size_info{},
-    atts = []
+    atts = [],
+    access = []
 }).
 
 -record (fabric_changes_acc, {
diff --git a/src/couch/src/couch_db_int.hrl b/src/couch/src/couch_db_int.hrl
index 7da0ce5..b67686f 100644
--- a/src/couch/src/couch_db_int.hrl
+++ b/src/couch/src/couch_db_int.hrl
@@ -37,7 +37,8 @@
     waiting_delayed_commit_deprecated,
 
     options = [],
-    compression
+    compression,
+    access = false
 }).
 
 
diff --git a/src/couch/src/couch_doc.erl b/src/couch/src/couch_doc.erl
index 33ad14f..21b62c9 100644
--- a/src/couch/src/couch_doc.erl
+++ b/src/couch/src/couch_doc.erl
@@ -25,7 +25,7 @@
 
 -export([with_ejson_body/1]).
 -export([is_deleted/1]).
-
+-export([has_access/1, has_no_access/1]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -41,15 +41,27 @@ to_branch(Doc, [RevId | Rest]) ->
     [{RevId, ?REV_MISSING, to_branch(Doc, Rest)}].
 
 % helpers used by to_json_obj
+reduce_access({Access}) -> Access;
+reduce_access(Access) -> Access.
+
 to_json_rev(0, []) ->
     [];
 to_json_rev(Start, [FirstRevId|_]) ->
     [{<<"_rev">>, ?l2b([integer_to_list(Start),"-",revid_to_str(FirstRevId)])}].
 
-to_json_body(true, {Body}) ->
+to_json_body(Del, Body) ->
+    to_json_body(Del, Body, []).
+
+to_json_body(true, {Body}, []) ->
     Body ++ [{<<"_deleted">>, true}];
-to_json_body(false, {Body}) ->
-    Body.
+to_json_body(false, {Body}, []) ->
+    Body;
+to_json_body(true, {Body}, Access0) ->
+    Access = reduce_access(Access0),
+    Body ++ [{<<"_deleted">>, true}] ++ [{<<"_access">>, {Access}}];
+to_json_body(false, {Body}, Access0) ->
+    Access = reduce_access(Access0),
+    Body ++ [{<<"_access">>, Access}].
 
 to_json_revisions(Options, Start, RevIds0) ->
     RevIds = case proplists:get_value(revs, Options) of
@@ -118,10 +130,10 @@ to_json_obj(Doc, Options) ->
     doc_to_json_obj(with_ejson_body(Doc), Options).
 
 doc_to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds},
-            meta=Meta}=Doc,Options)->
+            meta=Meta,access=Access}=Doc,Options)->
     {[{<<"_id">>, Id}]
         ++ to_json_rev(Start, RevIds)
-        ++ to_json_body(Del, Body)
+        ++ to_json_body(Del, Body, Access)
         ++ to_json_revisions(Options, Start, RevIds)
         ++ to_json_meta(Meta)
         ++ to_json_attachments(Doc#doc.atts, Options)
@@ -252,6 +264,10 @@ transfer_fields([{<<"_id">>, Id} | Rest], Doc, DbName) ->
     validate_docid(Id, DbName),
     transfer_fields(Rest, Doc#doc{id=Id}, DbName);
 
+transfer_fields([{<<"_access">>, Access} = Field | Rest], Doc, DbName) ->
+    % TODO: validate access as array strings, and optional arrays of strings
+    transfer_fields(Rest, Doc#doc{access=Access}, DbName);
+
 transfer_fields([{<<"_rev">>, Rev} | Rest], #doc{revs={0, []}}=Doc, DbName) ->
     {Pos, RevId} = parse_rev(Rev),
     transfer_fields(Rest,
@@ -351,7 +367,7 @@ max_seq(Tree, UpdateSeq) ->
     end,
     couch_key_tree:fold(FoldFun, UpdateSeq, Tree).
 
-to_doc_info_path(#full_doc_info{id=Id,rev_tree=Tree,update_seq=FDISeq}) ->
+to_doc_info_path(#full_doc_info{id=Id,rev_tree=Tree,update_seq=FDISeq,access=Access}) ->
     RevInfosAndPath = [
         {rev_info(Node), Path} || {_Leaf, Path} = Node <-
             couch_key_tree:get_all_leafs(Tree)
@@ -364,7 +380,7 @@ to_doc_info_path(#full_doc_info{id=Id,rev_tree=Tree,update_seq=FDISeq}) ->
         end, RevInfosAndPath),
     [{_RevInfo, WinPath}|_] = SortedRevInfosAndPath,
     RevInfos = [RevInfo || {RevInfo, _Path} <- SortedRevInfosAndPath],
-    {#doc_info{id=Id, high_seq=max_seq(Tree, FDISeq), revs=RevInfos}, WinPath}.
+    {#doc_info{id=Id, high_seq=max_seq(Tree, FDISeq), revs=RevInfos,access=Access}, WinPath}.
 
 rev_info({#leaf{} = Leaf, {Pos, [RevId | _]}}) ->
     #rev_info{
@@ -399,6 +415,19 @@ is_deleted(Tree) ->
         false
     end.
 
+get_access({Props}) ->
+    get_access(couch_doc:from_json_obj({Props}));
+get_access(#doc{access=Access}) ->
+    Access.
+
+has_access(Doc) ->
+    has_access1(get_access(Doc)).
+
+has_no_access(Doc) ->
+    not has_access1(get_access(Doc)).
+
+has_access1([]) -> false;
+has_access1(_) -> true.
 
 get_validate_doc_fun({Props}) ->
     get_validate_doc_fun(couch_doc:from_json_obj({Props}));


[couchdb] 09/17: feat(access): _users users now have a default _users role

Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 92d1c4399ebb4bd1a9c64b9919f6cbe752b50bd0
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:01:06 2020 +0200

    feat(access): _users users now have a default _users role
---
 src/couch/src/couch_httpd_auth.erl | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index 45a82bd..cc6715e 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -87,6 +87,13 @@ basic_name_pw(Req) ->
         nil
     end.
 
+extract_roles(UserProps) ->
+    Roles = couch_util:get_value(<<"roles">>, UserProps, []),
+    case lists:member(<<"_admin">>, Roles) of
+        true -> Roles;
+        _ -> Roles ++ [<<"_users">>]
+    end.
+
 default_authentication_handler(Req) ->
     default_authentication_handler(Req, couch_auth_cache).
 
@@ -104,7 +111,7 @@ default_authentication_handler(Req, AuthModule) ->
                     true ->
                         Req#httpd{user_ctx=#user_ctx{
                             name=UserName,
-                            roles=couch_util:get_value(<<"roles">>, UserProps, [])
+                            roles=extract_roles(UserProps)
                         }};
                     false ->
                         authentication_warning(Req, UserName),
@@ -167,7 +174,7 @@ proxy_auth_user(Req) ->
             Roles = case header_value(Req, XHeaderRoles) of
                 undefined -> [];
                 Else ->
-                    [?l2b(R) || R <- string:tokens(Else, ",")]
+                    [?l2b(R) || R <- string:tokens(Else, ",")] ++ [<<"_users">>]
             end,
             case config:get("couch_httpd_auth", "proxy_use_secret", "false") of
                 "true" ->
@@ -269,7 +276,7 @@ cookie_authentication_handler(#httpd{mochi_req=MochiReq}=Req, AuthModule) ->
                                                 [User]),
                                 Req#httpd{user_ctx=#user_ctx{
                                     name=?l2b(User),
-                                    roles=couch_util:get_value(<<"roles">>, UserProps, [])
+                                    roles=extract_roles(UserProps)
                                 }, auth={FullSecret, TimeLeft < Timeout*0.9}};
                             _Else ->
                                 Req