You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/12/06 21:00:18 UTC

[couchdb] branch prototype/fdb-layer-get-dbs-info updated (3e6c529 -> f4945ca)

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard 3e6c529  Delete attachments when possible
 discard 20cdcee  Track the size of data store in a database
 discard f8e1c2d  Support `GET /_dbs_info` endpoint
     new f387055  Support `GET /_dbs_info` endpoint
     new b38ab2b  Track the size of data stored in a database
     new 1613319  Delete attachments when possible
     new f4945ca  Add database size tests

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (3e6c529)
            \
             N -- N -- N   refs/heads/prototype/fdb-layer-get-dbs-info (f4945ca)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/chttpd/src/chttpd_misc.erl            |   4 +-
 src/fabric/src/fabric2_fdb.erl            |   9 +-
 src/fabric/src/fabric2_util.erl           |   2 +-
 src/fabric/test/fabric2_db_size_tests.erl | 198 ++++++++++++++++++++++++++++++
 4 files changed, 206 insertions(+), 7 deletions(-)
 create mode 100644 src/fabric/test/fabric2_db_size_tests.erl


[couchdb] 01/04: Support `GET /_dbs_info` endpoint

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f387055ed6092c13f156d315a35d9610bc62ebb2
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 13:44:35 2019 -0600

    Support `GET /_dbs_info` endpoint
    
    Previously only `POST` with a list of keys was supported. The new `GET`
    support just dumps all database info blobs in a single ordered response.
---
 src/chttpd/src/chttpd_misc.erl | 49 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 186ec9f..b181f3c 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -157,6 +157,37 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
     {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
     {ok, Acc#vacc{resp=Resp1}}.
 
+handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
+    ok = chttpd:verify_is_server_admin(Req),
+
+    #mrargs{
+        start_key = StartKey,
+        end_key = EndKey,
+        direction = Dir,
+        limit = Limit,
+        skip = Skip
+    } = couch_mrview_http:parse_params(Req, undefined),
+
+    Options = [
+        {start_key, StartKey},
+        {end_key, EndKey},
+        {dir, Dir},
+        {limit, Limit},
+        {skip, Skip}
+    ],
+
+    % TODO: Figure out if we can't calculate a valid
+    % ETag for this request. \xFFmetadataVersion won't
+    % work as we don't bump versions on size changes
+
+    {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, []),
+    Callback = fun dbs_info_callback/2,
+    Acc = #vacc{req = Req, resp = Resp},
+    {ok, Resp} = fabric2_db:list_dbs_info(Callback, Acc, Options),
+    case is_record(Resp, vacc) of
+        true -> {ok, Resp#vacc.resp};
+        _ -> {ok, Resp}
+    end;
 handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
     chttpd:validate_ctype(Req, "application/json"),
     Props = chttpd:json_body_obj(Req),
@@ -188,7 +219,23 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
     send_chunk(Resp, "]"),
     chttpd:end_json_response(Resp);
 handle_dbs_info_req(Req) ->
-    send_method_not_allowed(Req, "POST").
+    send_method_not_allowed(Req, "GET,HEAD,POST").
+
+dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
+    {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
+    {ok, Acc#vacc{resp = Resp1}};
+dbs_info_callback({row, Props}, #vacc{resp = Resp0} = Acc) ->
+    Prepend = couch_mrview_http:prepend_val(Acc),
+    Chunk = [Prepend, ?JSON_ENCODE({Props})],
+    {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+    {ok, Acc#vacc{prepend = ",", resp = Resp1}};
+dbs_info_callback(complete, #vacc{resp = Resp0} = Acc) ->
+    {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"),
+    {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+    {ok, Acc#vacc{resp = Resp2}};
+dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) ->
+    {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
+    {ok, Acc#vacc{resp = Resp1}}.
 
 handle_task_status_req(#httpd{method='GET'}=Req) ->
     ok = chttpd:verify_is_server_admin(Req),


[couchdb] 04/04: Add database size tests

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f4945ca658bd9918ac174e3dca9c56edd198e75d
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 6 14:59:30 2019 -0600

    Add database size tests
---
 src/fabric/test/fabric2_db_size_tests.erl | 198 ++++++++++++++++++++++++++++++
 1 file changed, 198 insertions(+)

diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
new file mode 100644
index 0000000..fc95e0e
--- /dev/null
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -0,0 +1,198 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_db_size_tests).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("fabric2_test.hrl").
+
+
+db_size_test_() ->
+    {
+        "Test document CRUD operations",
+        {
+            setup,
+            fun setup/0,
+            fun cleanup/1,
+            with([
+                ?TDEF(empty_size),
+                ?TDEF(new_doc),
+                ?TDEF(edit_doc),
+                ?TDEF(del_doc),
+                ?TDEF(conflicted_doc),
+                ?TDEF(del_conflict)
+            ])
+        }
+    }.
+
+
+setup() ->
+    Ctx = test_util:start_couch([fabric]),
+    {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+    {Db, Ctx}.
+
+
+cleanup({Db, Ctx}) ->
+    ok = fabric2_db:delete(fabric2_db:name(Db), []),
+    test_util:stop_couch(Ctx).
+
+
+empty_size({Db, _}) ->
+    ?assertEqual(2, db_size(Db)).
+
+
+new_doc({Db, _}) ->
+    increases(Db, fun() ->
+        create_doc(Db)
+    end).
+
+
+edit_doc({Db, _}) ->
+    DocId = fabric2_util:uuid(),
+    {ok, RevId1} = increases(Db, fun() ->
+        create_doc(Db, DocId)
+    end),
+    {ok, RevId2} = increases(Db, fun() ->
+        update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    decreases(Db, fun() ->
+        update_doc(Db, DocId, RevId2)
+    end).
+
+
+del_doc({Db, _}) ->
+    DocId = fabric2_util:uuid(),
+    {ok, RevId} = increases(Db, fun() ->
+        create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    % The change here is -11 becuase we're going from
+    % {"foo":"bar"} == 13 bytes to {} == 2 bytes.
+    % I.e., 2 - 13 == -11
+    diff(Db, fun() ->
+        delete_doc(Db, DocId, RevId)
+    end, -11).
+
+
+conflicted_doc({Db, _}) ->
+    DocId = fabric2_util:uuid(),
+    Before = db_size(Db),
+    {ok, RevId1} = increases(Db, fun() ->
+        create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    Between = db_size(Db),
+    increases(Db, fun() ->
+        create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    After = db_size(Db),
+    ?assertEqual(After - Between, Between - Before).
+
+
+del_conflict({Db, _}) ->
+    DocId = fabric2_util:uuid(),
+    {ok, RevId1} = increases(Db, fun() ->
+        create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    {ok, RevId2} = increases(Db, fun() ->
+        create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+    end),
+    decreases(Db, fun() ->
+        {ok, RevId3} = delete_doc(Db, DocId, RevId2),
+        ?debugFmt("~p ~p ~p", [RevId1, RevId2, RevId3])
+    end).
+
+
+create_doc(Db) ->
+    create_doc(Db, fabric2_util:uuid()).
+
+
+create_doc(Db, DocId) when is_binary(DocId) ->
+    create_doc(Db, DocId, {[]});
+create_doc(Db, {Props} = Body) when is_list(Props) ->
+    create_doc(Db, fabric2_util:uuid(), Body).
+
+
+create_doc(Db, DocId, Body) ->
+    Doc = #doc{
+        id = DocId,
+        body = Body
+    },
+    fabric2_db:update_doc(Db, Doc).
+
+
+create_conflict(Db, DocId, RevId) ->
+    create_conflict(Db, DocId, RevId, {[]}).
+
+
+create_conflict(Db, DocId, RevId, Body) ->
+    {Pos, _} = RevId,
+    % Only keep the first 16 bytes of the UUID
+    % so that we match the normal sized revs
+    <<NewRev:16/binary, _/binary>> = fabric2_util:uuid(),
+    Doc = #doc{
+        id = DocId,
+        revs = {Pos, [NewRev]},
+        body = Body
+    },
+    fabric2_db:update_doc(Db, Doc, [replicated_changes]).
+
+
+update_doc(Db, DocId, RevId) ->
+    update_doc(Db, DocId, RevId, {[]}).
+
+
+update_doc(Db, DocId, {Pos, Rev}, Body) ->
+    Doc = #doc{
+        id = DocId,
+        revs = {Pos, [Rev]},
+        body = Body
+    },
+    fabric2_db:update_doc(Db, Doc).
+
+
+delete_doc(Db, DocId, RevId) ->
+    delete_doc(Db, DocId, RevId, {[]}).
+
+
+delete_doc(Db, DocId, {Pos, Rev}, Body) ->
+    Doc = #doc{
+        id = DocId,
+        revs = {Pos, [Rev]},
+        deleted = true,
+        body = Body
+    },
+    fabric2_db:update_doc(Db, Doc).
+
+
+constant(Db, Fun) -> check(Db, Fun, fun erlang:'=='/2).
+increases(Db, Fun) -> check(Db, Fun, fun erlang:'>'/2).
+decreases(Db, Fun) -> check(Db, Fun, fun erlang:'<'/2).
+diff(Db, Fun, Change) -> check(Db, Fun, fun(A, B) -> (A - B) == Change end).
+
+check(Db, Fun, Cmp) ->
+    Before = db_size(Db),
+    Result = Fun(),
+    After = db_size(Db),
+    ?debugFmt("~p :: ~p ~p", [erlang:fun_info(Cmp), After, Before]),
+    ?assert(Cmp(After, Before)),
+    Result.
+
+
+db_size(Info) when is_list(Info) ->
+    {sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
+    {<<"external">>, External} = lists:keyfind(<<"external">>, 1, Sizes),
+    External;
+db_size(Db) when is_map(Db) ->
+    {ok, Info} = fabric2_db:get_db_info(Db),
+    db_size(Info).


[couchdb] 02/04: Track the size of data stored in a database

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit b38ab2b1c7d57e2ca5635a8da325f5a15a24c1a4
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Dec 4 11:38:48 2019 -0600

    Track the size of data stored in a database
    
    This tracks the number of bytes that would be required to store the
    contents of a database as flat files on disk. Currently the following
    items are tracked:
    
        * Doc ids
        * Revisions
        * Doc body as JSON
        * Attachment names
        * Attachment type
        * Attachment length
        * Attachment md5s
        * Attachment headers
        * Local doc id
        * Local doc revision
        * Local doc bodies
---
 src/couch/src/couch_att.erl                |  19 ++++
 src/fabric/include/fabric2.hrl             |   8 +-
 src/fabric/src/fabric2_db.erl              |   6 +-
 src/fabric/src/fabric2_fdb.erl             | 157 ++++++++++++++++++++++++-----
 src/fabric/test/fabric2_doc_crud_tests.erl |   5 +-
 5 files changed, 163 insertions(+), 32 deletions(-)

diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index 2c33362..90d498c 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -27,6 +27,7 @@
 ]).
 
 -export([
+    external_size/1,
     size_info/1,
     to_disk_term/1,
     from_disk_term/3
@@ -179,6 +180,24 @@ merge_stubs([], _, Merged) ->
     {ok, lists:reverse(Merged)}.
 
 
+external_size(Att) ->
+    NameSize = size(fetch(name, Att)),
+    TypeSize = case fetch(type, Att) of
+        undefined -> 0;
+        Type -> size(Type)
+    end,
+    AttSize = fetch(att_len, Att),
+    Md5Size = case fetch(md5, Att) of
+        undefined -> 0;
+        Md5 -> size(Md5)
+    end,
+    HeadersSize = case fetch(headers, Att) of
+        undefined -> 0;
+        Headers -> couch_ejson_size:encoded_size(Headers)
+    end,
+    NameSize + TypeSize + AttSize + Md5Size + HeadersSize.
+
+
 size_info([]) ->
     {ok, []};
 size_info(Atts) ->
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 189995d..fc07f33 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -41,8 +41,14 @@
 
 % Versions
 
--define(CURR_REV_FORMAT, 0).
+% 0 - Initial implementation
+% 1 - Added size information
 
+-define(CURR_REV_FORMAT, 1).
+
+% 0 - Adding local doc versions
+
+-define(CURR_LDOC_FORMAT, 0).
 
 % Misc constants
 
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 688f794..1bc53ca 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1420,7 +1420,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
         rev_id => {NewRevPos, NewRev},
         rev_path => NewRevPath,
         sequence => undefined,
-        branch_count => undefined
+        branch_count => undefined,
+        rev_size => null
     },
 
     % Gather the list of possible winnig revisions
@@ -1475,7 +1476,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
         rev_id => {RevPos, Rev},
         rev_path => RevPath,
         sequence => undefined,
-        branch_count => undefined
+        branch_count => undefined,
+        rev_size => null
     },
 
     AllRevInfos = fabric2_fdb:get_all_revs(Db, DocId),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index c13ef7c..a91f82a 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -36,6 +36,7 @@
 
     get_stat/2,
     incr_stat/3,
+    incr_stat/4,
 
     get_all_revs/2,
     get_winning_revs/3,
@@ -451,6 +452,19 @@ incr_stat(#{} = Db, StatKey, Increment) when is_integer(Increment) ->
     erlfdb:add(Tx, Key, Increment).
 
 
+incr_stat(_Db, _Section, _Key, 0) ->
+    ok;
+
+incr_stat(#{} = Db, Section, Key, Increment) when is_integer(Increment) ->
+    #{
+        tx := Tx,
+        db_prefix := DbPrefix
+    } = ensure_current(Db),
+
+    BinKey = erlfdb_tuple:pack({?DB_STATS, Section, Key}, DbPrefix),
+    erlfdb:add(Tx, BinKey, Increment).
+
+
 get_all_revs(#{} = Db, DocId) ->
     #{
         tx := Tx,
@@ -570,6 +584,15 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
 
 get_local_doc_rev(_Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId, Val) ->
     case Val of
+        <<255, RevBin/binary>> ->
+            % Versioned local docs
+            try
+                case erlfdb_tuple:unpack(RevBin) of
+                    {?CURR_LDOC_FORMAT, Rev, _Size} -> Rev
+                end
+            catch _:_ ->
+                erlang:error({invalid_local_doc_rev, DocId, Val})
+            end;
         <<131, _/binary>> ->
             % Compatibility clause for an older encoding format
             try binary_to_term(Val, [safe]) of
@@ -603,9 +626,16 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
         deleted = Deleted
     } = Doc,
 
+    % Doc body
+
+    {ok, RevSize} = write_doc_body(Db, Doc),
+
     % Revision tree
 
-    NewWinner = NewWinner0#{winner := true},
+    NewWinner = NewWinner0#{
+        winner := true,
+        rev_size := RevSize
+    },
     NewRevId = maps:get(rev_id, NewWinner),
 
     {WKey, WVal, WinnerVS} = revinfo_to_fdb(Tx, DbPrefix, DocId, NewWinner),
@@ -667,9 +697,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
     NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, NewRevId}),
     erlfdb:set_versionstamped_key(Tx, NewSeqKey, NewSeqVal),
 
-    % And all the rest...
-
-    ok = write_doc_body(Db, Doc),
+    % Bump db version on design doc changes
 
     IsDDoc = case Doc#doc.id of
         <<?DESIGN_DOC_PREFIX, _/binary>> -> true;
@@ -680,6 +708,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
         bump_db_version(Db)
     end,
 
+    % Update our document counts
+
     case UpdateStatus of
         created ->
             if not IsDDoc -> ok; true ->
@@ -706,6 +736,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
             ok
     end,
 
+    % Update database size
+    SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
+        Acc + case maps:get(rev_size, RI, null) of
+            null -> 0;
+            Size -> Size
+        end
+    end, 0, ToRemove),
+    incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+
     ok.
 
 
@@ -717,11 +756,18 @@ write_local_doc(#{} = Db0, Doc) ->
 
     Id = Doc#doc.id,
 
-    {LDocKey, LDocVal, Rows} = local_doc_to_fdb(Db, Doc),
+    {LDocKey, LDocVal, NewSize, Rows} = local_doc_to_fdb(Db, Doc),
 
-    WasDeleted = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
-        <<_/binary>> -> false;
-        not_found -> true
+    {WasDeleted, PrevSize} = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
+        <<255, RevBin/binary>> ->
+            case erlfdb_tuple:unpack(RevBin) of
+                {?CURR_LDOC_FORMAT, _Rev, Size} ->
+                    {false, Size}
+            end;
+        <<_/binary>> ->
+            {false, 0};
+        not_found ->
+            {true, 0}
     end,
 
     BPrefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, Id}, DbPrefix),
@@ -747,6 +793,8 @@ write_local_doc(#{} = Db0, Doc) ->
             ok
     end,
 
+    incr_stat(Db, <<"sizes">>, <<"external">>, NewSize - PrevSize),
+
     ok.
 
 
@@ -1010,9 +1058,11 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
         tx := Tx
     } = Db = ensure_current(Db0),
 
+    {Rows, RevSize} = doc_to_fdb(Db, Doc),
     lists:foreach(fun({Key, Value}) ->
         ok = erlfdb:set(Tx, Key, Value)
-    end, doc_to_fdb(Db, Doc)).
+    end, Rows),
+    {ok, RevSize}.
 
 
 clear_doc_body(_Db, _DocId, not_found) ->
@@ -1039,11 +1089,12 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
         deleted := Deleted,
         rev_id := {RevPos, Rev},
         rev_path := RevPath,
-        branch_count := BranchCount
+        branch_count := BranchCount,
+        rev_size := RevSize
     } = RevId,
     VS = new_versionstamp(Tx),
     Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
-    Val = {?CURR_REV_FORMAT, VS, BranchCount, list_to_tuple(RevPath)},
+    Val = {?CURR_REV_FORMAT, VS, BranchCount, list_to_tuple(RevPath), RevSize},
     KBin = erlfdb_tuple:pack(Key, DbPrefix),
     VBin = erlfdb_tuple:pack_vs(Val),
     {KBin, VBin, VS};
@@ -1052,38 +1103,49 @@ revinfo_to_fdb(_Tx, DbPrefix, DocId, #{} = RevId) ->
     #{
         deleted := Deleted,
         rev_id := {RevPos, Rev},
-        rev_path := RevPath
+        rev_path := RevPath,
+        rev_size := RevSize
     } = RevId,
     Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
-    Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath)},
+    Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), RevSize},
     KBin = erlfdb_tuple:pack(Key, DbPrefix),
     VBin = erlfdb_tuple:pack(Val),
     {KBin, VBin, undefined}.
 
 
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
     {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
-    {_RevFormat, Sequence, BranchCount, RevPath} = Val,
+    {_RevFormat, Sequence, BranchCount, RevPath, RevSize} = Val,
     #{
         winner => true,
         deleted => not NotDeleted,
         rev_id => {RevPos, Rev},
         rev_path => tuple_to_list(RevPath),
         sequence => Sequence,
-        branch_count => BranchCount
+        branch_count => BranchCount,
+        rev_size => RevSize
     };
 
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _} = Val)  ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val)  ->
     {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
-    {_RevFormat, RevPath} = Val,
+    {_RevFormat, RevPath, RevSize} = Val,
     #{
         winner => false,
         deleted => not NotDeleted,
         rev_id => {RevPos, Rev},
         rev_path => tuple_to_list(RevPath),
         sequence => undefined,
-        branch_count => undefined
-    }.
+        branch_count => undefined,
+        rev_size => RevSize
+    };
+
+fdb_to_revinfo(Key, {0, S, B, R}) ->
+    Val = {?CURR_REV_FORMAT, S, B, R, null},
+    fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {0, R}) ->
+    Val = {?CURR_REV_FORMAT, R, null},
+    fdb_to_revinfo(Key, Val).
 
 
 doc_to_fdb(Db, #doc{} = Doc) ->
@@ -1102,12 +1164,26 @@ doc_to_fdb(Db, #doc{} = Doc) ->
     DiskAtts = lists:map(fun couch_att:to_disk_term/1, Atts),
 
     Value = term_to_binary({Body, DiskAtts, Deleted}, [{minor_version, 1}]),
+    Chunks = chunkify_binary(Value),
 
     {Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
         Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev, ChunkId}, DbPrefix),
         {{Key, Chunk}, ChunkId + 1}
-    end, 0, chunkify_binary(Value)),
-    Rows.
+    end, 0, Chunks),
+
+    % Calculate the size of this revision
+    TotalSize = lists:sum([
+        size(Id),
+        size(erlfdb_tuple:pack({Start})),
+        size(Rev),
+        1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+        couch_ejson_size:encoded_size(Body),
+        lists:foldl(fun(Att, Acc) ->
+            couch_att:external_size(Att) + Acc
+        end, 0, Atts)
+    ]),
+
+    {Rows, TotalSize}.
 
 
 fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1157,8 +1233,28 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
         {{K, Chunk}, ChunkId + 1}
     end, 0, chunkify_binary(BVal)),
 
-    {Key, StoreRev, Rows}.
+    % Calculate size
+    TotalSize = case Doc#doc.deleted of
+        true ->
+            0;
+        false ->
+            lists:sum([
+                size(Id),
+                size(StoreRev),
+                couch_ejson_size:encoded_size(Body)
+            ])
+    end,
+
+    RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
 
+    % Prefix our tuple encoding to make upgrades easier
+    Value = <<255, RawValue/binary>>,
+
+    {Key, Value, TotalSize, Rows}.
+
+
+fdb_to_local_doc(_Db, _DocId, not_found, []) ->
+    {not_found, missing};
 
 fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
     % This is an upgrade clause for the old encoding. We allow reading the old
@@ -1171,18 +1267,25 @@ fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
         body = Body
     };
 
-fdb_to_local_doc(_Db, _DocId, not_found, []) ->
-    {not_found, missing};
+fdb_to_local_doc(_Db, DocId, <<255, RevBin/binary>>, Rows) when is_list(Rows) ->
+    Rev = case erlfdb_tuple:unpack(RevBin) of
+        {?CURR_LDOC_FORMAT, Rev0, _Size} -> Rev0
+    end,
 
-fdb_to_local_doc(_Db, DocId, Rev, Rows) when is_list(Rows), is_binary(Rev) ->
     BodyBin = iolist_to_binary(Rows),
     Body = binary_to_term(BodyBin, [safe]),
+
     #doc{
         id = DocId,
         revs = {0, [Rev]},
         deleted = false,
         body = Body
-    }.
+    };
+
+fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
+    BaseRev = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, RawRev, 0}),
+    Rev = <<255, BaseRev/binary>>,
+    fdb_to_local_doc(Db, DocId, Rev, Rows).
 
 
 chunkify_binary(Data) ->
diff --git a/src/fabric/test/fabric2_doc_crud_tests.erl b/src/fabric/test/fabric2_doc_crud_tests.erl
index 184eb4a..46cd4fc 100644
--- a/src/fabric/test/fabric2_doc_crud_tests.erl
+++ b/src/fabric/test/fabric2_doc_crud_tests.erl
@@ -884,11 +884,12 @@ local_doc_with_previous_encoding({Db, _}) ->
     ?assertEqual(NewBody, Doc3#doc.body),
 
     % Old doc now has only the rev number in it
-    OldDocBin = fabric2_fdb:transactional(Db, fun(TxDb) ->
+    <<255, OldDocBin/binary>> = fabric2_fdb:transactional(Db, fun(TxDb) ->
         #{tx := Tx} = TxDb,
         erlfdb:wait(erlfdb:get(Tx, Key))
     end),
-    ?assertEqual(<<"2">> , OldDocBin).
+    Unpacked = erlfdb_tuple:unpack(OldDocBin),
+    ?assertMatch({?CURR_LDOC_FORMAT, <<"2">>, _}, Unpacked).
 
 
 before_doc_update_skips_local_docs({Db0, _}) ->


[couchdb] 03/04: Delete attachments when possible

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 16133196fd31a96253f1857ea5b3c2653fe51e9a
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 6 14:40:55 2019 -0600

    Delete attachments when possible
---
 src/fabric/include/fabric2.hrl  |   4 +-
 src/fabric/src/fabric2_db.erl   |  12 +++--
 src/fabric/src/fabric2_fdb.erl  | 116 ++++++++++++++++++++++++++++++++++------
 src/fabric/src/fabric2_util.erl |  13 +++++
 4 files changed, 125 insertions(+), 20 deletions(-)

diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index fc07f33..78b59c3 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -37,14 +37,16 @@
 -define(DB_ATTS, 23).
 -define(DB_VIEWS, 24).
 -define(DB_LOCAL_DOC_BODIES, 25).
+-define(DB_ATT_NAMES, 26).
 
 
 % Versions
 
 % 0 - Initial implementation
 % 1 - Added size information
+% 2 - Added attachment hash
 
--define(CURR_REV_FORMAT, 1).
+-define(CURR_REV_FORMAT, 2).
 
 % 0 - Adding local doc versions
 
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 1bc53ca..c57d33d 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1411,7 +1411,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
 
     #doc{
         deleted = NewDeleted,
-        revs = {NewRevPos, [NewRev | NewRevPath]}
+        revs = {NewRevPos, [NewRev | NewRevPath]},
+        atts = Atts
     } = Doc4 = stem_revisions(Db, Doc3),
 
     NewRevInfo = #{
@@ -1421,7 +1422,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
         rev_path => NewRevPath,
         sequence => undefined,
         branch_count => undefined,
-        rev_size => null
+        rev_size => null,
+        att_hash => fabric2_util:hash_atts(Atts)
     },
 
     % Gather the list of possible winnig revisions
@@ -1467,7 +1469,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
     #doc{
         id = DocId,
         deleted = Deleted,
-        revs = {RevPos, [Rev | RevPath]}
+        revs = {RevPos, [Rev | RevPath]},
+        atts = Atts
     } = Doc0,
 
     DocRevInfo0 = #{
@@ -1477,7 +1480,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
         rev_path => RevPath,
         sequence => undefined,
         branch_count => undefined,
-        rev_size => null
+        rev_size => null,
+        att_hash => fabric2_util:hash_atts(Atts)
     },
 
     AllRevInfos = fabric2_fdb:get_all_revs(Db, DocId),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index a91f82a..a61e06d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -623,7 +623,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
 
     #doc{
         id = DocId,
-        deleted = Deleted
+        deleted = Deleted,
+        atts = Atts
     } = Doc,
 
     % Doc body
@@ -654,6 +655,31 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
         ok = clear_doc_body(Db, DocId, RI0)
     end, ToRemove),
 
+    % Attachment bookkeeping
+
+    % If a document's attachments have changed we have to scan
+    % for any attachments that may need to be deleted. The check
+    % for `>= 2` is a bit subtle. The important point is that
+    % one of the revisions will be from the new document so we
+    % have to find at least one more beyond that to assert that
+    % the attachments have not changed.
+    %
+    % This has to happen after we update the revision tree above
+    % or else we won't delete attachments until the second update
+    % after they have been removed.
+    AttHash = fabric2_util:hash_atts(Atts),
+    RevsToCheck = [NewWinner0] ++ ToUpdate ++ ToRemove,
+    AttHashCount = lists:foldl(fun(Att, Count) ->
+        #{att_hash := RevAttHash} = Att,
+        case RevAttHash == AttHash of
+            true -> Count + 1;
+            false -> Count
+        end
+    end, 0, RevsToCheck),
+    if AttHashCount >= 2 -> ok; true ->
+        scan_attachments(Db, DocId, Doc)
+    end,
+
     % _all_docs
 
     UpdateStatus = case {OldWinner, NewWinner} of
@@ -823,6 +849,9 @@ write_attachment(#{} = Db, DocId, Data) when is_binary(Data) ->
     AttId = fabric2_util:uuid(),
     Chunks = chunkify_binary(Data),
 
+    IdKey = erlfdb_tuple:pack({?DB_ATT_NAMES, DocId, AttId}, DbPrefix),
+    ok = erlfdb:set(Tx, IdKey, <<>>),
+
     lists:foldl(fun(Chunk, ChunkId) ->
         AttKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId, ChunkId}, DbPrefix),
         ok = erlfdb:set(Tx, AttKey, Chunk),
@@ -1084,17 +1113,63 @@ clear_doc_body(#{} = Db, DocId, #{} = RevInfo) ->
     ok = erlfdb:clear_range(Tx, StartKey, EndKey).
 
 
+scan_attachments(Db, DocId, NewDoc) ->
+    #{
+        tx := Tx,
+        db_prefix := DbPrefix
+    } = Db,
+
+    % Gather all known document revisions
+    {ok, DiskDocs} = fabric2_db:open_doc_revs(Db, DocId, all, []),
+    AllDocs = [{ok, NewDoc} | DiskDocs],
+
+    % Get referenced attachment ids
+    ActiveIdSet = list:foldl(fun({ok, Doc}, Acc) ->
+        lists:foldl(fun(Att, InnerAcc) ->
+            {loc, _Db, _DocId, AttId} = couch_att:fetch(data, Att),
+            sets:add_element(AttId, InnerAcc)
+        end, Acc, Doc#doc.atts)
+    end, sets:new(), AllDocs),
+
+    AttPrefix = erlfdb_tuple:pack({?DB_ATT_NAMES, DocId}, DbPrefix),
+    Options = [{streaming_mode, want_all}],
+    Future = erlfdb:get_range_startswith(Tx, AttPrefix, Options),
+
+    ExistingIdSet = lists:foldl(fun({K, _}, Acc) ->
+        {?DB_ATT_NAMES, DocId, AttId} = erlfdb_tuple:unpack(K, DbPrefix),
+        sets:add_element(AttId, Acc)
+    end, sets:new(), erlfdb:wait(Future)),
+
+    ToRemove = sets:subtract(ExistingIdSet, ActiveIdSet),
+
+    lists:foreach(fun(AttId) ->
+        IdKey = erlfdb_tuple:pack({?DB_ATT_NAMES, DocId, AttId}, DbPrefix),
+        erlfdb:clear(Tx, IdKey),
+
+        ChunkKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId}, DbPrefix),
+        erlfdb:clear_range_startswith(Tx, ChunkKey)
+    end, sets:to_list(ToRemove)).
+
+
 revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
     #{
         deleted := Deleted,
         rev_id := {RevPos, Rev},
         rev_path := RevPath,
         branch_count := BranchCount,
-        rev_size := RevSize
+        rev_size := RevSize,
+        att_hash := AttHash
     } = RevId,
     VS = new_versionstamp(Tx),
     Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
-    Val = {?CURR_REV_FORMAT, VS, BranchCount, list_to_tuple(RevPath), RevSize},
+    Val = {
+        ?CURR_REV_FORMAT,
+        VS,
+        BranchCount,
+        list_to_tuple(RevPath),
+        RevSize,
+        AttHash
+    },
     KBin = erlfdb_tuple:pack(Key, DbPrefix),
     VBin = erlfdb_tuple:pack_vs(Val),
     {KBin, VBin, VS};
@@ -1104,18 +1179,19 @@ revinfo_to_fdb(_Tx, DbPrefix, DocId, #{} = RevId) ->
         deleted := Deleted,
         rev_id := {RevPos, Rev},
         rev_path := RevPath,
-        rev_size := RevSize
+        rev_size := RevSize,
+        att_hash := AttHash
     } = RevId,
     Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
-    Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), RevSize},
+    Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), RevSize, AttHash},
     KBin = erlfdb_tuple:pack(Key, DbPrefix),
     VBin = erlfdb_tuple:pack(Val),
     {KBin, VBin, undefined}.
 
 
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
     {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
-    {_RevFormat, Sequence, BranchCount, RevPath, RevSize} = Val,
+    {_RevFormat, Sequence, BranchCount, RevPath, RevSize, AttHash} = Val,
     #{
         winner => true,
         deleted => not NotDeleted,
@@ -1123,12 +1199,13 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
         rev_path => tuple_to_list(RevPath),
         sequence => Sequence,
         branch_count => BranchCount,
-        rev_size => RevSize
+        rev_size => RevSize,
+        att_hash => AttHash
     };
 
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val)  ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val)  ->
     {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
-    {_RevFormat, RevPath, RevSize} = Val,
+    {_RevFormat, RevPath, RevSize, AttHash} = Val,
     #{
         winner => false,
         deleted => not NotDeleted,
@@ -1136,15 +1213,24 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val)  ->
         rev_path => tuple_to_list(RevPath),
         sequence => undefined,
         branch_count => undefined,
-        rev_size => RevSize
+        rev_size => RevSize,
+        att_hash => AttHash
     };
 
-fdb_to_revinfo(Key, {0, S, B, R}) ->
-    Val = {?CURR_REV_FORMAT, S, B, R, null},
+fdb_to_revinfo(Key, {0, Seq, BCount, RPath}) ->
+    Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, null, <<>>},
+    fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {0, RPath}) ->
+    Val = {?CURR_REV_FORMAT, RPath, null, <<>>},
+    fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {1, Seq, BCount, RPath, Size}) ->
+    Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, Size, <<>>},
     fdb_to_revinfo(Key, Val);
 
-fdb_to_revinfo(Key, {0, R}) ->
-    Val = {?CURR_REV_FORMAT, R, null},
+fdb_to_revinfo(Key, {1, RPath, Size}) ->
+    Val = {?CURR_REV_FORMAT, RPath, Size, <<>>},
     fdb_to_revinfo(Key, Val).
 
 
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 2b8e49e..6bd4324 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -25,6 +25,8 @@
 
     validate_security_object/1,
 
+    hash_atts/1,
+
     dbname_ends_with/2,
 
     get_value/2,
@@ -124,6 +126,17 @@ validate_json_list_of_strings(Member, Props) ->
     end.
 
 
+hash_atts([]) ->
+    <<>>;
+
+hash_atts(Atts) ->
+    Md5St = lists:foldl(fun(Att, Acc) ->
+        {loc, _Db, _DocId, AttId} = couch_att:fetch(data, Att),
+        couch_hash:md5_hash_update(Acc, AttId)
+    end, couch_hash:md5_hash_init(), Atts),
+    couch_hash:md5_hash_final(Md5St).
+
+
 dbname_ends_with(#{} = Db, Suffix) ->
     dbname_ends_with(fabric2_db:name(Db), Suffix);