[couchdb] 01/10: [01/10] Clustered Purge: Define new purge API

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 167e1a0bef515361c327551324aa617cb0adc86f
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Apr 24 12:23:16 2018 -0500

    [01/10] Clustered Purge: Define new purge API
    
    This is the first of a series of commits to implement the new clustered
    purge API. Currently purge is a single-node only API that allows for
    removing document revisions (and by extension entire documents)
    completely from a database. However, given our anti-entropy measures
    this API is extremely difficult to use in a cluster and requires
    significant operator intervention.
    
    Along with the operator intervention, this API is inherently unsafe with
    regards to accidentally triggering the rebuild of secondary indices. As
    such this patch set is aimed at creating a cluster aware API that is
    both easier to use and less likely to cause application downtime while
    secondary indices are rebuilt.
    
    There are four major areas that will be covered by this patch set:
    
      1. Single node APIs and behavior changes
      2. Cluster aware APIs
      3. Anti-entropy updates
      4. Cluster HTTP implementation
    
    This patch set is split up into a series of commits to aid in the review
    by other commiters that will hopefully allow for a logical and intuitive
    progression of implementation rather than landing as a single opaque
    commit covering a huge swath of the code base.
    
    COUCHDB-3326
    
    Co-authored-by: Mayya Sharipova <ma...@ca.ibm.com>
    Co-authored-by: jiangphcn <ji...@cn.ibm.com>
---
 src/couch/src/couch_db_engine.erl | 167 ++++++++++++++++++++++++++++++--------
 1 file changed, 134 insertions(+), 33 deletions(-)

diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index 2fe0b0d..ea30dbc 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -22,6 +22,8 @@
 -type rev() :: {non_neg_integer(), binary()}.
 -type revs() :: [rev()].
 -type json() :: any().
+-type uuid() :: binary().
+-type purge_seq() :: non_neg_integer().
 
 -type doc_pair() :: {
         #full_doc_info{} | not_found,
@@ -39,7 +41,7 @@
         sync
     ].
 
--type purge_info() :: [{docid(), revs()}].
+-type purge_info() :: {purge_seq(), uuid(), docid(), revs()}.
 -type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}].
 -type size_info() :: [{Name::atom(), Size::non_neg_integer()}].
 
@@ -62,6 +64,13 @@
         {dir, fwd | rev}
     ].
 
+-type purge_fold_options() :: [
+        {start_key, Key::any()} |
+        {end_key, Key::any()} |
+        {end_key_gt, Key::any()} |
+        {dir, fwd | rev}
+    ].
+
 -type db_handle() :: any().
 
 -type doc_fold_fun() :: fun((#full_doc_info{}, UserAcc::any()) ->
@@ -76,6 +85,10 @@
         {ok, NewUserAcc::any()} |
         {stop, NewUserAcc::any()}).
 
+-type purge_fold_fun() :: fun((purge_info(), UserAcc::any()) ->
+        {ok, NewUserAcc::any()} |
+        {stop, NewUserAcc::any()}).
+
 
 % This is called by couch_server to determine which
 % engine should be used for the given database. DbPath
@@ -206,13 +219,18 @@
 -callback get_epochs(DbHandle::db_handle()) -> Epochs::epochs().
 
 
-% Get the last purge request performed.
--callback get_last_purged(DbHandle::db_handle()) -> LastPurged::purge_info().
+% Get the current purge sequence known to the engine. This
+% value should be updated during calls to purge_docs.
+-callback get_purge_seq(DbHandle::db_handle()) -> purge_seq().
+
 
+% Get the oldest purge sequence known to the engine
+-callback get_oldest_purge_seq(DbHandle::db_handle()) -> purge_seq().
 
-% Get the current purge sequence. This should be incremented
-% for every purge operation.
--callback get_purge_seq(DbHandle::db_handle()) -> PurgeSeq::non_neg_integer().
+
+% Get the purged infos limit. This should just return the last
+% value that was passed to set_purged_docs_limit/2.
+-callback get_purge_infos_limit(DbHandle::db_handle()) -> pos_integer().
 
 
 % Get the revision limit. This should just return the last
@@ -261,6 +279,11 @@
 -callback set_revs_limit(DbHandle::db_handle(), RevsLimit::pos_integer()) ->
         {ok, NewDbHandle::db_handle()}.
 
+
+-callback set_purge_infos_limit(DbHandle::db_handle(), Limit::pos_integer()) ->
+        {ok, NewDbHandle::db_handle()}.
+
+
 -callback set_security(DbHandle::db_handle(), SecProps::any()) ->
         {ok, NewDbHandle::db_handle()}.
 
@@ -301,6 +324,15 @@
         doc().
 
 
+% This function will be called from many contexts concurrently.
+% If the storage engine has a purge_info() record for any of the
+% provided UUIDs, those purge_info() records should be returned. The
+% resulting list should have the same length as the input list of
+% UUIDs.
+-callback load_purge_infos(DbHandle::db_handle(), [uuid()]) ->
+        [purge_info() | not_found].
+
+
 % This function is called concurrently by any client process
 % that is writing a document. It should accept a #doc{}
 % record and return a #doc{} record with a mutated body it
@@ -341,31 +373,20 @@
 % #full_doc_info{} records. The first element of the pair is
 % the #full_doc_info{} that exists on disk. The second element
 % is the new version that should be written to disk. There are
-% three basic cases that should be followed:
+% two basic cases that should be followed:
 %
 %     1. {not_found, #full_doc_info{}} - A new document was created
 %     2. {#full_doc_info{}, #full_doc_info{}} - A document was updated
-%     3. {#full_doc_info{}, not_found} - A document was purged completely
 %
-% Number one and two are fairly straight forward as long as proper
-% accounting for moving entries in the udpate sequence are accounted
-% for. However, case 3 you'll notice is "purged completely" which
-% means it needs to be removed from the database including the
-% update sequence. Also, for engines that are not using append
-% only storage like the legacy engine, case 2 can be the result of
-% a purge so special care will be needed to see which revisions
-% should be removed.
+% The cases are fairly straight forward as long as proper
+% accounting for moving entries in the update sequence are accounted
+% for.
 %
 % The LocalDocs variable is applied separately. Its important to
 % note for new storage engine authors that these documents are
 % separate because they should *not* be included as part of the
 % changes index for the database.
 %
-% The PurgedDocIdRevs is the list of Ids and Revisions that were
-% purged during this update. While its not guaranteed by the API,
-% currently there will never be purge changes comingled with
-% standard updates.
-%
 % Traditionally an invocation of write_doc_infos should be all
 % or nothing in so much that if an error occurs (or the VM dies)
 % then the database doesn't retain any of the changes. However
@@ -376,8 +397,40 @@
 -callback write_doc_infos(
     DbHandle::db_handle(),
     Pairs::doc_pairs(),
-    LocalDocs::[#doc{}],
-    PurgedDocIdRevs::[{docid(), revs()}]) ->
+    LocalDocs::[#doc{}]) ->
+        {ok, NewDbHandle::db_handle()}.
+
+
+% This function is called from the context of couch_db_updater
+% and as such is guaranteed single threaded for the given
+% DbHandle.
+%
+% Each doc_pair() is a 2-tuple of #full_doc_info{} records. The
+% first element of the pair is the #full_doc_info{} that exists
+% on disk. The second element is the new version that should be
+% written to disk. There are three basic cases that should be considered:
+%
+%     1. {#full_doc_info{}, #full_doc_info{}} - A document was partially purged
+%     2. {#full_doc_info{}, not_found} - A document was completely purged
+%     3. {not_found, not_found} - A no-op purge
+%
+% In case 1, non-tail-append engines may have to remove revisions
+% specifically rather than rely on compaction to remove them. Also
+% note that the new #full_doc_info{} will have a different update_seq
+% that will need to be reflected in the changes feed.
+%
+% In case 2 you'll notice is "purged completely" which
+% means it needs to be removed from the database including the
+% update sequence.
+%
+% In case 3 we just need to store the purge_info() to know that it
+% was processed even though it produced no changes to the database.
+%
+% The purge_info() tuples contain the purge_seq, uuid, docid and
+% revisions that were requested to be purged. This should be persisted
+% in such a way that we can efficiently load purge_info() by its UUID
+% as well as iterate over purge_info() entries in order of their PurgeSeq.
+-callback purge_docs(DbHandle::db_handle(), [doc_pair()], [purge_info()]) ->
         {ok, NewDbHandle::db_handle()}.
 
 
@@ -518,6 +571,21 @@
 
 % This function may be called by many processes concurrently.
 %
+% This function is called to fold over purged requests in order of
+% their oldest purge (increasing purge_seq order)
+%
+% The StartPurgeSeq parameter indicates where the fold should start *after*.
+-callback fold_purge_infos(
+    DbHandle::db_handle(),
+    StartPurgeSeq::purge_seq(),
+    UserFold::purge_fold_fun(),
+    UserAcc::any(),
+    purge_fold_options()) ->
+        {ok, LastUserAcc::any()}.
+
+
+% This function may be called by many processes concurrently.
+%
 % This function is called to count the number of documents changed
 % since the given UpdateSeq (ie, not including the possible change
 % at exactly UpdateSeq). It is currently only used internally to
@@ -597,8 +665,9 @@
     get_disk_version/1,
     get_doc_count/1,
     get_epochs/1,
-    get_last_purged/1,
     get_purge_seq/1,
+    get_oldest_purge_seq/1,
+    get_purge_infos_limit/1,
     get_revs_limit/1,
     get_security/1,
     get_size_info/1,
@@ -607,14 +676,17 @@
 
     set_revs_limit/2,
     set_security/2,
+    set_purge_infos_limit/2,
 
     open_docs/2,
     open_local_docs/2,
     read_doc_body/2,
+    load_purge_infos/2,
 
     serialize_doc/2,
     write_doc_body/2,
-    write_doc_infos/4,
+    write_doc_infos/3,
+    purge_docs/3,
     commit_data/1,
 
     open_write_stream/2,
@@ -624,6 +696,7 @@
     fold_docs/4,
     fold_local_docs/4,
     fold_changes/5,
+    fold_purge_infos/5,
     count_changes_since/2,
 
     start_compaction/1,
@@ -738,14 +811,19 @@ get_epochs(#db{} = Db) ->
     Engine:get_epochs(EngineState).
 
 
-get_last_purged(#db{} = Db) ->
+get_purge_seq(#db{} = Db) ->
     #db{engine = {Engine, EngineState}} = Db,
-    Engine:get_last_purged(EngineState).
+    Engine:get_purge_seq(EngineState).
 
 
-get_purge_seq(#db{} = Db) ->
+get_oldest_purge_seq(#db{} = Db) ->
     #db{engine = {Engine, EngineState}} = Db,
-    Engine:get_purge_seq(EngineState).
+    Engine:get_oldest_purge_seq(EngineState).
+
+
+get_purge_infos_limit(#db{} = Db) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:get_purge_infos_limit(EngineState).
 
 
 get_revs_limit(#db{} = Db) ->
@@ -778,6 +856,12 @@ set_revs_limit(#db{} = Db, RevsLimit) ->
     {ok, Db#db{engine = {Engine, NewSt}}}.
 
 
+set_purge_infos_limit(#db{} = Db, PurgedDocsLimit) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    {ok, NewSt} = Engine:set_purge_infos_limit(EngineState, PurgedDocsLimit),
+    {ok, Db#db{engine = {Engine, NewSt}}}.
+
+
 set_security(#db{} = Db, SecProps) ->
     #db{engine = {Engine, EngineState}} = Db,
     {ok, NewSt} = Engine:set_security(EngineState, SecProps),
@@ -799,6 +883,11 @@ read_doc_body(#db{} = Db, RawDoc) ->
     Engine:read_doc_body(EngineState, RawDoc).
 
 
+load_purge_infos(#db{} = Db, UUIDs) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:load_purge_infos(EngineState, UUIDs).
+
+
 serialize_doc(#db{} = Db, #doc{} = Doc) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:serialize_doc(EngineState, Doc).
@@ -809,10 +898,16 @@ write_doc_body(#db{} = Db, #doc{} = Doc) ->
     Engine:write_doc_body(EngineState, Doc).
 
 
-write_doc_infos(#db{} = Db, DocUpdates, LocalDocs, PurgedDocIdRevs) ->
+write_doc_infos(#db{} = Db, DocUpdates, LocalDocs) ->
     #db{engine = {Engine, EngineState}} = Db,
-    {ok, NewSt} = Engine:write_doc_infos(
-            EngineState, DocUpdates, LocalDocs, PurgedDocIdRevs),
+    {ok, NewSt} = Engine:write_doc_infos(EngineState, DocUpdates, LocalDocs),
+    {ok, Db#db{engine = {Engine, NewSt}}}.
+
+
+purge_docs(#db{} = Db, DocUpdates, Purges) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    {ok, NewSt} = Engine:purge_docs(
+        EngineState, DocUpdates, Purges),
     {ok, Db#db{engine = {Engine, NewSt}}}.
 
 
@@ -852,6 +947,12 @@ fold_changes(#db{} = Db, StartSeq, UserFun, UserAcc, Options) ->
     Engine:fold_changes(EngineState, StartSeq, UserFun, UserAcc, Options).
 
 
+fold_purge_infos(#db{} = Db, StartPurgeSeq, UserFun, UserAcc, Options) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:fold_purge_infos(
+            EngineState, StartPurgeSeq, UserFun, UserAcc, Options).
+
+
 count_changes_since(#db{} = Db, StartSeq) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:count_changes_since(EngineState, StartSeq).
@@ -914,7 +1015,7 @@ get_ddocs(<<"shards/", _/binary>> = DbName) ->
 get_ddocs(DbName) ->
     couch_util:with_db(DbName, fun(Db) ->
         FoldFun = fun(FDI, Acc) ->
-            Doc = couch_db:open_doc_int(Db, FDI, []),
+            {ok, Doc} = couch_db:open_doc_int(Db, FDI, []),
             {ok, [Doc | Acc]}
         end,
         {ok, Docs} = couch_db:fold_design_docs(Db, FoldFun, [], []),