[couchdb] 01/01: Include shard uuids in db_info update sequences

[va...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vatamane pushed a commit to branch add-uuid-and-epochs-to-update-sequences
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 53077258dff9391728e9c517c0b5d020c886889a
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Tue Oct 19 12:27:47 2021 -0400

    Include shard uuids in db_info update sequences
    
    This means `update_seq` values from `GET $db` `last_seq` returned from ` GET
    $db/_changes?since=now&limit=` will be more reselient to change feed rewinds.
    Besides, those sequences will not be more consistent and users won't have to
    wonder why one opaque sequence works slightly differently than another opaque
    update sequence.
    
    Previously, when the sequences were returned only as numeric values, it was
    impossible to calculate replacements and change feeds had to always rewind back
    to 0 for those ranges. With uuids and epochs in play, it is possible to figure
    out that some shards might have moved to new nodes or find internal replication
    checkpoints to avoid streaming changes feeds from 0 on those ranges.
    
    Fixes: https://github.com/apache/couchdb/issues/3787
---
 src/fabric/src/fabric_db_info.erl              |  9 +++-
 src/fabric/src/fabric_view_changes.erl         |  9 ++++
 src/fabric/test/eunit/fabric_db_info_tests.erl | 68 ++++++++++++++++++++++++++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/src/fabric/src/fabric_db_info.erl b/src/fabric/src/fabric_db_info.erl
index 40da678..586f282 100644
--- a/src/fabric/src/fabric_db_info.erl
+++ b/src/fabric/src/fabric_db_info.erl
@@ -77,7 +77,7 @@ handle_message(Reason, Shard, {Counters, Resps, CInfo}) ->
 
 build_final_response(CInfo, DbName, Responses) ->
     AccF = fabric_dict:fold(fun(Shard, Info, {Seqs, PSeqs, Infos}) ->
-        Seq = couch_util:get_value(update_seq, Info),
+        Seq = build_seq(Shard, Info),
         PSeq = couch_util:get_value(purge_seq, Info),
         {[{Shard, Seq} | Seqs], [{Shard, PSeq} | PSeqs], [Info | Infos]}
     end, {[], [], []}, Responses),
@@ -89,6 +89,13 @@ build_final_response(CInfo, DbName, Responses) ->
     [{db_name, DbName}] ++ Sequences ++ MergedInfos.
 
 
+build_seq(#shard{node = Node}, Info) when is_list(Info) ->
+    Seq = couch_util:get_value(update_seq, Info),
+    Uuid = couch_util:get_value(uuid, Info),
+    PrefixLen = fabric_util:get_uuid_prefix_len(),
+    {Seq, binary:part(Uuid, {0, PrefixLen}), Node}.
+
+
 merge_results(Info) ->
     Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end,
         orddict:new(), Info),
diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl
index beeaece..0a9b33b 100644
--- a/src/fabric/src/fabric_view_changes.erl
+++ b/src/fabric/src/fabric_view_changes.erl
@@ -18,6 +18,9 @@
 %% exported for upgrade purposes.
 -export([keep_sending_changes/8]).
 
+%% exported for testing and remsh debugging
+-export([unpack_seqs/1]).
+
 -include_lib("fabric/include/fabric.hrl").
 -include_lib("mem3/include/mem3.hrl").
 -include_lib("couch/include/couch_db.hrl").
@@ -410,6 +413,12 @@ unpack_seq_decode_term(Opaque) ->
     binary_to_term(couch_util:decodeBase64Url(Opaque)).
 
 
+% This is used for testing and for remsh debugging
+unpack_seqs(Packed) ->
+    Opaque = unpack_seq_regex_match(Packed),
+    unpack_seq_decode_term(Opaque).
+
+
 unpack_seqs(0, DbName) ->
     fabric_dict:init(mem3:shards(DbName), 0);
 
diff --git a/src/fabric/test/eunit/fabric_db_info_tests.erl b/src/fabric/test/eunit/fabric_db_info_tests.erl
new file mode 100644
index 0000000..0d27a54
--- /dev/null
+++ b/src/fabric/test/eunit/fabric_db_info_tests.erl
@@ -0,0 +1,68 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_db_info_tests).
+
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+
+-define(TDEF(A), {atom_to_list(A), fun A/0}).
+
+
+main_test_() ->
+    {
+        setup,
+        fun setup/0,
+        fun teardown/1,
+        [
+            ?TDEF(t_update_seq_has_uuids)
+        ]
+    }.
+
+
+setup() ->
+    test_util:start_couch([fabric]).
+
+
+teardown(Ctx) ->
+    meck:unload(),
+    test_util:stop_couch(Ctx).
+
+
+t_update_seq_has_uuids() ->
+    DbName = ?tempdb(),
+    ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]),
+
+    {ok, Info} = fabric:get_db_info(DbName),
+    UpdateSeq = couch_util:get_value(update_seq, Info),
+    UnpackedSeq = fabric_view_changes:unpack_seqs(UpdateSeq),
+
+    ?assertMatch([{_, _, _}], UnpackedSeq),
+    [{Node, Range, Seq}] = UnpackedSeq,
+    ?assert(is_atom(Node)),
+    ?assertMatch([_, _], Range),
+    ?assertMatch({_, _, _}, Seq),
+    {SeqNum, SeqUuid, EpochNode} = Seq,
+    ?assert(is_integer(SeqNum)),
+    ?assert(is_binary(SeqUuid)),
+    ?assert(is_atom(EpochNode)),
+
+    {ok, UuidMap} = fabric:db_uuids(DbName),
+    PrefixLen = fabric_util:get_uuid_prefix_len(),
+    Uuids = [binary:part(Uuid, {0, PrefixLen}) || Uuid <- maps:keys(UuidMap)],
+    [UuidFromShard] = Uuids,
+    ?assertEqual(UuidFromShard, SeqUuid),
+
+    ok = fabric:delete_db(DbName, []).