You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2026/03/05 21:54:49 UTC
(couchdb) branch fix-mrview-purge-checkpoints created (now bb4e1982c)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch fix-mrview-purge-checkpoints
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at bb4e1982c Fix signatures in mrview purge checkpoints
This branch includes the following new commits:
new bb4e1982c Fix signatures in mrview purge checkpoints
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb) 01/01: Fix signatures in mrview purge checkpoints
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch fix-mrview-purge-checkpoints
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bb4e1982cbc042987e4329818cab504ac27f40bd
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Mar 5 16:35:28 2026 -0500
Fix signatures in mrview purge checkpoints
Switch from lists of integers (see example below) to a binary.
```json
{
"_id": "_local/purge-mrview-e758e0655818718ba98d8f53168f848f",
"_rev": "0-1",
"type": "mrview",
"purge_seq": 393,
"updated_on": 1772740238,
"ddoc_id": "_design/mr",
"signature": [
101,
55,
53,
56,
101,
48,
54,
53,
53,
56,
49,
56,
55,
49,
56,
98,
97,
57,
56,
100,
56,
102,
53,
51,
49,
54,
56,
102,
56,
52,
56,
102
]
}
```
Since the old format was written to disk leave in a compatibility clause. At
some point in the future maybe opt to auto-upgrade those docs, but for now, to
allow effortless downgrading keep them as is.
While at it, update the doc writer to use the newer time function and update
the tests to use the ?TDEF_FE macro and save two indentation levels that way.
---
src/couch_mrview/src/couch_mrview_index.erl | 26 +-
.../eunit/couch_mrview_purge_docs_fabric_tests.erl | 362 +++++++++++----------
2 files changed, 199 insertions(+), 189 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index a5080ed76..26b4f1aac 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -240,14 +240,10 @@ verify_index_exists(DbName, Props) ->
couch_util:with_db(DbName, fun(Db) ->
case couch_db:get_design_doc(Db, DDocId) of
{ok, #doc{} = DDoc} ->
- {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(
- DbName, DDoc
- ),
+ {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
IdxSig = IdxState#mrst.sig,
- SigInLocal = couch_util:get_value(
- <<"signature">>, Props
- ),
- couch_index_util:hexsig(IdxSig) == SigInLocal;
+ DocSig = couch_util:get_value(<<"signature">>, Props),
+ match_signatures(IdxSig, DocSig);
{not_found, _} ->
false
end
@@ -258,6 +254,16 @@ verify_index_exists(DbName, Props) ->
false
end.
+match_signatures(IdxSig, DocSig) when is_binary(IdxSig), is_list(DocSig) ->
+ % Compatibility clause. In versions =< 3.5.1 mvrview signatures in purge
+ % checkpoints where written as lists of integers instead of a binary. After
+ % a few versions 3.6, 3.7 consider automatically upgrading them to binaries
+ % on open if find the older format. For now we don't as we'd like to be
+ % able to roll back to a previous version on upgrade.
+ couch_index_util:hexsig(IdxSig) =:= DocSig;
+match_signatures(IdxSig, DocSig) when is_binary(IdxSig), is_binary(DocSig) ->
+ couch_util:to_hex_bin(IdxSig) =:= DocSig.
+
set_partitioned(Db, State) ->
#mrst{
design_opts = DesignOpts
@@ -305,16 +311,14 @@ update_local_purge_doc(Db, State) ->
update_local_purge_doc(Db, State, PSeq) ->
Sig = couch_index_util:hexsig(State#mrst.sig),
DocId = couch_mrview_util:get_local_purge_doc_id(Sig),
- {Mega, Secs, _} = os:timestamp(),
- NowSecs = Mega * 1000000 + Secs,
BaseDoc = couch_doc:from_json_obj(
{[
{<<"_id">>, DocId},
{<<"type">>, <<"mrview">>},
{<<"purge_seq">>, PSeq},
- {<<"updated_on">>, NowSecs},
+ {<<"updated_on">>, erlang:system_time(second)},
{<<"ddoc_id">>, get(idx_name, State)},
- {<<"signature">>, Sig}
+ {<<"signature">>, list_to_binary(Sig)}
]}
),
Doc =
diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl
index f3452b55a..8a963622f 100644
--- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl
+++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl
@@ -53,192 +53,198 @@ view_purge_fabric_test_() ->
fun setup/0,
fun teardown/1,
[
- fun test_purge_verify_index/1,
- fun test_purge_hook_before_compaction/1
+ ?TDEF_FE(test_purge_verify_index, ?TIMEOUT),
+ ?TDEF_FE(test_purge_hook_before_compaction, ?TIMEOUT)
]
}
}
}.
test_purge_verify_index(DbName) ->
- {timeout, ?TIMEOUT,
- ?_test(begin
- Docs1 = couch_mrview_test_util:make_docs(normal, 5),
- {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]),
- {ok, _} = fabric:update_doc(
- DbName,
- couch_mrview_test_util:ddoc(map),
- [?ADMIN_CTX]
- ),
-
- Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
- Expect1 =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect1, Result1),
-
- {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
- ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)),
- ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
- [ShardDbName | _Rest] = ShardNames,
- ?assertEqual(
- true,
- couch_mrview_index:verify_index_exists(
- ShardDbName, Props1
- )
- ),
-
- purge_docs(DbName, [<<"1">>]),
-
- Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
- Expect2 =
- {ok, [
- {meta, [{total, 4}, {offset, 0}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2),
-
- {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName),
- ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)),
- ?assertEqual(
- true,
- couch_mrview_index:verify_index_exists(
- ShardDbName, Props2
- )
- )
- end)}.
+ Docs1 = couch_mrview_test_util:make_docs(normal, 5),
+ {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]),
+ {ok, _} = fabric:update_doc(
+ DbName,
+ couch_mrview_test_util:ddoc(map),
+ [?ADMIN_CTX]
+ ),
+
+ Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+ Expect1 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+ {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+ {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect1, Result1),
+
+ {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
+ ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)),
+ ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
+ [ShardDbName | _Rest] = ShardNames,
+ ?assertEqual(
+ true,
+ couch_mrview_index:verify_index_exists(
+ ShardDbName, Props1
+ )
+ ),
+
+ purge_docs(DbName, [<<"1">>]),
+
+ Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 4}, {offset, 0}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+ {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+ {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2),
+
+ {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName),
+ ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)),
+ ?assertEqual(
+ true,
+ couch_mrview_index:verify_index_exists(
+ ShardDbName, Props2
+ )
+ ),
+
+ % Make sure we can verify an older format in versions =< 3.5.1
+ % where signatures were lists (arrays) of integers
+ DocSig = couch_util:get_value(<<"signature">>, Props2),
+ DocSig1 = binary_to_list(DocSig),
+ Props3 = lists:keyreplace(<<"signature">>, 1, Props2, {<<"signature">>, DocSig1}),
+ ?assertEqual(
+ true,
+ couch_mrview_index:verify_index_exists(
+ ShardDbName, Props3
+ )
+ ).
test_purge_hook_before_compaction(DbName) ->
- {timeout, ?TIMEOUT,
- ?_test(begin
- Docs1 = couch_mrview_test_util:make_docs(normal, 5),
- {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]),
- {ok, _} = fabric:update_doc(
- DbName,
- couch_mrview_test_util:ddoc(map),
- [?ADMIN_CTX]
- ),
-
- Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
- Expect1 =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect1, Result1),
-
- purge_docs(DbName, [<<"1">>]),
-
- Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
- Expect2 =
- {ok, [
- {meta, [{total, 4}, {offset, 0}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2),
-
- {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
- ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)),
-
- [ShardName | _] = local_shards(DbName),
- couch_util:with_db(ShardName, fun(Db) ->
- {ok, _} = couch_db:start_compact(Db)
- end),
- wait_compaction(ShardName, ?LINE),
-
- ?assertEqual(
- ok,
- meck:wait(
- 1,
- couch_mrview_index,
- ensure_local_purge_docs,
- '_',
- 5000
- )
- ),
-
- % Make sure compaction didn't change the update seq
- {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
- ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)),
-
- purge_docs(DbName, [<<"2">>]),
-
- couch_util:with_db(ShardName, fun(Db) ->
- {ok, _} = couch_db:start_compact(Db)
- end),
- wait_compaction(ShardName, ?LINE),
-
- ?assertEqual(
- ok,
- meck:wait(
- 2,
- couch_mrview_index,
- ensure_local_purge_docs,
- '_',
- 5000
- )
- ),
-
- % Make sure compaction after a purge didn't overwrite
- % the local purge doc for the index
- {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName),
- ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)),
-
- % Force another update to ensure that we update
- % the local doc appropriate after compaction
- Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
- Expect3 =
- {ok, [
- {meta, [{total, 3}, {offset, 0}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect3, Result3),
-
- {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName),
- ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)),
-
- % Check that if the local doc doesn't exist that one
- % is created for the index on compaction
- delete_local_purge_doc(DbName),
- ?assertMatch({not_found, _}, get_local_purge_doc(DbName)),
-
- couch_util:with_db(ShardName, fun(Db) ->
- {ok, _} = couch_db:start_compact(Db)
- end),
- wait_compaction(ShardName, ?LINE),
-
- ?assertEqual(
- ok,
- meck:wait(
- 3,
- couch_mrview_index,
- ensure_local_purge_docs,
- '_',
- 5000
- )
- ),
-
- {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName),
- ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4))
- end)}.
+ Docs1 = couch_mrview_test_util:make_docs(normal, 5),
+ {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]),
+ {ok, _} = fabric:update_doc(
+ DbName,
+ couch_mrview_test_util:ddoc(map),
+ [?ADMIN_CTX]
+ ),
+
+ Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+ Expect1 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+ {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+ {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect1, Result1),
+
+ purge_docs(DbName, [<<"1">>]),
+
+ Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 4}, {offset, 0}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+ {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+ {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2),
+
+ {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
+ ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)),
+
+ [ShardName | _] = local_shards(DbName),
+ couch_util:with_db(ShardName, fun(Db) ->
+ {ok, _} = couch_db:start_compact(Db)
+ end),
+ wait_compaction(ShardName, ?LINE),
+
+ ?assertEqual(
+ ok,
+ meck:wait(
+ 1,
+ couch_mrview_index,
+ ensure_local_purge_docs,
+ '_',
+ 5000
+ )
+ ),
+
+ % Make sure compaction didn't change the update seq
+ {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName),
+ ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)),
+
+ purge_docs(DbName, [<<"2">>]),
+
+ couch_util:with_db(ShardName, fun(Db) ->
+ {ok, _} = couch_db:start_compact(Db)
+ end),
+ wait_compaction(ShardName, ?LINE),
+
+ ?assertEqual(
+ ok,
+ meck:wait(
+ 2,
+ couch_mrview_index,
+ ensure_local_purge_docs,
+ '_',
+ 5000
+ )
+ ),
+
+ % Make sure compaction after a purge didn't overwrite
+ % the local purge doc for the index
+ {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName),
+ ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)),
+
+ % Force another update to ensure that we update
+ % the local doc appropriate after compaction
+ Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+ Expect3 =
+ {ok, [
+ {meta, [{total, 3}, {offset, 0}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+ {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+ {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect3, Result3),
+
+ {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName),
+ ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)),
+
+ % Check that if the local doc doesn't exist that one
+ % is created for the index on compaction
+ delete_local_purge_doc(DbName),
+ ?assertMatch({not_found, _}, get_local_purge_doc(DbName)),
+
+ couch_util:with_db(ShardName, fun(Db) ->
+ {ok, _} = couch_db:start_compact(Db)
+ end),
+ wait_compaction(ShardName, ?LINE),
+
+ ?assertEqual(
+ ok,
+ meck:wait(
+ 3,
+ couch_mrview_index,
+ ensure_local_purge_docs,
+ '_',
+ 5000
+ )
+ ),
+
+ {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName),
+ ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4)).
get_local_purge_doc(DbName) ->
{ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []),