You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2010/10/10 21:07:10 UTC
svn commit: r1006339 - in /couchdb/trunk: ./ share/www/script/test/
src/couchdb/
Author: davisp
Date: Sun Oct 10 19:07:10 2010
New Revision: 1006339
URL: http://svn.apache.org/viewvc?rev=1006339&view=rev
Log:
Fixes COUCHDB-799 - More granular ETags for views.
ETags for views now only change when their underlying view index
changes due to indexing or purges. ETags are also specific to each
view.
Thanks to Klaus Trainer for the patch.
Modified:
couchdb/trunk/THANKS
couchdb/trunk/share/www/script/test/design_docs.js
couchdb/trunk/share/www/script/test/etags_views.js
couchdb/trunk/src/couchdb/couch_db.hrl
couchdb/trunk/src/couchdb/couch_httpd_show.erl
couchdb/trunk/src/couchdb/couch_httpd_view.erl
couchdb/trunk/src/couchdb/couch_view_group.erl
couchdb/trunk/src/couchdb/couch_view_updater.erl
Modified: couchdb/trunk/THANKS
URL: http://svn.apache.org/viewvc/couchdb/trunk/THANKS?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/THANKS (original)
+++ couchdb/trunk/THANKS Sun Oct 10 19:07:10 2010
@@ -68,5 +68,6 @@ suggesting improvements or submitting ch
* David Rose <do...@gmail.com>
* Lim Yue Chuan <sh...@gmail.com>
* David Davis <xa...@xantus.org>
+ * Klaus Trainer <kl...@web.de>
For a list of authors see the `AUTHORS` file.
Modified: couchdb/trunk/share/www/script/test/design_docs.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/design_docs.js (original)
+++ couchdb/trunk/share/www/script/test/design_docs.js Sun Oct 10 19:07:10 2010
@@ -110,7 +110,7 @@ function() {
var vinfo = dinfo.view_index;
TEquals(51, vinfo.disk_size);
TEquals(false, vinfo.compact_running);
- TEquals("dc3264b45b74cc6d94666e3043e07154", vinfo.signature, 'ddoc sig');
+ TEquals("86e9b34892b4df35cd2f7c27da30c94d", vinfo.signature, 'ddoc sig');
db.bulkSave(makeDocs(1, numDocs + 1));
Modified: couchdb/trunk/share/www/script/test/etags_views.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/etags_views.js?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/etags_views.js (original)
+++ couchdb/trunk/share/www/script/test/etags_views.js Sun Oct 10 19:07:10 2010
@@ -11,23 +11,34 @@
// the License.
couchTests.etags_views = function(debug) {
- var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"});
+ var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"});
db.deleteDb();
db.createDb();
if (debug) debugger;
var designDoc = {
- _id:"_design/etags",
+ _id: "_design/etags",
language: "javascript",
views : {
+ fooView: {
+ map: stringFun(function(doc) {
+ if (doc.foo) {
+ emit("bar", 1);
+ }
+ }),
+ },
basicView : {
map : stringFun(function(doc) {
- emit(doc.integer, doc.string);
+ if(doc.integer && doc.string) {
+ emit(doc.integer, doc.string);
+ }
})
},
withReduce : {
map : stringFun(function(doc) {
- emit(doc.integer, doc.string);
+ if(doc.integer && doc.string) {
+ emit(doc.integer, doc.string);
+ }
}),
reduce : stringFun(function(keys, values, rereduce) {
if (rereduce) {
@@ -40,9 +51,9 @@ couchTests.etags_views = function(debug)
}
};
T(db.save(designDoc).ok);
+ db.bulkSave(makeDocs(0, 10));
+
var xhr;
- var docs = makeDocs(0, 10);
- db.bulkSave(docs);
// verify get w/Etag on map view
xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
@@ -52,17 +63,92 @@ couchTests.etags_views = function(debug)
headers: {"if-none-match": etag}
});
T(xhr.status == 304);
- // TODO GET with keys (when that is available)
+
+ // verify ETag doesn't change when an update
+ // doesn't change the view group's index
+ T(db.save({"_id":"doc1", "foo":"bar"}).ok);
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // Verify that purges affect etags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var foo_etag = xhr.getResponseHeader("etag");
+ var doc1 = db.open("doc1");
+ xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+ body: JSON.stringify({"doc1":[doc1._rev]})
+ });
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != foo_etag);
+
+ // Test that _purge didn't affect the other view etags.
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // verify different views in the same view group may have different ETags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 != etag2);
+
+ // verify ETag changes when an update changes the view group's index.
+ db.bulkSave(makeDocs(10, 20));
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != etag);
+
+ // verify ETag is the same after a restart
+ restartServer();
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 == etag2);
// reduce view
xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
T(xhr.status == 200);
var etag = xhr.getResponseHeader("etag");
- xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce", {
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce",{
headers: {"if-none-match": etag}
});
T(xhr.status == 304);
+ // verify ETag doesn't change when an update
+ // doesn't change the view group's index
+ T(db.save({"_id":"doc3", "foo":"bar"}).ok);
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+ // purge
+ var doc3 = db.open("doc3");
+ xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+ body: JSON.stringify({"doc3":[doc3._rev]})
+ });
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // verify different views in the same view group may have different ETags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 != etag2);
+
+ // verify ETag changes when an update changes the view group's index
+ db.bulkSave(makeDocs(20, 30));
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != etag);
+
+ // verify ETag is the same after a restart
+ restartServer();
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 == etag2);
+
// confirm ETag changes with different POST bodies
xhr = CouchDB.request("POST", "/test_suite_db/_design/etags/_view/basicView",
{body: JSON.stringify({keys:[1]})}
Modified: couchdb/trunk/src/couchdb/couch_db.hrl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.hrl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.hrl (original)
+++ couchdb/trunk/src/couchdb/couch_db.hrl Sun Oct 10 19:07:10 2010
@@ -238,6 +238,8 @@
-record(view,
{id_num,
+ update_seq=0,
+ purge_seq=0,
map_names=[],
def,
btree=nil,
Modified: couchdb/trunk/src/couchdb/couch_httpd_show.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_show.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_show.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_show.erl Sun Oct 10 19:07:10 2010
@@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) ->
handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
ViewDesignId = <<"_design/", ViewDesignName/binary>>,
{ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys),
- Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}),
+ Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}),
couch_httpd:etag_respond(Req, Etag, fun() ->
output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group)
end).
-list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) ->
+list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) ->
Accept = couch_httpd:header_value(Req, "Accept"),
- couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}).
+ couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}).
output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) ->
output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group);
Modified: couchdb/trunk/src/couchdb/couch_httpd_view.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_view.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_view.erl Sun Oct 10 19:07:10 2010
@@ -17,7 +17,7 @@
-export([parse_view_params/3]).
-export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]).
--export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]).
+-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]).
-export([design_doc_view/5, parse_bool_param/1, doc_member/2]).
-export([make_key_options/1, load_view/4]).
@@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, Qu
limit = Limit,
skip = SkipCount
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db),
+ CurrentEtag = view_etag(Db, Group, View),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, RowCount} = couch_view:get_row_count(View),
FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}),
@@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, Qu
limit = Limit,
skip = SkipCount
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db, Keys),
+ CurrentEtag = view_etag(Db, Group, View, Keys),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, RowCount} = couch_view:get_row_count(View),
FoldAccInit = {Limit, SkipCount, undefined, []},
@@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group,
skip = Skip,
group_level = GroupLevel
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db),
+ CurrentEtag = view_etag(Db, Group, View),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group,
skip = Skip,
group_level = GroupLevel
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db, Keys),
+ CurrentEtag = view_etag(Db, Group, View, Keys),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value},
send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})),
{ok, ",\r\n"}.
-view_group_etag(Group, Db) ->
- view_group_etag(Group, Db, nil).
+view_etag(Db, Group, View) ->
+ view_etag(Db, Group, View, nil).
-view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) ->
- % ?LOG_ERROR("Group ~p",[Group]),
- % This is not as granular as it could be.
- % If there are updates to the db that do not effect the view index,
- % they will change the Etag. For more granular Etags we'd need to keep
- % track of the last Db seq that caused an index change.
- couch_httpd:make_etag({Sig, CurrentSeq, Extra}).
+view_etag(Db, Group, {reduce, _, _, View}, Extra) ->
+ view_etag(Db, Group, View, Extra);
+view_etag(Db, Group, {temp_reduce, View}, Extra) ->
+ view_etag(Db, Group, View, Extra);
+view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) ->
+ couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}).
% the view row has an error
view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) ->
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Sun Oct 10 19:07:10 2010
@@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{s
get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq,
id_btree=IdBtree,views=Views}) ->
- ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views],
- #index_header{seq=Seq,
- purge_seq=PurgeSeq,
- id_btree_state=couch_btree:get_state(IdBtree),
- view_states=ViewStates}.
+ ViewStates = [
+ {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <- Views
+ ],
+ #index_header{
+ seq=Seq,
+ purge_seq=PurgeSeq,
+ id_btree_state=couch_btree:get_state(IdBtree),
+ view_states=ViewStates
+ }.
hex_sig(GroupSig) ->
couch_util:to_hex(?b2l(GroupSig)).
@@ -460,13 +464,15 @@ set_view_sig(#group{
lib={[]},
def_lang=Language,
design_options=DesignOptions}=G) ->
- G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))};
+ ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+ G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))};
set_view_sig(#group{
views=Views,
lib=Lib,
def_lang=Language,
design_options=DesignOptions}=G) ->
- G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}.
+ ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+ G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}.
sort_lib({Lib}) ->
sort_lib(Lib, []).
@@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, Group
init_group(Db, Fd, #group{views=Views}=Group, nil) ->
init_group(Db, Fd, Group,
#index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db),
- id_btree_state=nil, view_states=[nil || _ <- Views]});
+ id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]});
init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
Group, IndexHeader) ->
#index_header{seq=Seq, purge_seq=PurgeSeq,
id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader,
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd),
Views2 = lists:zipwith(
- fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) ->
+ fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
ReduceFun =
fun(reduce, KVs) ->
@@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,
<<"raw">> ->
Less = fun(A,B) -> A < B end
end,
- {ok, Btree} = couch_btree:open(BtreeState, Fd,
- [{less, Less},
- {reduce, ReduceFun}]),
- View#view{btree=Btree}
+ {ok, Btree} = couch_btree:open(BTState, Fd,
+ [{less, Less}, {reduce, ReduceFun}]
+ ),
+ View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq}
end,
ViewStates, Views),
Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq,
Modified: couchdb/trunk/src/couchdb/couch_view_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_updater.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_updater.erl Sun Oct 10 19:07:10 2010
@@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, i
end, dict:new(), Lookups),
% Now remove the values from the btrees
+ PurgeSeq = couch_db:get_purge_seq(Db),
Views2 = lists:map(
fun(#view{id_num=Num,btree=Btree}=View) ->
case dict:find(Num, ViewKeysToRemoveDict) of
{ok, RemoveKeys} ->
- {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
- View#view{btree=Btree2};
+ {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
+ case ViewBtree2 =/= Btree of
+ true ->
+ View#view{btree=ViewBtree2, purge_seq=PurgeSeq};
+ _ ->
+ View#view{btree=ViewBtree2}
+ end;
error -> % no keys to remove in this view
View
end
end, Views),
Group#group{id_btree=IdBtree2,
views=Views2,
- purge_seq=couch_db:get_purge_seq(Db)}.
+ purge_seq=PurgeSeq}.
load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) ->
@@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd,
Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) ->
KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []),
{ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove),
- View#view{btree = ViewBtree2}
+ case ViewBtree2 =/= View#view.btree of
+ true ->
+ View#view{btree=ViewBtree2, update_seq=NewSeq};
+ _ ->
+ View#view{btree=ViewBtree2}
+ end
end, Group#group.views, ViewKeyValuesToAdd),
Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}.