You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2009/03/11 23:40:11 UTC
svn commit: r752668 - in /couchdb/branches/rep_security:
share/www/script/couch.js share/www/script/test/bulk_docs.js
share/www/script/test/security_validation.js src/couchdb/couch_db.erl
src/couchdb/couch_httpd_db.erl
Author: damien
Date: Wed Mar 11 22:40:10 2009
New Revision: 752668
URL: http://svn.apache.org/viewvc?rev=752668&view=rev
Log:
Adding all_or_nothing option to bulk docs. If a doc doesn't pass validation or there is failure during update, then no docs are sacved. However, there is no conflict checking, if all docs validate but some or all docs are conflicts, they are saved as conflicts regardless.
Modified:
couchdb/branches/rep_security/share/www/script/couch.js
couchdb/branches/rep_security/share/www/script/test/bulk_docs.js
couchdb/branches/rep_security/share/www/script/test/security_validation.js
couchdb/branches/rep_security/src/couchdb/couch_db.erl
couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl
Modified: couchdb/branches/rep_security/share/www/script/couch.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/couch.js?rev=752668&r1=752667&r2=752668&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/couch.js [utf-8] (original)
+++ couchdb/branches/rep_security/share/www/script/couch.js [utf-8] Wed Mar 11 22:40:10 2009
@@ -98,15 +98,26 @@
if (docs[i]._id == undefined)
docs[i]._id = newUuids.pop();
}
- this.last_req = this.request("POST", this.uri + "_bulk_docs" + encodeOptions(options), {
- body: JSON.stringify({"docs": docs})
+ var json = {"docs": docs};
+ // put any options in the json
+ for (var option in options) {
+ json[option] = options[option];
+ }
+ this.last_req = this.request("POST", this.uri + "_bulk_docs", {
+ body: JSON.stringify(json)
});
- CouchDB.maybeThrowError(this.last_req);
- var results = JSON.parse(this.last_req.responseText);
- for (var i = 0; i < docs.length; i++) {
- docs[i]._rev = results[i].rev;
+ if (this.last_req.status == 417) {
+ return {errors: JSON.parse(this.last_req.responseText)};
+ }
+ else {
+ CouchDB.maybeThrowError(this.last_req);
+ var results = JSON.parse(this.last_req.responseText);
+ for (var i = 0; i < docs.length; i++) {
+ if(results[i].rev)
+ docs[i]._rev = results[i].rev;
+ }
+ return results;
}
- return results;
}
this.ensureFullCommit = function() {
Modified: couchdb/branches/rep_security/share/www/script/test/bulk_docs.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/bulk_docs.js?rev=752668&r1=752667&r2=752668&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/bulk_docs.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/bulk_docs.js Wed Mar 11 22:40:10 2009
@@ -40,22 +40,45 @@
}
// now test a bulk update with a conflict
-
+ // open and save
var doc = db.open("0");
db.save(doc);
- // Delete the docs
+ // Now bulk delete the docs
results = db.bulkSave(docs);
+
+ // doc "0" should be a conflict
T(results.length == 5);
T(results[0].id == "0");
- T(results[0].rev === undefined);
T(results[0].error == "conflict");
+ T(results[0].rev === undefined); // no rev member when a conflict
+ // but the rest are not
for (i = 1; i < 5; i++) {
T(results[i].id == i.toString());
T(results[i].rev)
T(db.open(docs[i]._id) == null);
}
+
+ // now force a conflict to to save
+
+ // save doc 0, this will cause a conflict when we save docs[0]
+ var doc = db.open("0");
+ docs[0] = db.open("0")
+ db.save(doc);
+
+ docs[0].shooby = "dooby";
+
+ // Now save the bulk docs, When we use all_or_nothing, we don't get conflict
+ // checking, all docs are saved regardless of conflict status, or none are
+ // saved.
+ results = db.bulkSave(docs,{all_or_nothing:true});
+ T(results.error === undefined);
+
+ var doc = db.open("0", {conflicts:true});
+ var docConflict = db.open("0", {rev:doc._conflicts[0]});
+
+ T(doc.shooby == "dooby" || docConflict.shooby == "dooby");
// verify creating a document with no id returns a new id
var req = CouchDB.request("POST", "/test_suite_db/_bulk_docs", {
Modified: couchdb/branches/rep_security/share/www/script/test/security_validation.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/security_validation.js?rev=752668&r1=752667&r2=752668&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/security_validation.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/security_validation.js Wed Mar 11 22:40:10 2009
@@ -148,9 +148,35 @@
}
// Now delete document
- T(user2Db.deleteDoc(doc).ok);
+ T(user2Db.deleteDoc(doc).ok);
+ // now test bulk docs
+ var docs = [{_id:"bahbah",author:"Damien Katz",foo:"bar"},{_id:"fahfah",foo:"baz"}];
+ // Create the docs
+ var results = db.bulkSave(docs);
+
+ T(results[0].rev)
+ T(results[0].error == undefined)
+ T(results[1].rev === undefined)
+ T(results[1].error == "forbidden")
+
+ T(db.open("bahbah"));
+ T(db.open("fahfah") == null);
+
+
+ // now all or nothing with a failure
+ var docs = [{_id:"booboo",author:"Damien Katz",foo:"bar"},{_id:"foofoo",foo:"baz"}];
+
+ // Create the docs
+ var results = db.bulkSave(docs, {all_or_nothing:true});
+
+ T(results.errors.length == 1);
+ T(results.errors[0].error == "forbidden");
+ T(db.open("booboo") == null);
+ T(db.open("foofoo") == null);
+
+
// Now test replication
var AuthHeaders = {"WWW-Authenticate": "X-Couch-Test-Auth Christopher Lenz:dog food"};
var host = CouchDB.host;
Modified: couchdb/branches/rep_security/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_db.erl?rev=752668&r1=752667&r2=752668&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_db.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_db.erl Wed Mar 11 22:40:10 2009
@@ -357,11 +357,11 @@
update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options, interactive_edit).
-validate_replicated_updates(_Db, [], [], AccPrepped, AccErrors) ->
+prep_and_validate_replicated_updates(_Db, [], [], AccPrepped, AccErrors) ->
Errors2 = [{{Id, {Pos, Rev}}, Error} ||
{#doc{id=Id,revs={Pos,[Rev|_]}}, Error} <- AccErrors],
{lists:reverse(AccPrepped), lists:reverse(Errors2)};
-validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldInfo], AccPrepped, AccErrors) ->
+prep_and_validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldInfo], AccPrepped, AccErrors) ->
case OldInfo of
not_found ->
{ValidatedBucket, AccErrors3} = lists:foldl(
@@ -374,7 +374,7 @@
end
end,
{[], AccErrors}, Bucket),
- validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3);
+ prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3);
{ok, #full_doc_info{rev_tree=OldTree}} ->
NewRevTree = lists:foldl(
fun(NewDoc, AccTree) ->
@@ -391,12 +391,32 @@
{ok, {Start, Path}} ->
% our unflushed doc is a leaf node. Go back on the path
% to find the previous rev that's on disk.
- LoadPrevRev = fun() ->
- make_first_doc_on_disk(Db, Id, Start - 1, tl(Path))
+ PrevRevResult =
+ case couch_doc:has_stubs(Doc) of
+ true ->
+ [_PrevRevFull | [PrevRevFull | _]=PrevPath] = Path,
+ case PrevRevFull of
+ {_RevId, ?REV_MISSING} ->
+ conflict;
+ {RevId, {IsDel, DiskSp}} ->
+ DiskDoc = make_doc(Db, Id, IsDel, DiskSp, PrevPath),
+ Doc2 = couch_doc:merge_stubs(Doc, DiskDoc),
+ {ok, Doc2, fun() -> DiskDoc end}
+ end;
+ false ->
+ {ok, Doc,
+ fun() ->
+ make_first_doc_on_disk(Db,Id,Start-1, tl(Path))
+ end}
end,
- case validate_doc_update(Db, Doc, LoadPrevRev) of
- ok ->
- {[Doc | AccValidated], AccErrors2};
+ case PrevRevResult of
+ {ok, NewDoc, LoadPrevRevFun} ->
+ case validate_doc_update(Db, NewDoc, LoadPrevRevFun) of
+ ok ->
+ {[NewDoc | AccValidated], AccErrors2};
+ Error ->
+ {AccValidated, [{NewDoc, Error} | AccErrors2]}
+ end;
Error ->
{AccValidated, [{Doc, Error} | AccErrors2]}
end;
@@ -407,7 +427,7 @@
end
end,
{[], AccErrors}, Bucket),
- validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3)
+ prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3)
end.
update_docs(Db, Docs, Options, replicated_changes) ->
@@ -424,7 +444,7 @@
ExistingDocs = get_full_doc_infos(Db, Ids),
{DocBuckets2, DocErrors} =
- validate_replicated_updates(Db, DocBuckets, ExistingDocs, [], []),
+ prep_and_validate_replicated_updates(Db, DocBuckets, ExistingDocs, [], []),
DocBuckets3 = [Bucket || [_|_]=Bucket <- DocBuckets2]; % remove empty buckets
false ->
DocErrors = [],
@@ -435,6 +455,7 @@
update_docs(Db, Docs, Options, interactive_edit) ->
couch_stats_collector:increment({couchdb, database_writes}),
+ AllOrNothing = lists:member(all_or_nothing, Options),
% go ahead and generate the new revision ids for the documents.
Docs2 = lists:map(
fun(#doc{id=Id,revs={Start, RevIds}}=Doc) ->
@@ -459,26 +480,41 @@
% lookup the doc by id and get the most recent
Ids = [Id || [#doc{id=Id}|_] <- DocBuckets],
ExistingDocInfos = get_full_doc_infos(Db, Ids),
-
- {DocBucketsPrepped, Failures} = prep_and_validate_updates(Db, DocBuckets, ExistingDocInfos, [], []),
+
+ {DocBucketsPrepped, Failures} =
+ case AllOrNothing of
+ true ->
+ prep_and_validate_replicated_updates(Db, DocBuckets,
+ ExistingDocInfos, [], []);
+ false ->
+ prep_and_validate_updates(Db, DocBuckets, ExistingDocInfos, [], [])
+ end,
+
% strip out any empty buckets
DocBuckets2 = [Bucket || [_|_] = Bucket <- DocBucketsPrepped];
false ->
Failures = [],
DocBuckets2 = DocBuckets
end,
- {ok, CommitFailures} = write_and_commit(Db, DocBuckets2, Options),
- FailDict = dict:from_list(CommitFailures ++ Failures),
- % the output for each is either {ok, NewRev} or Error
- {ok, lists:map(
- fun(#doc{id=Id,revs={Pos, [NewRevId|_]}}) ->
- case dict:find({Id, {Pos, NewRevId}}, FailDict) of
- {ok, Error} ->
- Error;
- error ->
- {ok, {Pos, NewRevId}}
- end
- end, Docs2)}.
+
+ if (AllOrNothing) and (Failures /= []) ->
+ {aborted, Failures};
+ true ->
+ Options2 = if AllOrNothing -> [merge_conflicts];
+ true -> [] end ++ Options,
+ {ok, CommitFailures} = write_and_commit(Db, DocBuckets2, Options2),
+ FailDict = dict:from_list(CommitFailures ++ Failures),
+ % the output for each is either {ok, NewRev} or Error
+ {ok, lists:map(
+ fun(#doc{id=Id,revs={Pos, [NewRevId|_]}}) ->
+ case dict:find({Id, {Pos, NewRevId}}, FailDict) of
+ {ok, Error} ->
+ Error;
+ error ->
+ {ok, {Pos, NewRevId}}
+ end
+ end, Docs2)}
+ end.
% Returns the first available document on disk. Input list is a full rev path
% for the doc.
Modified: couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl?rev=752668&r1=752667&r2=752668&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl Wed Mar 11 22:40:10 2009
@@ -130,20 +130,36 @@
Doc#doc{id=Id,revs=Revs}
end,
DocsArray),
- {ok, Results} = couch_db:update_docs(Db, Docs, Options),
-
- % output the results
- DocResults = lists:zipwith(
- fun(Doc, {ok, NewRev}) ->
- {[{<<"id">>, Doc#doc.id}, {<<"rev">>, couch_doc:rev_to_str(NewRev)}]};
- (Doc, Error) ->
- {_Code, Err, Msg} = couch_httpd:error_info(Error),
- % maybe we should add the http error code to the json?
- {[{<<"id">>, Doc#doc.id}, {<<"error">>, Err}, {"reason", Msg}]}
- end,
- Docs, Results),
- send_json(Req, 201, DocResults);
-
+ Options2 =
+ case proplists:get_value(<<"all_or_nothing">>, JsonProps) of
+ true -> [all_or_nothing|Options];
+ _ -> Options
+ end,
+ case couch_db:update_docs(Db, Docs, Options2) of
+ {ok, Results} ->
+ % output the results
+ DocResults = lists:zipwith(
+ fun(Doc, {ok, NewRev}) ->
+ {[{<<"id">>, Doc#doc.id}, {<<"rev">>, couch_doc:rev_to_str(NewRev)}]};
+ (Doc, Error) ->
+ {_Code, Err, Msg} = couch_httpd:error_info(Error),
+ % maybe we should add the http error code to the json?
+ {[{<<"id">>, Doc#doc.id}, {<<"error">>, Err}, {"reason", Msg}]}
+ end,
+ Docs, Results),
+ send_json(Req, 201, DocResults);
+ {aborted, Errors} ->
+ ErrorsJson =
+ lists:map(
+ fun({{Id, Rev}, Error}) ->
+ {_Code, Err, Msg} = couch_httpd:error_info(Error),
+ {[{<<"id">>, Id},
+ {<<"rev">>, couch_doc:rev_to_str(Rev)},
+ {<<"error">>, Err},
+ {"reason", Msg}]}
+ end, Errors),
+ send_json(Req, 417, ErrorsJson)
+ end;
false ->
Docs = [couch_doc:from_json_obj(JsonObj) || JsonObj <- DocsArray],
{ok, Errors} = couch_db:update_docs(Db, Docs, Options, replicated_changes),