You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by fd...@apache.org on 2010/07/22 19:33:04 UTC
svn commit: r966765 - in /couchdb/branches/new_replicator:
share/www/script/test/new_replication.js src/couchdb/couch_api_wrap.erl
src/couchdb/couch_doc.erl
Author: fdmanana
Date: Thu Jul 22 17:33:04 2010
New Revision: 966765
URL: http://svn.apache.org/viewvc?rev=966765&view=rev
Log:
Updated the new replicator test case to test for replicated attachments and correct an issue with attachment replication when doing a replication by doc IDs list.
Modified:
couchdb/branches/new_replicator/share/www/script/test/new_replication.js
couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
couchdb/branches/new_replicator/src/couchdb/couch_doc.erl
Modified: couchdb/branches/new_replicator/share/www/script/test/new_replication.js
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/share/www/script/test/new_replication.js?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/share/www/script/test/new_replication.js (original)
+++ couchdb/branches/new_replicator/share/www/script/test/new_replication.js Thu Jul 22 17:33:04 2010
@@ -37,6 +37,12 @@ couchTests.new_replication = function(de
}
];
+ var att1_data = CouchDB.request("GET", "/_utils/script/test/lorem.txt");
+ att1_data = att1_data.responseText;
+
+ var att2_data = CouchDB.request("GET", "/_utils/script/test/lorem_b64.txt");
+ att2_data = att2_data.responseText;
+
var sourceInfo, targetInfo;
var docs, doc, copy;
var repResult;
@@ -58,6 +64,47 @@ couchTests.new_replication = function(de
}
+ function addAtt(db, doc, attName, attData, type) {
+ var uri = "/" + db.name + "/" + encodeURIComponent(doc._id) + "/" + attName;
+
+ if (doc._rev) {
+ uri += "?rev=" + doc._rev;
+ }
+
+ var xhr = CouchDB.request("PUT", uri, {
+ headers: {
+ "Content-Type": type
+ },
+ body: attData
+ });
+
+ T(xhr.status === 201);
+ doc._rev = JSON.parse(xhr.responseText).rev;
+ }
+
+
+ function compareObjects(o1, o2) {
+ for (var p in o1) {
+ if (o1[p] === null && o2[p] !== null) {
+ return false;
+ } else if (typeof o1[p] === "object") {
+ if ((typeof o2[p] !== "object") || o2[p] === null) {
+ return false;
+ }
+ if (!arguments.callee(o1[p], o2[p])) {
+ return false;
+ }
+ } else {
+ if (o1[p] !== o2[p]) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+
+
// test simple replications (not continuous, not filtered), including
// conflict creation
docs = makeDocs(1, 21);
@@ -71,6 +118,11 @@ couchTests.new_replication = function(de
populateDb(sourceDb, docs);
populateDb(targetDb, []);
+ // add some attachments
+ for (j = 10; j < 15; j++) {
+ addAtt(sourceDb, docs[j], "readme.txt", att1_data, "text/plain");
+ }
+
repResult = CouchDB.new_replicate(dbPairs[i].source, dbPairs[i].target);
T(repResult.ok === true);
@@ -78,7 +130,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === targetInfo.update_seq);
T(typeof repResult.session_id === "string");
T(repResult.source_last_seq === sourceInfo.update_seq);
@@ -101,19 +152,38 @@ couchTests.new_replication = function(de
copy = targetDb.open(doc._id);
T(copy !== null);
- for (var p in doc) {
- T(copy[p] === doc[p]);
+ T(compareObjects(doc, copy) === true);
+
+ if (j >= 10 && j < 15) {
+ var atts = copy._attachments;
+ T(typeof atts === "object");
+ T(typeof atts["readme.txt"] === "object");
+ T(atts["readme.txt"].revpos === 2);
+ T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+ T(atts["readme.txt"].stub === true);
+
+ var att_copy = CouchDB.request(
+ "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+ ).responseText;
+ T(att_copy.length === att1_data.length);
+ T(att_copy === att1_data);
}
}
- // add one more doc to source and replicate again
+ // add one more doc to source, more attachments to some existing docs
+ // and replicate again
var newDoc = {
_id: "foo666",
value: "d"
};
T(sourceDb.save(newDoc).ok);
+ // add some more attachments
+ for (j = 10; j < 15; j++) {
+ addAtt(sourceDb, docs[j], "data.dat", att2_data, "application/binary");
+ }
+
repResult = CouchDB.new_replicate(dbPairs[i].source, dbPairs[i].target);
T(repResult.ok === true);
@@ -121,7 +191,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === targetInfo.update_seq);
T(typeof repResult.session_id === "string");
T(repResult.source_last_seq === sourceInfo.update_seq);
@@ -130,13 +199,13 @@ couchTests.new_replication = function(de
T(repResult.history[0].session_id === repResult.session_id);
T(typeof repResult.history[0].start_time === "string");
T(typeof repResult.history[0].end_time === "string");
- T(repResult.history[0].start_last_seq === (sourceInfo.update_seq - 1));
+ T(repResult.history[0].start_last_seq === (sourceInfo.update_seq - 6));
T(repResult.history[0].end_last_seq === sourceInfo.update_seq);
T(repResult.history[0].recorded_seq === sourceInfo.update_seq);
- T(repResult.history[0].missing_checked === 1);
- T(repResult.history[0].missing_found === 1);
- T(repResult.history[0].docs_read === 1);
- T(repResult.history[0].docs_written === 1);
+ T(repResult.history[0].missing_checked === 6);
+ T(repResult.history[0].missing_found === 6);
+ T(repResult.history[0].docs_read === 6);
+ T(repResult.history[0].docs_written === 6);
T(repResult.history[0].doc_write_failures === 0);
copy = targetDb.open(newDoc._id);
@@ -144,6 +213,38 @@ couchTests.new_replication = function(de
T(copy._id === newDoc._id);
T(copy.value === newDoc.value);
+ for (j = 10; j < 15; j++) {
+ doc = docs[j];
+ copy = targetDb.open(doc._id);
+
+ T(copy !== null);
+ T(compareObjects(doc, copy) === true);
+
+ var atts = copy._attachments;
+ T(typeof atts === "object");
+ T(typeof atts["readme.txt"] === "object");
+ T(atts["readme.txt"].revpos === 2);
+ T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+ T(atts["readme.txt"].stub === true);
+
+ var att1_copy = CouchDB.request(
+ "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+ ).responseText;
+ T(att1_copy.length === att1_data.length);
+ T(att1_copy === att1_data);
+
+ T(typeof atts["data.dat"] === "object");
+ T(atts["data.dat"].revpos === 3);
+ T(atts["data.dat"].content_type.indexOf("application/binary") === 0);
+ T(atts["data.dat"].stub === true);
+
+ var att2_copy = CouchDB.request(
+ "GET", "/" + targetDb.name + "/" + copy._id + "/data.dat"
+ ).responseText;
+ T(att2_copy.length === att2_data.length);
+ T(att2_copy === att2_data);
+ }
+
// test deletion is replicated
doc = sourceDb.open(docs[1]._id);
T(sourceDb.deleteDoc(doc).ok);
@@ -155,7 +256,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === targetInfo.update_seq);
T(sourceInfo.doc_del_count === targetInfo.doc_del_count);
T(targetInfo.doc_del_count === 1);
@@ -173,10 +273,10 @@ couchTests.new_replication = function(de
copy = targetDb.open(docs[1]._id);
T(copy === null);
- var changes = targetDb.changes({since: sourceInfo.update_seq - 1});
- T(changes.results[0].id === docs[1]._id);
- T(changes.results[0].seq === sourceInfo.update_seq);
- T(changes.results[0].deleted === true);
+ var changes = targetDb.changes({since: 0});
+ var idx = changes.results.length - 1;
+ T(changes.results[idx].id === docs[1]._id);
+ T(changes.results[idx].deleted === true);
// test conflict
doc = sourceDb.open(docs[0]._id);
@@ -194,7 +294,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === (targetInfo.update_seq - 1));
T(repResult.history instanceof Array);
T(repResult.history.length === 4);
@@ -225,7 +324,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === (targetInfo.update_seq - 1));
T(repResult.history instanceof Array);
T(repResult.history.length === 5);
@@ -259,7 +357,6 @@ couchTests.new_replication = function(de
targetInfo = targetDb.info();
T(sourceInfo.doc_count === targetInfo.doc_count);
- T(sourceInfo.update_seq === (targetInfo.update_seq - 2));
T(repResult.history instanceof Array);
T(repResult.history.length === 6);
@@ -340,9 +437,7 @@ couchTests.new_replication = function(de
if ((doc.integer && (doc.integer % 2 === 0)) || (doc.string === "7")) {
T(copy !== null);
- for (var p in doc) {
- T(copy[p] === doc[p]);
- }
+ T(compareObjects(doc, copy) === true);
} else {
T(copy === null);
}
@@ -390,9 +485,7 @@ couchTests.new_replication = function(de
if (doc.integer && (doc.integer % 2 === 0)) {
T(copy !== null);
- for (var p in doc) {
- T(copy[p] === doc[p]);
- }
+ T(compareObjects(doc, copy) === true);
} else {
T(copy === null);
}
@@ -466,9 +559,6 @@ couchTests.new_replication = function(de
T(repResult.docs_written === total);
T(repResult.doc_write_failures === 0);
- targetInfo = targetDb.info();
- T(targetInfo.doc_count === total);
-
for (k = 0; k < doc_ids.length; k++) {
id = doc_ids[k];
doc = sourceDb.open(id);
@@ -480,12 +570,26 @@ couchTests.new_replication = function(de
} else {
T(doc !== null);
T(copy !== null);
- for (var p in doc) {
- T(copy[p] === doc[p]);
- }
+ T(compareObjects(doc, copy) === true);
}
}
+ // be absolutely sure that other docs were not replicated
+ for (k = 0; k < docs.length; k++) {
+ id = docs[k]._id;
+ doc = targetDb.open(id);
+
+ if (doc_ids.indexOf(id) >= 0) {
+ T(doc !== null);
+ } else {
+ T(doc === null);
+ }
+ }
+
+ targetInfo = targetDb.info();
+ T(targetInfo.doc_count === total);
+
+
// add more docs throught replication by doc IDs
after_doc_ids = target_doc_ids[j].after;
after_num_inexistent_docs = 0;
@@ -515,9 +619,6 @@ couchTests.new_replication = function(de
T(repResult.docs_written === after_total);
T(repResult.doc_write_failures === 0);
- targetInfo = targetDb.info();
- T(targetInfo.doc_count === (total + after_total));
-
for (k = 0; k < after_doc_ids.length; k++) {
id = after_doc_ids[k];
doc = sourceDb.open(id);
@@ -529,18 +630,34 @@ couchTests.new_replication = function(de
} else {
T(doc !== null);
T(copy !== null);
- for (var p in doc) {
- T(copy[p] === doc[p]);
- }
+ T(compareObjects(doc, copy) === true);
}
}
+ // be absolutely sure that other docs were not replicated
+ for (k = 0; k < docs.length; k++) {
+ id = docs[k]._id;
+ doc = targetDb.open(id);
+
+ if ((doc_ids.indexOf(id) >= 0) || (after_doc_ids.indexOf(id) >= 0)) {
+ T(doc !== null);
+ } else {
+ T(doc === null);
+ }
+ }
+
+ targetInfo = targetDb.info();
+ T(targetInfo.doc_count === (total + after_total));
+
+
// replicate again the same doc after updated on source (no conflict)
id = target_doc_ids[j].conflict_id;
doc = sourceDb.open(id);
T(doc !== null);
- doc.integer += 100;
+ doc.integer = 666;
T(sourceDb.save(doc).ok);
+ addAtt(sourceDb, doc, "readme.txt", att1_data, "text/plain");
+ addAtt(sourceDb, doc, "data.dat", att2_data, "application/binary");
repResult = CouchDB.new_replicate(
dbPairs[i].source,
@@ -559,9 +676,35 @@ couchTests.new_replication = function(de
copy = targetDb.open(id, {conflicts: true});
- T(copy._rev.indexOf("2-") === 0);
+ T(copy.integer === 666);
+ T(copy._rev.indexOf("4-") === 0);
T(typeof copy._conflicts === "undefined");
+ var atts = copy._attachments;
+ T(typeof atts === "object");
+ T(typeof atts["readme.txt"] === "object");
+ T(atts["readme.txt"].revpos === 3);
+ T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+ T(atts["readme.txt"].stub === true);
+
+ var att1_copy = CouchDB.request(
+ "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+ ).responseText;
+ T(att1_copy.length === att1_data.length);
+ T(att1_copy === att1_data);
+
+ T(typeof atts["data.dat"] === "object");
+ T(atts["data.dat"].revpos === 4);
+ T(atts["data.dat"].content_type.indexOf("application/binary") === 0);
+ T(atts["data.dat"].stub === true);
+
+ var att2_copy = CouchDB.request(
+ "GET", "/" + targetDb.name + "/" + copy._id + "/data.dat"
+ ).responseText;
+ T(att2_copy.length === att2_data.length);
+ T(att2_copy === att2_data);
+
+
// generate a conflict throught replication by doc IDs
id = target_doc_ids[j].conflict_id;
doc = sourceDb.open(id);
@@ -590,10 +733,10 @@ couchTests.new_replication = function(de
copy = targetDb.open(id, {conflicts: true});
- T(copy._rev.indexOf("3-") === 0);
+ T(copy._rev.indexOf("5-") === 0);
T(copy._conflicts instanceof Array);
T(copy._conflicts.length === 1);
- T(copy._conflicts[0].indexOf("3-") === 0);
+ T(copy._conflicts[0].indexOf("5-") === 0);
}
}
Modified: couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl Thu Jul 22 17:33:04 2010
@@ -37,10 +37,10 @@
db_open/3,
db_close/1,
get_db_info/1,
- open_doc/3,
update_doc/3,
ensure_full_commit/1,
get_missing_revs/2,
+ open_doc/3,
open_doc/4,
open_doc_revs/6,
update_doc/4,
@@ -94,27 +94,6 @@ get_db_info(Db) ->
couch_db:get_db_info(Db).
-open_doc(#httpdb{url=Url, oauth=OAuth, headers=Headers}, DocId, Options) ->
- Url2 = Url ++ couch_util:url_encode(DocId),
- QArgs = options_to_query_args(Options, []),
- Headers2 = oauth_header(Url2, QArgs, get, OAuth) ++ Headers,
- #url{host=Host, port=Port} = ibrowse_lib:parse_url(Url),
- {ok, Worker} = ibrowse:spawn_link_worker_process(Host,Port),
- try ibrowse:send_req_direct(Worker, Url2 ++ query_args_to_string(QArgs, []),
- Headers2, get, [], [
- {response_format,binary}
- ], infinity) of
- {ok, "200", _RespHeaders, Body} ->
- {ok, couch_doc:from_json_obj(?JSON_DECODE(Body))};
- {ok, "404", _RespHeaders, _Body} ->
- {not_found, missing}
- after
- catch ibrowse:stop_worker_process(Worker)
- end;
-open_doc(Db, DocId, Options) ->
- couch_db:open_doc(Db, DocId, Options).
-
-
update_doc(Db, Doc, Options) ->
update_doc(Db,Doc,Options,interactive_edit).
@@ -203,8 +182,10 @@ open_doc_revs(Db, Id, Revs, Options, Fun
{ok, Results} = couch_db:open_doc_revs(Db, Id, Revs, Options),
{ok, lists:foldl(Fun, Acc, Results)}.
+open_doc(Db, Id, Options, Fun) ->
+ Fun(open_doc(Db, Id, Options)).
-open_doc(#httpdb{} = HttpDb, Id, Options, Fun) ->
+open_doc(#httpdb{} = HttpDb, Id, Options) ->
#httpdb{url=Url, oauth=OAuth, headers=Headers} = HttpDb,
QArgs = [
{"attachments", "true"},
@@ -229,39 +210,48 @@ open_doc(#httpdb{} = HttpDb, Id, Options
[{response_format, binary}, {stream_to, {self(), once}}],
infinity),
- DataFun = fun() -> stream_data_self(ReqId) end,
receive
- {ibrowse_async_headers, ReqId, "200", RespHeaders} ->
- case couch_util:get_value("Content-Type", RespHeaders) of
- ("multipart/related;" ++ _) = CType ->
- {ok, Doc} = couch_doc:doc_from_multi_part_stream(
- CType, DataFun);
+ {ibrowse_async_headers, ReqId, Code, RespHeaders} ->
+ CType = couch_util:get_value("Content-Type", RespHeaders),
+ Self ! {self(), CType},
+ case CType of
"application/json" ->
- Doc = couch_doc:from_json_obj(
- json_stream_parse:to_ejson(DataFun))
- end,
- receive
- {get_doc, From} ->
- From ! {doc, Doc}
- end;
- {ibrowse_async_headers, ReqId, _ErrorCode, _RespHeaders} ->
- receive
- {get_doc, From} ->
- From ! {error, json_stream_parse:to_ejson(DataFun)}
+ receive
+ {get, From} ->
+ EJson = json_stream_parse:to_ejson(
+ fun() -> stream_data_self(ReqId) end),
+ case Code of
+ "200" ->
+ Doc = couch_doc:from_json_obj(EJson),
+ From ! {data, self(), Doc};
+ _ErrorCode ->
+ From ! {data, self(), EJson}
+ end
+ end;
+ "multipart/related;" ++ _ ->
+ couch_httpd:parse_multipart_request(
+ CType,
+ fun() -> stream_data_self(ReqId) end,
+ fun(Ev)-> couch_doc:mp_parse_doc(Ev, []) end)
end
end,
catch ibrowse:stop_worker_process(Worker),
unlink(Self)
end),
- Streamer ! {get_doc, self()},
receive
- {doc, Doc} ->
- Fun({ok, Doc});
- {error, Error} ->
- Fun(Error)
+ {Streamer, "application/json"} ->
+ Streamer ! {get, self()},
+ receive
+ {data, Streamer, #doc{} = Doc} ->
+ {ok, Doc};
+ {data, Streamer, Error} ->
+ Error
+ end;
+ {Streamer, "multipart/related;" ++ _} ->
+ couch_doc:doc_from_multi_part_stream(Streamer)
end;
-open_doc(Db, Id, Options, Fun) ->
- Fun(couch_db:open_doc(Db, Id, Options)).
+open_doc(Db, Id, Options) ->
+ couch_db:open_doc(Db, Id, Options).
update_doc(#httpdb{} = HttpDb, Doc, Options, Type) ->
Modified: couchdb/branches/new_replicator/src/couchdb/couch_doc.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_doc.erl?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_doc.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_doc.erl Thu Jul 22 17:33:04 2010
@@ -16,8 +16,9 @@
-export([att_foldl/3,att_foldl_decode/3,get_validate_doc_fun/1]).
-export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]).
-export([validate_docid/1]).
--export([doc_from_multi_part_stream/2]).
+-export([doc_from_multi_part_stream/1, doc_from_multi_part_stream/2]).
-export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
+-export([mp_parse_doc/2]).
-include("couch_db.hrl").
@@ -441,13 +442,7 @@ atts_to_mp([Att | RestAtts], Boundary, W
atts_to_mp(RestAtts, Boundary, WriteFun, SendEncodedAtts).
-doc_from_multi_part_stream(ContentType, DataFun) ->
- Self = self(),
- Parser = spawn_link(fun() ->
- couch_httpd:parse_multipart_request(ContentType, DataFun,
- fun(Next)-> mp_parse_doc(Next, []) end),
- unlink(Self)
- end),
+doc_from_multi_part_stream(Parser) ->
Parser ! {get_doc_bytes, self()},
receive
{doc_bytes, DocBytes} ->
@@ -467,6 +462,15 @@ doc_from_multi_part_stream(ContentType,
{ok, Doc#doc{atts=Atts2}}
end.
+doc_from_multi_part_stream(ContentType, DataFun) ->
+ Self = self(),
+ Parser = spawn_link(fun() ->
+ couch_httpd:parse_multipart_request(ContentType, DataFun,
+ fun(Next)-> mp_parse_doc(Next, []) end),
+ unlink(Self)
+ end),
+ doc_from_multi_part_stream(Parser).
+
mp_parse_doc({headers, H}, []) ->
case couch_util:get_value("content-type", H) of
{"application/json", _} ->