You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2009/07/16 22:54:46 UTC
svn commit: r794851 - in /couchdb/trunk:
src/couchdb/couch_rep_changes_feed.erl
test/etap/110-replication-changes-feed.t
Author: kocolosk
Date: Thu Jul 16 20:54:45 2009
New Revision: 794851
URL: http://svn.apache.org/viewvc?rev=794851&view=rev
Log:
second cut at _changes feed consumer. still not used by replicator
Modified:
couchdb/trunk/src/couchdb/couch_rep_changes_feed.erl
couchdb/trunk/test/etap/110-replication-changes-feed.t
Modified: couchdb/trunk/src/couchdb/couch_rep_changes_feed.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_rep_changes_feed.erl?rev=794851&r1=794850&r2=794851&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_rep_changes_feed.erl (original)
+++ couchdb/trunk/src/couchdb/couch_rep_changes_feed.erl Thu Jul 16 20:54:45 2009
@@ -22,13 +22,22 @@
-include("couch_db.hrl").
-include("../ibrowse/ibrowse.hrl").
--record (state, {
+-record (remote, {
conn = nil,
reqid = nil,
+ complete = false,
count = 0,
reply_to = nil,
- rows = queue:new(),
- complete = false
+ rows = queue:new()
+}).
+
+-record (local, {
+ changes_from = nil,
+ changes_pid = nil,
+ complete = false,
+ count = 0,
+ reply_to = nil,
+ rows = queue:new()
}).
start(Url, Options) ->
@@ -38,7 +47,6 @@
gen_server:start_link(?MODULE, [Url, Options], []).
next(Server) ->
- ?LOG_DEBUG("~p at ~p received next_change call", [?MODULE, Server]),
gen_server:call(Server, next_change, infinity).
stop(Server) ->
@@ -49,64 +57,110 @@
Continuous = proplists:get_value(continuous, Options, false),
{Pid, ReqId} = start_http_request(lists:concat([Url, "/_changes",
"?style=all_docs", "&since=", Since, "&continuous=", Continuous])),
- {ok, #state{conn=Pid, reqid=ReqId}};
+ {ok, #remote{conn=Pid, reqid=ReqId}};
+init([{local, DbName}, Options]) when is_list(DbName) ->
+ init([{local, ?l2b(DbName)}, Options]);
init([{local, DbName}, Options]) ->
- init([{remote, "http://" ++ couch_config:get("httpd", "bind_address") ++ ":"
- ++ couch_config:get("httpd", "port") ++ "/" ++ DbName}, Options]).
+ ?LOG_DEBUG("initializing local changes feed for ~s with ~p", [DbName, Options]),
+ process_flag(trap_exit, true),
+ Server = self(),
+ Since = proplists:get_value(since, Options, 0),
+ ChangesPid =
+ case proplists:get_value(continuous, Options, false) of
+ false ->
+ spawn_link(fun() -> send_local_changes_once(Server, DbName, Since) end);
+ true ->
+ spawn_link(fun() -> send_local_changes_forever(Server, DbName, Since) end)
+ end,
+ {ok, #local{changes_pid=ChangesPid}}.
+
+handle_call({add, Row}, _From, #local{count=Count, rows=Rows}=State)
+ when Count < ?MIN_BUFFER_SIZE->
+ case State of
+ #local{reply_to=nil} ->
+ {reply, ok, State#local{count=Count+1, rows = queue:in(Row, Rows)}};
+ #local{count=0, reply_to=Requestor}->
+ gen_server:reply(Requestor, Row),
+ {reply, ok, State#local{reply_to=nil}}
+ end;
+handle_call({add, Row}, From, #local{}=State) ->
+ #local{
+ count = Count,
+ rows = Rows
+ } = State,
+ {noreply, State#local{count=Count+1, changes_from=From, rows=queue:in(Row,Rows)}};
+
+handle_call(next_change, From, #local{count=0}=State) ->
+ if State#local.complete ->
+ {stop, normal, complete, State};
+ true ->
+ {noreply, State#local{reply_to=From}}
+ end;
+handle_call(next_change, _From, #local{}=State) ->
+ #local{
+ count = Count,
+ changes_from = ChangesFrom,
+ rows = Rows
+ } = State,
+ {{value, Row}, NewRows} = queue:out(Rows),
+ if Count =:= ?MIN_BUFFER_SIZE, ChangesFrom =/= nil ->
+ gen_server:reply(ChangesFrom, ok),
+ {reply, Row, State#local{count=Count-1, changes_from=nil, rows=NewRows}};
+ true ->
+ {reply, Row, State#local{count=Count-1, rows=NewRows}}
+ end;
-handle_call(next_change, From, State) ->
- #state{
+handle_call(next_change, From, #remote{count=0}=State) ->
+ if State#remote.complete ->
+ {stop, normal, complete, State};
+ true ->
+ {noreply, State#remote{reply_to=From}}
+ end;
+handle_call(next_change, _From, #remote{}=State) ->
+ #remote{
reqid = Id,
complete = Complete,
count = Count,
rows = Rows
} = State,
-
ok = maybe_stream_next(Complete, Count, Id),
-
- case queue:out(Rows) of
- {{value, Row}, NewRows} ->
- {reply, Row, State#state{count=Count-1, rows=NewRows}};
- {empty, Rows} ->
- if State#state.complete ->
- {stop, normal, complete, State};
- % State#state.waiting_on_headers ->
- % {noreply, State#state{reply_to=From}};
- true ->
- {noreply, State#state{reply_to=From}}
- end
- end;
+ {{value, Row}, NewRows} = queue:out(Rows),
+ {reply, Row, State#remote{count=Count-1, rows=NewRows}};
-handle_call(stop, _From, State) ->
- catch ibrowse:stop_worker_process(State#state.conn),
+handle_call(stop, _From, #local{changes_pid=ChangesPid} = State) ->
+ exit(ChangesPid, stop),
+ {stop, normal, ok, State};
+
+handle_call(stop, _From, #remote{conn=Conn} = State) ->
+ catch ibrowse:stop_worker_process(Conn),
{stop, normal, ok, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
-handle_info({ibrowse_async_headers, Id, "200", _}, #state{reqid=Id}=State) ->
- #state{
+handle_info({ibrowse_async_headers, Id, "200", _}, #remote{reqid=Id}=State) ->
+ #remote{
complete = Complete,
count = Count
} = State,
?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 200", [?MODULE, Id]),
ok = maybe_stream_next(Complete, Count, Id),
{noreply, State};
-handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #state{reqid=Id}=State) ->
+handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #remote{reqid=Id}=State) ->
?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 301", [?MODULE, Id]),
- catch ibrowse:stop_worker_process(State#state.conn),
+ catch ibrowse:stop_worker_process(State#remote.conn),
Url = mochiweb_headers:get_value("Location", mochiweb_headers:make(Hdrs)),
{Pid, ReqId} = start_http_request(Url),
- {noreply, State#state{conn=Pid, reqid=ReqId}};
-handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #state{reqid=Id}=State) ->
+ {noreply, State#remote{conn=Pid, reqid=ReqId}};
+handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #remote{reqid=Id}=State) ->
?LOG_ERROR("replicator changes feed failed with code ~s and Headers ~n~p",
[Code,Hdrs]),
{stop, {error, list_to_integer(Code)}, State};
-handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
+handle_info({ibrowse_async_response, Id, Msg}, #remote{reqid=Id} = State) ->
?LOG_DEBUG("~p reqid ~p ibrowse_async_response ~p", [?MODULE, Id, Msg]),
- #state{
+ #remote{
complete = Complete,
count = Count,
rows = Rows
@@ -114,11 +168,11 @@
try
Row = decode_row(Msg),
case State of
- #state{reply_to=nil} ->
- {noreply, State#state{count=Count+1, rows = queue:in(Row, Rows)}};
- #state{count=0, reply_to=From}->
+ #remote{reply_to=nil} ->
+ {noreply, State#remote{count=Count+1, rows = queue:in(Row, Rows)}};
+ #remote{count=0, reply_to=From}->
gen_server:reply(From, Row),
- {noreply, State#state{reply_to=nil}}
+ {noreply, State#remote{reply_to=nil}}
end
catch
throw:{invalid_json, Msg} ->
@@ -127,21 +181,32 @@
{noreply, State}
end;
-handle_info({ibrowse_async_response_end, Id}, #state{reqid=Id} = State) ->
- ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#state.reply_to]),
+handle_info({ibrowse_async_response_end, Id}, #remote{reqid=Id} = State) ->
+ ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#remote.reply_to]),
case State of
- #state{reply_to=nil} ->
- {noreply, State#state{complete=true}};
- #state{count=0, reply_to=From}->
+ #remote{reply_to=nil} ->
+ {noreply, State#remote{complete=true}};
+ #remote{count=0, reply_to=From}->
gen_server:reply(From, complete),
{stop, normal, State}
end;
+handle_info({'EXIT', From, normal}, #local{changes_pid=From} = State) ->
+ if State#local.reply_to =/= nil ->
+ gen_server:reply(State#local.reply_to, complete),
+ {stop, normal, State};
+ true ->
+ {noreply, State#local{complete=true}}
+ end;
+handle_info({'EXIT', From, Reason}, #local{changes_pid=From} = State) ->
+ ?LOG_ERROR("changes_pid died with reason ~p", [Reason]),
+ {stop, changes_pid_died, State};
+
handle_info(Msg, State) ->
?LOG_INFO("unexpected message ~p", [Msg]),
{noreply, State}.
-terminate(_Reason, #state{conn=Pid}) when is_pid(Pid) ->
+terminate(_Reason, #remote{conn=Pid}) when is_pid(Pid) ->
catch ibrowse:stop_worker_process(Pid),
ok;
terminate(_Reason, _State) ->
@@ -150,6 +215,8 @@
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
+%internal funs
+
decode_row([$,, $\n | Rest]) ->
decode_row(Rest);
decode_row(Row) ->
@@ -162,6 +229,35 @@
?LOG_DEBUG("~p reqid ~p not streaming", [?MODULE, Id]),
ok.
+send_local_changes_forever(Server, DbName, Since) ->
+ {ok, NewSeq} = send_local_changes_once(Server, DbName, Since),
+ timer:sleep(5000),
+ send_local_changes_forever(Server, DbName, NewSeq).
+
+send_local_changes_once(Server, DbName, Since) ->
+ {ok, Db} = couch_db:open(DbName, []),
+
+ FilterFun =
+ fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
+ {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}
+ end,
+
+ ChangesFun =
+ fun([#doc_info{id=Id, high_seq=Seq}|_]=DocInfos, _) ->
+ Results0 = [FilterFun(DocInfo) || DocInfo <- DocInfos],
+ Results = [Result || Result <- Results0, Result /= null],
+ if Results /= [] ->
+ Change = {[{<<"seq">>,Seq}, {<<"id">>,Id}, {<<"changes">>,Results}]},
+ gen_server:call(Server, {add, Change}, infinity);
+ true ->
+ ?LOG_DEBUG("Results was empty ~p", [Results0]),
+ ok
+ end,
+ {ok, Seq}
+ end,
+
+ couch_db:changes_since(Db, all_docs, Since, ChangesFun, Since).
+
start_http_request(RawUrl) ->
Url = ibrowse_lib:parse_url(RawUrl),
{ok, Pid} = ibrowse:spawn_link_worker_process(Url#url.host, Url#url.port),
Modified: couchdb/trunk/test/etap/110-replication-changes-feed.t
URL: http://svn.apache.org/viewvc/couchdb/trunk/test/etap/110-replication-changes-feed.t?rev=794851&r1=794850&r2=794851&view=diff
==============================================================================
--- couchdb/trunk/test/etap/110-replication-changes-feed.t (original)
+++ couchdb/trunk/test/etap/110-replication-changes-feed.t Thu Jul 16 20:54:45 2009
@@ -22,7 +22,7 @@
code:add_pathz("src/ibrowse"),
code:add_pathz("src/mochiweb"),
- etap:plan(6),
+ etap:plan(12),
case (catch test()) of
ok ->
etap:end_tests();
@@ -40,47 +40,59 @@
crypto:start(),
couch_server:delete(<<"etap-test-db">>, []),
- {ok, Db} = couch_db:create(<<"etap-test-db">>, []),
-
- test_unchanged_db(),
- test_simple_change(),
- test_since_parameter(),
- test_continuous_parameter(),
- test_conflicts(),
- test_deleted_conflicts(),
+ {ok, Db1} = couch_db:create(<<"etap-test-db">>, []),
+ test_all(local),
+ couch_db:close(Db1),
+ couch_server:delete(<<"etap-test-db">>, []),
- couch_db:close(Db),
couch_server:delete(<<"etap-test-db">>, []),
+ {ok, Db2} = couch_db:create(<<"etap-test-db">>, []),
+ test_all(remote),
+ couch_db:close(Db2),
+ couch_server:delete(<<"etap-test-db">>, []),
+
ok.
-test_unchanged_db() ->
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []),
+test_all(Type) ->
+ test_unchanged_db(Type),
+ test_simple_change(Type),
+ test_since_parameter(Type),
+ test_continuous_parameter(Type),
+ test_conflicts(Type),
+ test_deleted_conflicts(Type).
+
+test_unchanged_db(Type) ->
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []),
etap:is(
couch_rep_changes_feed:next(Pid),
complete,
- "changes feed for unchanged DB is automatically complete"
+ io_lib:format(
+ "(~p) changes feed for unchanged DB is automatically complete",
+ [Type])
).
-test_simple_change() ->
+test_simple_change(Type) ->
Expect = generate_change(),
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []),
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []),
etap:is(
{couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
{Expect, complete},
- "change one document, get one row"
+ io_lib:format("(~p) change one document, get one row", [Type])
).
-test_since_parameter() ->
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"},
+test_since_parameter(Type) ->
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)},
[{since, get_update_seq()}]),
etap:is(
couch_rep_changes_feed:next(Pid),
complete,
- "since query-string parameter allows us to skip changes"
+ io_lib:format(
+ "(~p) since query-string parameter allows us to skip changes",
+ [Type])
).
-test_continuous_parameter() ->
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"},
+test_continuous_parameter(Type) ->
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)},
[{since, get_update_seq()}, {continuous, true}]),
% make the changes_feed request before the next update
@@ -94,23 +106,25 @@
etap:is(
receive {actual, Actual} -> Actual end,
Expect,
- "continuous query-string parameter picks up new changes"
+ io_lib:format(
+ "(~p) continuous query-string parameter picks up new changes",
+ [Type])
),
ok = couch_rep_changes_feed:stop(Pid).
-test_conflicts() ->
+test_conflicts(Type) ->
Since = get_update_seq(),
Expect = generate_conflict(),
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"},
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)},
[{since, Since}]),
etap:is(
{couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
{Expect, complete},
- "conflict revisions show up in feed"
+ io_lib:format("(~p) conflict revisions show up in feed", [Type])
).
-test_deleted_conflicts() ->
+test_deleted_conflicts(Type) ->
Since = get_update_seq(),
{ExpectProps} = generate_conflict(),
@@ -132,12 +146,12 @@
{<<"changes">>, [Win, {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}]}
]},
- {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"},
+ {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)},
[{since, Since}]),
etap:is(
{couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
{Expect, complete},
- "deleted conflict revisions show up in feed"
+ io_lib:format("(~p) deleted conflict revisions show up in feed", [Type])
).
generate_change() ->
@@ -178,6 +192,11 @@
{ok, Db} = couch_db:open(<<"etap-test-db">>, []),
Db.
+get_dbname(local) ->
+ "etap-test-db";
+get_dbname(remote) ->
+ "http://127.0.0.1:5984/etap-test-db".
+
get_update_seq() ->
Db = get_db(),
Seq = couch_db:get_update_seq(Db),