You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by jc...@apache.org on 2009/07/20 06:11:37 UTC
svn commit: r795687 - in /couchdb/trunk: share/ share/server/
share/www/script/test/ src/couchdb/ test/
Author: jchris
Date: Mon Jul 20 04:11:36 2009
New Revision: 795687
URL: http://svn.apache.org/viewvc?rev=795687&view=rev
Log:
Initial checkin of _changes filters. The prime weak-spot for this approach is that it maintains an OS-process per connected filtered _changes consumer. I'm pretty sure we'll be able to work around this without changing the API, but it'll involve a lot of OS-process bookkeeping. Those enhancements should generally improve show & list performance as well. Punting on them for now, first wanted to get _changes filters implemented so people could give feedback.
Added:
couchdb/trunk/share/server/filter.js (with props)
Modified:
couchdb/trunk/share/Makefile.am
couchdb/trunk/share/server/loop.js
couchdb/trunk/share/www/script/test/changes.js
couchdb/trunk/src/couchdb/couch_db.erl
couchdb/trunk/src/couchdb/couch_httpd_db.erl
couchdb/trunk/src/couchdb/couch_httpd_show.erl
couchdb/trunk/src/couchdb/couch_query_servers.erl
couchdb/trunk/src/couchdb/couch_util.erl
couchdb/trunk/test/query_server_spec.rb
Modified: couchdb/trunk/share/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/Makefile.am?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/share/Makefile.am (original)
+++ couchdb/trunk/share/Makefile.am Mon Jul 20 04:11:36 2009
@@ -13,6 +13,7 @@
JS_FILE = server/main.js
JS_FILE_COMPONENTS = \
+ server/filter.js \
server/render.js \
server/state.js \
server/util.js \
Added: couchdb/trunk/share/server/filter.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/filter.js?rev=795687&view=auto
==============================================================================
--- couchdb/trunk/share/server/filter.js (added)
+++ couchdb/trunk/share/server/filter.js Mon Jul 20 04:11:36 2009
@@ -0,0 +1,25 @@
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+var Filter = {
+ filter : function(docs, req, userCtx) {
+ var results = [];
+ try {
+ for (var i=0; i < docs.length; i++) {
+ results.push((funs[0](docs[i], req, userCtx) && true) || false);
+ };
+ respond([true, results]);
+ } catch (error) {
+ respond(error);
+ }
+ }
+};
Propchange: couchdb/trunk/share/server/filter.js
------------------------------------------------------------------------------
svn:eol-style = native
Modified: couchdb/trunk/share/server/loop.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/loop.js?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/share/server/loop.js (original)
+++ couchdb/trunk/share/server/loop.js Mon Jul 20 04:11:36 2009
@@ -41,7 +41,8 @@
"rereduce" : Views.rereduce,
"validate" : Validate.validate,
"show" : Render.show,
- "list" : Render.list
+ "list" : Render.list,
+ "filter" : Filter.filter
};
while (line = eval(readline())) {
Modified: couchdb/trunk/share/www/script/test/changes.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/changes.js?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/changes.js (original)
+++ couchdb/trunk/share/www/script/test/changes.js Mon Jul 20 04:11:36 2009
@@ -110,4 +110,66 @@
T(str.charAt(str.length - 1) == "\n")
T(str.charAt(str.length - 2) == "\n")
}
+
+ // test the filtered changes
+ var ddoc = {
+ _id : "_design/changes_filter",
+ "filters" : {
+ "bop" : "function(doc, req, userCtx) { return (doc.bop);}",
+ "dynamic" : stringFun(function(doc, req, userCtx) {
+ var field = req.query.field;
+ return doc[field];
+ }),
+ "userCtx" : stringFun(function(doc, req, userCtx) {
+ return doc.user && (doc.user == userCtx.name);
+ })
+ }
+ }
+
+ db.save(ddoc);
+
+ var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/bop");
+ var resp = JSON.parse(req.responseText);
+ T(resp.results.length == 0);
+
+ db.save({"bop" : "foom"});
+
+ var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/bop");
+ var resp = JSON.parse(req.responseText);
+ T(resp.results.length == 1);
+
+ req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/dynamic&field=woox");
+ resp = JSON.parse(req.responseText);
+ T(resp.results.length == 0);
+
+ req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/dynamic&field=bop");
+ resp = JSON.parse(req.responseText);
+ T(resp.results.length == 1);
+
+ // test for userCtx
+ run_on_modified_server(
+ [{section: "httpd",
+ key: "authentication_handler",
+ value: "{couch_httpd, special_test_authentication_handler}"},
+ {section:"httpd",
+ key: "WWW-Authenticate",
+ value: "X-Couch-Test-Auth"}],
+
+ function() {
+ var authOpts = {"headers":{"WWW-Authenticate": "X-Couch-Test-Auth Chris Anderson:mp3"}};
+
+ T(db.save({"user" : "Noah Slater"}).ok);
+ var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/userCtx", authOpts);
+ var resp = JSON.parse(req.responseText);
+ T(resp.results.length == 0);
+
+ var docResp = db.save({"user" : "Chris Anderson"});
+ T(docResp.ok);
+ req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/userCtx", authOpts);
+ resp = JSON.parse(req.responseText);
+ T(resp.results.length == 1);
+ T(resp.results[0].id == docResp.id);
+ });
+
+ // todo implement adhoc filters...
};
Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Mon Jul 20 04:11:36 2009
@@ -279,11 +279,9 @@
ok;
validate_doc_update(_Db, #doc{id= <<"_local/",_/binary>>}, _GetDiskDocFun) ->
ok;
-validate_doc_update(#db{name=DbName,user_ctx=Ctx}=Db, Doc, GetDiskDocFun) ->
+validate_doc_update(Db, Doc, GetDiskDocFun) ->
DiskDoc = GetDiskDocFun(),
- JsonCtx = {[{<<"db">>, DbName},
- {<<"name">>,Ctx#user_ctx.name},
- {<<"roles">>,Ctx#user_ctx.roles}]},
+ JsonCtx = couch_util:json_user_ctx(Db),
try [case Fun(Doc, DiskDoc, JsonCtx) of
ok -> ok;
Error -> throw(Error)
Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Mon Jul 20 04:11:36 2009
@@ -66,33 +66,38 @@
handle_changes_req(#httpd{method='GET',path_parts=[DbName|_]}=Req, Db) ->
StartSeq = list_to_integer(couch_httpd:qs_value(Req, "since", "0")),
+ Filter = couch_httpd:qs_value(Req, "filter", nil),
+ {ok, FilterFun, EndFilterFun} = make_filter_funs(Req, Db, Filter),
+ try
+ {ok, Resp} = start_json_response(Req, 200),
+ send_chunk(Resp, "{\"results\":[\n"),
+ case couch_httpd:qs_value(Req, "continuous", "false") of
+ "true" ->
+ Self = self(),
+ {ok, Notify} = couch_db_update_notifier:start_link(
+ fun({_, DbName0}) when DbName0 == DbName ->
+ Self ! db_updated;
+ (_) ->
+ ok
+ end),
+ {Timeout, TimeoutFun} = get_changes_timeout(Req, Resp),
+ couch_stats_collector:track_process_count(Self,
+ {httpd, clients_requesting_changes}),
+ try
+ keep_sending_changes(Req, Resp, Db, StartSeq, <<"">>, Timeout, TimeoutFun, FilterFun)
+ after
+ couch_db_update_notifier:stop(Notify),
+ get_rest_db_updated() % clean out any remaining update messages
+ end;
- {ok, Resp} = start_json_response(Req, 200),
- send_chunk(Resp, "{\"results\":[\n"),
- case couch_httpd:qs_value(Req, "continuous", "false") of
- "true" ->
- Self = self(),
- {ok, Notify} = couch_db_update_notifier:start_link(
- fun({_, DbName0}) when DbName0 == DbName ->
- Self ! db_updated;
- (_) ->
- ok
- end),
- {Timeout, TimeoutFun} = get_changes_timeout(Req, Resp),
- couch_stats_collector:track_process_count(Self,
- {httpd, clients_requesting_changes}),
- try
- keep_sending_changes(Req, Resp, Db, StartSeq, <<"">>, Timeout, TimeoutFun)
- after
- couch_db_update_notifier:stop(Notify),
- get_rest_db_updated() % clean out any remaining update messages
- end;
-
- "false" ->
- {ok, {LastSeq, _Prepend}} =
- send_changes(Req, Resp, Db, StartSeq, <<"">>),
- send_chunk(Resp, io_lib:format("\n],\n\"last_seq\":~w}\n", [LastSeq])),
- send_chunk(Resp, "")
+ "false" ->
+ {ok, {LastSeq, _Prepend}} =
+ send_changes(Req, Resp, Db, StartSeq, <<"">>, FilterFun),
+ send_chunk(Resp, io_lib:format("\n],\n\"last_seq\":~w}\n", [LastSeq])),
+ send_chunk(Resp, "")
+ end
+ after
+ EndFilterFun()
end;
handle_changes_req(#httpd{path_parts=[_,<<"_changes">>]}=Req, _Db) ->
@@ -113,27 +118,23 @@
after 0 -> updated
end.
-keep_sending_changes(#httpd{user_ctx=UserCtx,path_parts=[DbName|_]}=Req, Resp, Db, StartSeq, Prepend, Timeout, TimeoutFun) ->
- {ok, {EndSeq, Prepend2}} = send_changes(Req, Resp, Db, StartSeq, Prepend),
+keep_sending_changes(#httpd{user_ctx=UserCtx,path_parts=[DbName|_]}=Req, Resp, Db, StartSeq, Prepend, Timeout, TimeoutFun, FilterFun) ->
+ {ok, {EndSeq, Prepend2}} = send_changes(Req, Resp, Db, StartSeq, Prepend, FilterFun),
couch_db:close(Db),
case wait_db_updated(Timeout, TimeoutFun) of
updated ->
{ok, Db2} = couch_db:open(DbName, [{user_ctx, UserCtx}]),
- keep_sending_changes(Req, Resp, Db2, EndSeq, Prepend2, Timeout, TimeoutFun);
+ keep_sending_changes(Req, Resp, Db2, EndSeq, Prepend2, Timeout, TimeoutFun, FilterFun);
stop ->
send_chunk(Resp, io_lib:format("\n],\n\"last_seq\":~w}\n", [EndSeq])),
send_chunk(Resp, "")
end.
-send_changes(Req, Resp, Db, StartSeq, Prepend0) ->
+send_changes(Req, Resp, Db, StartSeq, Prepend0, FilterFun) ->
Style = list_to_existing_atom(
couch_httpd:qs_value(Req, "style", "main_only")),
couch_db:changes_since(Db, Style, StartSeq,
fun([#doc_info{id=Id, high_seq=Seq}|_]=DocInfos, {_, Prepend}) ->
- FilterFun =
- fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
- {[{rev, couch_doc:rev_to_str(Rev)}]}
- end,
Results0 = [FilterFun(DocInfo) || DocInfo <- DocInfos],
Results = [Result || Result <- Results0, Result /= null],
case Results of
@@ -147,6 +148,39 @@
end
end, {StartSeq, Prepend0}).
+make_filter_funs(_Req, _Db, nil) ->
+ {ok, fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
+ {[{rev, couch_doc:rev_to_str(Rev)}]}
+ end,
+ fun() -> ok end};
+make_filter_funs(Req, Db, Filter) ->
+ case [list_to_binary(couch_httpd:unquote(Part))
+ || Part <- string:tokens(Filter, "/")] of
+ [DName, FName] ->
+ DesignId = <<"_design/", DName/binary>>,
+ #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
+ FilterSrc = couch_util:get_nested_json_value({Props}, [<<"filters">>, FName]),
+ {ok, Pid} = couch_query_servers:start_filter(Lang, FilterSrc),
+ FilterFun = fun(DInfo = #doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
+ {ok, Doc} = couch_db:open_doc(Db, DInfo),
+ {ok, Pass} = couch_query_servers:filter_doc(Pid, Doc, Req, Db),
+ case Pass of
+ true ->
+ {[{rev, couch_doc:rev_to_str(Rev)}]};
+ false ->
+ null
+ end
+ end,
+ EndFilterFun = fun() ->
+ couch_query_servers:end_filter(Pid)
+ end,
+ {ok, FilterFun, EndFilterFun};
+ _Else ->
+ throw({bad_request,
+ "filter parameter must be of the form `designname/filtername`"})
+ end.
+
handle_compact_req(#httpd{method='POST',path_parts=[DbName,_,Id|_]}=Req, _Db) ->
ok = couch_view_compactor:start_compact(DbName, Id),
send_json(Req, 202, {[{ok, true}]});
@@ -747,7 +781,7 @@
% Useful for debugging
% couch_doc_open(Db, DocId) ->
-% couch_doc_open(Db, DocId, [], []).
+% couch_doc_open(Db, DocId, nil, []).
couch_doc_open(Db, DocId, Rev, Options) ->
case Rev of
Modified: couchdb/trunk/src/couchdb/couch_httpd_show.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_show.erl?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_show.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_show.erl Mon Jul 20 04:11:36 2009
@@ -45,7 +45,7 @@
DesignId = <<"_design/", DesignName/binary>>,
#doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
- ShowSrc = get_nested_json_value({Props}, [<<"shows">>, ShowName]),
+ ShowSrc = couch_util:get_nested_json_value({Props}, [<<"shows">>, ShowName]),
Doc = case DocId of
nil -> nil;
_ ->
@@ -78,18 +78,10 @@
DesignId = <<"_design/", DesignName/binary>>,
#doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
- ListSrc = get_nested_json_value({Props}, [<<"lists">>, ListName]),
+ ListSrc = couch_util:get_nested_json_value({Props}, [<<"lists">>, ListName]),
send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db, Keys).
-get_nested_json_value({Props}, [Key|Keys]) ->
- case proplists:get_value(Key, Props, nil) of
- nil -> throw({not_found, <<"missing json key: ", Key/binary>>});
- Value -> get_nested_json_value(Value, Keys)
- end;
-get_nested_json_value(Value, []) ->
- Value;
-get_nested_json_value(_NotJSONObj, _) ->
- throw({not_found, json_mismatch}).
+
send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db, Keys) ->
Stale = couch_httpd_view:get_stale_type(Req),
Modified: couchdb/trunk/src/couchdb/couch_query_servers.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_query_servers.erl?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_query_servers.erl (original)
+++ couchdb/trunk/src/couchdb/couch_query_servers.erl Mon Jul 20 04:11:36 2009
@@ -20,6 +20,7 @@
-export([reduce/3, rereduce/3,validate_doc_update/5]).
-export([render_doc_show/6, start_view_list/2,
render_list_head/4, render_list_row/3, render_list_tail/1]).
+-export([start_filter/2, filter_doc/4, end_filter/1]).
% -export([test/0]).
-include("couch_db.hrl").
@@ -211,8 +212,22 @@
ok = ret_os_process(Lang, Pid),
JsonResp.
+start_filter(Lang, FilterSrc) ->
+ Pid = get_os_process(Lang),
+ true = couch_os_process:prompt(Pid, [<<"add_fun">>, FilterSrc]),
+ {ok, {Lang, Pid}}.
+filter_doc({_Lang, Pid}, Doc, Req, Db) ->
+ JsonReq = couch_httpd_external:json_req_obj(Req, Db),
+ JsonDoc = couch_doc:to_json_obj(Doc, [revs]),
+ JsonCtx = couch_util:json_user_ctx(Db),
+ [true, [Pass]] = couch_os_process:prompt(Pid,
+ [<<"filter">>, [JsonDoc], JsonReq, JsonCtx]),
+ {ok, Pass}.
+end_filter({Lang, Pid}) ->
+ ok = ret_os_process(Lang, Pid).
+
init([]) ->
Modified: couchdb/trunk/src/couchdb/couch_util.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_util.erl?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_util.erl (original)
+++ couchdb/trunk/src/couchdb/couch_util.erl Mon Jul 20 04:11:36 2009
@@ -17,7 +17,7 @@
-export([new_uuid/0, rand32/0, implode/2, collate/2, collate/3]).
-export([abs_pathname/1,abs_pathname/2, trim/1, ascii_lower/1]).
-export([encodeBase64/1, decodeBase64/1, to_hex/1,parse_term/1,dict_find/3]).
--export([file_read_size/1]).
+-export([file_read_size/1, get_nested_json_value/2, json_user_ctx/1]).
-export([to_binary/1, to_list/1]).
-include("couch_db.hrl").
@@ -75,6 +75,22 @@
erl_parse:parse_term(Tokens).
+get_nested_json_value({Props}, [Key|Keys]) ->
+ case proplists:get_value(Key, Props, nil) of
+ nil -> throw({not_found, <<"missing json key: ", Key/binary>>});
+ Value -> get_nested_json_value(Value, Keys)
+ end;
+get_nested_json_value(Value, []) ->
+ Value;
+get_nested_json_value(_NotJSONObj, _) ->
+ throw({not_found, json_mismatch}).
+
+json_user_ctx(#db{name=DbName, user_ctx=Ctx}) ->
+ {[{<<"db">>, DbName},
+ {<<"name">>,Ctx#user_ctx.name},
+ {<<"roles">>,Ctx#user_ctx.roles}]}.
+
+
% returns a random integer
rand32() ->
crypto:rand_uniform(0, 16#100000000).
Modified: couchdb/trunk/test/query_server_spec.rb
URL: http://svn.apache.org/viewvc/couchdb/trunk/test/query_server_spec.rb?rev=795687&r1=795686&r2=795687&view=diff
==============================================================================
--- couchdb/trunk/test/query_server_spec.rb (original)
+++ couchdb/trunk/test/query_server_spec.rb Mon Jul 20 04:11:36 2009
@@ -239,6 +239,15 @@
return "tail";
};
JS
+ },
+ "filter-basic" => {
+ "js" => <<-JS
+ function(doc, req, userCtx) {
+ if (doc.good) {
+ return true;
+ }
+ }
+ JS
}
}
@@ -420,6 +429,18 @@
should == true
end
end
+
+ describe "changes filter" do
+ before(:all) do
+ @fun = functions["filter-basic"][LANGUAGE]
+ @qs.reset!
+ @qs.add_fun(@fun).should == true
+ end
+ it "should only return true for good docs" do
+ @qs.run(["filter", [{"key"=>"bam", "good" => true}, {"foo" => "bar"}, {"good" => true}]]).
+ should == [true, [true, false, true]]
+ end
+ end
end
def should_have_exited qs