You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2015/04/02 10:00:11 UTC
couchdb-mango git commit: Enable bulk deletes
Repository: couchdb-mango
Updated Branches:
refs/heads/2651-delete-bulk-docs 9642daa37 -> 8ce9bf4ba (forced update)
Enable bulk deletes
Add _bulk_delete path to _index so that users can bulk delete
indexes via POST. Users will pass in a list of docids via the POST body.
We delete each index one by one until an error is thrown or all indexes
in the list are deleted.
Fixes COUCHDB-2651
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/8ce9bf4b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/8ce9bf4b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/8ce9bf4b
Branch: refs/heads/2651-delete-bulk-docs
Commit: 8ce9bf4baf28c35924bde049f7f2c3edc1b5cf11
Parents: f6d4901
Author: Tony Sun <to...@cloudant.com>
Authored: Wed Apr 1 15:30:51 2015 -0700
Committer: Tony Sun <to...@cloudant.com>
Committed: Thu Apr 2 01:01:55 2015 -0700
----------------------------------------------------------------------
src/mango_error.erl | 7 +++++
src/mango_httpd.erl | 69 +++++++++++++++++++++++++----------------
src/mango_idx.erl | 25 ++++++++++++++-
src/mango_opts.erl | 27 +++++++++++++++-
test/01-index-crud-test.py | 26 ++++++++++++++++
test/mango.py | 9 ++++++
6 files changed, 135 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index babf564..6dcf7c8 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -110,6 +110,13 @@ info(mango_idx_view, {index_not_found, BadIdx}) ->
fmt("JSON index ~s not found in this design doc.", [BadIdx])
};
+info(mango_opts, {invalid_bulk_docs, Val}) ->
+ {
+ 400,
+ <<"invalid_bulk_docs">>,
+ fmt("Bulk Delete requires an array of non-null docids. Docids: ~w",
+ [Val])
+ };
info(mango_opts, {invalid_ejson, Val}) ->
{
400,
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index 28a1578..3ea5520 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -65,7 +65,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, DDoc} ->
<<"exists">>;
{ok, NewDDoc} ->
- CreateOpts = get_idx_create_opts(Opts),
+ CreateOpts = get_idx_w_opts(Opts),
case mango_crud:insert(Db, NewDDoc, CreateOpts) of
{ok, [{RespProps}]} ->
case lists:keyfind(error, 1, RespProps) of
@@ -80,6 +80,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
end,
chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
+%% Essentially we just iterate through the list of ddoc ids passed in and
+%% delete one by one. If an error occurs, all previous documents will be
+%% deleted, but an error will be thrown for the current ddoc id.
+handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
+ <<"_bulk_delete">>]}=Req, Db) ->
+ {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
+ Idxs = mango_idx:list(Db),
+ DDocs = get_bulk_delete_ddocs(Opts),
+ DelOpts = get_idx_w_opts(Opts),
+ lists:foreach(fun(DDocId0) ->
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) ->
+ IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+ JsonType = mango_idx:type(Idx) == <<"json">>,
+ TextType = mango_idx:name(Idx) == <<"text">>,
+ IsDDoc andalso (JsonType orelse TextType)
+ end,
+ mango_idx:delete(Filt, Db, Idxs, DelOpts)
+ end, DDocs),
+ chttpd:send_json(Req, {[{result, <<"Indexes Deleted">>}]});
+
handle_index_req(#httpd{method='DELETE',
path_parts=[A, B, <<"_design">>, DDocId0, Type, Name]}=Req, Db) ->
PathParts = [A, B, <<"_design/", DDocId0/binary>>, Type, Name],
@@ -87,37 +108,17 @@ handle_index_req(#httpd{method='DELETE',
handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
- DDocId = case DDocId0 of
- <<"_design/", _/binary>> -> DDocId0;
- _ -> <<"_design/", DDocId0/binary>>
- end,
Idxs = mango_idx:list(Db),
+ DDocId = convert_to_design_id(DDocId0),
+ DelOpts = get_idx_del_opts(Req),
Filt = fun(Idx) ->
IsDDoc = mango_idx:ddoc(Idx) == DDocId,
IsType = mango_idx:type(Idx) == Type,
IsName = mango_idx:name(Idx) == Name,
IsDDoc andalso IsType andalso IsName
end,
- case lists:filter(Filt, Idxs) of
- [Idx] ->
- {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
- {ok, NewDDoc} = mango_idx:remove(DDoc, Idx),
- FinalDDoc = case NewDDoc#doc.body of
- {[{<<"language">>, <<"query">>}]} ->
- NewDDoc#doc{deleted = true, body = {[]}};
- _ ->
- NewDDoc
- end,
- DelOpts = get_idx_del_opts(Req),
- case mango_crud:insert(Db, FinalDDoc, DelOpts) of
- {ok, _} ->
- chttpd:send_json(Req, {[{ok, true}]});
- _ ->
- ?MANGO_ERROR(error_saving_ddoc)
- end;
- [] ->
- throw({not_found, missing})
- end;
+ Result = mango_idx:delete(Filt, Db, Idxs, DelOpts),
+ chttpd:send_json(Req, {[{ok, Result}]});
handle_index_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "GET,POST,DELETE").
@@ -148,7 +149,7 @@ set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
Db#db{user_ctx=Ctx}.
-get_idx_create_opts(Opts) ->
+get_idx_w_opts(Opts) ->
case lists:keyfind(w, 1, Opts) of
{w, N} when is_integer(N), N > 0 ->
[{w, integer_to_list(N)}];
@@ -157,6 +158,15 @@ get_idx_create_opts(Opts) ->
end.
+get_bulk_delete_ddocs(Opts) ->
+ case lists:keyfind(docids, 1, Opts) of
+ {docids, DDocs} when is_list(DDocs) ->
+ DDocs;
+ _ ->
+ []
+ end.
+
+
get_idx_del_opts(Req) ->
try
WStr = chttpd:qs_value(Req, "w", "2"),
@@ -167,6 +177,13 @@ get_idx_del_opts(Req) ->
end.
+convert_to_design_id(DDocId) ->
+ case DDocId of
+ <<"_design/", _/binary>> -> DDocId;
+ _ -> <<"_design/", DDocId/binary>>
+ end.
+
+
start_find_resp(Req) ->
chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":[").
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 1c15894..31b29f2 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -41,7 +41,8 @@
end_key/2,
cursor_mod/1,
idx_mod/1,
- to_json/1
+ to_json/1,
+ delete/4
]).
@@ -134,6 +135,28 @@ remove(DDoc, Idx) ->
{ok, NewDDoc#doc{body = Body}}.
+delete(Filt, Db, Indexes, DelOpts) ->
+ case lists:filter(Filt, Indexes) of
+ [Idx] ->
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
+ {ok, NewDDoc} = mango_idx:remove(DDoc, Idx),
+ FinalDDoc = case NewDDoc#doc.body of
+ {[{<<"language">>, <<"query">>}]} ->
+ NewDDoc#doc{deleted = true, body = {[]}};
+ _ ->
+ NewDDoc
+ end,
+ case mango_crud:insert(Db, FinalDDoc, DelOpts) of
+ {ok, _} ->
+ true;
+ _ ->
+ ?MANGO_ERROR(error_saving_ddoc)
+ end;
+ [] ->
+ throw({not_found, missing})
+ end.
+
+
from_ddoc(Db, {Props}) ->
DbName = db_to_name(Db),
DDoc = proplists:get_value(<<"_id">>, Props),
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/src/mango_opts.erl
----------------------------------------------------------------------
diff --git a/src/mango_opts.erl b/src/mango_opts.erl
index f7874a6..0c66a21 100644
--- a/src/mango_opts.erl
+++ b/src/mango_opts.erl
@@ -31,7 +31,8 @@
validate_use_index/1,
validate_bookmark/1,
validate_sort/1,
- validate_fields/1
+ validate_fields/1,
+ validate_bulk_delete/1
]).
@@ -129,6 +130,22 @@ validate_find({Props}) ->
validate(Props, Opts).
+validate_bulk_delete({Props}) ->
+ Opts = [
+ {<<"docids">>, [
+ {tag, docids},
+ {validator, fun validate_bulk_docs/1}
+ ]},
+ {<<"w">>, [
+ {tag, w},
+ {optional, true},
+ {default, 2},
+ {validator, fun is_pos_integer/1}
+ ]}
+ ],
+ validate(Props, Opts).
+
+
validate(Props, Opts) ->
case mango_util:assert_ejson({Props}) of
true ->
@@ -192,6 +209,14 @@ validate_selector(Else) ->
?MANGO_ERROR({invalid_selector_json, Else}).
+%% We re-use validate_use_index to make sure the index names are valid
+validate_bulk_docs(Docs) when is_list(Docs) ->
+ lists:foreach(fun validate_use_index/1, Docs),
+ {ok, Docs};
+validate_bulk_docs(Else) ->
+ ?MANGO_ERROR({invalid_bulk_docs, Else}).
+
+
validate_use_index(IndexName) when is_binary(IndexName) ->
case binary:split(IndexName, <<"/">>) of
[DesignId] ->
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/test/01-index-crud-test.py
----------------------------------------------------------------------
diff --git a/test/01-index-crud-test.py b/test/01-index-crud-test.py
index 459566b..adc0ecb 100644
--- a/test/01-index-crud-test.py
+++ b/test/01-index-crud-test.py
@@ -150,6 +150,32 @@ class IndexCrudTests(mango.DbPerClass):
post_indexes = self.db.list_indexes()
assert pre_indexes == post_indexes
+ def test_bulk_delete(self):
+ fields = ["field1"]
+ ret = self.db.create_index(fields, name="idx_01")
+ assert ret is True
+
+ fields = ["field2"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+
+ fields = ["field3"]
+ ret = self.db.create_index(fields, name="idx_03")
+ assert ret is True
+
+ docids = []
+
+ for idx in self.db.list_indexes():
+ if idx["ddoc"] is not None:
+ docids.append(idx["ddoc"])
+
+ ret = self.db.bulk_delete(docids)
+ assert ret is True
+
+ for idx in self.db.list_indexes():
+ assert idx["type"] != "json"
+ assert idx["type"] != "text"
+
def test_recreate_index(self):
pre_indexes = self.db.list_indexes()
for i in range(5):
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/8ce9bf4b/test/mango.py
----------------------------------------------------------------------
diff --git a/test/mango.py b/test/mango.py
index 57dfffb..d1da9d7 100644
--- a/test/mango.py
+++ b/test/mango.py
@@ -135,6 +135,15 @@ class Database(object):
r = self.sess.delete(self.path(path), params={"w":"3"})
r.raise_for_status()
+ def bulk_delete(self, docs):
+ body = {
+ "docids" : docs,
+ "w": 3
+ }
+ body = json.dumps(body)
+ r = self.sess.post(self.path("_index/_bulk_delete"), data=body)
+ return r.json()["result"] == "Indexes Deleted"
+
def find(self, selector, limit=25, skip=0, sort=None, fields=None,
r=1, conflicts=False, use_index=None, explain=False,
bookmark=None, return_raw=False):