You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by jc...@apache.org on 2009/07/04 17:47:47 UTC
svn commit: r791140 - in /couchdb/trunk: etc/couchdb/ share/www/script/
share/www/script/test/ src/couchdb/
Author: jchris
Date: Sat Jul 4 15:47:47 2009
New Revision: 791140
URL: http://svn.apache.org/viewvc?rev=791140&view=rev
Log:
Name view index files by their function hashes for no downtime deploys. Closes COUCHDB-218
Adds ability to switch view indexes on the fly by building the index from a "staging" design doc, and then COPYing the staging doc to the production doc's id. Since indexes are referenced by view definition, the new version of the production design doc will point immediately to the index files already built in staging.
Please use and give feedback.
Modified:
couchdb/trunk/etc/couchdb/default.ini.tpl.in
couchdb/trunk/share/www/script/couch.js
couchdb/trunk/share/www/script/test/design_docs.js
couchdb/trunk/src/couchdb/couch_db.erl
couchdb/trunk/src/couchdb/couch_db.hrl
couchdb/trunk/src/couchdb/couch_db_updater.erl
couchdb/trunk/src/couchdb/couch_httpd_db.erl
couchdb/trunk/src/couchdb/couch_task_status.erl
couchdb/trunk/src/couchdb/couch_view.erl
couchdb/trunk/src/couchdb/couch_view_group.erl
couchdb/trunk/src/couchdb/couch_view_updater.erl
Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Sat Jul 4 15:47:47 2009
@@ -65,6 +65,7 @@
_whoami = {couch_httpd_misc_handlers, handle_whoami_req}
[httpd_db_handlers]
+_view_cleanup = {couch_httpd_db, handle_view_cleanup_req}
_compact = {couch_httpd_db, handle_compact_req}
_design = {couch_httpd_db, handle_design_req}
_temp_view = {couch_httpd_view, handle_temp_view_req}
@@ -80,3 +81,4 @@
_view = {couch_httpd_view, handle_view_req}
_show = {couch_httpd_show, handle_doc_show_req}
_list = {couch_httpd_show, handle_view_list_req}
+_info = {couch_httpd_db, handle_design_info_req}
Modified: couchdb/trunk/share/www/script/couch.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/couch.js?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/couch.js [utf-8] (original)
+++ couchdb/trunk/share/www/script/couch.js [utf-8] Sat Jul 4 15:47:47 2009
@@ -177,6 +177,19 @@
return JSON.parse(this.last_req.responseText);
}
+ // gets information about a design doc
+ this.designInfo = function(docid) {
+ this.last_req = this.request("GET", this.uri + docid + "/_info");
+ CouchDB.maybeThrowError(this.last_req);
+ return JSON.parse(this.last_req.responseText);
+ }
+
+ this.viewCleanup = function() {
+ this.last_req = this.request("POST", this.uri + "_view_cleanup");
+ CouchDB.maybeThrowError(this.last_req);
+ return JSON.parse(this.last_req.responseText);
+ }
+
this.allDocs = function(options,keys) {
if(!keys) {
this.last_req = this.request("GET", this.uri + "_all_docs" + encodeOptions(options));
Modified: couchdb/trunk/share/www/script/test/design_docs.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/design_docs.js (original)
+++ couchdb/trunk/share/www/script/test/design_docs.js Sat Jul 4 15:47:47 2009
@@ -49,6 +49,14 @@
}
T(db.save(designDoc).ok);
+ // test that we get design doc info back
+ var dinfo = db.designInfo("_design/test");
+ TEquals("test", dinfo.name);
+ var vinfo = dinfo.view_index;
+ TEquals(51, vinfo.disk_size);
+ TEquals(false, vinfo.compact_running);
+ TEquals("64625dce94960fd5ca116e42aa9d011a", vinfo.signature);
+
db.bulkSave(makeDocs(1, numDocs + 1));
// test that the _all_docs view returns correctly with keys
@@ -66,7 +74,7 @@
T(db.ensureFullCommit().ok);
restartServer();
};
-
+
// test when language not specified, Javascript is implied
var designDoc2 = {
_id:"_design/test2",
@@ -110,5 +118,9 @@
restartServer();
T(db.open(designDoc._id) == null);
T(db.view("test/no_docs") == null);
+
+ // trigger ddoc cleanup
+ T(db.viewCleanup().ok);
+
});
};
Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Sat Jul 4 15:47:47 2009
@@ -13,7 +13,7 @@
-module(couch_db).
-behaviour(gen_server).
--export([open/2,close/1,create/2,start_compact/1,get_db_info/1]).
+-export([open/2,close/1,create/2,start_compact/1,get_db_info/1,get_design_docs/1]).
-export([open_ref_counted/2,is_idle/1,monitor/1,count_changes_since/2]).
-export([update_doc/3,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]).
-export([get_doc_info/2,open_doc/2,open_doc/3,open_doc_revs/4]).
@@ -192,6 +192,16 @@
],
{ok, InfoList}.
+get_design_docs(#db{fulldocinfo_by_id_btree=Btree}=Db) ->
+ couch_btree:foldl(Btree, <<"_design/">>,
+ fun(#full_doc_info{id= <<"_design/",_/binary>>}=FullDocInfo, _Reds, AccDocs) ->
+ {ok, Doc} = couch_db:open_doc_int(Db, FullDocInfo, []),
+ {ok, [Doc | AccDocs]};
+ (_, _Reds, AccDocs) ->
+ {stop, AccDocs}
+ end,
+ []).
+
check_is_admin(#db{admins=Admins, user_ctx=#user_ctx{name=Name,roles=Roles}}) ->
DbAdmins = [<<"_admin">> | Admins],
case DbAdmins -- [Name | Roles] of
Modified: couchdb/trunk/src/couchdb/couch_db.hrl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.hrl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.hrl (original)
+++ couchdb/trunk/src/couchdb/couch_db.hrl Sat Jul 4 15:47:47 2009
@@ -190,8 +190,7 @@
headers = []
}).
--record(group,
- {type=view, % can also be temp_view
+-record(group, {
sig=nil,
db=nil,
fd=nil,
Modified: couchdb/trunk/src/couchdb/couch_db_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db_updater.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db_updater.erl Sat Jul 4 15:47:47 2009
@@ -350,7 +350,7 @@
refresh_validate_doc_funs(Db) ->
- {ok, DesignDocs} = get_design_docs(Db),
+ {ok, DesignDocs} = couch_db:get_design_docs(Db),
ProcessDocFuns = lists:flatmap(
fun(DesignDoc) ->
case couch_doc:get_validate_doc_fun(DesignDoc) of
@@ -360,16 +360,6 @@
end, DesignDocs),
Db#db{validate_doc_funs=ProcessDocFuns}.
-get_design_docs(#db{fulldocinfo_by_id_btree=Btree}=Db) ->
- couch_btree:foldl(Btree, <<"_design/">>,
- fun(#full_doc_info{id= <<"_design/",_/binary>>}=FullDocInfo, _Reds, AccDocs) ->
- {ok, Doc} = couch_db:open_doc_int(Db, FullDocInfo, []),
- {ok, [Doc | AccDocs]};
- (_, _Reds, AccDocs) ->
- {stop, AccDocs}
- end,
- []).
-
% rev tree functions
flush_trees(_Db, [], AccFlushedTrees) ->
Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Sat Jul 4 15:47:47 2009
@@ -15,7 +15,8 @@
-export([handle_request/1, handle_compact_req/2, handle_design_req/2,
db_req/2, couch_doc_open/4,handle_changes_req/2,
- update_doc_result_to_json/1, update_doc_result_to_json/2]).
+ update_doc_result_to_json/1, update_doc_result_to_json/2,
+ handle_design_info_req/2, handle_view_cleanup_req/2]).
-import(couch_httpd,
[send_json/2,send_json/3,send_json/4,send_method_not_allowed/2,
@@ -156,6 +157,15 @@
handle_compact_req(Req, _Db) ->
send_method_not_allowed(Req, "POST").
+handle_view_cleanup_req(#httpd{method='POST'}=Req, Db) ->
+ % delete unreferenced index files
+ ok = couch_view:cleanup_index_files(Db),
+ send_json(Req, 202, {[{ok, true}]});
+
+handle_view_cleanup_req(Req, _Db) ->
+ send_method_not_allowed(Req, "POST").
+
+
handle_design_req(#httpd{
path_parts=[_DbName,_Design,_DesName, <<"_",_/binary>> = Action | _Rest],
design_url_handlers = DesignUrlHandlers
@@ -166,6 +176,23 @@
handle_design_req(Req, Db) ->
db_req(Req, Db).
+handle_design_info_req(#httpd{
+ method='GET',
+ path_parts=[_DbName, _Design, DesignName, _]
+ }=Req, Db) ->
+ DesignId = <<"_design/", DesignName/binary>>,
+ ?LOG_ERROR("DesignId ~p",[DesignId]),
+ {ok, GroupInfoList} = couch_view:get_group_info(Db, DesignId),
+ ?LOG_ERROR("GroupInfoList ~p",[GroupInfoList]),
+ send_json(Req, 200, {[
+ {name, DesignName},
+ {view_index, {GroupInfoList}}
+ ]});
+
+handle_design_info_req(Req, _Db) ->
+ send_method_not_allowed(Req, "GET").
+
+
create_db_req(#httpd{user_ctx=UserCtx}=Req, DbName) ->
ok = couch_httpd:verify_is_server_admin(Req),
case couch_server:create(DbName, [{user_ctx, UserCtx}]) of
Modified: couchdb/trunk/src/couchdb/couch_task_status.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_task_status.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_task_status.erl (original)
+++ couchdb/trunk/src/couchdb/couch_task_status.erl Sat Jul 4 15:47:47 2009
@@ -107,12 +107,12 @@
handle_cast({update_status, Pid, StatusText}, Server) ->
[{Pid, {Type, TaskName, _StatusText}}] = ets:lookup(?MODULE, Pid),
+ ?LOG_DEBUG("New task status for ~s: ~s",[TaskName, StatusText]),
true = ets:insert(?MODULE, {Pid, {Type, TaskName, StatusText}}),
{noreply, Server};
handle_cast(stop, State) ->
{stop, normal, State}.
-
handle_info({'DOWN', _MonitorRef, _Type, Pid, _Info}, Server) ->
%% should we also erlang:demonitor(_MonitorRef), ?
ets:delete(?MODULE, Pid),
Modified: couchdb/trunk/src/couchdb/couch_view.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view.erl Sat Jul 4 15:47:47 2009
@@ -17,7 +17,7 @@
detuple_kvs/2,init/1,terminate/2,handle_call/3,handle_cast/2,handle_info/2,
code_change/3,get_reduce_view/4,get_temp_reduce_view/5,get_temp_map_view/4,
get_map_view/4,get_row_count/1,reduce_to_count/1,fold_reduce/7,
- extract_map_view/1,get_group_server/2]).
+ extract_map_view/1,get_group_server/2,get_group_info/2,cleanup_index_files/1]).
-include("couch_db.hrl").
@@ -28,18 +28,32 @@
start_link() ->
gen_server:start_link({local, couch_view}, couch_view, [], []).
-get_temp_updater(DbName, Type, DesignOptions, MapSrc, RedSrc) ->
- {ok, Pid} = gen_server:call(couch_view,
- {start_temp_updater, DbName, Type, DesignOptions, MapSrc, RedSrc}),
- Pid.
-
-get_group_server(DbName, GroupId) ->
- case gen_server:call(couch_view, {start_group_server, DbName, GroupId}) of
+get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
+ % make temp group
+ % do we need to close this db?
+ {ok, _Db, Group} =
+ couch_view_group:open_temp_group(DbName, Language, DesignOptions, MapSrc, RedSrc),
+ case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
{ok, Pid} ->
Pid;
Error ->
throw(Error)
end.
+
+get_group_server(DbName, GroupId) ->
+ % get signature for group
+ case couch_view_group:open_db_group(DbName, GroupId) of
+ % do we need to close this db?
+ {ok, _Db, Group} ->
+ case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
+ {ok, Pid} ->
+ Pid;
+ Error ->
+ throw(Error)
+ end;
+ Error ->
+ throw(Error)
+ end.
get_group(Db, GroupId, Stale) ->
MinUpdateSeq = case Stale of
@@ -50,18 +64,52 @@
get_group_server(couch_db:name(Db), GroupId),
MinUpdateSeq).
-
-get_temp_group(Db, Type, DesignOptions, MapSrc, RedSrc) ->
+get_temp_group(Db, Language, DesignOptions, MapSrc, RedSrc) ->
couch_view_group:request_group(
- get_temp_updater(couch_db:name(Db), Type, DesignOptions, MapSrc, RedSrc),
- couch_db:get_update_seq(Db)).
+ get_temp_updater(couch_db:name(Db), Language, DesignOptions, MapSrc, RedSrc),
+ couch_db:get_update_seq(Db)).
+
+get_group_info(Db, GroupId) ->
+ couch_view_group:request_group_info(
+ get_group_server(couch_db:name(Db), GroupId)).
+
+cleanup_index_files(Db) ->
+ % load all ddocs
+ {ok, DesignDocs} = couch_db:get_design_docs(Db),
+
+ % make unique list of group sigs
+ Sigs = lists:map(fun(#doc{id = GroupId} = DDoc) ->
+ {ok, Info} = get_group_info(Db, GroupId),
+ ?b2l(proplists:get_value(signature, Info))
+ end, [DD||DD <- DesignDocs, DD#doc.deleted == false]),
+
+ FileList = list_index_files(Db),
+
+ % regex that matches all ddocs
+ RegExp = "("++ string:join(Sigs, "|") ++")",
+
+ % filter out the ones in use
+ DeleteFiles = lists:filter(fun(FilePath) ->
+ regexp:first_match(FilePath, RegExp)==nomatch
+ end, FileList),
+ % delete unused files
+ ?LOG_DEBUG("deleting unused view index files: ~p",[DeleteFiles]),
+ [file:delete(File)||File <- DeleteFiles],
+ ok.
+
+list_index_files(Db) ->
+ % call server to fetch the index files
+ RootDir = couch_config:get("couchdb", "view_index_dir"),
+ Files = filelib:wildcard(RootDir ++ "/." ++ ?b2l(couch_db:name(Db)) ++ "_design"++"/*").
+
get_row_count(#view{btree=Bt}) ->
{ok, {Count, _Reds}} = couch_btree:full_reduce(Bt),
{ok, Count}.
-get_temp_reduce_view(Db, Type, DesignOptions, MapSrc, RedSrc) ->
- {ok, #group{views=[View]}=Group} = get_temp_group(Db, Type, DesignOptions, MapSrc, RedSrc),
+get_temp_reduce_view(Db, Language, DesignOptions, MapSrc, RedSrc) ->
+ {ok, #group{views=[View]}=Group} =
+ get_temp_group(Db, Language, DesignOptions, MapSrc, RedSrc),
{ok, {temp_reduce, View}, Group}.
@@ -141,8 +189,8 @@
get_key_pos(Key, Rest, N+1).
-get_temp_map_view(Db, Type, DesignOptions, Src) ->
- {ok, #group{views=[View]}=Group} = get_temp_group(Db, Type, DesignOptions, Src, []),
+get_temp_map_view(Db, Language, DesignOptions, Src) ->
+ {ok, #group{views=[View]}=Group} = get_temp_group(Db, Language, DesignOptions, Src, []),
{ok, View, Group}.
get_map_view(Db, GroupId, Name, Stale) ->
@@ -220,9 +268,8 @@
ok
end),
ets:new(couch_groups_by_db, [bag, private, named_table]),
- ets:new(group_servers_by_name, [set, protected, named_table]),
+ ets:new(group_servers_by_sig, [set, protected, named_table]),
ets:new(couch_groups_by_updater, [set, private, named_table]),
- ets:new(couch_temp_group_fd_by_db, [set, protected, named_table]),
process_flag(trap_exit, true),
{ok, #server{root_dir=RootDir}}.
@@ -232,37 +279,15 @@
ok.
-handle_call({start_temp_updater, DbName, Lang, DesignOptions, MapSrc, RedSrc},
- _From, #server{root_dir=Root}=Server) ->
- <<SigInt:128/integer>> = erlang:md5(term_to_binary({Lang, DesignOptions, MapSrc, RedSrc})),
- Name = lists:flatten(io_lib:format("_temp_~.36B",[SigInt])),
- Pid =
- case ets:lookup(group_servers_by_name, {DbName, Name}) of
+handle_call({get_group_server, DbName,
+ #group{name=GroupId,sig=Sig}=Group}, _From, #server{root_dir=Root}=Server) ->
+ case ets:lookup(group_servers_by_sig, {DbName, Sig}) of
[] ->
- case ets:lookup(couch_temp_group_fd_by_db, DbName) of
- [] ->
- FileName = Root ++ "/." ++ binary_to_list(DbName) ++ "_temp",
- {ok, Fd} = couch_file:open(FileName, [create, overwrite]),
- Count = 0;
- [{_, Fd, Count}] ->
- ok
- end,
- ?LOG_DEBUG("Spawning new temp update process for db ~s.", [DbName]),
- {ok, NewPid} = couch_view_group:start_link({slow_view, DbName, Fd, Lang, DesignOptions, MapSrc, RedSrc}),
- true = ets:insert(couch_temp_group_fd_by_db, {DbName, Fd, Count + 1}),
- add_to_ets(NewPid, DbName, Name),
- NewPid;
- [{_, ExistingPid0}] ->
- ExistingPid0
- end,
- {reply, {ok, Pid}, Server};
-handle_call({start_group_server, DbName, GroupId}, _From, #server{root_dir=Root}=Server) ->
- case ets:lookup(group_servers_by_name, {DbName, GroupId}) of
- [] ->
- ?LOG_DEBUG("Spawning new group server for view group ~s in database ~s.", [GroupId, DbName]),
- case (catch couch_view_group:start_link({view, Root, DbName, GroupId})) of
+ ?LOG_DEBUG("Spawning new group server for view group ~s in database ~s.",
+ [GroupId, DbName]),
+ case (catch couch_view_group:start_link({Root, DbName, Group})) of
{ok, NewPid} ->
- add_to_ets(NewPid, DbName, GroupId),
+ add_to_ets(NewPid, DbName, Sig),
{reply, {ok, NewPid}, Server};
Error ->
{reply, Error, Server}
@@ -272,22 +297,22 @@
end.
handle_cast({reset_indexes, DbName}, #server{root_dir=Root}=Server) ->
- % shutdown all the updaters
+ % shutdown all the updaters and clear the files, the db got changed
Names = ets:lookup(couch_groups_by_db, DbName),
lists:foreach(
- fun({_DbName, GroupId}) ->
- ?LOG_DEBUG("Killing update process for view group ~s. in database ~s.", [GroupId, DbName]),
- [{_, Pid}] = ets:lookup(group_servers_by_name, {DbName, GroupId}),
+ fun({_DbName, Sig}) ->
+ ?LOG_DEBUG("Killing update process for view group ~s. in database ~s.", [Sig, DbName]),
+ [{_, Pid}] = ets:lookup(group_servers_by_sig, {DbName, Sig}),
exit(Pid, kill),
receive {'EXIT', Pid, _} ->
- delete_from_ets(Pid, DbName, GroupId)
+ delete_from_ets(Pid, DbName, Sig)
end
end, Names),
delete_index_dir(Root, DbName),
file:delete(Root ++ "/." ++ binary_to_list(DbName) ++ "_temp"),
{noreply, Server}.
-handle_info({'EXIT', FromPid, Reason}, #server{root_dir=RootDir}=Server) ->
+handle_info({'EXIT', FromPid, Reason}, Server) ->
case ets:lookup(couch_groups_by_updater, FromPid) of
[] ->
if Reason /= normal ->
@@ -296,40 +321,27 @@
exit(Reason);
true -> ok
end;
- [{_, {DbName, "_temp_" ++ _ = GroupId}}] ->
- delete_from_ets(FromPid, DbName, GroupId),
- [{_, Fd, Count}] = ets:lookup(couch_temp_group_fd_by_db, DbName),
- case Count of
- 1 -> % Last ref
- couch_file:close(Fd),
- file:delete(RootDir ++ "/." ++ binary_to_list(DbName) ++ "_temp"),
- true = ets:delete(couch_temp_group_fd_by_db, DbName);
- _ ->
- true = ets:insert(couch_temp_group_fd_by_db, {DbName, Fd, Count - 1})
- end;
[{_, {DbName, GroupId}}] ->
delete_from_ets(FromPid, DbName, GroupId)
end,
{noreply, Server}.
-add_to_ets(Pid, DbName, GroupId) ->
- true = ets:insert(couch_groups_by_updater, {Pid, {DbName, GroupId}}),
- true = ets:insert(group_servers_by_name, {{DbName, GroupId}, Pid}),
- true = ets:insert(couch_groups_by_db, {DbName, GroupId}).
+add_to_ets(Pid, DbName, Sig) ->
+ true = ets:insert(couch_groups_by_updater, {Pid, {DbName, Sig}}),
+ true = ets:insert(group_servers_by_sig, {{DbName, Sig}, Pid}),
+ true = ets:insert(couch_groups_by_db, {DbName, Sig}).
-delete_from_ets(Pid, DbName, GroupId) ->
+delete_from_ets(Pid, DbName, Sig) ->
true = ets:delete(couch_groups_by_updater, Pid),
- true = ets:delete(group_servers_by_name, {DbName, GroupId}),
- true = ets:delete_object(couch_groups_by_db, {DbName, GroupId}).
+ true = ets:delete(group_servers_by_sig, {DbName, Sig}),
+ true = ets:delete_object(couch_groups_by_db, {DbName, Sig}).
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-
-
delete_index_dir(RootDir, DbName) ->
- nuke_dir(RootDir ++ "/." ++ binary_to_list(DbName) ++ "_design").
+ nuke_dir(RootDir ++ "/." ++ ?b2l(DbName) ++ "_design").
nuke_dir(Dir) ->
case file:list_dir(Dir) of
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Sat Jul 4 15:47:47 2009
@@ -14,8 +14,8 @@
-behaviour(gen_server).
%% API
--export([start_link/1, request_group/2]).
--export([design_doc_to_view_group/1]).
+-export([start_link/1, request_group/2, request_group_info/1]).
+-export([open_db_group/2, open_temp_group/5, design_doc_to_view_group/1,design_root/2]).
%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -47,6 +47,22 @@
throw(Error)
end.
+request_group_info(Pid) ->
+ case gen_server:call(Pid, request_group_info) of
+ {ok, GroupInfoList} ->
+ {ok, GroupInfoList};
+ Error ->
+ throw(Error)
+ end.
+
+request_index_files(Pid) ->
+ case gen_server:call(Pid, request_index_files) of
+ {ok, Filelist} ->
+ {ok, Filelist};
+ Error ->
+ throw(Error)
+ end.
+
% from template
start_link(InitArgs) ->
@@ -67,9 +83,7 @@
Error
end.
-% init differentiates between temp and design_doc views. It creates a closure
-% which spawns the appropriate view_updater. (It might also spawn the first
-% view_updater run.)
+% init creates a closure which spawns the appropriate view_updater.
init({InitArgs, ReturnPid, Ref}) ->
process_flag(trap_exit, true),
case prepare_group(InitArgs, false) of
@@ -95,8 +109,7 @@
% view group, and the couch_view_updater, which when spawned, updates the
% group and sends it back here. We employ a caching mechanism, so that between
% database writes, we don't have to spawn a couch_view_updater with every view
-% request. This should give us more control, and the ability to request view
-% statuses eventually.
+% request.
% The caching mechanism: each request is submitted with a seq_id for the
% database at the time it was read. We guarantee to return a view from that
@@ -137,8 +150,14 @@
#group_state{waiting_list=WaitList}=State) ->
{noreply, State#group_state{
waiting_list=[{From, RequestSeq}|WaitList]
- }, infinity}.
+ }, infinity};
+handle_call(request_group_info, _From, #group_state{
+ group = Group,
+ compactor_pid = CompactorPid
+ } = State) ->
+ GroupInfo = get_group_info(Group, CompactorPid),
+ {reply, {ok, GroupInfo}, State}.
handle_cast({start_compact, CompactFun}, #group_state{ compactor_pid=nil,
group=Group, init_args={view, RootDir, DbName, GroupId} } = State) ->
@@ -154,15 +173,14 @@
handle_cast({compact_done, #group{fd=NewFd, current_seq=NewSeq} = NewGroup},
#group_state{
- group = #group{current_seq=OldSeq} = Group,
- init_args = {view, RootDir, DbName, GroupId},
+ group = #group{current_seq=OldSeq, sig=GroupSig} = Group,
+ init_args = {view, RootDir, DbName, _GroupId},
updater_pid = nil,
ref_counter = RefCounter
} = State) when NewSeq >= OldSeq ->
?LOG_INFO("View Group compaction complete", []),
- BaseName = RootDir ++ "/." ++ ?b2l(DbName) ++ ?b2l(GroupId),
- FileName = BaseName ++ ".view",
- CompactName = BaseName ++".compact.view",
+ FileName = index_file_name(RootDir, DbName, GroupSig),
+ CompactName = index_file_name(compact, RootDir, DbName, GroupSig),
file:delete(FileName),
ok = file:rename(CompactName, FileName),
@@ -224,8 +242,7 @@
waiting_list=WaitList,
waiting_commit=WaitingCommit}=State) when UpPid == FromPid ->
ok = couch_db:close(Db),
-
- if Group#group.type == view andalso not WaitingCommit ->
+ if not WaitingCommit ->
erlang:send_after(1000, self(), delayed_commit);
true -> ok
end,
@@ -306,12 +323,13 @@
[catch gen_server:reply(Pid, Reply) || {Pid, _} <- WaitList],
State#group_state{waiting_list=[]}.
-prepare_group({view, RootDir, DbName, GroupId}, ForceReset)->
- case open_db_group(DbName, GroupId) of
- {ok, Db, #group{sig=Sig}=Group} ->
- case open_index_file(RootDir, DbName, GroupId) of
+prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)->
+ case couch_db:open(DbName, []) of
+ {ok, Db} ->
+ case open_index_file(RootDir, DbName, Sig) of
{ok, Fd} ->
if ForceReset ->
+ % this can happen if we missed a purge
{ok, reset_file(Db, Fd, DbName, Group)};
true ->
% 09 UPGRADE CODE
@@ -321,32 +339,18 @@
% sigs match!
{ok, init_group(Db, Fd, Group, HeaderInfo)};
_ ->
+ % this happens on a new file
{ok, reset_file(Db, Fd, DbName, Group)}
end
end;
Error ->
- catch delete_index_file(RootDir, DbName, GroupId),
+ catch delete_index_file(RootDir, DbName, Sig),
Error
end;
- Error ->
- catch delete_index_file(RootDir, DbName, GroupId),
- Error
- end;
-prepare_group({slow_view, DbName, Fd, Lang, DesignOptions, MapSrc, RedSrc}, _ForceReset) ->
- case couch_db:open(DbName, []) of
- {ok, Db} ->
- View = #view{map_names=[<<"_temp">>],
- id_num=0,
- btree=nil,
- def=MapSrc,
- reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}] end},
- {ok, init_group(Db, Fd, #group{type=slow_view, name= <<"_temp">>, db=Db,
- views=[View], def_lang=Lang, design_options=DesignOptions}, nil)};
- Error ->
- Error
+ Else ->
+ Else
end.
-
get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq,
id_btree=IdBtree,views=Views}) ->
ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views],
@@ -355,14 +359,47 @@
id_btree_state=couch_btree:get_state(IdBtree),
view_states=ViewStates}.
+hex_sig(GroupSig) ->
+ couch_util:to_hex(?b2l(GroupSig)).
+
+design_root(RootDir, DbName) ->
+ RootDir ++ "/." ++ ?b2l(DbName) ++ "_design/".
+
+index_file_name(RootDir, DbName, GroupSig) ->
+ design_root(RootDir, DbName) ++ hex_sig(GroupSig) ++".view".
-open_index_file(RootDir, DbName, GroupId) ->
- FileName = RootDir ++ "/." ++ ?b2l(DbName) ++ ?b2l(GroupId) ++".view",
+index_file_name(compact, RootDir, DbName, GroupSig) ->
+ design_root(RootDir, DbName) ++ hex_sig(GroupSig) ++".compact.view".
+
+
+open_index_file(RootDir, DbName, GroupSig) ->
+ FileName = index_file_name(RootDir, DbName, GroupSig),
case couch_file:open(FileName) of
{ok, Fd} -> {ok, Fd};
{error, enoent} -> couch_file:open(FileName, [create]);
Error -> Error
end.
+
+open_temp_group(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
+ case couch_db:open(DbName, []) of
+ {ok, Db} ->
+ View = #view{map_names=[<<"_temp">>],
+ id_num=0,
+ btree=nil,
+ def=MapSrc,
+ reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}] end},
+
+ {ok, Db, #group{
+ name = <<"_temp">>,
+ db=Db,
+ views=[View],
+ def_lang=Language,
+ design_options=DesignOptions,
+ sig = erlang:md5(term_to_binary({[View], Language, DesignOptions}))
+ }};
+ Error ->
+ Error
+ end.
open_db_group(DbName, GroupId) ->
case couch_db:open(DbName, []) of
@@ -378,12 +415,28 @@
Else
end.
+get_group_info(#group{
+ fd = Fd,
+ sig = GroupSig,
+ def_lang = Lang
+ }, CompactorPid) ->
+ {ok, Size} = couch_file:bytes(Fd),
+ [
+ {signature, ?l2b(hex_sig(GroupSig))},
+ {language, Lang},
+ {disk_size, Size},
+ {compact_running, CompactorPid /= nil}
+ ].
+
% maybe move to another module
design_doc_to_view_group(#doc{id=Id,body={Fields}}) ->
Language = proplists:get_value(<<"language">>, Fields, <<"javascript">>),
{DesignOptions} = proplists:get_value(<<"options">>, Fields, {[]}),
{RawViews} = proplists:get_value(<<"views">>, Fields, {[]}),
-
+ % sort the views by name to avoid spurious signature changes
+ SortedRawViews = lists:sort(fun({Name1, _}, {Name2, _}) ->
+ Name1 >= Name2
+ end, RawViews),
% add the views to a dictionary object, with the map source as the key
DictBySrc =
lists:foldl(
@@ -402,7 +455,7 @@
View#view{reduce_funs=[{Name,RedSrc}|View#view.reduce_funs]}
end,
dict:store(MapSrc, View2, DictBySrcAcc)
- end, dict:new(), RawViews),
+ end, dict:new(), SortedRawViews),
% number the views
{Views, _N} = lists:mapfoldl(
fun({_Src, View}, N) ->
@@ -410,7 +463,7 @@
end, 0, dict:to_list(DictBySrc)),
Group = #group{name=Id, views=Views, def_lang=Language, design_options=DesignOptions},
- Group#group{sig=erlang:md5(term_to_binary(Group))}.
+ Group#group{sig=erlang:md5(term_to_binary({Views, Language, DesignOptions}))}.
reset_group(#group{views=Views}=Group) ->
Views2 = [View#view{btree=nil} || View <- Views],
@@ -423,9 +476,8 @@
ok = couch_file:write_header(Fd, {Sig, nil}),
init_group(Db, Fd, reset_group(Group), nil).
-delete_index_file(RootDir, DbName, GroupId) ->
- file:delete(RootDir ++ "/." ++ binary_to_list(DbName)
- ++ binary_to_list(GroupId) ++ ".view").
+delete_index_file(RootDir, DbName, GroupSig) ->
+ file:delete(index_file_name(RootDir, DbName, GroupSig)).
init_group(Db, Fd, #group{views=Views}=Group, nil) ->
init_group(Db, Fd, Group,
Modified: couchdb/trunk/src/couchdb/couch_view_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_updater.erl?rev=791140&r1=791139&r2=791140&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_updater.erl Sat Jul 4 15:47:47 2009
@@ -106,26 +106,6 @@
[conflicts, deleted_conflicts]
end,
case {IncludeDesign, DocId} of
- {_, GroupId} ->
- % uh oh. this is the design doc with our definitions. See if
- % anything in the definition changed.
- case couch_db:open_doc_int(Db, DocInfo, DocOpts) of
- {ok, Doc} ->
- case couch_view_group:design_doc_to_view_group(Doc) of
- #group{sig=Sig} ->
- % The same md5 signature, keep on computing
- case IncludeDesign of
- true ->
- {[Doc | Docs], Group, ViewKVs, DocIdViewIdKeys};
- _ ->
- {Docs, Group, ViewKVs, DocIdViewIdKeys}
- end;
- _ ->
- exit(reset)
- end;
- {not_found, missing} ->
- exit(reset)
- end;
{false, <<?DESIGN_DOC_PREFIX, _/binary>>} -> % we skip design docs
{Docs, Group, ViewKVs, DocIdViewIdKeys};
_ ->