You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/02/05 00:44:23 UTC
[10/44] Remove src/couch
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_emsort.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_emsort.erl b/src/couch/src/couch_emsort.erl
deleted file mode 100644
index 2a25a23..0000000
--- a/src/couch/src/couch_emsort.erl
+++ /dev/null
@@ -1,318 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_emsort).
-
-% This is an implementation of an external N-way merge sort. It's primary
-% purpose is to be used during database compaction as an optimization for
-% managing the docid btree.
-%
-% Trunk currently writes the docid btree as its compacting the database but
-% this is quite inneficient as its written out of order in the general case
-% as writes are ordered by update_seq.
-%
-% The general design of this module is a very standard merge sort with one
-% caveat due to append only files. This is described in more detail in the
-% sorting phase.
-%
-% The basic algorithm is in two halves. The first half stores KV pairs to disk
-% which is then followed by the actual sorting phase that streams KV's back
-% to the client using a fold-like function. After some basic definitions we'll
-% describe both phases.
-%
-% Key/Value apairs (aka, KV pairs, or KVs) are simply lists of two-tuples with
-% a key as the first element and an arbitrary value as the second. The key of
-% this pair is what used to determine the sort order based on native Erlang
-% term comparison.
-%
-% Internally, KVs are stored as lists with a max size defined by
-% #ems.chain_chunk. These lists are then chained together on disk using disk
-% offsets as a poor man's linked list. The basic format of a list looks like
-% {KVs, DiskOffset} where DiskOffset is either the atom nil which means "end
-% of the list" or an integer that is a file position offset that is the
-% location of another {KVs, DiskOffset} term. The head of each list is
-% referred to with a single DiskOffset. The set of terms that extend from
-% this initial DiskOffset to the last {KVs, nil} term is referred to in the
-% code as a chain. Two important facts are that one call to couch_emsort:add/2
-% creates a single chain, and that a chain is always sorted on disk (though its
-% possible to be sorted in descending order which will be discussed later).
-%
-% The second major internal structure is the back bone. This is a list of
-% chains that has a quite similar structure to chains but contains different
-% data types and has no guarantee on ordering. The back bone is merely the
-% list of all head DiskOffsets. The structure has the similar structure of
-% {DiskOffsets, DiskOffset} that we use for chains, except that DiskOffsets is
-% a list of integers that refer to the heads of chains. The maximum size of
-% DiskOffsets is defined by #ems.bb_chunk. It is important to note that the
-% backbone has no defined ordering. The other thing of note is that the RAM
-% bounds are loosely defined as:
-%
-% #ems.bb_chunk * #ems.chain_chunk * avg_size(KV).
-%
-% Build Phase
-% -----------
-%
-% As mentioned, each call to couch_emsort:add/2 creates a chain from the
-% list of KVs that are passed in. This list is first sorted and then the
-% chain is created by foldr-ing (note: r) across the list to build the
-% chain on disk. It is important to note that the final chain is then
-% sorted in ascending order on disk.
-%
-%
-% Sort Phase
-% ----------
-%
-% The sort phase is where the merge sort kicks in. This is generally your
-% average merge sort with a caveat for append only storage. First the
-% general outline.
-%
-% The general outline for this sort is that it iteratively merges chains
-% in the backbone until less than #ems.bb_chunk chains exist. At this
-% point it switches to the last merge sort phase where it just streams
-% the sorted KVs back to the client using a fold function.
-%
-% The general chain merging is a pretty standard merge sort. You load up
-% the initial KVs from each phase, pick the next one in sort order and
-% then when you run out of KVs you're left with a single DiskOffset for
-% the head of a single chain that represents the merge. These new
-% DiskOffsets are used to build the new back bone.
-%
-% The one caveat here is that we're using append only storage. This is
-% important because once we make a pass we've effectively reversed the
-% sort order of each chain. Ie, the first merge results in chains that
-% are ordered in descending order. Since, one pass reverses the list
-% the trick is that each phase does two passes. The first phase picks
-% the smallest KV to write next and the second phase picks the largest.
-% In this manner each time we do a back bone merge we end up with chains
-% that are always sorted in an ascending order.
-%
-% The one downfall is that in the interest of simplicity the sorting is
-% restricted to Erlang's native term sorting. A possible extension would
-% be to allow two comparison functions to be used, but this module is
-% currently only used for docid sorting which is hardcoded to be raw
-% Erlang ordering.
-%
-% Diagram
-% -------
-%
-% If it helps, this is a general diagram of the internal structures. A
-% couple points to note since this is ASCII art. The BB pointers across
-% the top are lists of chains going down. Each BBN item is one of the
-% {DiskOffsets, DiskOffset} structures discussed earlier. Going down,
-% the CMN nodes are actually representing #ems.bb_chunk chains in parallel
-% going off the back bone. It is important and not represented in this
-% diagram that within these groups the chains don't have to be the same
-% length. That's just a limitiationg of my ASCII artistic abilities.
-%
-% The BBN* node is marked with a * to denote that it is the only state
-% that we store when writing headeres to disk as it has pointers that
-% lead us to all data in the tree.
-%
-% BB1 <- BB2 <- BB3 <- BBN*
-% | | | |
-% v v v v
-% CA1 CB1 CC1 CD1
-% | | |
-% v v v
-% CA2 CC2 CD2
-% | |
-% v v
-% CA3 CD3
-%
-
--export([open/1, open/2, get_fd/1, get_state/1]).
--export([add/2, merge/1, sort/1, iter/1, next/1]).
-
-
--record(ems, {
- fd,
- root,
- bb_chunk = 10,
- chain_chunk = 100
-}).
-
-
-open(Fd) ->
- {ok, #ems{fd=Fd}}.
-
-
-open(Fd, Options) ->
- {ok, set_options(#ems{fd=Fd}, Options)}.
-
-
-set_options(Ems, []) ->
- Ems;
-set_options(Ems, [{root, Root} | Rest]) ->
- set_options(Ems#ems{root=Root}, Rest);
-set_options(Ems, [{chain_chunk, Count} | Rest]) when is_integer(Count) ->
- set_options(Ems#ems{chain_chunk=Count}, Rest);
-set_options(Ems, [{back_bone_chunk, Count} | Rest]) when is_integer(Count) ->
- set_options(Ems#ems{bb_chunk=Count}, Rest).
-
-
-get_fd(#ems{fd=Fd}) ->
- Fd.
-
-
-get_state(#ems{root=Root}) ->
- Root.
-
-
-add(Ems, []) ->
- {ok, Ems};
-add(Ems, KVs) ->
- Pos = write_kvs(Ems, KVs),
- {ok, add_bb_pos(Ems, Pos)}.
-
-
-sort(#ems{}=Ems) ->
- {ok, Ems1} = merge(Ems),
- iter(Ems1).
-
-
-merge(#ems{root=undefined}=Ems) ->
- {ok, Ems};
-merge(#ems{}=Ems) ->
- {ok, decimate(Ems)}.
-
-
-iter(#ems{root=undefined}=Ems) ->
- {ok, {Ems, []}};
-iter(#ems{root={BB, nil}}=Ems) ->
- Chains = init_chains(Ems, small, BB),
- {ok, {Ems, Chains}};
-iter(#ems{root={_, _}}) ->
- {error, not_merged}.
-
-
-next({_Ems, []}) ->
- finished;
-next({Ems, Chains}) ->
- {KV, RestChains} = choose_kv(small, Ems, Chains),
- {ok, KV, {Ems, RestChains}}.
-
-
-add_bb_pos(#ems{root=undefined}=Ems, Pos) ->
- Ems#ems{root={[Pos], nil}};
-add_bb_pos(#ems{root={BB, Prev}}=Ems, Pos) ->
- {NewBB, NewPrev} = append_item(Ems, {BB, Prev}, Pos, Ems#ems.bb_chunk),
- Ems#ems{root={NewBB, NewPrev}}.
-
-
-write_kvs(Ems, KVs) ->
- % Write the list of KV's to disk in sorted order in chunks
- % of 100. Also make sure that the order is so that they
- % can be streamed in asscending order.
- {LastKVs, LastPos} =
- lists:foldr(fun(KV, Acc) ->
- append_item(Ems, Acc, KV, Ems#ems.chain_chunk)
- end, {[], nil}, lists:sort(KVs)),
- {ok, Final, _} = couch_file:append_term(Ems#ems.fd, {LastKVs, LastPos}),
- Final.
-
-
-decimate(#ems{root={_BB, nil}}=Ems) ->
- % We have less than bb_chunk backbone pointers so we're
- % good to start streaming KV's back to the client.
- Ems;
-decimate(#ems{root={BB, NextBB}}=Ems) ->
- % To make sure we have a bounded amount of data in RAM
- % at any given point we first need to decimate the data
- % by performing the first couple iterations of a merge
- % sort writing the intermediate results back to disk.
-
- % The first pass gives us a sort with pointers linked from
- % largest to smallest.
- {RevBB, RevNextBB} = merge_back_bone(Ems, small, BB, NextBB),
-
- % We have to run a second pass so that links are pointed
- % back from smallest to largest.
- {FwdBB, FwdNextBB} = merge_back_bone(Ems, big, RevBB, RevNextBB),
-
- % Continue deicmating until we have an acceptable bound on
- % the number of keys to use.
- decimate(Ems#ems{root={FwdBB, FwdNextBB}}).
-
-
-merge_back_bone(Ems, Choose, BB, NextBB) ->
- BBPos = merge_chains(Ems, Choose, BB),
- merge_rest_back_bone(Ems, Choose, NextBB, {[BBPos], nil}).
-
-
-merge_rest_back_bone(_Ems, _Choose, nil, Acc) ->
- Acc;
-merge_rest_back_bone(Ems, Choose, BBPos, Acc) ->
- {ok, {BB, NextBB}} = couch_file:pread_term(Ems#ems.fd, BBPos),
- NewPos = merge_chains(Ems, Choose, BB),
- {NewBB, NewPrev} = append_item(Ems, Acc, NewPos, Ems#ems.bb_chunk),
- merge_rest_back_bone(Ems, Choose, NextBB, {NewBB, NewPrev}).
-
-
-merge_chains(Ems, Choose, BB) ->
- Chains = init_chains(Ems, Choose, BB),
- merge_chains(Ems, Choose, Chains, {[], nil}).
-
-
-merge_chains(Ems, _Choose, [], ChainAcc) ->
- {ok, CPos, _} = couch_file:append_term(Ems#ems.fd, ChainAcc),
- CPos;
-merge_chains(#ems{chain_chunk=CC}=Ems, Choose, Chains, Acc) ->
- {KV, RestChains} = choose_kv(Choose, Ems, Chains),
- {NewKVs, NewPrev} = append_item(Ems, Acc, KV, CC),
- merge_chains(Ems, Choose, RestChains, {NewKVs, NewPrev}).
-
-
-init_chains(Ems, Choose, BB) ->
- Chains = lists:map(fun(CPos) ->
- {ok, {KVs, NextKVs}} = couch_file:pread_term(Ems#ems.fd, CPos),
- {KVs, NextKVs}
- end, BB),
- order_chains(Choose, Chains).
-
-
-order_chains(small, Chains) -> lists:sort(Chains);
-order_chains(big, Chains) -> lists:reverse(lists:sort(Chains)).
-
-
-choose_kv(_Choose, _Ems, [{[KV], nil} | Rest]) ->
- {KV, Rest};
-choose_kv(Choose, Ems, [{[KV], Pos} | RestChains]) ->
- {ok, Chain} = couch_file:pread_term(Ems#ems.fd, Pos),
- case Choose of
- small -> {KV, ins_small_chain(RestChains, Chain, [])};
- big -> {KV, ins_big_chain(RestChains, Chain, [])}
- end;
-choose_kv(Choose, _Ems, [{[KV | RestKVs], Prev} | RestChains]) ->
- case Choose of
- small -> {KV, ins_small_chain(RestChains, {RestKVs, Prev}, [])};
- big -> {KV, ins_big_chain(RestChains, {RestKVs, Prev}, [])}
- end.
-
-
-ins_small_chain([{[{K1,_}|_],_}=C1|Rest], {[{K2,_}|_],_}=C2, Acc) when K1<K2 ->
- ins_small_chain(Rest, C2, [C1 | Acc]);
-ins_small_chain(Rest, Chain, Acc) ->
- lists:reverse(Acc, [Chain | Rest]).
-
-
-ins_big_chain([{[{K1,_}|_],_}=C1|Rest], {[{K2,_}|_],_}=C2, Acc) when K1>K2 ->
- ins_big_chain(Rest, C2, [C1 | Acc]);
-ins_big_chain(Rest, Chain, Acc) ->
- lists:reverse(Acc, [Chain | Rest]).
-
-
-append_item(Ems, {List, Prev}, Pos, Size) when length(List) >= Size ->
- {ok, PrevList, _} = couch_file:append_term(Ems#ems.fd, {List, Prev}),
- {[Pos], PrevList};
-append_item(_Ems, {List, Prev}, Pos, _Size) ->
- {[Pos | List], Prev}.
-
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_event_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_event_sup.erl b/src/couch/src/couch_event_sup.erl
deleted file mode 100644
index e9e7214..0000000
--- a/src/couch/src/couch_event_sup.erl
+++ /dev/null
@@ -1,73 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-%% The purpose of this module is to allow event handlers to particpate in Erlang
-%% supervisor trees. It provide a monitorable process that crashes if the event
-%% handler fails. The process, when shutdown, deregisters the event handler.
-
--module(couch_event_sup).
--behaviour(gen_server).
-
--include_lib("couch/include/couch_db.hrl").
-
--export([start_link/3,start_link/4, stop/1]).
--export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3]).
-
-%
-% Instead calling the
-% ok = gen_event:add_sup_handler(error_logger, my_log, Args)
-%
-% do this:
-% {ok, LinkedPid} = couch_event_sup:start_link(error_logger, my_log, Args)
-%
-% The benefit is the event is now part of the process tree, and can be
-% started, restarted and shutdown consistently like the rest of the server
-% components.
-%
-% And now if the "event" crashes, the supervisor is notified and can restart
-% the event handler.
-%
-% Use this form to named process:
-% {ok, LinkedPid} = couch_event_sup:start_link({local, my_log}, error_logger, my_log, Args)
-%
-
-start_link(EventMgr, EventHandler, Args) ->
- gen_server:start_link(couch_event_sup, {EventMgr, EventHandler, Args}, []).
-
-start_link(ServerName, EventMgr, EventHandler, Args) ->
- gen_server:start_link(ServerName, couch_event_sup, {EventMgr, EventHandler, Args}, []).
-
-stop(Pid) ->
- gen_server:cast(Pid, stop).
-
-init({EventMgr, EventHandler, Args}) ->
- case gen_event:add_sup_handler(EventMgr, EventHandler, Args) of
- ok ->
- {ok, {EventMgr, EventHandler}};
- {stop, Error} ->
- {stop, Error}
- end.
-
-terminate(_Reason, _State) ->
- ok.
-
-handle_call(_Whatever, _From, State) ->
- {ok, State}.
-
-handle_cast(stop, State) ->
- {stop, normal, State}.
-
-handle_info({gen_event_EXIT, _Handler, Reason}, State) ->
- {stop, Reason, State}.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_external_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_external_manager.erl b/src/couch/src/couch_external_manager.erl
deleted file mode 100644
index a8cad63..0000000
--- a/src/couch/src/couch_external_manager.erl
+++ /dev/null
@@ -1,115 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_external_manager).
--behaviour(gen_server).
--behaviour(config_listener).
-
--export([start_link/0, execute/2]).
--export([init/1, terminate/2, code_change/3, handle_call/3, handle_cast/2, handle_info/2]).
-
-% config_listener api
--export([handle_config_change/5]).
-
--include_lib("couch/include/couch_db.hrl").
-
-start_link() ->
- gen_server:start_link({local, couch_external_manager},
- couch_external_manager, [], []).
-
-execute(UrlName, JsonReq) ->
- Pid = gen_server:call(couch_external_manager, {get, UrlName}),
- case Pid of
- {error, Reason} ->
- Reason;
- _ ->
- couch_external_server:execute(Pid, JsonReq)
- end.
-
-handle_config_change("external", UrlName, _, _, _) ->
- {ok, gen_server:call(couch_external_manager, {config, UrlName})};
-handle_config_change(_, _, _, _, _) ->
- {ok, nil}.
-
-% gen_server API
-
-init([]) ->
- process_flag(trap_exit, true),
- Handlers = ets:new(couch_external_manager_handlers, [set, private]),
- ok = config:listen_for_changes(?MODULE, nil),
- {ok, Handlers}.
-
-terminate(_Reason, Handlers) ->
- ets:foldl(fun({_UrlName, Pid}, nil) ->
- couch_external_server:stop(Pid),
- nil
- end, nil, Handlers),
- ok.
-
-handle_call({get, UrlName}, _From, Handlers) ->
- case ets:lookup(Handlers, UrlName) of
- [] ->
- case config:get("external", UrlName, nil) of
- nil ->
- Msg = lists:flatten(
- io_lib:format("No server configured for ~p.", [UrlName])),
- {reply, {error, {unknown_external_server, ?l2b(Msg)}}, Handlers};
- Command ->
- {ok, NewPid} = couch_external_server:start_link(UrlName, Command),
- true = ets:insert(Handlers, {UrlName, NewPid}),
- {reply, NewPid, Handlers}
- end;
- [{UrlName, Pid}] ->
- {reply, Pid, Handlers}
- end;
-handle_call({config, UrlName}, _From, Handlers) ->
- % A newly added handler and a handler that had it's command
- % changed are treated exactly the same.
-
- % Shutdown the old handler.
- case ets:lookup(Handlers, UrlName) of
- [{UrlName, Pid}] ->
- couch_external_server:stop(Pid);
- [] ->
- ok
- end,
- % Wait for next request to boot the handler.
- {reply, ok, Handlers}.
-
-handle_cast(_Whatever, State) ->
- {noreply, State}.
-
-handle_info({'EXIT', Pid, normal}, Handlers) ->
- ?LOG_INFO("EXTERNAL: Server ~p terminated normally", [Pid]),
- % The process terminated normally without us asking - Remove Pid from the
- % handlers table so we don't attempt to reuse it
- ets:match_delete(Handlers, {'_', Pid}),
- {noreply, Handlers};
-
-handle_info({'EXIT', Pid, Reason}, Handlers) ->
- ?LOG_INFO("EXTERNAL: Server ~p died. (reason: ~p)", [Pid, Reason]),
- % Remove Pid from the handlers table so we don't try closing
- % it a second time in terminate/2.
- ets:match_delete(Handlers, {'_', Pid}),
- {stop, normal, Handlers};
-
-handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
- erlang:send_after(5000, self(), restart_config_listener),
- {noreply, State};
-
-handle_info(restart_config_listener, State) ->
- ok = config:listen_for_changes(?MODULE, nil),
- {noreply, State}.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_external_server.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_external_server.erl b/src/couch/src/couch_external_server.erl
deleted file mode 100644
index 56406bb..0000000
--- a/src/couch/src/couch_external_server.erl
+++ /dev/null
@@ -1,84 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_external_server).
--behaviour(gen_server).
--behaviour(config_listener).
-
--export([start_link/2, stop/1, execute/2]).
--export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2, code_change/3]).
-
-% config_listener api
--export([handle_config_change/5]).
-
--include_lib("couch/include/couch_db.hrl").
-
-% External API
-
-start_link(Name, Command) ->
- gen_server:start_link(couch_external_server, [Name, Command], []).
-
-stop(Pid) ->
- gen_server:cast(Pid, stop).
-
-execute(Pid, JsonReq) ->
- {json, Json} = gen_server:call(Pid, {execute, JsonReq}, infinity),
- ?JSON_DECODE(Json).
-
-% Gen Server Handlers
-
-init([Name, Command]) ->
- ?LOG_INFO("EXTERNAL: Starting process for: ~s", [Name]),
- ?LOG_INFO("COMMAND: ~s", [Command]),
- process_flag(trap_exit, true),
- Timeout = list_to_integer(config:get("couchdb", "os_process_timeout",
- "5000")),
- {ok, Pid} = couch_os_process:start_link(Command, [{timeout, Timeout}]),
- ok = config:listen_for_changes(?MODULE, {Name, Command, Pid}, Pid),
- {ok, {Name, Command, Pid}}.
-
-terminate(_Reason, {_Name, _Command, Pid}) ->
- couch_os_process:stop(Pid),
- ok.
-
-handle_call({execute, JsonReq}, _From, {Name, Command, Pid}) ->
- {reply, couch_os_process:prompt(Pid, JsonReq), {Name, Command, Pid}}.
-
-handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
- erlang:send_after(5000, self(), restart_config_listener),
- {noreply, State};
-handle_info(restart_config_listener, State) ->
- ok = config:listen_for_changes(?MODULE, State),
- {noreply, State};
-handle_info({'EXIT', _Pid, normal}, State) ->
- {noreply, State};
-handle_info({'EXIT', Pid, Reason}, {Name, Command, Pid}) ->
- ?LOG_INFO("EXTERNAL: Process for ~s exiting. (reason: ~w)", [Name, Reason]),
- {stop, Reason, {Name, Command, Pid}}.
-
-handle_cast(stop, {Name, Command, Pid}) ->
- ?LOG_INFO("EXTERNAL: Shutting down ~s", [Name]),
- exit(Pid, normal),
- {stop, normal, {Name, Command, Pid}};
-handle_cast(_Whatever, State) ->
- {noreply, State}.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-
-handle_config_change("couchdb", "os_process_timeout", NewTimeout, _, Pid) ->
- couch_os_process:set_timeout(Pid, list_to_integer(NewTimeout)),
- {ok, Pid};
-handle_config_change(_, _, _, _, Pid) ->
- {ok, Pid}.
-
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
deleted file mode 100644
index 7528091..0000000
--- a/src/couch/src/couch_file.erl
+++ /dev/null
@@ -1,587 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_file).
--behaviour(gen_server).
-
--include_lib("couch/include/couch_db.hrl").
-
-
--define(INITIAL_WAIT, 60000).
--define(MONITOR_CHECK, 10000).
--define(SIZE_BLOCK, 16#1000). % 4 KiB
-
-
--record(file, {
- fd,
- is_sys,
- eof = 0,
- db_pid
-}).
-
-% public API
--export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]).
--export([pread_term/2, pread_iolist/2, pread_binary/2]).
--export([append_binary/2, append_binary_md5/2]).
--export([append_raw_chunk/2, assemble_file_chunk/1, assemble_file_chunk/2]).
--export([append_term/2, append_term/3, append_term_md5/2, append_term_md5/3]).
--export([write_header/2, read_header/1]).
--export([delete/2, delete/3, nuke_dir/2, init_delete_dir/1]).
-
-% gen_server callbacks
--export([init/1, terminate/2, code_change/3]).
--export([handle_call/3, handle_cast/2, handle_info/2]).
-
-
-%%----------------------------------------------------------------------
-%% Args: Valid Options are [create] and [create,overwrite].
-%% Files are opened in read/write mode.
-%% Returns: On success, {ok, Fd}
-%% or {error, Reason} if the file could not be opened.
-%%----------------------------------------------------------------------
-
-open(Filepath) ->
- open(Filepath, []).
-
-open(Filepath, Options) ->
- case gen_server:start_link(couch_file,
- {Filepath, Options, self(), Ref = make_ref()}, []) of
- {ok, Fd} ->
- {ok, Fd};
- ignore ->
- % get the error
- receive
- {Ref, Pid, {error, Reason} = Error} ->
- case process_info(self(), trap_exit) of
- {trap_exit, true} -> receive {'EXIT', Pid, _} -> ok end;
- {trap_exit, false} -> ok
- end,
- case {lists:member(nologifmissing, Options), Reason} of
- {true, enoent} -> ok;
- _ ->
- ?LOG_ERROR("Could not open file ~s: ~s",
- [Filepath, file:format_error(Reason)])
- end,
- Error
- end;
- Error ->
- % We can't say much here, because it could be any kind of error.
- % Just let it bubble and an encapsulating subcomponent can perhaps
- % be more informative. It will likely appear in the SASL log, anyway.
- Error
- end.
-
-
-set_db_pid(Fd, Pid) ->
- gen_server:call(Fd, {set_db_pid, Pid}).
-
-
-%%----------------------------------------------------------------------
-%% Purpose: To append an Erlang term to the end of the file.
-%% Args: Erlang term to serialize and append to the file.
-%% Returns: {ok, Pos, NumBytesWritten} where Pos is the file offset to
-%% the beginning the serialized term. Use pread_term to read the term
-%% back.
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-append_term(Fd, Term) ->
- append_term(Fd, Term, []).
-
-append_term(Fd, Term, Options) ->
- Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION),
- append_binary(Fd, couch_compress:compress(Term, Comp)).
-
-append_term_md5(Fd, Term) ->
- append_term_md5(Fd, Term, []).
-
-append_term_md5(Fd, Term, Options) ->
- Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION),
- append_binary_md5(Fd, couch_compress:compress(Term, Comp)).
-
-%%----------------------------------------------------------------------
-%% Purpose: To append an Erlang binary to the end of the file.
-%% Args: Erlang term to serialize and append to the file.
-%% Returns: {ok, Pos, NumBytesWritten} where Pos is the file offset to the
-%% beginning the serialized term. Use pread_term to read the term back.
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-append_binary(Fd, Bin) ->
- gen_server:call(Fd, {append_bin, assemble_file_chunk(Bin)}, infinity).
-
-append_binary_md5(Fd, Bin) ->
- gen_server:call(Fd,
- {append_bin, assemble_file_chunk(Bin, couch_util:md5(Bin))}, infinity).
-
-append_raw_chunk(Fd, Chunk) ->
- gen_server:call(Fd, {append_bin, Chunk}, infinity).
-
-
-assemble_file_chunk(Bin) ->
- [<<0:1/integer, (iolist_size(Bin)):31/integer>>, Bin].
-
-assemble_file_chunk(Bin, Md5) ->
- [<<1:1/integer, (iolist_size(Bin)):31/integer>>, Md5, Bin].
-
-%%----------------------------------------------------------------------
-%% Purpose: Reads a term from a file that was written with append_term
-%% Args: Pos, the offset into the file where the term is serialized.
-%% Returns: {ok, Term}
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-
-pread_term(Fd, Pos) ->
- {ok, Bin} = pread_binary(Fd, Pos),
- {ok, couch_compress:decompress(Bin)}.
-
-
-%%----------------------------------------------------------------------
-%% Purpose: Reads a binrary from a file that was written with append_binary
-%% Args: Pos, the offset into the file where the term is serialized.
-%% Returns: {ok, Term}
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-pread_binary(Fd, Pos) ->
- {ok, L} = pread_iolist(Fd, Pos),
- {ok, iolist_to_binary(L)}.
-
-
-pread_iolist(Fd, Pos) ->
- case gen_server:call(Fd, {pread_iolist, Pos}, infinity) of
- {ok, IoList, <<>>} ->
- {ok, IoList};
- {ok, IoList, Md5} ->
- case couch_util:md5(IoList) of
- Md5 ->
- {ok, IoList};
- _ ->
- twig:log(emerg, "File corruption in ~p at position ~B",
- [Fd, Pos]),
- exit({file_corruption, <<"file corruption">>})
- end;
- Error ->
- Error
- end.
-
-%%----------------------------------------------------------------------
-%% Purpose: The length of a file, in bytes.
-%% Returns: {ok, Bytes}
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-% length in bytes
-bytes(Fd) ->
- gen_server:call(Fd, bytes, infinity).
-
-%%----------------------------------------------------------------------
-%% Purpose: Truncate a file to the number of bytes.
-%% Returns: ok
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-truncate(Fd, Pos) ->
- gen_server:call(Fd, {truncate, Pos}, infinity).
-
-%%----------------------------------------------------------------------
-%% Purpose: Ensure all bytes written to the file are flushed to disk.
-%% Returns: ok
-%% or {error, Reason}.
-%%----------------------------------------------------------------------
-
-sync(Filepath) when is_list(Filepath) ->
- {ok, Fd} = file:open(Filepath, [append, raw]),
- try ok = file:sync(Fd) after ok = file:close(Fd) end;
-sync(Fd) ->
- gen_server:call(Fd, sync, infinity).
-
-%%----------------------------------------------------------------------
-%% Purpose: Close the file.
-%% Returns: ok
-%%----------------------------------------------------------------------
-close(Fd) ->
- gen_server:call(Fd, close, infinity).
-
-
-delete(RootDir, Filepath) ->
- delete(RootDir, Filepath, true).
-
-
-delete(RootDir, Filepath, Async) ->
- DelFile = filename:join([RootDir,".delete", ?b2l(couch_uuids:random())]),
- case file:rename(Filepath, DelFile) of
- ok ->
- if (Async) ->
- spawn(file, delete, [DelFile]),
- ok;
- true ->
- file:delete(DelFile)
- end;
- Error ->
- Error
- end.
-
-
-nuke_dir(RootDelDir, Dir) ->
- FoldFun = fun(File) ->
- Path = Dir ++ "/" ++ File,
- case filelib:is_dir(Path) of
- true ->
- ok = nuke_dir(RootDelDir, Path),
- file:del_dir(Path);
- false ->
- delete(RootDelDir, Path, false)
- end
- end,
- case file:list_dir(Dir) of
- {ok, Files} ->
- lists:foreach(FoldFun, Files),
- ok = file:del_dir(Dir);
- {error, enoent} ->
- ok
- end.
-
-
-init_delete_dir(RootDir) ->
- Dir = filename:join(RootDir,".delete"),
- % note: ensure_dir requires an actual filename companent, which is the
- % reason for "foo".
- filelib:ensure_dir(filename:join(Dir,"foo")),
- filelib:fold_files(Dir, ".*", true,
- fun(Filename, _) ->
- ok = file:delete(Filename)
- end, ok).
-
-
-read_header(Fd) ->
- case gen_server:call(Fd, find_header, infinity) of
- {ok, Bin} ->
- {ok, binary_to_term(Bin)};
- Else ->
- Else
- end.
-
-write_header(Fd, Data) ->
- Bin = term_to_binary(Data),
- Md5 = couch_util:md5(Bin),
- % now we assemble the final header binary and write to disk
- FinalBin = <<Md5/binary, Bin/binary>>,
- gen_server:call(Fd, {write_header, FinalBin}, infinity).
-
-
-init_status_error(ReturnPid, Ref, Error) ->
- ReturnPid ! {Ref, self(), Error},
- ignore.
-
-% server functions
-
-init({Filepath, Options, ReturnPid, Ref}) ->
- process_flag(trap_exit, true),
- OpenOptions = file_open_options(Options),
- case lists:member(create, Options) of
- true ->
- filelib:ensure_dir(Filepath),
- case file:open(Filepath, OpenOptions) of
- {ok, Fd} ->
- {ok, Length} = file:position(Fd, eof),
- case Length > 0 of
- true ->
- % this means the file already exists and has data.
- % FYI: We don't differentiate between empty files and non-existant
- % files here.
- case lists:member(overwrite, Options) of
- true ->
- {ok, 0} = file:position(Fd, 0),
- ok = file:truncate(Fd),
- ok = file:sync(Fd),
- maybe_track_open_os_files(Options),
- erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd=Fd, is_sys=lists:member(sys_db, Options)}};
- false ->
- ok = file:close(Fd),
- init_status_error(ReturnPid, Ref, {error, eexist})
- end;
- false ->
- maybe_track_open_os_files(Options),
- erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd=Fd, is_sys=lists:member(sys_db, Options)}}
- end;
- Error ->
- init_status_error(ReturnPid, Ref, Error)
- end;
- false ->
- % open in read mode first, so we don't create the file if it doesn't exist.
- case file:open(Filepath, [read, raw]) of
- {ok, Fd_Read} ->
- {ok, Fd} = file:open(Filepath, OpenOptions),
- ok = file:close(Fd_Read),
- maybe_track_open_os_files(Options),
- {ok, Eof} = file:position(Fd, eof),
- erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd=Fd, eof=Eof, is_sys=lists:member(sys_db, Options)}};
- Error ->
- init_status_error(ReturnPid, Ref, Error)
- end
- end.
-
-file_open_options(Options) ->
- [read, raw, binary] ++ case lists:member(read_only, Options) of
- true ->
- [];
- false ->
- [append]
- end.
-
-maybe_track_open_os_files(Options) ->
- case not lists:member(sys_db, Options) of
- true ->
- couch_stats_collector:track_process_count({couchdb, open_os_files});
- false ->
- ok
- end.
-
-terminate(_Reason, #file{fd = nil}) ->
- ok;
-terminate(_Reason, #file{fd = Fd}) ->
- ok = file:close(Fd).
-
-handle_call(close, _From, #file{fd=Fd}=File) ->
- {stop, normal, file:close(Fd), File#file{fd = nil}};
-
-handle_call({pread_iolist, Pos}, _From, File) ->
- {RawData, NextPos} = try
- % up to 8Kbs of read ahead
- read_raw_iolist_int(File, Pos, 2 * ?SIZE_BLOCK - (Pos rem ?SIZE_BLOCK))
- catch
- _:_ ->
- read_raw_iolist_int(File, Pos, 4)
- end,
- <<Prefix:1/integer, Len:31/integer, RestRawData/binary>> =
- iolist_to_binary(RawData),
- case Prefix of
- 1 ->
- {Md5, IoList} = extract_md5(
- maybe_read_more_iolist(RestRawData, 16 + Len, NextPos, File)),
- {reply, {ok, IoList, Md5}, File};
- 0 ->
- IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, File),
- {reply, {ok, IoList, <<>>}, File}
- end;
-
-handle_call(bytes, _From, #file{fd = Fd} = File) ->
- {reply, file:position(Fd, eof), File};
-
-handle_call({set_db_pid, Pid}, _From, #file{db_pid=OldPid}=File) ->
- case is_pid(OldPid) of
- true -> unlink(OldPid);
- false -> ok
- end,
- link(Pid),
- {reply, ok, File#file{db_pid=Pid}};
-
-handle_call(sync, _From, #file{fd=Fd}=File) ->
- {reply, file:sync(Fd), File};
-
-handle_call({truncate, Pos}, _From, #file{fd=Fd}=File) ->
- {ok, Pos} = file:position(Fd, Pos),
- case file:truncate(Fd) of
- ok ->
- {reply, ok, File#file{eof = Pos}};
- Error ->
- {reply, Error, File}
- end;
-
-handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
- Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin),
- Size = iolist_size(Blocks),
- case file:write(Fd, Blocks) of
- ok ->
- {reply, {ok, Pos, Size}, File#file{eof = Pos + Size}};
- Error ->
- {reply, Error, File}
- end;
-
-handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
- BinSize = byte_size(Bin),
- case Pos rem ?SIZE_BLOCK of
- 0 ->
- Padding = <<>>;
- BlockOffset ->
- Padding = <<0:(8*(?SIZE_BLOCK-BlockOffset))>>
- end,
- FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])],
- case file:write(Fd, FinalBin) of
- ok ->
- {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}};
- Error ->
- {reply, Error, File}
- end;
-
-handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) ->
- {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}.
-
-handle_cast(close, Fd) ->
- {stop,normal,Fd}.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-handle_info(maybe_close, File) ->
- case is_idle(File) of
- true ->
- {stop, normal, File};
- false ->
- erlang:send_after(?MONITOR_CHECK, self(), maybe_close),
- {noreply, File}
- end;
-
-handle_info({'EXIT', Pid, _}, #file{db_pid=Pid}=File) ->
- case is_idle(File) of
- true -> {stop, normal, File};
- false -> {noreply, File}
- end;
-
-handle_info({'EXIT', _, normal}, Fd) ->
- {noreply, Fd};
-handle_info({'EXIT', _, Reason}, Fd) ->
- {stop, Reason, Fd}.
-
-
-find_header(_Fd, -1) ->
- no_valid_header;
-find_header(Fd, Block) ->
- case (catch load_header(Fd, Block)) of
- {ok, Bin} ->
- {ok, Bin};
- _Error ->
- find_header(Fd, Block -1)
- end.
-
-load_header(Fd, Block) ->
- {ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} =
- file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK),
- TotalBytes = calculate_total_read_len(5, HeaderLen),
- case TotalBytes > byte_size(RestBlock) of
- false ->
- <<RawBin:TotalBytes/binary, _/binary>> = RestBlock;
- true ->
- {ok, Missing} = file:pread(
- Fd, (Block * ?SIZE_BLOCK) + 5 + byte_size(RestBlock),
- TotalBytes - byte_size(RestBlock)),
- RawBin = <<RestBlock/binary, Missing/binary>>
- end,
- <<Md5Sig:16/binary, HeaderBin/binary>> =
- iolist_to_binary(remove_block_prefixes(5, RawBin)),
- Md5Sig = couch_util:md5(HeaderBin),
- {ok, HeaderBin}.
-
-maybe_read_more_iolist(Buffer, DataSize, _, _)
- when DataSize =< byte_size(Buffer) ->
- <<Data:DataSize/binary, _/binary>> = Buffer,
- [Data];
-maybe_read_more_iolist(Buffer, DataSize, NextPos, File) ->
- {Missing, _} =
- read_raw_iolist_int(File, NextPos, DataSize - byte_size(Buffer)),
- [Buffer, Missing].
-
--spec read_raw_iolist_int(#file{}, Pos::non_neg_integer(), Len::non_neg_integer()) ->
- {Data::iolist(), CurPos::non_neg_integer()}.
-read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE
- read_raw_iolist_int(Fd, Pos, Len);
-read_raw_iolist_int(#file{fd = Fd}, Pos, Len) ->
- BlockOffset = Pos rem ?SIZE_BLOCK,
- TotalBytes = calculate_total_read_len(BlockOffset, Len),
- {ok, <<RawBin:TotalBytes/binary>>} = file:pread(Fd, Pos, TotalBytes),
- {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}.
-
--spec extract_md5(iolist()) -> {binary(), iolist()}.
-extract_md5(FullIoList) ->
- {Md5List, IoList} = split_iolist(FullIoList, 16, []),
- {iolist_to_binary(Md5List), IoList}.
-
-calculate_total_read_len(0, FinalLen) ->
- calculate_total_read_len(1, FinalLen) + 1;
-calculate_total_read_len(BlockOffset, FinalLen) ->
- case ?SIZE_BLOCK - BlockOffset of
- BlockLeft when BlockLeft >= FinalLen ->
- FinalLen;
- BlockLeft ->
- FinalLen + ((FinalLen - BlockLeft) div (?SIZE_BLOCK -1)) +
- if ((FinalLen - BlockLeft) rem (?SIZE_BLOCK -1)) =:= 0 -> 0;
- true -> 1 end
- end.
-
-remove_block_prefixes(_BlockOffset, <<>>) ->
- [];
-remove_block_prefixes(0, <<_BlockPrefix,Rest/binary>>) ->
- remove_block_prefixes(1, Rest);
-remove_block_prefixes(BlockOffset, Bin) ->
- BlockBytesAvailable = ?SIZE_BLOCK - BlockOffset,
- case size(Bin) of
- Size when Size > BlockBytesAvailable ->
- <<DataBlock:BlockBytesAvailable/binary,Rest/binary>> = Bin,
- [DataBlock | remove_block_prefixes(0, Rest)];
- _Size ->
- [Bin]
- end.
-
-make_blocks(_BlockOffset, []) ->
- [];
-make_blocks(0, IoList) ->
- [<<0>> | make_blocks(1, IoList)];
-make_blocks(BlockOffset, IoList) ->
- case split_iolist(IoList, (?SIZE_BLOCK - BlockOffset), []) of
- {Begin, End} ->
- [Begin | make_blocks(0, End)];
- _SplitRemaining ->
- IoList
- end.
-
-%% @doc Returns a tuple where the first element contains the leading SplitAt
-%% bytes of the original iolist, and the 2nd element is the tail. If SplitAt
-%% is larger than byte_size(IoList), return the difference.
--spec split_iolist(IoList::iolist(), SplitAt::non_neg_integer(), Acc::list()) ->
- {iolist(), iolist()} | non_neg_integer().
-split_iolist(List, 0, BeginAcc) ->
- {lists:reverse(BeginAcc), List};
-split_iolist([], SplitAt, _BeginAcc) ->
- SplitAt;
-split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) when SplitAt > byte_size(Bin) ->
- split_iolist(Rest, SplitAt - byte_size(Bin), [Bin | BeginAcc]);
-split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) ->
- <<Begin:SplitAt/binary,End/binary>> = Bin,
- split_iolist([End | Rest], 0, [Begin | BeginAcc]);
-split_iolist([Sublist| Rest], SplitAt, BeginAcc) when is_list(Sublist) ->
- case split_iolist(Sublist, SplitAt, BeginAcc) of
- {Begin, End} ->
- {Begin, [End | Rest]};
- SplitRemaining ->
- split_iolist(Rest, SplitAt - (SplitAt - SplitRemaining), [Sublist | BeginAcc])
- end;
-split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) ->
- split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]).
-
-
-% System dbs aren't monitored by couch_stats_collector
-is_idle(#file{is_sys=true}) ->
- case process_info(self(), monitored_by) of
- {monitored_by, []} -> true;
- _ -> false
- end;
-is_idle(#file{is_sys=false}) ->
- case process_info(self(), monitored_by) of
- {monitored_by, []} -> true;
- {monitored_by, [_]} -> true;
- _ -> false
- end.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ed98610c/src/couch/src/couch_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch/src/couch_httpd.erl b/src/couch/src/couch_httpd.erl
deleted file mode 100644
index db90129..0000000
--- a/src/couch/src/couch_httpd.erl
+++ /dev/null
@@ -1,1082 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_httpd).
--include_lib("couch/include/couch_db.hrl").
-
--export([start_link/0, start_link/1, stop/0, handle_request/5]).
-
--export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,qs_json_value/3]).
--export([path/1,absolute_uri/2,body_length/1]).
--export([verify_is_server_admin/1,unquote/1,quote/1,recv/2,recv_chunked/4,error_info/1]).
--export([make_fun_spec_strs/1]).
--export([make_arity_1_fun/1, make_arity_2_fun/1, make_arity_3_fun/1]).
--export([parse_form/1,json_body/1,json_body_obj/1,body/1]).
--export([doc_etag/1, make_etag/1, etag_match/2, etag_respond/3, etag_maybe/2]).
--export([primary_header_value/2,partition/1,serve_file/3,serve_file/4, server_header/0]).
--export([start_chunked_response/3,send_chunk/2,log_request/2]).
--export([start_response_length/4, start_response/3, send/2]).
--export([start_json_response/2, start_json_response/3, end_json_response/1]).
--export([send_response/4,send_method_not_allowed/2,send_error/4, send_redirect/2,send_chunked_error/2]).
--export([send_json/2,send_json/3,send_json/4,last_chunk/1,parse_multipart_request/3]).
--export([accepted_encodings/1,handle_request_int/5,validate_referer/1,validate_ctype/2]).
--export([http_1_0_keep_alive/2]).
-
-start_link() ->
- start_link(http).
-start_link(http) ->
- Port = config:get("httpd", "port", "5984"),
- start_link(?MODULE, [{port, Port}]);
-start_link(https) ->
- Port = config:get("ssl", "port", "6984"),
- CertFile = config:get("ssl", "cert_file", nil),
- KeyFile = config:get("ssl", "key_file", nil),
- Options = case CertFile /= nil andalso KeyFile /= nil of
- true ->
- SslOpts = [{certfile, CertFile}, {keyfile, KeyFile}],
-
- %% set password if one is needed for the cert
- SslOpts1 = case config:get("ssl", "password", nil) of
- nil -> SslOpts;
- Password ->
- SslOpts ++ [{password, Password}]
- end,
- % do we verify certificates ?
- FinalSslOpts = case config:get("ssl",
- "verify_ssl_certificates", "false") of
- "false" -> SslOpts1;
- "true" ->
- case config:get("ssl",
- "cacert_file", nil) of
- nil ->
- io:format("Verify SSL certificate "
- ++"enabled but file containing "
- ++"PEM encoded CA certificates is "
- ++"missing", []),
- throw({error, missing_cacerts});
- CaCertFile ->
- Depth = list_to_integer(config:get("ssl",
- "ssl_certificate_max_depth",
- "1")),
- FinalOpts = [
- {cacertfile, CaCertFile},
- {depth, Depth},
- {verify, verify_peer}],
- % allows custom verify fun.
- case config:get("ssl",
- "verify_fun", nil) of
- nil -> FinalOpts;
- SpecStr ->
- FinalOpts
- ++ [{verify_fun, make_arity_3_fun(SpecStr)}]
- end
- end
- end,
-
- [{port, Port},
- {ssl, true},
- {ssl_opts, FinalSslOpts}];
- false ->
- io:format("SSL enabled but PEM certificates are missing.", []),
- throw({error, missing_certs})
- end,
- start_link(https, Options).
-start_link(Name, Options) ->
- BindAddress = config:get("httpd", "bind_address", any),
- validate_bind_address(BindAddress),
- DefaultSpec = "{couch_httpd_db, handle_request}",
- DefaultFun = make_arity_1_fun(
- config:get("httpd", "default_handler", DefaultSpec)
- ),
-
- UrlHandlersList = lists:map(
- fun({UrlKey, SpecStr}) ->
- {?l2b(UrlKey), make_arity_1_fun(SpecStr)}
- end, config:get("httpd_global_handlers")),
-
- DbUrlHandlersList = lists:map(
- fun({UrlKey, SpecStr}) ->
- {?l2b(UrlKey), make_arity_2_fun(SpecStr)}
- end, config:get("httpd_db_handlers")),
-
- DesignUrlHandlersList = lists:map(
- fun({UrlKey, SpecStr}) ->
- {?l2b(UrlKey), make_arity_3_fun(SpecStr)}
- end, config:get("httpd_design_handlers")),
-
- UrlHandlers = dict:from_list(UrlHandlersList),
- DbUrlHandlers = dict:from_list(DbUrlHandlersList),
- DesignUrlHandlers = dict:from_list(DesignUrlHandlersList),
- {ok, ServerOptions} = couch_util:parse_term(
- config:get("httpd", "server_options", "[]")),
- {ok, SocketOptions} = couch_util:parse_term(
- config:get("httpd", "socket_options", "[]")),
-
- set_auth_handlers(),
-
- % ensure uuid is set so that concurrent replications
- % get the same value.
- couch_server:get_uuid(),
-
- Loop = fun(Req)->
- case SocketOptions of
- [] ->
- ok;
- _ ->
- ok = mochiweb_socket:setopts(Req:get(socket), SocketOptions)
- end,
- apply(?MODULE, handle_request, [
- Req, DefaultFun, UrlHandlers, DbUrlHandlers, DesignUrlHandlers
- ])
- end,
-
- % set mochiweb options
- FinalOptions = lists:append([Options, ServerOptions, [
- {loop, Loop},
- {name, Name},
- {ip, BindAddress}]]),
-
- % launch mochiweb
- case mochiweb_http:start(FinalOptions) of
- {ok, MochiPid} ->
- {ok, MochiPid};
- {error, Reason} ->
- io:format("Failure to start Mochiweb: ~s~n",[Reason]),
- throw({error, Reason})
- end.
-
-
-stop() ->
- mochiweb_http:stop(couch_httpd),
- mochiweb_http:stop(https).
-
-
-set_auth_handlers() ->
- AuthenticationSrcs = make_fun_spec_strs(
- config:get("httpd", "authentication_handlers", "")),
- AuthHandlers = lists:map(
- fun(A) -> {make_arity_1_fun(A), ?l2b(A)} end, AuthenticationSrcs),
- ok = application:set_env(couch, auth_handlers, AuthHandlers).
-
-% SpecStr is a string like "{my_module, my_fun}"
-% or "{my_module, my_fun, <<"my_arg">>}"
-make_arity_1_fun(SpecStr) ->
- case couch_util:parse_term(SpecStr) of
- {ok, {Mod, Fun, SpecArg}} ->
- fun(Arg) -> Mod:Fun(Arg, SpecArg) end;
- {ok, {Mod, Fun}} ->
- fun(Arg) -> Mod:Fun(Arg) end
- end.
-
-make_arity_2_fun(SpecStr) ->
- case couch_util:parse_term(SpecStr) of
- {ok, {Mod, Fun, SpecArg}} ->
- fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2, SpecArg) end;
- {ok, {Mod, Fun}} ->
- fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2) end
- end.
-
-make_arity_3_fun(SpecStr) ->
- case couch_util:parse_term(SpecStr) of
- {ok, {Mod, Fun, SpecArg}} ->
- fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3, SpecArg) end;
- {ok, {Mod, Fun}} ->
- fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3) end
- end.
-
-% SpecStr is "{my_module, my_fun}, {my_module2, my_fun2}"
-make_fun_spec_strs(SpecStr) ->
- re:split(SpecStr, "(?<=})\\s*,\\s*(?={)", [{return, list}]).
-
-handle_request(MochiReq, DefaultFun, UrlHandlers, DbUrlHandlers,
- DesignUrlHandlers) ->
-
- MochiReq1 = couch_httpd_vhost:dispatch_host(MochiReq),
-
- handle_request_int(MochiReq1, DefaultFun,
- UrlHandlers, DbUrlHandlers, DesignUrlHandlers).
-
-handle_request_int(MochiReq, DefaultFun,
- UrlHandlers, DbUrlHandlers, DesignUrlHandlers) ->
- Begin = now(),
- % for the path, use the raw path with the query string and fragment
- % removed, but URL quoting left intact
- RawUri = MochiReq:get(raw_path),
- {"/" ++ Path, _, _} = mochiweb_util:urlsplit_path(RawUri),
-
- Headers = MochiReq:get(headers),
-
- % get requested path
- RequestedPath = case MochiReq:get_header_value("x-couchdb-vhost-path") of
- undefined ->
- case MochiReq:get_header_value("x-couchdb-requested-path") of
- undefined -> RawUri;
- R -> R
- end;
- P -> P
- end,
-
- HandlerKey =
- case mochiweb_util:partition(Path, "/") of
- {"", "", ""} ->
- <<"/">>; % Special case the root url handler
- {FirstPart, _, _} ->
- list_to_binary(FirstPart)
- end,
- ?LOG_DEBUG("~p ~s ~p from ~p~nHeaders: ~p", [
- MochiReq:get(method),
- RawUri,
- MochiReq:get(version),
- MochiReq:get(peer),
- mochiweb_headers:to_list(MochiReq:get(headers))
- ]),
-
- Method1 =
- case MochiReq:get(method) of
- % already an atom
- Meth when is_atom(Meth) -> Meth;
-
- % Non standard HTTP verbs aren't atoms (COPY, MOVE etc) so convert when
- % possible (if any module references the atom, then it's existing).
- Meth -> couch_util:to_existing_atom(Meth)
- end,
- increment_method_stats(Method1),
-
- % allow broken HTTP clients to fake a full method vocabulary with an X-HTTP-METHOD-OVERRIDE header
- MethodOverride = MochiReq:get_primary_header_value("X-HTTP-Method-Override"),
- Method2 = case lists:member(MethodOverride, ["GET", "HEAD", "POST",
- "PUT", "DELETE",
- "TRACE", "CONNECT",
- "COPY"]) of
- true ->
- ?LOG_INFO("MethodOverride: ~s (real method was ~s)", [MethodOverride, Method1]),
- case Method1 of
- 'POST' -> couch_util:to_existing_atom(MethodOverride);
- _ ->
- % Ignore X-HTTP-Method-Override when the original verb isn't POST.
- % I'd like to send a 406 error to the client, but that'd require a nasty refactor.
- % throw({not_acceptable, <<"X-HTTP-Method-Override may only be used with POST requests.">>})
- Method1
- end;
- _ -> Method1
- end,
-
- % alias HEAD to GET as mochiweb takes care of stripping the body
- Method = case Method2 of
- 'HEAD' -> 'GET';
- Other -> Other
- end,
-
- HttpReq = #httpd{
- mochi_req = MochiReq,
- peer = MochiReq:get(peer),
- method = Method,
- requested_path_parts =
- [?l2b(unquote(Part)) || Part <- string:tokens(RequestedPath, "/")],
- path_parts = [?l2b(unquote(Part)) || Part <- string:tokens(Path, "/")],
- db_url_handlers = DbUrlHandlers,
- design_url_handlers = DesignUrlHandlers,
- default_fun = DefaultFun,
- url_handlers = UrlHandlers,
- user_ctx = erlang:erase(pre_rewrite_user_ctx)
- },
-
- HandlerFun = couch_util:dict_find(HandlerKey, UrlHandlers, DefaultFun),
- {ok, AuthHandlers} = application:get_env(couch, auth_handlers),
-
- {ok, Resp} =
- try
- case couch_httpd_cors:is_preflight_request(HttpReq) of
- #httpd{} ->
- case authenticate_request(HttpReq, AuthHandlers) of
- #httpd{} = Req ->
- HandlerFun(Req);
- Response ->
- Response
- end;
- Response ->
- Response
- end
- catch
- throw:{http_head_abort, Resp0} ->
- {ok, Resp0};
- throw:{invalid_json, S} ->
- ?LOG_ERROR("attempted upload of invalid JSON (set log_level to debug to log it)", []),
- ?LOG_DEBUG("Invalid JSON: ~p",[S]),
- send_error(HttpReq, {bad_request, invalid_json});
- throw:unacceptable_encoding ->
- ?LOG_ERROR("unsupported encoding method for the response", []),
- send_error(HttpReq, {not_acceptable, "unsupported encoding"});
- throw:bad_accept_encoding_value ->
- ?LOG_ERROR("received invalid Accept-Encoding header", []),
- send_error(HttpReq, bad_request);
- exit:normal ->
- exit(normal);
- exit:snappy_nif_not_loaded ->
- ErrorReason = "To access the database or view index, Apache CouchDB"
- " must be built with Erlang OTP R13B04 or higher.",
- ?LOG_ERROR("~s", [ErrorReason]),
- send_error(HttpReq, {bad_otp_release, ErrorReason});
- exit:{body_too_large, _} ->
- send_error(HttpReq, request_entity_too_large);
- throw:Error ->
- Stack = erlang:get_stacktrace(),
- ?LOG_DEBUG("Minor error in HTTP request: ~p",[Error]),
- ?LOG_DEBUG("Stacktrace: ~p",[Stack]),
- send_error(HttpReq, Error);
- error:badarg ->
- Stack = erlang:get_stacktrace(),
- ?LOG_ERROR("Badarg error in HTTP request",[]),
- ?LOG_INFO("Stacktrace: ~p",[Stack]),
- send_error(HttpReq, badarg);
- error:function_clause ->
- Stack = erlang:get_stacktrace(),
- ?LOG_ERROR("function_clause error in HTTP request",[]),
- ?LOG_INFO("Stacktrace: ~p",[Stack]),
- send_error(HttpReq, function_clause);
- Tag:Error ->
- Stack = erlang:get_stacktrace(),
- ?LOG_ERROR("Uncaught error in HTTP request: ~p",[{Tag, Error}]),
- ?LOG_INFO("Stacktrace: ~p",[Stack]),
- send_error(HttpReq, Error)
- end,
- RequestTime = round(timer:now_diff(now(), Begin)/1000),
- couch_stats_collector:record({couchdb, request_time}, RequestTime),
- couch_stats_collector:increment({httpd, requests}),
- {ok, Resp}.
-
-% Try authentication handlers in order until one sets a user_ctx
-% the auth funs also have the option of returning a response
-% move this to couch_httpd_auth?
-authenticate_request(#httpd{user_ctx=#user_ctx{}} = Req, _AuthHandlers) ->
- Req;
-authenticate_request(#httpd{} = Req, []) ->
- case config:get("couch_httpd_auth", "require_valid_user", "false") of
- "true" ->
- throw({unauthorized, <<"Authentication required.">>});
- "false" ->
- Req#httpd{user_ctx=#user_ctx{}}
- end;
-authenticate_request(#httpd{} = Req, [{AuthFun, AuthSrc} | RestAuthHandlers]) ->
- R = case AuthFun(Req) of
- #httpd{user_ctx=#user_ctx{}=UserCtx}=Req2 ->
- Req2#httpd{user_ctx=UserCtx#user_ctx{handler=AuthSrc}};
- Else -> Else
- end,
- authenticate_request(R, RestAuthHandlers);
-authenticate_request(Response, _AuthSrcs) ->
- Response.
-
-increment_method_stats(Method) ->
- couch_stats_collector:increment({httpd_request_methods, Method}).
-
-validate_referer(Req) ->
- Host = host_for_request(Req),
- Referer = header_value(Req, "Referer", fail),
- case Referer of
- fail ->
- throw({bad_request, <<"Referer header required.">>});
- Referer ->
- {_,RefererHost,_,_,_} = mochiweb_util:urlsplit(Referer),
- if
- RefererHost =:= Host -> ok;
- true -> throw({bad_request, <<"Referer header must match host.">>})
- end
- end.
-
-validate_ctype(Req, Ctype) ->
- case header_value(Req, "Content-Type") of
- undefined ->
- throw({bad_ctype, "Content-Type must be "++Ctype});
- ReqCtype ->
- case string:tokens(ReqCtype, ";") of
- [Ctype] -> ok;
- [Ctype, _Rest] -> ok;
- _Else ->
- throw({bad_ctype, "Content-Type must be "++Ctype})
- end
- end.
-
-% Utilities
-
-partition(Path) ->
- mochiweb_util:partition(Path, "/").
-
-header_value(#httpd{mochi_req=MochiReq}, Key) ->
- MochiReq:get_header_value(Key).
-
-header_value(#httpd{mochi_req=MochiReq}, Key, Default) ->
- case MochiReq:get_header_value(Key) of
- undefined -> Default;
- Value -> Value
- end.
-
-primary_header_value(#httpd{mochi_req=MochiReq}, Key) ->
- MochiReq:get_primary_header_value(Key).
-
-accepted_encodings(#httpd{mochi_req=MochiReq}) ->
- case MochiReq:accepted_encodings(["gzip", "identity"]) of
- bad_accept_encoding_value ->
- throw(bad_accept_encoding_value);
- [] ->
- throw(unacceptable_encoding);
- EncList ->
- EncList
- end.
-
-serve_file(Req, RelativePath, DocumentRoot) ->
- serve_file(Req, RelativePath, DocumentRoot, []).
-
-serve_file(#httpd{mochi_req=MochiReq}=Req, RelativePath, DocumentRoot,
- ExtraHeaders) ->
- log_request(Req, 200),
- ResponseHeaders = server_header()
- ++ couch_httpd_auth:cookie_auth_header(Req, [])
- ++ ExtraHeaders,
- {ok, MochiReq:serve_file(RelativePath, DocumentRoot,
- couch_httpd_cors:cors_headers(Req, ResponseHeaders))}.
-
-qs_value(Req, Key) ->
- qs_value(Req, Key, undefined).
-
-qs_value(Req, Key, Default) ->
- couch_util:get_value(Key, qs(Req), Default).
-
-qs_json_value(Req, Key, Default) ->
- case qs_value(Req, Key, Default) of
- Default ->
- Default;
- Result ->
- ?JSON_DECODE(Result)
- end.
-
-qs(#httpd{mochi_req=MochiReq}) ->
- MochiReq:parse_qs().
-
-path(#httpd{mochi_req=MochiReq}) ->
- MochiReq:get(path).
-
-host_for_request(#httpd{mochi_req=MochiReq}) ->
- XHost = config:get("httpd", "x_forwarded_host", "X-Forwarded-Host"),
- case MochiReq:get_header_value(XHost) of
- undefined ->
- case MochiReq:get_header_value("Host") of
- undefined ->
- {ok, {Address, Port}} = case MochiReq:get(socket) of
- {ssl, SslSocket} -> ssl:sockname(SslSocket);
- Socket -> inet:sockname(Socket)
- end,
- inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port);
- Value1 ->
- Value1
- end;
- Value -> Value
- end.
-
-absolute_uri(#httpd{mochi_req=MochiReq}=Req, Path) ->
- Host = host_for_request(Req),
- XSsl = config:get("httpd", "x_forwarded_ssl", "X-Forwarded-Ssl"),
- Scheme = case MochiReq:get_header_value(XSsl) of
- "on" -> "https";
- _ ->
- XProto = config:get("httpd", "x_forwarded_proto", "X-Forwarded-Proto"),
- case MochiReq:get_header_value(XProto) of
- %% Restrict to "https" and "http" schemes only
- "https" -> "https";
- _ -> case MochiReq:get(scheme) of
- https -> "https";
- http -> "http"
- end
- end
- end,
- Scheme ++ "://" ++ Host ++ Path.
-
-unquote(UrlEncodedString) ->
- mochiweb_util:unquote(UrlEncodedString).
-
-quote(UrlDecodedString) ->
- mochiweb_util:quote_plus(UrlDecodedString).
-
-parse_form(#httpd{mochi_req=MochiReq}) ->
- mochiweb_multipart:parse_form(MochiReq).
-
-recv(#httpd{mochi_req=MochiReq}, Len) ->
- MochiReq:recv(Len).
-
-recv_chunked(#httpd{mochi_req=MochiReq}, MaxChunkSize, ChunkFun, InitState) ->
- % Fun is called once with each chunk
- % Fun({Length, Binary}, State)
- % called with Length == 0 on the last time.
- MochiReq:stream_body(MaxChunkSize, ChunkFun, InitState).
-
-body_length(#httpd{mochi_req=MochiReq}) ->
- MochiReq:get(body_length).
-
-body(#httpd{mochi_req=MochiReq, req_body=undefined}) ->
- MaxSize = list_to_integer(
- config:get("couchdb", "max_document_size", "4294967296")),
- MochiReq:recv_body(MaxSize);
-body(#httpd{req_body=ReqBody}) ->
- ReqBody.
-
-json_body(Httpd) ->
- ?JSON_DECODE(body(Httpd)).
-
-json_body_obj(Httpd) ->
- case json_body(Httpd) of
- {Props} -> {Props};
- _Else ->
- throw({bad_request, "Request body must be a JSON object"})
- end.
-
-
-
-doc_etag(#doc{revs={Start, [DiskRev|_]}}) ->
- "\"" ++ ?b2l(couch_doc:rev_to_str({Start, DiskRev})) ++ "\"".
-
-make_etag(Term) ->
- <<SigInt:128/integer>> = couch_util:md5(term_to_binary(Term)),
- iolist_to_binary([$", io_lib:format("~.36B", [SigInt]), $"]).
-
-etag_match(Req, CurrentEtag) when is_binary(CurrentEtag) ->
- etag_match(Req, binary_to_list(CurrentEtag));
-
-etag_match(Req, CurrentEtag) ->
- EtagsToMatch = string:tokens(
- header_value(Req, "If-None-Match", ""), ", "),
- lists:member(CurrentEtag, EtagsToMatch).
-
-etag_respond(Req, CurrentEtag, RespFun) ->
- case etag_match(Req, CurrentEtag) of
- true ->
- % the client has this in their cache.
- send_response(Req, 304, [{"ETag", CurrentEtag}], <<>>);
- false ->
- % Run the function.
- RespFun()
- end.
-
-etag_maybe(Req, RespFun) ->
- try
- RespFun()
- catch
- throw:{etag_match, ETag} ->
- send_response(Req, 304, [{"ETag", ETag}], <<>>)
- end.
-
-verify_is_server_admin(#httpd{user_ctx=UserCtx}) ->
- verify_is_server_admin(UserCtx);
-verify_is_server_admin(#user_ctx{roles=Roles}) ->
- case lists:member(<<"_admin">>, Roles) of
- true -> ok;
- false -> throw({unauthorized, <<"You are not a server admin.">>})
- end.
-
-log_request(#httpd{mochi_req=MochiReq,peer=Peer}, Code) ->
- ?LOG_INFO("~s - - ~s ~s ~B", [
- Peer,
- MochiReq:get(method),
- MochiReq:get(raw_path),
- Code
- ]).
-
-start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Length) ->
- log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
- Headers1 = Headers ++ server_header() ++
- couch_httpd_auth:cookie_auth_header(Req, Headers),
- Headers2 = couch_httpd_cors:cors_headers(Req, Headers1),
- Resp = MochiReq:start_response_length({Code, Headers2, Length}),
- case MochiReq:get(method) of
- 'HEAD' -> throw({http_head_abort, Resp});
- _ -> ok
- end,
- {ok, Resp}.
-
-start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) ->
- log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
- CookieHeader = couch_httpd_auth:cookie_auth_header(Req, Headers),
- Headers1 = Headers ++ server_header() ++ CookieHeader,
- Headers2 = couch_httpd_cors:cors_headers(Req, Headers1),
- Resp = MochiReq:start_response({Code, Headers2}),
- case MochiReq:get(method) of
- 'HEAD' -> throw({http_head_abort, Resp});
- _ -> ok
- end,
- {ok, Resp}.
-
-send(Resp, Data) ->
- Resp:send(Data),
- {ok, Resp}.
-
-no_resp_conn_header([]) ->
- true;
-no_resp_conn_header([{Hdr, _}|Rest]) ->
- case string:to_lower(Hdr) of
- "connection" -> false;
- _ -> no_resp_conn_header(Rest)
- end.
-
-http_1_0_keep_alive(Req, Headers) ->
- KeepOpen = Req:should_close() == false,
- IsHttp10 = Req:get(version) == {1, 0},
- NoRespHeader = no_resp_conn_header(Headers),
- case KeepOpen andalso IsHttp10 andalso NoRespHeader of
- true -> [{"Connection", "Keep-Alive"} | Headers];
- false -> Headers
- end.
-
-start_chunked_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) ->
- log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
- Headers1 = http_1_0_keep_alive(MochiReq, Headers),
- Headers2 = Headers1 ++ server_header() ++
- couch_httpd_auth:cookie_auth_header(Req, Headers1),
- Headers3 = couch_httpd_cors:cors_headers(Req, Headers2),
- Resp = MochiReq:respond({Code, Headers3, chunked}),
- case MochiReq:get(method) of
- 'HEAD' -> throw({http_head_abort, Resp});
- _ -> ok
- end,
- {ok, Resp}.
-
-send_chunk(Resp, Data) ->
- case iolist_size(Data) of
- 0 -> ok; % do nothing
- _ -> Resp:write_chunk(Data)
- end,
- {ok, Resp}.
-
-last_chunk(Resp) ->
- Resp:write_chunk([]),
- {ok, Resp}.
-
-send_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Body) ->
- log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
- Headers1 = http_1_0_keep_alive(MochiReq, Headers),
- if Code >= 500 ->
- ?LOG_ERROR("httpd ~p error response:~n ~s", [Code, Body]);
- Code >= 400 ->
- ?LOG_DEBUG("httpd ~p error response:~n ~s", [Code, Body]);
- true -> ok
- end,
- Headers2 = Headers1 ++ server_header() ++
- couch_httpd_auth:cookie_auth_header(Req, Headers1),
- Headers3 = couch_httpd_cors:cors_headers(Req, Headers2),
-
- {ok, MochiReq:respond({Code, Headers3, Body})}.
-
-send_method_not_allowed(Req, Methods) ->
- send_error(Req, 405, [{"Allow", Methods}], <<"method_not_allowed">>, ?l2b("Only " ++ Methods ++ " allowed")).
-
-send_json(Req, Value) ->
- send_json(Req, 200, Value).
-
-send_json(Req, Code, Value) ->
- send_json(Req, Code, [], Value).
-
-send_json(Req, Code, Headers, Value) ->
- initialize_jsonp(Req),
- DefaultHeaders = [
- {"Content-Type", negotiate_content_type(Req)},
- {"Cache-Control", "must-revalidate"}
- ],
- Body = [start_jsonp(), ?JSON_ENCODE(Value), end_jsonp(), $\n],
- send_response(Req, Code, DefaultHeaders ++ Headers, Body).
-
-start_json_response(Req, Code) ->
- start_json_response(Req, Code, []).
-
-start_json_response(Req, Code, Headers) ->
- initialize_jsonp(Req),
- DefaultHeaders = [
- {"Content-Type", negotiate_content_type(Req)},
- {"Cache-Control", "must-revalidate"}
- ],
- {ok, Resp} = start_chunked_response(Req, Code, DefaultHeaders ++ Headers),
- case start_jsonp() of
- [] -> ok;
- Start -> send_chunk(Resp, Start)
- end,
- {ok, Resp}.
-
-end_json_response(Resp) ->
- send_chunk(Resp, end_jsonp() ++ [$\n]),
- last_chunk(Resp).
-
-initialize_jsonp(Req) ->
- case get(jsonp) of
- undefined -> put(jsonp, qs_value(Req, "callback", no_jsonp));
- _ -> ok
- end,
- case get(jsonp) of
- no_jsonp -> [];
- [] -> [];
- CallBack ->
- try
- % make sure jsonp is configured on (default off)
- case config:get("httpd", "allow_jsonp", "false") of
- "true" ->
- validate_callback(CallBack);
- _Else ->
- put(jsonp, no_jsonp)
- end
- catch
- Error ->
- put(jsonp, no_jsonp),
- throw(Error)
- end
- end.
-
-start_jsonp() ->
- case get(jsonp) of
- no_jsonp -> [];
- [] -> [];
- CallBack -> ["/* CouchDB */", CallBack, "("]
- end.
-
-end_jsonp() ->
- case erlang:erase(jsonp) of
- no_jsonp -> [];
- [] -> [];
- _ -> ");"
- end.
-
-validate_callback(CallBack) when is_binary(CallBack) ->
- validate_callback(binary_to_list(CallBack));
-validate_callback([]) ->
- ok;
-validate_callback([Char | Rest]) ->
- case Char of
- _ when Char >= $a andalso Char =< $z -> ok;
- _ when Char >= $A andalso Char =< $Z -> ok;
- _ when Char >= $0 andalso Char =< $9 -> ok;
- _ when Char == $. -> ok;
- _ when Char == $_ -> ok;
- _ when Char == $[ -> ok;
- _ when Char == $] -> ok;
- _ ->
- throw({bad_request, invalid_callback})
- end,
- validate_callback(Rest).
-
-
-error_info({Error, Reason}) when is_list(Reason) ->
- error_info({Error, ?l2b(Reason)});
-error_info(bad_request) ->
- {400, <<"bad_request">>, <<>>};
-error_info({bad_request, Reason}) ->
- {400, <<"bad_request">>, Reason};
-error_info({query_parse_error, Reason}) ->
- {400, <<"query_parse_error">>, Reason};
-% Prior art for md5 mismatch resulting in a 400 is from AWS S3
-error_info(md5_mismatch) ->
- {400, <<"content_md5_mismatch">>, <<"Possible message corruption.">>};
-error_info(not_found) ->
- {404, <<"not_found">>, <<"missing">>};
-error_info({not_found, Reason}) ->
- {404, <<"not_found">>, Reason};
-error_info({not_acceptable, Reason}) ->
- {406, <<"not_acceptable">>, Reason};
-error_info(conflict) ->
- {409, <<"conflict">>, <<"Document update conflict.">>};
-error_info({forbidden, Msg}) ->
- {403, <<"forbidden">>, Msg};
-error_info({unauthorized, Msg}) ->
- {401, <<"unauthorized">>, Msg};
-error_info(file_exists) ->
- {412, <<"file_exists">>, <<"The database could not be "
- "created, the file already exists.">>};
-error_info(request_entity_too_large) ->
- {413, <<"too_large">>, <<"the request entity is too large">>};
-error_info({bad_ctype, Reason}) ->
- {415, <<"bad_content_type">>, Reason};
-error_info(requested_range_not_satisfiable) ->
- {416, <<"requested_range_not_satisfiable">>, <<"Requested range not satisfiable">>};
-error_info({error, illegal_database_name, Name}) ->
- Message = "Name: '" ++ Name ++ "'. Only lowercase characters (a-z), "
- ++ "digits (0-9), and any of the characters _, $, (, ), +, -, and / "
- ++ "are allowed. Must begin with a letter.",
- {400, <<"illegal_database_name">>, couch_util:to_binary(Message)};
-error_info({missing_stub, Reason}) ->
- {412, <<"missing_stub">>, Reason};
-error_info({Error, Reason}) ->
- {500, couch_util:to_binary(Error), couch_util:to_binary(Reason)};
-error_info(Error) ->
- {500, <<"unknown_error">>, couch_util:to_binary(Error)}.
-
-error_headers(#httpd{mochi_req=MochiReq}=Req, Code, ErrorStr, ReasonStr) ->
- if Code == 401 ->
- % this is where the basic auth popup is triggered
- case MochiReq:get_header_value("X-CouchDB-WWW-Authenticate") of
- undefined ->
- case config:get("httpd", "WWW-Authenticate", nil) of
- nil ->
- % If the client is a browser and the basic auth popup isn't turned on
- % redirect to the session page.
- case ErrorStr of
- <<"unauthorized">> ->
- case config:get("couch_httpd_auth", "authentication_redirect", nil) of
- nil -> {Code, []};
- AuthRedirect ->
- case config:get("couch_httpd_auth", "require_valid_user", "false") of
- "true" ->
- % send the browser popup header no matter what if we are require_valid_user
- {Code, [{"WWW-Authenticate", "Basic realm=\"server\""}]};
- _False ->
- case MochiReq:accepts_content_type("application/json") of
- true ->
- {Code, []};
- false ->
- case MochiReq:accepts_content_type("text/html") of
- true ->
- % Redirect to the path the user requested, not
- % the one that is used internally.
- UrlReturnRaw = case MochiReq:get_header_value("x-couchdb-vhost-path") of
- undefined ->
- MochiReq:get(path);
- VHostPath ->
- VHostPath
- end,
- RedirectLocation = lists:flatten([
- AuthRedirect,
- "?return=", couch_util:url_encode(UrlReturnRaw),
- "&reason=", couch_util:url_encode(ReasonStr)
- ]),
- {302, [{"Location", absolute_uri(Req, RedirectLocation)}]};
- false ->
- {Code, []}
- end
- end
- end
- end;
- _Else ->
- {Code, []}
- end;
- Type ->
- {Code, [{"WWW-Authenticate", Type}]}
- end;
- Type ->
- {Code, [{"WWW-Authenticate", Type}]}
- end;
- true ->
- {Code, []}
- end.
-
-send_error(_Req, {already_sent, Resp, _Error}) ->
- {ok, Resp};
-
-send_error(Req, Error) ->
- {Code, ErrorStr, ReasonStr} = error_info(Error),
- {Code1, Headers} = error_headers(Req, Code, ErrorStr, ReasonStr),
- send_error(Req, Code1, Headers, ErrorStr, ReasonStr).
-
-send_error(Req, Code, ErrorStr, ReasonStr) ->
- send_error(Req, Code, [], ErrorStr, ReasonStr).
-
-send_error(Req, Code, Headers, ErrorStr, ReasonStr) ->
- send_json(Req, Code, Headers,
- {[{<<"error">>, ErrorStr},
- {<<"reason">>, ReasonStr}]}).
-
-% give the option for list functions to output html or other raw errors
-send_chunked_error(Resp, {_Error, {[{<<"body">>, Reason}]}}) ->
- send_chunk(Resp, Reason),
- last_chunk(Resp);
-
-send_chunked_error(Resp, Error) ->
- {Code, ErrorStr, ReasonStr} = error_info(Error),
- JsonError = {[{<<"code">>, Code},
- {<<"error">>, ErrorStr},
- {<<"reason">>, ReasonStr}]},
- send_chunk(Resp, ?l2b([$\n,?JSON_ENCODE(JsonError),$\n])),
- last_chunk(Resp).
-
-send_redirect(Req, Path) ->
- send_response(Req, 301, [{"Location", absolute_uri(Req, Path)}], <<>>).
-
-negotiate_content_type(Req) ->
- case get(jsonp) of
- no_jsonp -> negotiate_content_type1(Req);
- [] -> negotiate_content_type1(Req);
- _Callback -> "text/javascript"
- end.
-
-negotiate_content_type1(#httpd{mochi_req=MochiReq}) ->
- %% Determine the appropriate Content-Type header for a JSON response
- %% depending on the Accept header in the request. A request that explicitly
- %% lists the correct JSON MIME type will get that type, otherwise the
- %% response will have the generic MIME type "text/plain"
- AcceptedTypes = case MochiReq:get_header_value("Accept") of
- undefined -> [];
- AcceptHeader -> string:tokens(AcceptHeader, ", ")
- end,
- case lists:member("application/json", AcceptedTypes) of
- true -> "application/json";
- false -> "text/plain; charset=utf-8"
- end.
-
-server_header() ->
- [{"Server", "CouchDB/" ++ couch_server:get_version() ++
- " (Erlang OTP/" ++ erlang:system_info(otp_release) ++ ")"}].
-
-
--record(mp, {boundary, buffer, data_fun, callback}).
-
-
-parse_multipart_request(ContentType, DataFun, Callback) ->
- Boundary0 = iolist_to_binary(get_boundary(ContentType)),
- Boundary = <<"\r\n--", Boundary0/binary>>,
- Mp = #mp{boundary= Boundary,
- buffer= <<>>,
- data_fun=DataFun,
- callback=Callback},
- {Mp2, _NilCallback} = read_until(Mp, <<"--", Boundary0/binary>>,
- fun nil_callback/1),
- #mp{buffer=Buffer, data_fun=DataFun2, callback=Callback2} =
- parse_part_header(Mp2),
- {Buffer, DataFun2, Callback2}.
-
-nil_callback(_Data)->
- fun nil_callback/1.
-
-get_boundary({"multipart/" ++ _, Opts}) ->
- case couch_util:get_value("boundary", Opts) of
- S when is_list(S) ->
- S
- end;
-get_boundary(ContentType) ->
- {"multipart/" ++ _ , Opts} = mochiweb_util:parse_header(ContentType),
- get_boundary({"multipart/", Opts}).
-
-
-
-split_header(<<>>) ->
- [];
-split_header(Line) ->
- {Name, [$: | Value]} = lists:splitwith(fun (C) -> C =/= $: end,
- binary_to_list(Line)),
- [{string:to_lower(string:strip(Name)),
- mochiweb_util:parse_header(Value)}].
-
-read_until(#mp{data_fun=DataFun, buffer=Buffer}=Mp, Pattern, Callback) ->
- case find_in_binary(Pattern, Buffer) of
- not_found ->
- Callback2 = Callback(Buffer),
- {Buffer2, DataFun2} = DataFun(),
- Buffer3 = iolist_to_binary(Buffer2),
- read_until(Mp#mp{data_fun=DataFun2,buffer=Buffer3}, Pattern, Callback2);
- {partial, 0} ->
- {NewData, DataFun2} = DataFun(),
- read_until(Mp#mp{data_fun=DataFun2,
- buffer= iolist_to_binary([Buffer,NewData])},
- Pattern, Callback);
- {partial, Skip} ->
- <<DataChunk:Skip/binary, Rest/binary>> = Buffer,
- Callback2 = Callback(DataChunk),
- {NewData, DataFun2} = DataFun(),
- read_until(Mp#mp{data_fun=DataFun2,
- buffer= iolist_to_binary([Rest | NewData])},
- Pattern, Callback2);
- {exact, 0} ->
- PatternLen = size(Pattern),
- <<_:PatternLen/binary, Rest/binary>> = Buffer,
- {Mp#mp{buffer= Rest}, Callback};
- {exact, Skip} ->
- PatternLen = size(Pattern),
- <<DataChunk:Skip/binary, _:PatternLen/binary, Rest/binary>> = Buffer,
- Callback2 = Callback(DataChunk),
- {Mp#mp{buffer= Rest}, Callback2}
- end.
-
-
-parse_part_header(#mp{callback=UserCallBack}=Mp) ->
- {Mp2, AccCallback} = read_until(Mp, <<"\r\n\r\n">>,
- fun(Next) -> acc_callback(Next, []) end),
- HeaderData = AccCallback(get_data),
-
- Headers =
- lists:foldl(fun(Line, Acc) ->
- split_header(Line) ++ Acc
- end, [], re:split(HeaderData,<<"\r\n">>, [])),
- NextCallback = UserCallBack({headers, Headers}),
- parse_part_body(Mp2#mp{callback=NextCallback}).
-
-parse_part_body(#mp{boundary=Prefix, callback=Callback}=Mp) ->
- {Mp2, WrappedCallback} = read_until(Mp, Prefix,
- fun(Data) -> body_callback_wrapper(Data, Callback) end),
- Callback2 = WrappedCallback(get_callback),
- Callback3 = Callback2(body_end),
- case check_for_last(Mp2#mp{callback=Callback3}) of
- {last, #mp{callback=Callback3}=Mp3} ->
- Mp3#mp{callback=Callback3(eof)};
- {more, Mp3} ->
- parse_part_header(Mp3)
- end.
-
-acc_callback(get_data, Acc)->
- iolist_to_binary(lists:reverse(Acc));
-acc_callback(Data, Acc)->
- fun(Next) -> acc_callback(Next, [Data | Acc]) end.
-
-body_callback_wrapper(get_callback, Callback) ->
- Callback;
-body_callback_wrapper(Data, Callback) ->
- Callback2 = Callback({body, Data}),
- fun(Next) -> body_callback_wrapper(Next, Callback2) end.
-
-
-check_for_last(#mp{buffer=Buffer, data_fun=DataFun}=Mp) ->
- case Buffer of
- <<"--",_/binary>> -> {last, Mp};
- <<_, _, _/binary>> -> {more, Mp};
- _ -> % not long enough
- {Data, DataFun2} = DataFun(),
- check_for_last(Mp#mp{buffer= <<Buffer/binary, Data/binary>>,
- data_fun = DataFun2})
- end.
-
-find_in_binary(_B, <<>>) ->
- not_found;
-
-find_in_binary(B, Data) ->
- case binary:match(Data, [B], []) of
- nomatch ->
- partial_find(binary:part(B, {0, byte_size(B) - 1}),
- binary:part(Data, {byte_size(Data), -byte_size(Data) + 1}), 1);
- {Pos, _Len} ->
- {exact, Pos}
- end.
-
-partial_find(<<>>, _Data, _Pos) ->
- not_found;
-
-partial_find(B, Data, N) when byte_size(Data) > 0 ->
- case binary:match(Data, [B], []) of
- nomatch ->
- partial_find(binary:part(B, {0, byte_size(B) - 1}),
- binary:part(Data, {byte_size(Data), -byte_size(Data) + 1}), N + 1);
- {Pos, _Len} ->
- {partial, N + Pos}
- end;
-
-partial_find(_B, _Data, _N) ->
- not_found.
-
-
-validate_bind_address(Address) ->
- case inet_parse:address(Address) of
- {ok, _} -> ok;
- _ -> throw({error, invalid_bind_address})
- end.