You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/02/12 07:21:12 UTC
[23/33] ibrowse commit: updated refs/heads/import-master to 1167b0e
Updated ibrowse to version 2.1.0. It contains fixes for the following important issues:
- https://github.com/cmullaparthi/ibrowse/issues/closed#issue/17
- https://github.com/cmullaparthi/ibrowse/issues/closed#issue/15
- https://github.com/cmullaparthi/ibrowse/issues/closed#issue/19
git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1033456 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/commit/0db80d3e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/tree/0db80d3e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/diff/0db80d3e
Branch: refs/heads/import-master
Commit: 0db80d3e8bdd81f451fbeb002ef8bd107f653f31
Parents: a284c87
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Wed Nov 10 13:34:16 2010 +0000
Committer: Filipe David Borba Manana <fd...@apache.org>
Committed: Wed Nov 10 13:34:16 2010 +0000
----------------------------------------------------------------------
Makefile.am | 2 +-
ibrowse.app.in | 2 +-
ibrowse.erl | 33 ++++++-
ibrowse_http_client.erl | 211 ++++++++++++++++++++++++++++---------------
ibrowse_lib.erl | 2 +-
ibrowse_test.erl | 45 ++++++---
6 files changed, 201 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 39878f0..8c5d3f8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,7 +10,7 @@
## License for the specific language governing permissions and limitations under
## the License.
-ibrowseebindir = $(localerlanglibdir)/ibrowse-2.0.1/ebin
+ibrowseebindir = $(localerlanglibdir)/ibrowse-2.1.0/ebin
ibrowse_file_collection = \
ibrowse.app.in \
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/ibrowse.app.in
----------------------------------------------------------------------
diff --git a/ibrowse.app.in b/ibrowse.app.in
index 8fc2066..e8580d1 100644
--- a/ibrowse.app.in
+++ b/ibrowse.app.in
@@ -1,6 +1,6 @@
{application, ibrowse,
[{description, "HTTP client application"},
- {vsn, "2.0.1"},
+ {vsn, "2.1.0"},
{modules, [ ibrowse,
ibrowse_http_client,
ibrowse_app,
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/ibrowse.erl
----------------------------------------------------------------------
diff --git a/ibrowse.erl b/ibrowse.erl
index 7f8d8bc..1a42f4b 100644
--- a/ibrowse.erl
+++ b/ibrowse.erl
@@ -7,8 +7,8 @@
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
%% @copyright 2005-2010 Chandrashekhar Mullaparthi
-%% @version 2.0.1
-%% @doc The ibrowse application implements an HTTP 1.1 client. This
+%% @version 2.1.0
+%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
%% one process to handle one TCP connection to a webserver
@@ -87,6 +87,7 @@
send_req_direct/6,
send_req_direct/7,
stream_next/1,
+ stream_close/1,
set_max_sessions/3,
set_max_pipeline_size/3,
set_dest/3,
@@ -201,7 +202,11 @@ send_req(Url, Headers, Method, Body) ->
%% dealing with large response bodies and/or slow links. In these
%% cases, it might be hard to estimate how long a request will take to
%% complete. In such cases, the client might want to timeout if no
-%% data has been received on the link for a certain time interval.</li>
+%% data has been received on the link for a certain time interval.
+%%
+%% This value is also used to close connections which are not in use for
+%% the specified timeout value.
+%% </li>
%%
%% <li>
%% The <code>connect_timeout</code> option is to specify how long the
@@ -458,6 +463,8 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body.
spawn_worker_process(Url) ->
ibrowse_http_client:start(Url).
+%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port
+%% instead of a URL.
%% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
spawn_worker_process(Host, Port) ->
ibrowse_http_client:start({Host, Port}).
@@ -468,6 +475,8 @@ spawn_worker_process(Host, Port) ->
spawn_link_worker_process(Url) ->
ibrowse_http_client:start_link(Url).
+%% @doc Same as spawn_worker_process/2 except the the calling process
+%% is linked to the worker process which is spawned.
%% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
spawn_link_worker_process(Host, Port) ->
ibrowse_http_client:start_link({Host, Port}).
@@ -524,6 +533,21 @@ stream_next(Req_id) ->
ok
end.
+%% @doc Tell ibrowse to close the connection associated with the
+%% specified stream. Should be used in conjunction with the
+%% <code>stream_to</code> option. Note that all requests in progress on
+%% the connection which is serving this Req_id will be aborted, and an
+%% error returned.
+%% @spec stream_close(Req_id :: req_id()) -> ok | {error, unknown_req_id}
+stream_close(Req_id) ->
+ case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of
+ [] ->
+ {error, unknown_req_id};
+ [{_, Pid}] ->
+ catch Pid ! {stream_close, Req_id},
+ ok
+ end.
+
%% @doc Turn tracing on for the ibrowse process
trace_on() ->
ibrowse ! {trace, true}.
@@ -553,6 +577,9 @@ all_trace_off() ->
ibrowse ! all_trace_off,
ok.
+%% @doc Shows some internal information about load balancing. Info
+%% about workers spawned using spawn_worker_process/2 or
+%% spawn_link_worker_process/2 is not included.
show_dest_status() ->
Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
is_integer(Port) ->
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/ibrowse_http_client.erl
----------------------------------------------------------------------
diff --git a/ibrowse_http_client.erl b/ibrowse_http_client.erl
index 2dd209d..5c3d5c9 100644
--- a/ibrowse_http_client.erl
+++ b/ibrowse_http_client.erl
@@ -37,6 +37,7 @@
-include("ibrowse.hrl").
-record(state, {host, port, connect_timeout,
+ inactivity_timer_ref,
use_proxy = false, proxy_auth_digest,
ssl_options = [], is_ssl = false, socket,
proxy_tunnel_setup = false,
@@ -192,6 +193,12 @@ handle_info({stream_next, Req_id}, #state{socket = Socket,
handle_info({stream_next, _Req_id}, State) ->
{noreply, State};
+handle_info({stream_close, _Req_id}, State) ->
+ shutting_down(State),
+ do_close(State),
+ do_error_reply(State, closing_on_request),
+ {stop, normal, ok, State};
+
handle_info({tcp_closed, _Sock}, State) ->
do_trace("TCP connection closed by peer!~n", []),
handle_sock_closed(State),
@@ -221,6 +228,7 @@ handle_info({req_timedout, From}, State) ->
end;
handle_info(timeout, State) ->
+ do_trace("Inactivity timeout triggered. Shutting down connection~n", []),
shutting_down(State),
do_error_reply(State, req_timedout),
{stop, normal, State};
@@ -273,8 +281,8 @@ handle_sock_data(Data, #state{status = get_header}=State) ->
{stop, normal, State};
State_1 ->
active_once(State_1),
- set_inac_timer(State_1),
- {noreply, State_1}
+ State_2 = set_inac_timer(State_1),
+ {noreply, State_2}
end;
handle_sock_data(Data, #state{status = get_body,
@@ -293,8 +301,8 @@ handle_sock_data(Data, #state{status = get_body,
{stop, normal, State};
State_1 ->
active_once(State_1),
- set_inac_timer(State_1),
- {noreply, State_1}
+ State_2 = set_inac_timer(State_1),
+ {noreply, State_2}
end;
_ ->
case parse_11_response(Data, State) of
@@ -314,12 +322,12 @@ handle_sock_data(Data, #state{status = get_body,
active_once(State_1)
end,
State_2 = State_1#state{interim_reply_sent = false},
- set_inac_timer(State_2),
- {noreply, State_2};
+ State_3 = set_inac_timer(State_2),
+ {noreply, State_3};
State_1 ->
active_once(State_1),
- set_inac_timer(State_1),
- {noreply, State_1}
+ State_2 = set_inac_timer(State_1),
+ {noreply, State_2}
end
end.
@@ -507,29 +515,37 @@ do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req).
%% {fun_arity_0} |
%% {fun_arity_1, term()}
%% error() = term()
-do_send_body(Source, State) when is_function(Source) ->
- do_send_body({Source}, State);
-do_send_body({Source}, State) when is_function(Source) ->
- do_send_body1(Source, Source(), State);
-do_send_body({Source, Source_state}, State) when is_function(Source) ->
- do_send_body1(Source, Source(Source_state), State);
-do_send_body(Body, State) ->
+do_send_body(Source, State, TE) when is_function(Source) ->
+ do_send_body({Source}, State, TE);
+do_send_body({Source}, State, TE) when is_function(Source) ->
+ do_send_body1(Source, Source(), State, TE);
+do_send_body({Source, Source_state}, State, TE) when is_function(Source) ->
+ do_send_body1(Source, Source(Source_state), State, TE);
+do_send_body(Body, State, _TE) ->
do_send(Body, State).
-do_send_body1(Source, Resp, State) ->
+do_send_body1(Source, Resp, State, TE) ->
case Resp of
{ok, Data} ->
- do_send(Data, State),
- do_send_body({Source}, State);
+ do_send(maybe_chunked_encode(Data, TE), State),
+ do_send_body({Source}, State, TE);
{ok, Data, New_source_state} ->
- do_send(Data, State),
- do_send_body({Source, New_source_state}, State);
+ do_send(maybe_chunked_encode(Data, TE), State),
+ do_send_body({Source, New_source_state}, State, TE);
+ eof when TE == true ->
+ do_send(<<"0\r\n\r\n">>, State),
+ ok;
eof ->
ok;
Err ->
Err
end.
+maybe_chunked_encode(Data, false) ->
+ Data;
+maybe_chunked_encode(Data, true) ->
+ [ibrowse_lib:dec2hex(4, size(to_binary(Data))), "\r\n", Data, "\r\n"].
+
do_close(#state{socket = undefined}) -> ok;
do_close(#state{socket = Sock,
is_ssl = true,
@@ -619,11 +635,13 @@ send_req_1(From,
{Req, Body_1} = make_request(connect, Pxy_auth_headers,
Path, Path,
[], Options, State_1),
+ TE = is_chunked_encoding_specified(Options),
trace_request(Req),
case do_send(Req, State) of
ok ->
- case do_send_body(Body_1, State_1) of
+ case do_send_body(Body_1, State_1, TE) of
ok ->
+ trace_request_body(Body_1),
active_once(State_1),
Ref = case Timeout of
infinity ->
@@ -636,8 +654,8 @@ send_req_1(From,
send_timer = Ref,
proxy_tunnel_setup = in_progress,
tunnel_setup_queue = [{From, Url, Headers, Method, Body, Options, Timeout}]},
- set_inac_timer(State_1),
- {noreply, State_2};
+ State_3 = set_inac_timer(State_2),
+ {noreply, State_3};
Err ->
shutting_down(State_1),
do_trace("Send failed... Reason: ~p~n", [Err]),
@@ -706,10 +724,12 @@ send_req_1(From,
AbsPath, RelPath, Body, Options, State_1),
trace_request(Req),
do_setopts(Socket, Caller_socket_options, Is_ssl),
+ TE = is_chunked_encoding_specified(Options),
case do_send(Req, State_1) of
ok ->
- case do_send_body(Body_1, State_1) of
+ case do_send_body(Body_1, State_1, TE) of
ok ->
+ trace_request_body(Body_1),
State_2 = inc_pipeline_counter(State_1),
active_once(State_2),
Ref = case Timeout of
@@ -732,8 +752,8 @@ send_req_1(From,
_ ->
gen_server:reply(From, {ibrowse_req_id, ReqId})
end,
- set_inac_timer(State_1),
- {noreply, State_3};
+ State_4 = set_inac_timer(State_3),
+ {noreply, State_4};
Err ->
shutting_down(State_1),
do_trace("Send failed... Reason: ~p~n", [Err]),
@@ -759,6 +779,7 @@ maybe_modify_headers(#url{host = Host, port = Port} = Url,
false ->
case Port of
80 -> Host;
+ 443 -> Host;
_ -> [Host, ":", integer_to_list(Port)]
end;
{value, {_, Host_h_val}} ->
@@ -802,31 +823,42 @@ http_auth_digest(Username, Password) ->
make_request(Method, Headers, AbsPath, RelPath, Body, Options,
#state{use_proxy = UseProxy, is_ssl = Is_ssl}) ->
HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})),
+ Fun1 = fun({X, Y}) when is_atom(X) ->
+ {to_lower(atom_to_list(X)), X, Y};
+ ({X, Y}) when is_list(X) ->
+ {to_lower(X), X, Y}
+ end,
+ Headers_0 = [Fun1(X) || X <- Headers],
Headers_1 =
- case get_value(content_length, Headers, false) of
- false when (Body == []) or
- (Body == <<>>) or
- is_tuple(Body) or
- is_function(Body) ->
- Headers;
+ case lists:keysearch("content-length", 1, Headers_0) of
+ false when (Body == []) orelse
+ (Body == <<>>) orelse
+ is_tuple(Body) orelse
+ is_function(Body) ->
+ Headers_0;
false when is_binary(Body) ->
- [{"content-length", integer_to_list(size(Body))} | Headers];
- false ->
- [{"content-length", integer_to_list(length(Body))} | Headers];
+ [{"content-length", "content-length", integer_to_list(size(Body))} | Headers_0];
+ false when is_list(Body) ->
+ [{"content-length", "content-length", integer_to_list(length(Body))} | Headers_0];
_ ->
- Headers
+ %% Content-Length is already specified
+ Headers_0
end,
{Headers_2, Body_1} =
- case get_value(transfer_encoding, Options, false) of
+ case is_chunked_encoding_specified(Options) of
false ->
- {Headers_1, Body};
- {chunked, ChunkSize} ->
- {[{X, Y} || {X, Y} <- Headers_1,
- X /= "Content-Length",
- X /= "content-length",
- X /= content_length] ++
+ {[{Y, Z} || {_, Y, Z} <- Headers_1], Body};
+ true ->
+ Chunk_size_1 = case get_value(transfer_encoding, Options) of
+ chunked ->
+ 5120;
+ {chunked, Chunk_size} ->
+ Chunk_size
+ end,
+ {[{Y, Z} || {X, Y, Z} <- Headers_1,
+ X /= "content-length"] ++
[{"Transfer-Encoding", "chunked"}],
- chunk_request_body(Body, ChunkSize)}
+ chunk_request_body(Body, Chunk_size_1)}
end,
Headers_3 = cons_headers(Headers_2),
Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of
@@ -842,6 +874,16 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options,
end,
{[method(Method), " ", Uri, " ", HttpVsn, crnl(), Headers_3, crnl()], Body_1}.
+is_chunked_encoding_specified(Options) ->
+ case get_value(transfer_encoding, Options, false) of
+ false ->
+ false;
+ {chunked, _} ->
+ true;
+ chunked ->
+ true
+ end.
+
http_vsn_string({0,9}) -> "HTTP/0.9";
http_vsn_string({1,0}) -> "HTTP/1.0";
http_vsn_string({1,1}) -> "HTTP/1.1".
@@ -873,6 +915,9 @@ encode_headers([{Name,Val} | T], Acc) when is_atom(Name) ->
encode_headers([], Acc) ->
lists:reverse(Acc).
+chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse
+ is_function(Body) ->
+ Body;
chunk_request_body(Body, ChunkSize) ->
chunk_request_body(Body, ChunkSize, []).
@@ -1060,7 +1105,7 @@ upgrade_to_ssl(#state{socket = Socket,
send_queued_requests([], State) ->
do_trace("Sent all queued requests via SSL connection~n", []),
- State#state{tunnel_setup_queue = done};
+ State#state{tunnel_setup_queue = []};
send_queued_requests([{From, Url, Headers, Method, Body, Options, Timeout} | Q],
State) ->
case send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State) of
@@ -1217,7 +1262,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
reply_buffer = RepBuf,
recvd_headers = RespHeaders}=State) when SaveResponseToFile /= false ->
Body = RepBuf,
- State_1 = set_cur_request(State),
file:close(Fd),
ResponseBody = case TmpFilename of
undefined ->
@@ -1232,9 +1276,9 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
false ->
{ok, SCode, Resp_headers_1, ResponseBody}
end,
- State_2 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
+ State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
- State_2;
+ set_cur_request(State_1);
handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
options = Options},
@@ -1245,7 +1289,6 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
reply_buffer = RepBuf,
send_timer = ReqTimer} = State) ->
Body = RepBuf,
-%% State_1 = set_cur_request(State),
{Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options),
Reply = case get_value(give_raw_headers, Options, false) of
true ->
@@ -1253,15 +1296,8 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
false ->
{ok, SCode, Resp_headers_1, Body}
end,
- State_1 = case get(conn_close) of
- "close" ->
- do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
- exit(normal);
- _ ->
- State_1_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
- cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
- State_1_1
- end,
+ State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
+ cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
set_cur_request(State_1).
reset_state(State) ->
@@ -1353,6 +1389,8 @@ parse_status_line([32 | T], get_prot_vsn, ProtVsn, StatCode) ->
parse_status_line(T, get_status_code, ProtVsn, StatCode);
parse_status_line([32 | T], get_status_code, ProtVsn, StatCode) ->
{ok, lists:reverse(ProtVsn), lists:reverse(StatCode), T};
+parse_status_line([], get_status_code, ProtVsn, StatCode) ->
+ {ok, lists:reverse(ProtVsn), lists:reverse(StatCode), []};
parse_status_line([H | T], get_prot_vsn, ProtVsn, StatCode) ->
parse_status_line(T, get_prot_vsn, [H|ProtVsn], StatCode);
parse_status_line([H | T], get_status_code, ProtVsn, StatCode) ->
@@ -1710,36 +1748,61 @@ get_stream_chunk_size(Options) ->
end.
set_inac_timer(State) ->
- set_inac_timer(State, get_inac_timeout(State)).
-
-set_inac_timer(_State, Timeout) when is_integer(Timeout) ->
- TimerRef = erlang:send_after(Timeout, self(), timeout),
- case erlang:put(inac_timer, TimerRef) of
- OldTimer when is_reference(OldTimer) ->
- erlang:cancel_timer(OldTimer),
- receive timeout -> ok after 0 -> ok end;
- _ ->
- ok
- end,
- TimerRef;
-set_inac_timer(_, _) ->
- undefined.
+ cancel_timer(State#state.inactivity_timer_ref),
+ set_inac_timer(State#state{inactivity_timer_ref = undefined},
+ get_inac_timeout(State)).
+
+set_inac_timer(State, Timeout) when is_integer(Timeout) ->
+ Ref = erlang:send_after(Timeout, self(), timeout),
+ State#state{inactivity_timer_ref = Ref};
+set_inac_timer(State, _) ->
+ State.
get_inac_timeout(#state{cur_req = #request{options = Opts}}) ->
get_value(inactivity_timeout, Opts, infinity);
get_inac_timeout(#state{cur_req = undefined}) ->
- infinity.
+ case ibrowse:get_config_value(inactivity_timeout, undefined) of
+ Val when is_integer(Val) ->
+ Val;
+ _ ->
+ case application:get_env(ibrowse, inactivity_timeout) of
+ {ok, Val} when is_integer(Val), Val > 0 ->
+ Val;
+ _ ->
+ 10000
+ end
+ end.
trace_request(Req) ->
case get(my_trace_flag) of
true ->
%%Avoid the binary operations if trace is not on...
- NReq = binary_to_list(list_to_binary(Req)),
+ NReq = to_binary(Req),
do_trace("Sending request: ~n"
"--- Request Begin ---~n~s~n"
"--- Request End ---~n", [NReq]);
_ -> ok
end.
+trace_request_body(Body) ->
+ case get(my_trace_flag) of
+ true ->
+ %%Avoid the binary operations if trace is not on...
+ NBody = to_binary(Body),
+ case size(NBody) > 1024 of
+ true ->
+ ok;
+ false ->
+ do_trace("Sending request body: ~n"
+ "--- Request Body Begin ---~n~s~n"
+ "--- Request Body End ---~n", [NBody])
+ end;
+ false ->
+ ok
+ end.
+
to_integer(X) when is_list(X) -> list_to_integer(X);
to_integer(X) when is_integer(X) -> X.
+
+to_binary(X) when is_list(X) -> list_to_binary(X);
+to_binary(X) when is_binary(X) -> X.
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/ibrowse_lib.erl
----------------------------------------------------------------------
diff --git a/ibrowse_lib.erl b/ibrowse_lib.erl
index fbb9c34..c463c7b 100644
--- a/ibrowse_lib.erl
+++ b/ibrowse_lib.erl
@@ -208,7 +208,7 @@ parse_url(Url) ->
parse_url([$:, $/, $/ | _], get_protocol, Url, []) ->
{invalid_uri_1, Url};
parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) ->
- Prot = list_to_atom(lists:reverse(TmpAcc)),
+ Prot = list_to_existing_atom(lists:reverse(TmpAcc)),
parse_url(T, get_username,
Url#url{protocol = Prot},
[]);
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/0db80d3e/ibrowse_test.erl
----------------------------------------------------------------------
diff --git a/ibrowse_test.erl b/ibrowse_test.erl
index e7d6e59..3ad7660 100644
--- a/ibrowse_test.erl
+++ b/ibrowse_test.erl
@@ -217,14 +217,18 @@ dump_errors(Key, Iod) ->
{"http://jigsaw.w3.org/HTTP/300/", get},
{"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
{"http://jigsaw.w3.org/HTTP/CL/", get},
- {"http://www.httpwatch.com/httpgallery/chunked/", get}
+ {"http://www.httpwatch.com/httpgallery/chunked/", get},
+ {"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
]).
unit_tests() ->
unit_tests([]).
unit_tests(Options) ->
+ application:start(crypto),
+ application:start(public_key),
application:start(ssl),
+ ibrowse:start(),
Options_1 = Options ++ [{connect_timeout, 5000}],
{Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]),
receive
@@ -249,32 +253,45 @@ verify_chunked_streaming() ->
verify_chunked_streaming([]).
verify_chunked_streaming(Options) ->
+ io:format("~nVerifying that chunked streaming is working...~n", []),
Url = "http://www.httpwatch.com/httpgallery/chunked/",
- io:format("URL: ~s~n", [Url]),
- io:format("Fetching data without streaming...~n", []),
+ io:format(" URL: ~s~n", [Url]),
+ io:format(" Fetching data without streaming...~n", []),
Result_without_streaming = ibrowse:send_req(
Url, [], get, [],
[{response_format, binary} | Options]),
- io:format("Fetching data with streaming as list...~n", []),
+ io:format(" Fetching data with streaming as list...~n", []),
Async_response_list = do_async_req_list(
Url, get, [{response_format, list} | Options]),
- io:format("Fetching data with streaming as binary...~n", []),
+ io:format(" Fetching data with streaming as binary...~n", []),
Async_response_bin = do_async_req_list(
Url, get, [{response_format, binary} | Options]),
- io:format("Fetching data with streaming as binary, {active, once}...~n", []),
+ io:format(" Fetching data with streaming as binary, {active, once}...~n", []),
Async_response_bin_once = do_async_req_list(
Url, get, [once, {response_format, binary} | Options]),
- compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
- compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once).
+ Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin),
+ Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once),
+ case {Res1, Res2} of
+ {success, success} ->
+ io:format(" Chunked streaming working~n", []);
+ _ ->
+ ok
+ end.
test_chunked_streaming_once() ->
test_chunked_streaming_once([]).
test_chunked_streaming_once(Options) ->
+ io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []),
Url = "http://www.httpwatch.com/httpgallery/chunked/",
- io:format("URL: ~s~n", [Url]),
- io:format("Fetching data with streaming as binary, {active, once}...~n", []),
- do_async_req_list(Url, get, [once, {response_format, binary} | Options]).
+ io:format(" URL: ~s~n", [Url]),
+ io:format(" Fetching data with streaming as binary, {active, once}...~n", []),
+ case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of
+ {ok, _, _, _} ->
+ io:format(" Success!~n", []);
+ Err ->
+ io:format(" Fail: ~p~n", [Err])
+ end.
compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) ->
success;
@@ -310,7 +327,7 @@ do_async_req_list(Url, Method, Options) ->
{Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list,
[self(), Url, Method,
Options ++ [{stream_chunk_size, 1000}]]),
- io:format("Spawned process ~p~n", [Pid]),
+%% io:format("Spawned process ~p~n", [Pid]),
wait_for_resp(Pid).
wait_for_resp(Pid) ->
@@ -354,7 +371,7 @@ wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) ->
maybe_stream_next(Req_id, Options),
wait_for_async_resp(Req_id, Options, StatCode, Headers, Body);
{ibrowse_async_response_end, Req_id} ->
- io:format("Recvd end of response.~n", []),
+ %% io:format("Recvd end of response.~n", []),
Body_1 = list_to_binary(lists:reverse(Body)),
{ok, Acc_Stat_code, Acc_Headers, Body_1};
{ibrowse_async_response, Req_id, Data} ->
@@ -384,7 +401,7 @@ execute_req(Url, Method, Options) ->
{ok, SCode, _H, _B} ->
io:format("Status code: ~p~n", [SCode]);
Err ->
- io:format("Err -> ~p~n", [Err])
+ io:format("~p~n", [Err])
end.
drv_ue_test() ->