You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2014/08/15 22:24:46 UTC
[19/50] [abbrv] couch commit: updated refs/heads/1963-eunit-bigcouch
to 95bfc03
Port 140-attachments-comp.t etap test suite to eunit
- Merge into couchdb_attachments_tests suite;
- Add PUT requests to test_request util;
- Remove dependency from files outside fixtures directory;
- Group test cases to reduce amount of duplicate code;
- Fix hidden issue with gzip encoding: for encoding_length stub info
check using zlib:gzip on 2KiB+ files leads to mismatch by 2-4 bytes
and this difference grows with file size. Using gzip fun code from
couch_stream solves the issue.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/31b76762
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/31b76762
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/31b76762
Branch: refs/heads/1963-eunit-bigcouch
Commit: 31b76762682011c6b52067437cb380705f7eeb31
Parents: 60ed292
Author: Alexander Shorin <kx...@apache.org>
Authored: Tue Jun 3 01:51:46 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Fri Aug 15 13:23:44 2014 -0700
----------------------------------------------------------------------
test/couchdb/couchdb_attachments_tests.erl | 380 +++++++++++++++++++++++-
test/couchdb/fixtures/logo.png | Bin 0 -> 3010 bytes
test/couchdb/test_request.erl | 9 +-
3 files changed, 385 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/31b76762/test/couchdb/couchdb_attachments_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb/couchdb_attachments_tests.erl b/test/couchdb/couchdb_attachments_tests.erl
index aef1873..cf59785 100644
--- a/test/couchdb/couchdb_attachments_tests.erl
+++ b/test/couchdb/couchdb_attachments_tests.erl
@@ -15,13 +15,23 @@
-include("couch_eunit.hrl").
-include_lib("couchdb/couch_db.hrl").
+-define(COMPRESSION_LEVEL, 8).
+-define(ATT_BIN_NAME, <<"logo.png">>).
+-define(ATT_TXT_NAME, <<"file.erl">>).
+-define(FIXTURE_PNG, filename:join([?FIXTURESDIR, "logo.png"])).
+-define(FIXTURE_TXT, ?FILE).
-define(TIMEOUT, 1000).
+-define(TIMEOUT_EUNIT, 10).
-define(TIMEWAIT, 100).
-define(i2l(I), integer_to_list(I)).
start() ->
{ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ % ensure in default compression settings for attachments_compression_tests
+ couch_config:set("attachments", "compression_level",
+ ?i2l(?COMPRESSION_LEVEL), false),
+ couch_config:set("attachments", "compressible_types", "text/*", false),
Pid.
stop(Pid) ->
@@ -43,7 +53,35 @@ setup() ->
Host = Addr ++ ":" ++ ?i2l(Port),
{Host, ?b2l(DbName)}.
+setup({binary, standalone}) ->
+ {Host, DbName} = setup(),
+ setup_att(fun create_standalone_png_att/2, Host, DbName, ?FIXTURE_PNG);
+setup({text, standalone}) ->
+ {Host, DbName} = setup(),
+ setup_att(fun create_standalone_text_att/2, Host, DbName, ?FIXTURE_TXT);
+setup({binary, inline}) ->
+ {Host, DbName} = setup(),
+ setup_att(fun create_inline_png_att/2, Host, DbName, ?FIXTURE_PNG);
+setup({text, inline}) ->
+ {Host, DbName} = setup(),
+ setup_att(fun create_inline_text_att/2, Host, DbName, ?FIXTURE_TXT);
+setup(compressed) ->
+ {Host, DbName} = setup(),
+ setup_att(fun create_already_compressed_att/2, Host, DbName, ?FIXTURE_TXT).
+setup_att(Fun, Host, DbName, File) ->
+ HttpHost = "http://" ++ Host,
+ AttUrl = Fun(HttpHost, DbName),
+ {ok, Data} = file:read_file(File),
+ DocUrl = string:join([HttpHost, DbName, "doc"], "/"),
+ Helpers = {DbName, DocUrl, AttUrl},
+ {Data, Helpers}.
+
+teardown(_, {_, {DbName, _, _}}) ->
+ teardown(DbName).
+
teardown({_, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
ok = couch_server:delete(?l2b(DbName), []),
ok.
@@ -55,7 +93,8 @@ attachments_test_() ->
setup,
fun start/0, fun stop/1,
[
- attachments_md5_tests()
+ attachments_md5_tests(),
+ attachments_compression_tests()
]
}
}.
@@ -79,6 +118,67 @@ attachments_md5_tests() ->
}
}.
+attachments_compression_tests() ->
+ Funs = [
+ fun should_get_att_without_accept_gzip_encoding/2,
+ fun should_get_att_with_accept_gzip_encoding/2,
+ fun should_get_att_with_accept_deflate_encoding/2,
+ fun should_return_406_response_on_unsupported_encoding/2,
+ fun should_get_doc_with_att_data/2,
+ fun should_get_doc_with_att_data_stub/2
+ ],
+ {
+ "Attachments compression tests",
+ [
+ {
+ "Created via Attachments API",
+ created_attachments_compression_tests(standalone, Funs)
+ },
+ {
+ "Created inline via Document API",
+ created_attachments_compression_tests(inline, Funs)
+ },
+ {
+ "Created already been compressed via Attachments API",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{compressed, Fun} || Fun <- Funs]
+ }
+ },
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun should_not_create_compressed_att_with_deflate_encoding/1,
+ fun should_not_create_compressed_att_with_compress_encoding/1,
+ fun should_create_compressible_att_with_ctype_params/1
+ ]
+ }
+ ]
+ }.
+
+created_attachments_compression_tests(Mod, Funs) ->
+ [
+ {
+ "Compressiable attachments",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{{text, Mod}, Fun} || Fun <- Funs]
+ }
+ },
+ {
+ "Uncompressiable attachments",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{{binary, Mod}, Fun} || Fun <- Funs]
+ }
+ }
+ ].
+
+
should_upload_attachment_without_md5({Host, DbName}) ->
?_test(begin
@@ -212,10 +312,218 @@ should_reject_chunked_attachment_with_invalid_md5_trailer({Host, DbName}) ->
],
{ok, Code, Json} = request("PUT", AttUrl, Headers, Body),
?assertEqual(400, Code),
- ?assertEqual(<<"content_md5_mismatch">>,
- get_json(Json, [<<"error">>]))
+ ?assertEqual(<<"content_md5_mismatch">>, get_json(Json, [<<"error">>]))
+ end).
+
+should_get_att_without_accept_gzip_encoding(_, {Data, {_, _, AttUrl}}) ->
+ ?_test(begin
+ {ok, Code, Headers, Body} = test_request:get(AttUrl),
+ ?assertEqual(200, Code),
+ ?assertNot(lists:member({"Content-Encoding", "gzip"}, Headers)),
+ ?assertEqual(Data, iolist_to_binary(Body))
+ end).
+
+should_get_att_with_accept_gzip_encoding(compressed, {Data, {_, _, AttUrl}}) ->
+ ?_test(begin
+ {ok, Code, Headers, Body} = test_request:get(
+ AttUrl, [{"Accept-Encoding", "gzip"}]),
+ ?assertEqual(200, Code),
+ ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)),
+ ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body)))
+ end);
+should_get_att_with_accept_gzip_encoding({text, _}, {Data, {_, _, AttUrl}}) ->
+ ?_test(begin
+ {ok, Code, Headers, Body} = test_request:get(
+ AttUrl, [{"Accept-Encoding", "gzip"}]),
+ ?assertEqual(200, Code),
+ ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)),
+ ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body)))
+ end);
+should_get_att_with_accept_gzip_encoding({binary, _}, {Data, {_, _, AttUrl}}) ->
+ ?_test(begin
+ {ok, Code, Headers, Body} = test_request:get(
+ AttUrl, [{"Accept-Encoding", "gzip"}]),
+ ?assertEqual(200, Code),
+ ?assertEqual(undefined,
+ couch_util:get_value("Content-Encoding", Headers)),
+ ?assertEqual(Data, iolist_to_binary(Body))
+ end).
+
+should_get_att_with_accept_deflate_encoding(_, {Data, {_, _, AttUrl}}) ->
+ ?_test(begin
+ {ok, Code, Headers, Body} = test_request:get(
+ AttUrl, [{"Accept-Encoding", "deflate"}]),
+ ?assertEqual(200, Code),
+ ?assertEqual(undefined,
+ couch_util:get_value("Content-Encoding", Headers)),
+ ?assertEqual(Data, iolist_to_binary(Body))
+ end).
+
+should_return_406_response_on_unsupported_encoding(_, {_, {_, _, AttUrl}}) ->
+ ?_assertEqual(406,
+ begin
+ {ok, Code, _, _} = test_request:get(
+ AttUrl, [{"Accept-Encoding", "deflate, *;q=0"}]),
+ Code
+ end).
+
+should_get_doc_with_att_data(compressed, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?attachments=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ AttJson = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+ AttData = couch_util:get_nested_json_value(
+ AttJson, [<<"data">>]),
+ ?assertEqual(
+ <<"text/plain">>,
+ couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+ ?assertEqual(Data, base64:decode(AttData))
+ end);
+should_get_doc_with_att_data({text, _}, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?attachments=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ AttJson = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+ AttData = couch_util:get_nested_json_value(
+ AttJson, [<<"data">>]),
+ ?assertEqual(
+ <<"text/plain">>,
+ couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+ ?assertEqual(Data, base64:decode(AttData))
+ end);
+should_get_doc_with_att_data({binary, _}, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?attachments=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ AttJson = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_BIN_NAME]),
+ AttData = couch_util:get_nested_json_value(
+ AttJson, [<<"data">>]),
+ ?assertEqual(
+ <<"image/png">>,
+ couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+ ?assertEqual(Data, base64:decode(AttData))
+ end).
+
+should_get_doc_with_att_data_stub(compressed, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?att_encoding_info=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ {AttJson} = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+ ?assertEqual(<<"gzip">>,
+ couch_util:get_value(<<"encoding">>, AttJson)),
+ AttLength = couch_util:get_value(<<"length">>, AttJson),
+ EncLength = couch_util:get_value(<<"encoded_length">>, AttJson),
+ ?assertEqual(AttLength, EncLength),
+ ?assertEqual(iolist_size(zlib:gzip(Data)), AttLength)
+ end);
+should_get_doc_with_att_data_stub({text, _}, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?att_encoding_info=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ {AttJson} = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+ ?assertEqual(<<"gzip">>,
+ couch_util:get_value(<<"encoding">>, AttJson)),
+ AttEncLength = iolist_size(gzip(Data)),
+ ?assertEqual(AttEncLength,
+ couch_util:get_value(<<"encoded_length">>, AttJson)),
+ ?assertEqual(byte_size(Data),
+ couch_util:get_value(<<"length">>, AttJson))
+ end);
+should_get_doc_with_att_data_stub({binary, _}, {Data, {_, DocUrl, _}}) ->
+ ?_test(begin
+ Url = DocUrl ++ "?att_encoding_info=true",
+ {ok, Code, _, Body} = test_request:get(
+ Url, [{"Accept", "application/json"}]),
+ ?assertEqual(200, Code),
+ Json = ejson:decode(Body),
+ {AttJson} = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_BIN_NAME]),
+ ?assertEqual(undefined,
+ couch_util:get_value(<<"encoding">>, AttJson)),
+ ?assertEqual(undefined,
+ couch_util:get_value(<<"encoded_length">>, AttJson)),
+ ?assertEqual(byte_size(Data),
+ couch_util:get_value(<<"length">>, AttJson))
end).
+should_not_create_compressed_att_with_deflate_encoding({Host, DbName}) ->
+ ?_assertEqual(415,
+ begin
+ HttpHost = "http://" ++ Host,
+ AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"),
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Body = zlib:compress(Data),
+ Headers = [
+ {"Content-Encoding", "deflate"},
+ {"Content-Type", "text/plain"}
+ ],
+ {ok, Code, _, _} = test_request:put(AttUrl, Headers, Body),
+ Code
+ end).
+
+should_not_create_compressed_att_with_compress_encoding({Host, DbName}) ->
+ % Note: As of OTP R13B04, it seems there's no LZW compression
+ % (i.e. UNIX compress utility implementation) lib in OTP.
+ % However there's a simple working Erlang implementation at:
+ % http://scienceblogs.com/goodmath/2008/01/simple_lempelziv_compression_i.php
+ ?_assertEqual(415,
+ begin
+ HttpHost = "http://" ++ Host,
+ AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"),
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Headers = [
+ {"Content-Encoding", "compress"},
+ {"Content-Type", "text/plain"}
+ ],
+ {ok, Code, _, _} = test_request:put(AttUrl, Headers, Data),
+ Code
+ end).
+
+should_create_compressible_att_with_ctype_params({Host, DbName}) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ HttpHost = "http://" ++ Host,
+ DocUrl = string:join([HttpHost, DbName, ?docid()], "/"),
+ AttUrl = string:join([DocUrl, ?b2l(?ATT_TXT_NAME)], "/"),
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Headers = [{"Content-Type", "text/plain; charset=UTF-8"}],
+ {ok, Code0, _, _} = test_request:put(AttUrl, Headers, Data),
+ ?assertEqual(201, Code0),
+
+ {ok, Code1, _, Body} = test_request:get(
+ DocUrl ++ "?att_encoding_info=true"),
+ ?assertEqual(200, Code1),
+ Json = ejson:decode(Body),
+ {AttJson} = couch_util:get_nested_json_value(
+ Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+ ?assertEqual(<<"gzip">>,
+ couch_util:get_value(<<"encoding">>, AttJson)),
+ AttEncLength = iolist_size(gzip(Data)),
+ ?assertEqual(AttEncLength,
+ couch_util:get_value(<<"encoded_length">>, AttJson)),
+ ?assertEqual(byte_size(Data),
+ couch_util:get_value(<<"length">>, AttJson))
+ end)}.
+
get_json(Json, Path) ->
couch_util:get_nested_json_value(Json, Path).
@@ -262,3 +570,69 @@ request(Method, Url, Headers, Body) ->
erlang:decode_packet(http, Header, []),
Json = ejson:decode(Body1),
{ok, Code, Json}.
+
+create_standalone_text_att(Host, DbName) ->
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"),
+ {ok, Code, _Headers, _Body} = test_request:put(
+ Url, [{"Content-Type", "text/plain"}], Data),
+ ?assertEqual(201, Code),
+ Url.
+
+create_standalone_png_att(Host, DbName) ->
+ {ok, Data} = file:read_file(?FIXTURE_PNG),
+ Url = string:join([Host, DbName, "doc", ?b2l(?ATT_BIN_NAME)], "/"),
+ {ok, Code, _Headers, _Body} = test_request:put(
+ Url, [{"Content-Type", "image/png"}], Data),
+ ?assertEqual(201, Code),
+ Url.
+
+create_inline_text_att(Host, DbName) ->
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Url = string:join([Host, DbName, "doc"], "/"),
+ Doc = {[
+ {<<"_attachments">>, {[
+ {?ATT_TXT_NAME, {[
+ {<<"content_type">>, <<"text/plain">>},
+ {<<"data">>, base64:encode(Data)}
+ ]}
+ }]}}
+ ]},
+ {ok, Code, _Headers, _Body} = test_request:put(
+ Url, [{"Content-Type", "application/json"}], ejson:encode(Doc)),
+ ?assertEqual(201, Code),
+ string:join([Url, ?b2l(?ATT_TXT_NAME)], "/").
+
+create_inline_png_att(Host, DbName) ->
+ {ok, Data} = file:read_file(?FIXTURE_PNG),
+ Url = string:join([Host, DbName, "doc"], "/"),
+ Doc = {[
+ {<<"_attachments">>, {[
+ {?ATT_BIN_NAME, {[
+ {<<"content_type">>, <<"image/png">>},
+ {<<"data">>, base64:encode(Data)}
+ ]}
+ }]}}
+ ]},
+ {ok, Code, _Headers, _Body} = test_request:put(
+ Url, [{"Content-Type", "application/json"}], ejson:encode(Doc)),
+ ?assertEqual(201, Code),
+ string:join([Url, ?b2l(?ATT_BIN_NAME)], "/").
+
+create_already_compressed_att(Host, DbName) ->
+ {ok, Data} = file:read_file(?FIXTURE_TXT),
+ Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"),
+ {ok, Code, _Headers, _Body} = test_request:put(
+ Url, [{"Content-Type", "text/plain"}, {"Content-Encoding", "gzip"}],
+ zlib:gzip(Data)),
+ ?assertEqual(201, Code),
+ Url.
+
+gzip(Data) ->
+ Z = zlib:open(),
+ ok = zlib:deflateInit(Z, ?COMPRESSION_LEVEL, deflated, 16 + 15, 8, default),
+ zlib:deflate(Z, Data),
+ Last = zlib:deflate(Z, [], finish),
+ ok = zlib:deflateEnd(Z),
+ ok = zlib:close(Z),
+ Last.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/31b76762/test/couchdb/fixtures/logo.png
----------------------------------------------------------------------
diff --git a/test/couchdb/fixtures/logo.png b/test/couchdb/fixtures/logo.png
new file mode 100644
index 0000000..d21ac02
Binary files /dev/null and b/test/couchdb/fixtures/logo.png differ
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/31b76762/test/couchdb/test_request.erl
----------------------------------------------------------------------
diff --git a/test/couchdb/test_request.erl b/test/couchdb/test_request.erl
index 7abb92f..d6725ea 100644
--- a/test/couchdb/test_request.erl
+++ b/test/couchdb/test_request.erl
@@ -12,7 +12,7 @@
-module(test_request).
--export([get/1, get/2]).
+-export([get/1, get/2, put/2, put/3]).
-export([request/3, request/4]).
get(Url) ->
@@ -22,6 +22,13 @@ get(Url, Headers) ->
request(get, Url, Headers).
+put(Url, Body) ->
+ request(put, Url, [], Body).
+
+put(Url, Headers, Body) ->
+ request(put, Url, Headers, Body).
+
+
request(Method, Url, Headers) ->
request(Method, Url, Headers, []).