You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by kx...@apache.org on 2014/06/17 00:53:10 UTC
[29/50] couchdb commit: updated refs/heads/1963-eunit to bfb7eb9
Port 220-compaction-daemon.t etap test suite to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/4b01190b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/4b01190b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/4b01190b
Branch: refs/heads/1963-eunit
Commit: 4b01190b7864aec0df0d3d977e08179624fd3ebb
Parents: bde290a
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 9 20:15:37 2014 +0400
Committer: Alexander Shorin <kx...@apache.org>
Committed: Tue Jun 17 01:41:45 2014 +0400
----------------------------------------------------------------------
test/couchdb/Makefile.am | 1 +
test/couchdb/couchdb_compaction_daemon.erl | 231 ++++++++++++++++++++++++
test/etap/220-compaction-daemon.t | 225 -----------------------
test/etap/Makefile.am | 1 -
4 files changed, 232 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4b01190b/test/couchdb/Makefile.am
----------------------------------------------------------------------
diff --git a/test/couchdb/Makefile.am b/test/couchdb/Makefile.am
index f1728bb..0403559 100644
--- a/test/couchdb/Makefile.am
+++ b/test/couchdb/Makefile.am
@@ -39,6 +39,7 @@ eunit_files = \
couch_uuids_tests.erl \
couch_work_queue_tests.erl \
couchdb_attachments_tests.erl \
+ couchdb_compaction_daemon.erl \
couchdb_file_compression_tests.erl \
couchdb_http_proxy_tests.erl \
couchdb_modules_load_tests.erl \
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4b01190b/test/couchdb/couchdb_compaction_daemon.erl
----------------------------------------------------------------------
diff --git a/test/couchdb/couchdb_compaction_daemon.erl b/test/couchdb/couchdb_compaction_daemon.erl
new file mode 100644
index 0000000..725a97b
--- /dev/null
+++ b/test/couchdb/couchdb_compaction_daemon.erl
@@ -0,0 +1,231 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couchdb_compaction_daemon).
+
+-include("couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
+-define(DELAY, 100).
+-define(TIMEOUT, 30000).
+-define(TIMEOUT_S, ?TIMEOUT div 1000).
+
+
+start() ->
+ {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ couch_config:set("compaction_daemon", "check_interval", "3", false),
+ couch_config:set("compaction_daemon", "min_file_size", "100000", false),
+ Pid.
+
+stop(Pid) ->
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT ->
+ throw({timeout, server_stop})
+ end.
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ create_design_doc(Db),
+ ok = couch_db:close(Db),
+ DbName.
+
+teardown(DbName) ->
+ Configs = couch_config:get("compactions"),
+ lists:foreach(
+ fun({Key, _}) ->
+ ok = couch_config:delete("compactions", Key, false)
+ end,
+ Configs),
+ couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+
+compaction_daemon_test_() ->
+ {
+ "Compaction daemon tests",
+ {
+ setup,
+ fun start/0, fun stop/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun should_compact_by_default_rule/1,
+ fun should_compact_by_dbname_rule/1
+ ]
+ }
+ }
+ }.
+
+
+should_compact_by_default_rule(DbName) ->
+ {timeout, ?TIMEOUT_S, ?_test(begin
+ {ok, Db} = couch_db:open_int(DbName, []),
+ populate(DbName, 70, 70, 200 * 1024),
+
+ {_, DbFileSize} = get_db_frag(DbName),
+ {_, ViewFileSize} = get_view_frag(DbName),
+
+ ok = couch_config:set("compactions", "_default",
+ "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
+ false),
+
+ ok = timer:sleep(4000), % something >= check_interval
+ wait_compaction_finished(DbName),
+ ok = couch_config:delete("compactions", "_default", false),
+
+ {DbFrag2, DbFileSize2} = get_db_frag(DbName),
+ {ViewFrag2, ViewFileSize2} = get_view_frag(DbName),
+
+ ?assert(DbFrag2 < 70),
+ ?assert(ViewFrag2 < 70),
+
+ ?assert(DbFileSize > DbFileSize2),
+ ?assert(ViewFileSize > ViewFileSize2),
+
+ ?assert(couch_db:is_idle(Db)),
+ ok = couch_db:close(Db)
+ end)}.
+
+should_compact_by_dbname_rule(DbName) ->
+ {timeout, ?TIMEOUT_S, ?_test(begin
+ {ok, Db} = couch_db:open_int(DbName, []),
+ populate(DbName, 70, 70, 200 * 1024),
+
+ {_, DbFileSize} = get_db_frag(DbName),
+ {_, ViewFileSize} = get_view_frag(DbName),
+
+ ok = couch_config:set("compactions", ?b2l(DbName),
+ "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
+ false),
+
+ ok = timer:sleep(4000), % something >= check_interval
+ wait_compaction_finished(DbName),
+ ok = couch_config:delete("compactions", ?b2l(DbName), false),
+
+ {DbFrag2, DbFileSize2} = get_db_frag(DbName),
+ {ViewFrag2, ViewFileSize2} = get_view_frag(DbName),
+
+ ?assert(DbFrag2 < 70),
+ ?assert(ViewFrag2 < 70),
+
+ ?assert(DbFileSize > DbFileSize2),
+ ?assert(ViewFileSize > ViewFileSize2),
+
+ ?assert(couch_db:is_idle(Db)),
+ ok = couch_db:close(Db)
+ end)}.
+
+
+create_design_doc(Db) ->
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/foo">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"foo">>, {[
+ {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+ ]}},
+ {<<"foo2">>, {[
+ {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+ ]}},
+ {<<"foo3">>, {[
+ {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+ ]}}
+ ]}}
+ ]}),
+ {ok, _} = couch_db:update_docs(Db, [DDoc]),
+ {ok, _} = couch_db:ensure_full_commit(Db),
+ ok.
+
+populate(DbName, DbFrag, ViewFrag, MinFileSize) ->
+ {CurDbFrag, DbFileSize} = get_db_frag(DbName),
+ {CurViewFrag, ViewFileSize} = get_view_frag(DbName),
+ populate(DbName, DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
+ lists:min([DbFileSize, ViewFileSize])).
+
+populate(_Db, DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag, FileSize)
+ when CurDbFrag >= DbFrag, CurViewFrag >= ViewFrag, FileSize >= MinFileSize ->
+ ok;
+populate(DbName, DbFrag, ViewFrag, MinFileSize, _, _, _) ->
+ update(DbName),
+ {CurDbFrag, DbFileSize} = get_db_frag(DbName),
+ {CurViewFrag, ViewFileSize} = get_view_frag(DbName),
+ populate(DbName, DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
+ lists:min([DbFileSize, ViewFileSize])).
+
+update(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ lists:foreach(fun(_) ->
+ Doc = couch_doc:from_json_obj({[{<<"_id">>, couch_uuids:new()}]}),
+ {ok, _} = couch_db:update_docs(Db, [Doc]),
+ query_view(Db#db.name)
+ end, lists:seq(1, 200)),
+ couch_db:close(Db).
+
+db_url(DbName) ->
+ Addr = couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ ?b2l(DbName).
+
+query_view(DbName) ->
+ {ok, Code, _Headers, _Body} = test_request:get(
+ db_url(DbName) ++ "/_design/foo/_view/foo"),
+ ?assertEqual(200, Code).
+
+get_db_frag(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Info} = couch_db:get_db_info(Db),
+ couch_db:close(Db),
+ FileSize = couch_util:get_value(disk_size, Info),
+ DataSize = couch_util:get_value(data_size, Info),
+ {round((FileSize - DataSize) / FileSize * 100), FileSize}.
+
+get_view_frag(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Info} = couch_mrview:get_info(Db, <<"_design/foo">>),
+ couch_db:close(Db),
+ FileSize = couch_util:get_value(disk_size, Info),
+ DataSize = couch_util:get_value(data_size, Info),
+ {round((FileSize - DataSize) / FileSize * 100), FileSize}.
+
+wait_compaction_finished(DbName) ->
+ Parent = self(),
+ Loop = spawn_link(fun() -> wait_loop(DbName, Parent) end),
+ receive
+ {done, Loop} ->
+ ok
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, "Compaction timeout"}]})
+ end.
+
+wait_loop(DbName, Parent) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, DbInfo} = couch_db:get_db_info(Db),
+ {ok, ViewInfo} = couch_mrview:get_info(Db, <<"_design/foo">>),
+ couch_db:close(Db),
+ case (couch_util:get_value(compact_running, ViewInfo) =:= true) orelse
+ (couch_util:get_value(compact_running, DbInfo) =:= true) of
+ false ->
+ Parent ! {done, self()};
+ true ->
+ ok = timer:sleep(?DELAY),
+ wait_loop(DbName, Parent)
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4b01190b/test/etap/220-compaction-daemon.t
----------------------------------------------------------------------
diff --git a/test/etap/220-compaction-daemon.t b/test/etap/220-compaction-daemon.t
deleted file mode 100755
index 4c63b66..0000000
--- a/test/etap/220-compaction-daemon.t
+++ /dev/null
@@ -1,225 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
-test_db_name() ->
- <<"couch_test_compaction_daemon">>.
-
-main(_) ->
- test_util:init_code_path(),
-
- etap:plan(10),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
-
-test() ->
- couch_server_sup:start_link(test_util:config_files()),
- timer:sleep(1000),
- put(addr, couch_config:get("httpd", "bind_address", "127.0.0.1")),
- put(port, integer_to_list(mochiweb_socket_server:get(couch_httpd, port))),
-
- disable_compact_daemon(),
-
- delete_db(),
- {ok, Db} = create_db(),
-
- add_design_doc(Db),
- couch_db:close(Db),
- populate(70, 70, 200 * 1024),
-
- {_, DbFileSize} = get_db_frag(),
- {_, ViewFileSize} = get_view_frag(),
-
- % enable automatic compaction
- ok = couch_config:set("compaction_daemon", "check_interval", "3", false),
- ok = couch_config:set("compaction_daemon", "min_file_size", "100000", false),
- ok = couch_config:set(
- "compactions",
- binary_to_list(test_db_name()),
- "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
- false),
-
- ok = timer:sleep(4000), % something >= check_interval
- wait_compaction_finished(),
-
- {DbFrag2, DbFileSize2} = get_db_frag(),
- {ViewFrag2, ViewFileSize2} = get_view_frag(),
-
- etap:is(true, (DbFrag2 < 70), "Database fragmentation is < 70% after compaction"),
- etap:is(true, (ViewFrag2 < 70), "View fragmentation is < 70% after compaction"),
- etap:is(true, (DbFileSize2 < DbFileSize), "Database file size decreased"),
- etap:is(true, (ViewFileSize2 < ViewFileSize), "View file size decreased"),
-
- disable_compact_daemon(),
- ok = timer:sleep(6000), % 2 times check_interval
- etap:is(couch_db:is_idle(Db), true, "Database is idle"),
- populate(70, 70, 200 * 1024),
- {_, DbFileSize3} = get_db_frag(),
- {_, ViewFileSize3} = get_view_frag(),
-
- % enable automatic compaction
- ok = couch_config:set(
- "compactions",
- "_default",
- "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
- false),
-
- ok = timer:sleep(4000), % something >= check_interval
- wait_compaction_finished(),
-
- {DbFrag4, DbFileSize4} = get_db_frag(),
- {ViewFrag4, ViewFileSize4} = get_view_frag(),
-
- etap:is(true, (DbFrag4 < 70), "Database fragmentation is < 70% after compaction"),
- etap:is(true, (ViewFrag4 < 70), "View fragmentation is < 70% after compaction"),
- etap:is(true, (DbFileSize4 < DbFileSize3), "Database file size decreased again"),
- etap:is(true, (ViewFileSize4 < ViewFileSize3), "View file size decreased again"),
-
- ok = timer:sleep(6000), % 2 times check_interval
- etap:is(couch_db:is_idle(Db), true, "Database is idle"),
-
- delete_db(),
- couch_server_sup:stop(),
- ok.
-
-disable_compact_daemon() ->
- Configs = couch_config:get("compactions"),
- lists:foreach(
- fun({DbName, _}) ->
- ok = couch_config:delete("compactions", DbName, false)
- end,
- Configs).
-
-admin_user_ctx() ->
- {user_ctx, #user_ctx{roles = [<<"_admin">>]}}.
-
-create_db() ->
- {ok, _} = couch_db:create(test_db_name(), [admin_user_ctx()]).
-
-delete_db() ->
- couch_server:delete(test_db_name(), [admin_user_ctx()]).
-
-add_design_doc(Db) ->
- DDoc = couch_doc:from_json_obj({[
- {<<"_id">>, <<"_design/foo">>},
- {<<"language">>, <<"javascript">>},
- {<<"views">>, {[
- {<<"foo">>, {[
- {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
- ]}},
- {<<"foo2">>, {[
- {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
- ]}},
- {<<"foo3">>, {[
- {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
- ]}}
- ]}}
- ]}),
- {ok, _} = couch_db:update_docs(Db, [DDoc]),
- {ok, _} = couch_db:ensure_full_commit(Db),
- ok.
-
-populate(DbFrag, ViewFrag, MinFileSize) ->
- {CurDbFrag, DbFileSize} = get_db_frag(),
- {CurViewFrag, ViewFileSize} = get_view_frag(),
- populate(
- DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
- lists:min([DbFileSize, ViewFileSize])).
-
-populate(DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag, FileSize)
- when CurDbFrag >= DbFrag, CurViewFrag >= ViewFrag, FileSize >= MinFileSize ->
- ok;
-populate(DbFrag, ViewFrag, MinFileSize, _, _, _) ->
- update(),
- {CurDbFrag, DbFileSize} = get_db_frag(),
- {CurViewFrag, ViewFileSize} = get_view_frag(),
- populate(
- DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
- lists:min([DbFileSize, ViewFileSize])).
-
-update() ->
- {ok, Db} = couch_db:open_int(test_db_name(), []),
- lists:foreach(fun(_) ->
- Doc = couch_doc:from_json_obj({[{<<"_id">>, couch_uuids:new()}]}),
- {ok, _} = couch_db:update_docs(Db, [Doc]),
- query_view()
- end, lists:seq(1, 100)),
- couch_db:close(Db).
-
-db_url() ->
- "http://" ++ get(addr) ++ ":" ++ get(port) ++ "/" ++
- binary_to_list(test_db_name()).
-
-query_view() ->
- {ok, Code, _Headers, _Body} = test_util:request(
- db_url() ++ "/_design/foo/_view/foo", [], get),
- case Code of
- 200 ->
- ok;
- _ ->
- etap:bail("error querying view")
- end.
-
-get_db_frag() ->
- {ok, Db} = couch_db:open_int(test_db_name(), []),
- {ok, Info} = couch_db:get_db_info(Db),
- couch_db:close(Db),
- FileSize = couch_util:get_value(disk_size, Info),
- DataSize = couch_util:get_value(data_size, Info),
- {round((FileSize - DataSize) / FileSize * 100), FileSize}.
-
-get_view_frag() ->
- {ok, Db} = couch_db:open_int(test_db_name(), []),
- {ok, Info} = couch_mrview:get_info(Db, <<"_design/foo">>),
- couch_db:close(Db),
- FileSize = couch_util:get_value(disk_size, Info),
- DataSize = couch_util:get_value(data_size, Info),
- {round((FileSize - DataSize) / FileSize * 100), FileSize}.
-
-
-wait_compaction_finished() ->
- Parent = self(),
- Loop = spawn_link(fun() -> wait_loop(Parent) end),
- receive
- {done, Loop} ->
- etap:diag("Database and view compaction have finished")
- after 60000 ->
- etap:bail("Compaction not triggered")
- end.
-
-wait_loop(Parent) ->
- {ok, Db} = couch_db:open_int(test_db_name(), []),
- {ok, DbInfo} = couch_db:get_db_info(Db),
- {ok, ViewInfo} = couch_mrview:get_info(Db, <<"_design/foo">>),
- couch_db:close(Db),
- case (couch_util:get_value(compact_running, ViewInfo) =:= true) orelse
- (couch_util:get_value(compact_running, DbInfo) =:= true) of
- false ->
- Parent ! {done, self()};
- true ->
- ok = timer:sleep(500),
- wait_loop(Parent)
- end.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/4b01190b/test/etap/Makefile.am
----------------------------------------------------------------------
diff --git a/test/etap/Makefile.am b/test/etap/Makefile.am
index 28876d3..e05a0ad 100644
--- a/test/etap/Makefile.am
+++ b/test/etap/Makefile.am
@@ -32,7 +32,6 @@ fixture_files = \
fixtures/test.couch
tap_files = \
- 220-compaction-daemon.t \
230-pbkfd2.t \
231-cors.t \
250-upgrade-legacy-view-files.t