You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2012/02/06 19:48:40 UTC
[1/3] git commit: Ensure compaction respects compression setting
Updated Branches:
refs/heads/1.2.x 506deab47 -> ee2b9218c
Ensure compaction respects compression setting
For a database with file format version 6 (CouchDB 1.2+), if
the file_compression setting was changed, compaction ignored
it, making it impossible to change the compression method of
a database and its indexes.
Closes COUCHDB-1394
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/71a0f583
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/71a0f583
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/71a0f583
Branch: refs/heads/1.2.x
Commit: 71a0f583b586914c2009cceafb0102aa6fb52fb0
Parents: 506deab
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Thu Jan 26 14:40:39 2012 +0000
Committer: Jan Lehnardt <ja...@apache.org>
Committed: Mon Feb 6 19:47:54 2012 +0100
----------------------------------------------------------------------
src/couchdb/couch_compress.erl | 25 +++--
src/couchdb/couch_db_updater.erl | 7 +-
test/etap/076-file-compression.t | 185 +++++++++++++++++++++++++++++++++
test/etap/Makefile.am | 1 +
4 files changed, 208 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/src/couchdb/couch_compress.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_compress.erl b/src/couchdb/couch_compress.erl
index 523108f..ac386fd 100644
--- a/src/couchdb/couch_compress.erl
+++ b/src/couchdb/couch_compress.erl
@@ -12,16 +12,17 @@
-module(couch_compress).
--export([compress/2, decompress/1, is_compressed/1]).
+-export([compress/2, decompress/1, is_compressed/2]).
-export([get_compression_method/0]).
-include("couch_db.hrl").
% binaries compressed with snappy have their first byte set to this value
-define(SNAPPY_PREFIX, 1).
-% binaries that are a result of an erlang:term_to_binary/1,2 call have this
-% value as their first byte
+% Term prefixes documented at:
+% http://www.erlang.org/doc/apps/erts/erl_ext_dist.html
-define(TERM_PREFIX, 131).
+-define(COMPRESSED_TERM_PREFIX, 131, 80).
get_compression_method() ->
@@ -38,6 +39,12 @@ get_compression_method() ->
end.
+compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, snappy) ->
+ Bin;
+compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, Method) ->
+ compress(decompress(Bin), Method);
+compress(<<?TERM_PREFIX, _/binary>> = Bin, Method) ->
+ compress(decompress(Bin), Method);
compress(Term, none) ->
?term_to_bin(Term);
compress(Term, {deflate, Level}) ->
@@ -64,10 +71,14 @@ decompress(<<?TERM_PREFIX, _/binary>> = Bin) ->
binary_to_term(Bin).
-is_compressed(<<?SNAPPY_PREFIX, _/binary>>) ->
+is_compressed(<<?SNAPPY_PREFIX, _/binary>>, Method) ->
+ Method =:= snappy;
+is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, {deflate, _Level}) ->
true;
-is_compressed(<<?TERM_PREFIX, _/binary>>) ->
- true;
-is_compressed(Term) when not is_binary(Term) ->
+is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, _Method) ->
+ false;
+is_compressed(<<?TERM_PREFIX, _/binary>>, Method) ->
+ Method =:= none;
+is_compressed(Term, _Method) when not is_binary(Term) ->
false.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/src/couchdb/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl
index b87ffcd..eaf1bd3 100644
--- a/src/couchdb/couch_db_updater.erl
+++ b/src/couchdb/couch_db_updater.erl
@@ -897,7 +897,8 @@ copy_docs(Db, #db{updater_fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
copy_compact(Db, NewDb0, Retry) ->
FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header],
- NewDb = NewDb0#db{fsync_options=FsyncOptions},
+ Compression = couch_compress:get_compression_method(),
+ NewDb = NewDb0#db{fsync_options=FsyncOptions, compression=Compression},
TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq),
BufferSize = list_to_integer(
couch_config:get("database_compaction", "doc_buffer_size", "524288")),
@@ -1016,14 +1017,14 @@ update_compact_task(NumChanges) ->
couch_task_status:update([{changes_done, Changes2}, {progress, Progress}]).
make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
- Body = case couch_compress:is_compressed(Body0) of
+ Body = case couch_compress:is_compressed(Body0, Comp) of
true ->
Body0;
false ->
% pre 1.2 database file format
couch_compress:compress(Body0, Comp)
end,
- Atts = case couch_compress:is_compressed(Atts0) of
+ Atts = case couch_compress:is_compressed(Atts0, Comp) of
true ->
Atts0;
false ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/test/etap/076-file-compression.t
----------------------------------------------------------------------
diff --git a/test/etap/076-file-compression.t b/test/etap/076-file-compression.t
new file mode 100755
index 0000000..d316d96
--- /dev/null
+++ b/test/etap/076-file-compression.t
@@ -0,0 +1,185 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(user_ctx, {
+ name = null,
+ roles = [],
+ handler
+}).
+
+test_db_name() -> <<"couch_test_file_compression">>.
+ddoc_id() -> <<"_design/test">>.
+num_docs() -> 5000.
+
+
+main(_) ->
+ test_util:init_code_path(),
+
+ etap:plan(10),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ ok.
+
+
+test() ->
+ couch_server_sup:start_link(test_util:config_files()),
+ couch_config:set("couchdb", "file_compression", "none", false),
+
+ create_database(),
+ compact_db(),
+ compact_view(),
+ DbDiskSize1 = db_disk_size(),
+ ViewDiskSize1 = view_disk_size(),
+
+ couch_config:set("couchdb", "file_compression", "snappy", false),
+ compact_db(),
+ compact_view(),
+ DbDiskSize2 = db_disk_size(),
+ ViewDiskSize2 = view_disk_size(),
+
+ etap:is(DbDiskSize2 < DbDiskSize1, true, "Database disk size decreased"),
+ etap:is(ViewDiskSize2 < ViewDiskSize1, true, "Index disk size decreased"),
+
+ couch_config:set("couchdb", "file_compression", "deflate_9", false),
+ compact_db(),
+ compact_view(),
+ DbDiskSize3 = db_disk_size(),
+ ViewDiskSize3 = view_disk_size(),
+
+ etap:is(DbDiskSize3 < DbDiskSize2, true, "Database disk size decreased again"),
+ etap:is(ViewDiskSize3 < ViewDiskSize2, true, "Index disk size decreased again"),
+
+ couch_config:set("couchdb", "file_compression", "deflate_1", false),
+ compact_db(),
+ compact_view(),
+ DbDiskSize4 = db_disk_size(),
+ ViewDiskSize4 = view_disk_size(),
+
+ etap:is(DbDiskSize4 > DbDiskSize3, true, "Database disk size increased"),
+ etap:is(ViewDiskSize4 > ViewDiskSize3, true, "Index disk size increased"),
+
+ couch_config:set("couchdb", "file_compression", "snappy", false),
+ compact_db(),
+ compact_view(),
+ DbDiskSize5 = db_disk_size(),
+ ViewDiskSize5 = view_disk_size(),
+
+ etap:is(DbDiskSize5 > DbDiskSize4, true, "Database disk size increased again"),
+ etap:is(ViewDiskSize5 > ViewDiskSize4, true, "Index disk size increased again"),
+
+ couch_config:set("couchdb", "file_compression", "none", false),
+ compact_db(),
+ compact_view(),
+ DbDiskSize6 = db_disk_size(),
+ ViewDiskSize6 = view_disk_size(),
+
+ etap:is(DbDiskSize6 > DbDiskSize5, true, "Database disk size increased again"),
+ etap:is(ViewDiskSize6 > ViewDiskSize5, true, "Index disk size increased again"),
+
+ delete_db(),
+ couch_server_sup:stop(),
+ ok.
+
+
+create_database() ->
+ {ok, Db} = couch_db:create(
+ test_db_name(),
+ [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]),
+ ok = populate_db(Db, num_docs()),
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, ddoc_id()},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"view1">>, {[
+ {<<"map">>, <<"function(doc) { emit(doc._id, doc.string); }">>}
+ ]}}
+ ]}
+ }
+ ]}),
+ {ok, _} = couch_db:update_doc(Db, DDoc, []),
+ refresh_index(),
+ ok = couch_db:close(Db).
+
+
+populate_db(_Db, NumDocs) when NumDocs =< 0 ->
+ ok;
+populate_db(Db, NumDocs) ->
+ Docs = lists:map(
+ fun(_) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, couch_uuids:random()},
+ {<<"string">>, list_to_binary(lists:duplicate(1000, $X))}
+ ]})
+ end,
+ lists:seq(1, 500)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []),
+ populate_db(Db, NumDocs - 500).
+
+
+refresh_index() ->
+ {ok, Db} = couch_db:open_int(test_db_name(), []),
+ {ok, _, _} = couch_view:get_map_view(Db, ddoc_id(), <<"view1">>, false),
+ ok = couch_db:close(Db).
+
+
+compact_db() ->
+ {ok, Db} = couch_db:open_int(test_db_name(), []),
+ {ok, CompactPid} = couch_db:start_compact(Db),
+ MonRef = erlang:monitor(process, CompactPid),
+ receive
+ {'DOWN', MonRef, process, CompactPid, normal} ->
+ ok;
+ {'DOWN', MonRef, process, CompactPid, Reason} ->
+ etap:bail("Error compacting database: " ++ couch_util:to_list(Reason))
+ after 120000 ->
+ etap:bail("Timeout waiting for database compaction")
+ end,
+ ok = couch_db:close(Db).
+
+
+compact_view() ->
+ {ok, CompactPid} = couch_view_compactor:start_compact(test_db_name(), <<"test">>),
+ MonRef = erlang:monitor(process, CompactPid),
+ receive
+ {'DOWN', MonRef, process, CompactPid, normal} ->
+ ok;
+ {'DOWN', MonRef, process, CompactPid, Reason} ->
+ etap:bail("Error compacting view group: " ++ couch_util:to_list(Reason))
+ after 120000 ->
+ etap:bail("Timeout waiting for view group compaction")
+ end.
+
+
+db_disk_size() ->
+ {ok, Db} = couch_db:open_int(test_db_name(), []),
+ {ok, Info} = couch_db:get_db_info(Db),
+ ok = couch_db:close(Db),
+ couch_util:get_value(disk_size, Info).
+
+
+view_disk_size() ->
+ {ok, Db} = couch_db:open_int(test_db_name(), []),
+ {ok, Info} = couch_view:get_group_info(Db, ddoc_id()),
+ ok = couch_db:close(Db),
+ couch_util:get_value(disk_size, Info).
+
+
+delete_db() ->
+ ok = couch_server:delete(
+ test_db_name(), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).
http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/test/etap/Makefile.am
----------------------------------------------------------------------
diff --git a/test/etap/Makefile.am b/test/etap/Makefile.am
index 47f5626..277eedb 100644
--- a/test/etap/Makefile.am
+++ b/test/etap/Makefile.am
@@ -57,6 +57,7 @@ EXTRA_DIST = \
072-cleanup.t \
073-changes.t \
075-auth-cache.t \
+ 076-file-compression.t \
080-config-get-set.t \
081-config-override.1.ini \
081-config-override.2.ini \