You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by fd...@apache.org on 2011/05/02 15:28:23 UTC
svn commit: r1098558 [1/2] - in /couchdb/trunk: ./ etc/couchdb/ src/
src/couchdb/ src/snappy/ src/snappy/google-snappy/ test/etap/ utils/
Author: fdmanana
Date: Mon May 2 13:28:20 2011
New Revision: 1098558
URL: http://svn.apache.org/viewvc?rev=1098558&view=rev
Log:
Add configurable file compression (snappy, deflate or none)
Not only this makes database and view index files smaller it also increases
database read/write performance, view index generation (specially for large
documents and/or documents with nested JSON structures) and compaction.
Closes COUCHDB-1120.
Added:
couchdb/trunk/src/couchdb/couch_compress.erl
couchdb/trunk/src/snappy/
couchdb/trunk/src/snappy/Makefile.am
couchdb/trunk/src/snappy/erl_nif_compat.h
couchdb/trunk/src/snappy/google-snappy/
couchdb/trunk/src/snappy/google-snappy/AUTHORS
couchdb/trunk/src/snappy/google-snappy/COPYING
couchdb/trunk/src/snappy/google-snappy/config.h.in
couchdb/trunk/src/snappy/google-snappy/snappy-internal.h
couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.cc
couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.h
couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.cc
couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.h
couchdb/trunk/src/snappy/google-snappy/snappy-stubs-public.h.in
couchdb/trunk/src/snappy/google-snappy/snappy.cc
couchdb/trunk/src/snappy/google-snappy/snappy.h
couchdb/trunk/src/snappy/snappy.app.in
couchdb/trunk/src/snappy/snappy.erl
couchdb/trunk/src/snappy/snappy_nif.cc
Modified:
couchdb/trunk/.gitignore
couchdb/trunk/LICENSE
couchdb/trunk/NOTICE
couchdb/trunk/configure.ac
couchdb/trunk/etc/couchdb/default.ini.tpl.in
couchdb/trunk/license.skip
couchdb/trunk/src/Makefile.am
couchdb/trunk/src/couchdb/Makefile.am
couchdb/trunk/src/couchdb/couch_api_wrap.erl
couchdb/trunk/src/couchdb/couch_btree.erl
couchdb/trunk/src/couchdb/couch_changes.erl
couchdb/trunk/src/couchdb/couch_db.erl
couchdb/trunk/src/couchdb/couch_db.hrl
couchdb/trunk/src/couchdb/couch_db_updater.erl
couchdb/trunk/src/couchdb/couch_doc.erl
couchdb/trunk/src/couchdb/couch_file.erl
couchdb/trunk/src/couchdb/couch_httpd.erl
couchdb/trunk/src/couchdb/couch_httpd_db.erl
couchdb/trunk/src/couchdb/couch_query_servers.erl
couchdb/trunk/src/couchdb/couch_replication_manager.erl
couchdb/trunk/src/couchdb/couch_replicator.erl
couchdb/trunk/src/couchdb/couch_replicator_utils.erl
couchdb/trunk/src/couchdb/couch_server.erl
couchdb/trunk/src/couchdb/couch_view_group.erl
couchdb/trunk/test/etap/010-file-basics.t
couchdb/trunk/test/etap/020-btree-basics.t
couchdb/trunk/test/etap/200-view-group-no-db-leaks.t
couchdb/trunk/test/etap/test_util.erl.in
couchdb/trunk/utils/Makefile.am
Modified: couchdb/trunk/.gitignore
URL: http://svn.apache.org/viewvc/couchdb/trunk/.gitignore?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/.gitignore (original)
+++ couchdb/trunk/.gitignore Mon May 2 13:28:20 2011
@@ -69,6 +69,13 @@ src/ejson/.deps/
src/ejson/.libs/
src/ejson/priv
src/mochiweb/mochiweb.app
+src/snappy/.deps/
+src/snappy/.libs/
+src/snappy/priv
+src/snappy/snappy.app
+src/snappy/google-snappy/snappy-stubs-public.h
+src/snappy/google-snappy/stamp-h2
+src/snappy/google-snappy/.deps/
test/local.ini
test/etap/.deps/
test/etap/run
Modified: couchdb/trunk/LICENSE
URL: http://svn.apache.org/viewvc/couchdb/trunk/LICENSE?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/LICENSE (original)
+++ couchdb/trunk/LICENSE Mon May 2 13:28:20 2011
@@ -448,3 +448,33 @@ For the src/ejson/erl_nif_compat.h file
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
+
+For the src/snappy/google-snappy component
+
+ Copyright 2005 and onwards Google Inc.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: couchdb/trunk/NOTICE
URL: http://svn.apache.org/viewvc/couchdb/trunk/NOTICE?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/NOTICE (original)
+++ couchdb/trunk/NOTICE Mon May 2 13:28:20 2011
@@ -57,3 +57,11 @@ This product also includes the following
* yajl (http://lloyd.github.com/yajl/)
Copyright 2010, Lloyd Hilaiel
+
+ * snappy (http://code.google.com/p/snappy/)
+
+ Copyright 2005 and onwards Google Inc.
+
+ * snappy-erlang-nif (https://github.com/fdmanana/snappy-erlang-nif)
+
+ Copyright 2011, Filipe David Manana <fd...@apache.org>
Modified: couchdb/trunk/configure.ac
URL: http://svn.apache.org/viewvc/couchdb/trunk/configure.ac?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/configure.ac (original)
+++ couchdb/trunk/configure.ac Mon May 2 13:28:20 2011
@@ -19,6 +19,7 @@ AC_CONFIG_AUX_DIR([build-aux])
AC_CONFIG_MACRO_DIR([m4])
AM_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([src/snappy/google-snappy/config.h])
AM_INIT_AUTOMAKE([1.6.3 foreign])
@@ -30,6 +31,67 @@ AC_PROG_CC
AC_PROG_LIBTOOL
AC_PROG_LN_S
+dnl Config for google snappy
+m4_define([snappy_major], [1])
+m4_define([snappy_minor], [0])
+m4_define([snappy_patchlevel], [1])
+
+AC_PROG_CXX
+AC_LANG([C++])
+AC_C_BIGENDIAN
+AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h])
+AC_CHECK_FUNC([mmap])
+
+AC_MSG_CHECKING([if the compiler supports __builtin_expect])
+
+AC_TRY_COMPILE(, [
+ return __builtin_expect(1, 1) ? 1 : 0
+], [
+ snappy_have_builtin_expect=yes
+ AC_MSG_RESULT([yes])
+], [
+ snappy_have_builtin_expect=no
+ AC_MSG_RESULT([no])
+])
+if test x$snappy_have_builtin_expect = xyes ; then
+ AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
+fi
+
+AC_MSG_CHECKING([if the compiler supports __builtin_ctzll])
+
+AC_TRY_COMPILE(, [
+ return (__builtin_ctzll(0x100000000LL) == 32) ? 1 : 0
+], [
+ snappy_have_builtin_ctz=yes
+ AC_MSG_RESULT([yes])
+], [
+ snappy_have_builtin_ctz=no
+ AC_MSG_RESULT([no])
+])
+if test x$snappy_have_builtin_ctz = xyes ; then
+ AC_DEFINE([HAVE_BUILTIN_CTZ], [1], [Define to 1 if the compiler supports __builtin_ctz and friends.])
+fi
+
+if test "$ac_cv_header_stdint_h" = "yes"; then
+ AC_SUBST([ac_cv_have_stdint_h], [1])
+else
+ AC_SUBST([ac_cv_have_stdint_h], [0])
+fi
+if test "$ac_cv_header_stddef_h" = "yes"; then
+ AC_SUBST([ac_cv_have_stddef_h], [1])
+else
+ AC_SUBST([ac_cv_have_stddef_h], [0])
+fi
+
+SNAPPY_MAJOR="snappy_major"
+SNAPPY_MINOR="snappy_minor"
+SNAPPY_PATCHLEVEL="snappy_patchlevel"
+
+AC_SUBST([SNAPPY_MAJOR])
+AC_SUBST([SNAPPY_MINOR])
+AC_SUBST([SNAPPY_PATCHLEVEL])
+dnl End of google snappy specific config
+
AC_MSG_CHECKING([for pthread_create in -lpthread])
original_LIBS="$LIBS"
@@ -423,6 +485,8 @@ AC_CONFIG_FILES([src/erlang-oauth/Makefi
AC_CONFIG_FILES([src/etap/Makefile])
AC_CONFIG_FILES([src/ibrowse/Makefile])
AC_CONFIG_FILES([src/mochiweb/Makefile])
+AC_CONFIG_FILES([src/snappy/Makefile])
+AC_CONFIG_FILES([src/snappy/google-snappy/snappy-stubs-public.h])
AC_CONFIG_FILES([src/ejson/Makefile])
AC_CONFIG_FILES([test/Makefile])
AC_CONFIG_FILES([test/bench/Makefile])
Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Mon May 2 13:28:20 2011
@@ -11,6 +11,14 @@ os_process_timeout = 5000 ; 5 seconds. f
max_dbs_open = 100
delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
uri_file = %localstaterundir%/couch.uri
+; Method used to compress everything that is appended to database and view index files, except
+; for attachments (see the attachments section). Available methods are:
+;
+; none - no compression
+; snappy - use google snappy, a very fast compressor/decompressor
+; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest,
+; lowest compression ratio) to 9 (slowest, highest compression ratio)
+file_compression = snappy
[httpd]
port = 5984
Modified: couchdb/trunk/license.skip
URL: http://svn.apache.org/viewvc/couchdb/trunk/license.skip?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/license.skip (original)
+++ couchdb/trunk/license.skip Mon May 2 13:28:20 2011
@@ -88,6 +88,7 @@
^src/etap/*
^src/ibrowse/*
^src/mochiweb/*
+^src/snappy/*
^stamp-h1
^test/Makefile
^test/Makefile.in
Modified: couchdb/trunk/src/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/Makefile.am?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/Makefile.am (original)
+++ couchdb/trunk/src/Makefile.am Mon May 2 13:28:20 2011
@@ -10,4 +10,4 @@
## License for the specific language governing permissions and limitations under
## the License.
-SUBDIRS = couchdb ejson erlang-oauth etap ibrowse mochiweb
+SUBDIRS = couchdb ejson erlang-oauth etap ibrowse mochiweb snappy
Modified: couchdb/trunk/src/couchdb/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/Makefile.am?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/Makefile.am (original)
+++ couchdb/trunk/src/couchdb/Makefile.am Mon May 2 13:28:20 2011
@@ -34,6 +34,7 @@ source_files = \
couch_auth_cache.erl \
couch_btree.erl \
couch_changes.erl \
+ couch_compress.erl \
couch_config.erl \
couch_config_writer.erl \
couch_db.erl \
@@ -102,6 +103,7 @@ compiled_files = \
couch_auth_cache.beam \
couch_btree.beam \
couch_changes.beam \
+ couch_compress.beam \
couch_config.beam \
couch_config_writer.beam \
couch_db.beam \
Modified: couchdb/trunk/src/couchdb/couch_api_wrap.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_api_wrap.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/trunk/src/couchdb/couch_api_wrap.erl Mon May 2 13:28:20 2011
@@ -420,6 +420,8 @@ options_to_query_args(HttpDb, Path, Opti
options_to_query_args([], Acc) ->
lists:reverse(Acc);
+options_to_query_args([ejson_body | Rest], Acc) ->
+ options_to_query_args(Rest, Acc);
options_to_query_args([delay_commit | Rest], Acc) ->
options_to_query_args(Rest, Acc);
options_to_query_args([revs | Rest], Acc) ->
Modified: couchdb/trunk/src/couchdb/couch_btree.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_btree.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_btree.erl (original)
+++ couchdb/trunk/src/couchdb/couch_btree.erl Mon May 2 13:28:20 2011
@@ -41,7 +41,9 @@ set_options(Bt, [{join, Assemble}|Rest])
set_options(Bt, [{less, Less}|Rest]) ->
set_options(Bt#btree{less=Less}, Rest);
set_options(Bt, [{reduce, Reduce}|Rest]) ->
- set_options(Bt#btree{reduce=Reduce}, Rest).
+ set_options(Bt#btree{reduce=Reduce}, Rest);
+set_options(Bt, [{compression, Comp}|Rest]) ->
+ set_options(Bt#btree{compression=Comp}, Rest).
open(State, Fd, Options) ->
{ok, set_options(#btree{root=State, fd=Fd}, Options)}.
@@ -274,26 +276,26 @@ complete_root(Bt, KPs) ->
% written. Plus with the "case byte_size(term_to_binary(InList)) of" code
% it's probably really inefficient.
-chunkify(InList) ->
- case byte_size(term_to_binary(InList)) of
+chunkify(#btree{compression = Comp} = Bt, InList) ->
+ case byte_size(couch_compress:compress(InList, Comp)) of
Size when Size > ?CHUNK_THRESHOLD ->
NumberOfChunksLikely = ((Size div ?CHUNK_THRESHOLD) + 1),
ChunkThreshold = Size div NumberOfChunksLikely,
- chunkify(InList, ChunkThreshold, [], 0, []);
+ chunkify(Bt, InList, ChunkThreshold, [], 0, []);
_Else ->
[InList]
end.
-chunkify([], _ChunkThreshold, [], 0, OutputChunks) ->
+chunkify(_Bt, [], _ChunkThreshold, [], 0, OutputChunks) ->
lists:reverse(OutputChunks);
-chunkify([], _ChunkThreshold, OutList, _OutListSize, OutputChunks) ->
+chunkify(_Bt, [], _ChunkThreshold, OutList, _OutListSize, OutputChunks) ->
lists:reverse([lists:reverse(OutList) | OutputChunks]);
-chunkify([InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) ->
- case byte_size(term_to_binary(InElement)) of
+chunkify(Bt, [InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) ->
+ case byte_size(couch_compress:compress(InElement, Bt#btree.compression)) of
Size when (Size + OutListSize) > ChunkThreshold andalso OutList /= [] ->
- chunkify(RestInList, ChunkThreshold, [], 0, [lists:reverse([InElement | OutList]) | OutputChunks]);
+ chunkify(Bt, RestInList, ChunkThreshold, [], 0, [lists:reverse([InElement | OutList]) | OutputChunks]);
Size ->
- chunkify(RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks)
+ chunkify(Bt, RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks)
end.
modify_node(Bt, RootPointerInfo, Actions, QueryOutput) ->
@@ -346,13 +348,14 @@ get_node(#btree{fd = Fd}, NodePos) ->
{ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos),
{NodeType, NodeList}.
-write_node(Bt, NodeType, NodeList) ->
+write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) ->
% split up nodes into smaller sizes
- NodeListList = chunkify(NodeList),
+ NodeListList = chunkify(Bt, NodeList),
% now write out each chunk and return the KeyPointer pairs for those nodes
ResultList = [
begin
- {ok, Pointer, Size} = couch_file:append_term(Bt#btree.fd, {NodeType, ANodeList}),
+ {ok, Pointer, Size} = couch_file:append_term(
+ Fd, {NodeType, ANodeList}, [{compression, Comp}]),
{LastKey, _} = lists:last(ANodeList),
SubTreeSize = reduce_tree_size(NodeType, Size, ANodeList),
{LastKey, {Pointer, reduce_node(Bt, NodeType, ANodeList), SubTreeSize}}
Modified: couchdb/trunk/src/couchdb/couch_changes.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_changes.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_changes.erl (original)
+++ couchdb/trunk/src/couchdb/couch_changes.erl Mon May 2 13:28:20 2011
@@ -105,7 +105,7 @@ os_filter_fun(FilterName, Style, Req, Db
end;
[DName, FName] ->
DesignId = <<"_design/", DName/binary>>,
- DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]),
% validate that the ddoc has the filter fun
#doc{body={Props}} = DDoc,
couch_util:get_nested_json_value({Props}, [<<"filters">>, FName]),
@@ -178,7 +178,7 @@ filter_view(ViewName, Style, Db) ->
throw({bad_request, "Invalid `view` parameter."});
[DName, VName] ->
DesignId = <<"_design/", DName/binary>>,
- DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]),
% validate that the ddoc has the filter fun
#doc{body={Props}} = DDoc,
couch_util:get_nested_json_value({Props}, [<<"views">>, VName]),
Added: couchdb/trunk/src/couchdb/couch_compress.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_compress.erl?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/couchdb/couch_compress.erl (added)
+++ couchdb/trunk/src/couchdb/couch_compress.erl Mon May 2 13:28:20 2011
@@ -0,0 +1,73 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_compress).
+
+-export([compress/2, decompress/1, is_compressed/1]).
+-export([get_compression_method/0]).
+
+-include("couch_db.hrl").
+
+% binaries compressed with snappy have their first byte set to this value
+-define(SNAPPY_PREFIX, 1).
+% binaries that are a result of an erlang:term_to_binary/1,2 call have this
+% value as their first byte
+-define(TERM_PREFIX, 131).
+
+
+get_compression_method() ->
+ case couch_config:get("couchdb", "file_compression") of
+ undefined ->
+ ?DEFAULT_COMPRESSION;
+ Method1 ->
+ case string:tokens(Method1, "_") of
+ [Method] ->
+ list_to_existing_atom(Method);
+ [Method, Level] ->
+ {list_to_existing_atom(Method), list_to_integer(Level)}
+ end
+ end.
+
+
+compress(Term, none) ->
+ ?term_to_bin(Term);
+compress(Term, {deflate, Level}) ->
+ term_to_binary(Term, [{minor_version, 1}, {compressed, Level}]);
+compress(Term, snappy) ->
+ Bin = ?term_to_bin(Term),
+ try
+ {ok, CompressedBin} = snappy:compress(Bin),
+ case byte_size(CompressedBin) < byte_size(Bin) of
+ true ->
+ <<?SNAPPY_PREFIX, CompressedBin/binary>>;
+ false ->
+ Bin
+ end
+ catch exit:snappy_nif_not_loaded ->
+ Bin
+ end.
+
+
+decompress(<<?SNAPPY_PREFIX, Rest/binary>>) ->
+ {ok, TermBin} = snappy:decompress(Rest),
+ binary_to_term(TermBin);
+decompress(<<?TERM_PREFIX, _/binary>> = Bin) ->
+ binary_to_term(Bin).
+
+
+is_compressed(<<?SNAPPY_PREFIX, _/binary>>) ->
+ true;
+is_compressed(<<?TERM_PREFIX, _/binary>>) ->
+ true;
+is_compressed(Term) when not is_binary(Term) ->
+ false.
+
Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Mon May 2 13:28:20 2011
@@ -153,6 +153,8 @@ apply_open_options2(#doc{atts=Atts,revs=
apply_open_options2(Doc#doc{atts=[A#att{data=
if AttPos>RevPos -> Data; true -> stub end}
|| #att{revpos=AttPos,data=Data}=A <- Atts]}, Rest);
+apply_open_options2(Doc, [ejson_body | Rest]) ->
+ apply_open_options2(couch_doc:with_ejson_body(Doc), Rest);
apply_open_options2(Doc,[_|Rest]) ->
apply_open_options2(Doc,Rest).
@@ -295,7 +297,7 @@ sum_tree_sizes(Acc, [T | Rest]) ->
get_design_docs(Db) ->
{ok,_, Docs} = couch_btree:fold(by_id_btree(Db),
fun(#full_doc_info{id= <<"_design/",_/binary>>}=FullDocInfo, _Reds, AccDocs) ->
- {ok, Doc} = couch_db:open_doc_int(Db, FullDocInfo, []),
+ {ok, Doc} = open_doc_int(Db, FullDocInfo, [ejson_body]),
{ok, [Doc | AccDocs]};
(_, _Reds, AccDocs) ->
{stop, AccDocs}
@@ -803,8 +805,9 @@ collect_results(UpdatePid, MRef, Results
exit(Reason)
end.
-write_and_commit(#db{update_pid=UpdatePid}=Db, DocBuckets,
+write_and_commit(#db{update_pid=UpdatePid}=Db, DocBuckets1,
NonRepDocs, Options0) ->
+ DocBuckets = prepare_doc_summaries(Db, DocBuckets1),
Options = set_commit_option(Options0),
MergeConflicts = lists:member(merge_conflicts, Options),
FullCommit = lists:member(full_commit, Options),
@@ -819,11 +822,12 @@ write_and_commit(#db{update_pid=UpdatePi
{ok, Db2} = open_ref_counted(Db#db.main_pid, self()),
DocBuckets2 = [
[doc_flush_atts(Doc, Db2#db.updater_fd) || Doc <- Bucket] ||
- Bucket <- DocBuckets
+ Bucket <- DocBuckets1
],
% We only retry once
+ DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2),
close(Db2),
- UpdatePid ! {update_docs, self(), DocBuckets2, NonRepDocs, MergeConflicts, FullCommit},
+ UpdatePid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts, FullCommit},
case collect_results(UpdatePid, MRef, []) of
{ok, Results} -> {ok, Results};
retry -> throw({update_error, compaction_retry})
@@ -834,6 +838,24 @@ write_and_commit(#db{update_pid=UpdatePi
end.
+prepare_doc_summaries(Db, BucketList) ->
+ [lists:map(
+ fun(#doc{body = Body, atts = Atts} = Doc) ->
+ DiskAtts = [{N, T, P, AL, DL, R, M, E} ||
+ #att{name = N, type = T, data = {_, P}, md5 = M, revpos = R,
+ att_len = AL, disk_len = DL, encoding = E} <- Atts],
+ AttsFd = case Atts of
+ [#att{data = {Fd, _}} | _] ->
+ Fd;
+ [] ->
+ nil
+ end,
+ SummaryChunk = couch_db_updater:make_doc_summary(Db, {Body, DiskAtts}),
+ Doc#doc{body = {summary, SummaryChunk, AttsFd}}
+ end,
+ Bucket) || Bucket <- BucketList].
+
+
set_new_att_revpos(#doc{revs={RevPos,_Revs},atts=Atts}=Doc) ->
Doc#doc{atts= lists:map(fun(#att{data={_Fd,_Sp}}=Att) ->
% already commited to disk, do not set new rev
@@ -1128,23 +1150,26 @@ open_doc_revs_int(Db, IdRevs, Options) -
end,
IdRevs, LookupResults).
-open_doc_int(Db, <<?LOCAL_DOC_PREFIX, _/binary>> = Id, _Options) ->
+open_doc_int(Db, <<?LOCAL_DOC_PREFIX, _/binary>> = Id, Options) ->
case couch_btree:lookup(local_btree(Db), [Id]) of
[{ok, {_, {Rev, BodyData}}}] ->
- {ok, #doc{id=Id, revs={0, [list_to_binary(integer_to_list(Rev))]}, body=BodyData}};
+ Doc = #doc{id=Id, revs={0, [?l2b(integer_to_list(Rev))]}, body=BodyData},
+ apply_open_options({ok, Doc}, Options);
[not_found] ->
{not_found, missing}
end;
open_doc_int(Db, #doc_info{id=Id,revs=[RevInfo|_]}=DocInfo, Options) ->
#rev_info{deleted=IsDeleted,rev={Pos,RevId},body_sp=Bp} = RevInfo,
Doc = make_doc(Db, Id, IsDeleted, Bp, {Pos,[RevId]}),
- {ok, Doc#doc{meta=doc_meta_info(DocInfo, [], Options)}};
+ apply_open_options(
+ {ok, Doc#doc{meta=doc_meta_info(DocInfo, [], Options)}}, Options);
open_doc_int(Db, #full_doc_info{id=Id,rev_tree=RevTree}=FullDocInfo, Options) ->
#doc_info{revs=[#rev_info{deleted=IsDeleted,rev=Rev,body_sp=Bp}|_]} =
DocInfo = couch_doc:to_doc_info(FullDocInfo),
{[{_, RevPath}], []} = couch_key_tree:get(RevTree, [Rev]),
Doc = make_doc(Db, Id, IsDeleted, Bp, RevPath),
- {ok, Doc#doc{meta=doc_meta_info(DocInfo, RevTree, Options)}};
+ apply_open_options(
+ {ok, Doc#doc{meta=doc_meta_info(DocInfo, RevTree, Options)}}, Options);
open_doc_int(Db, Id, Options) ->
case get_full_doc_info(Db, Id) of
{ok, FullDocInfo} ->
@@ -1203,7 +1228,14 @@ make_doc(#db{updater_fd = Fd} = Db, Id,
nil ->
{[], []};
_ ->
- {ok, {BodyData0, Atts0}} = read_doc(Db, Bp),
+ {ok, {BodyData0, Atts00}} = read_doc(Db, Bp),
+ Atts0 = case Atts00 of
+ _ when is_binary(Atts00) ->
+ couch_compress:decompress(Atts00);
+ _ when is_list(Atts00) ->
+ % pre 1.2 format
+ Atts00
+ end,
{BodyData0,
lists:map(
fun({Name,Type,Sp,AttLen,DiskLen,RevPos,Md5,Enc}) ->
Modified: couchdb/trunk/src/couchdb/couch_db.hrl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.hrl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.hrl (original)
+++ couchdb/trunk/src/couchdb/couch_db.hrl Mon May 2 13:28:20 2011
@@ -13,6 +13,7 @@
-define(LOCAL_DOC_PREFIX, "_local/").
-define(DESIGN_DOC_PREFIX0, "_design").
-define(DESIGN_DOC_PREFIX, "_design/").
+-define(DEFAULT_COMPRESSION, snappy).
-define(MIN_STR, <<"">>).
-define(MAX_STR, <<255>>). % illegal utf string
@@ -25,6 +26,7 @@
-define(b2l(V), binary_to_list(V)).
-define(l2b(V), list_to_binary(V)).
+-define(term_to_bin(T), term_to_binary(T, [{minor_version, 1}])).
-define(DEFAULT_ATTACHMENT_CONTENT_TYPE, <<"application/octet-stream">>).
@@ -167,7 +169,8 @@
waiting_delayed_commit = nil,
revs_limit = 1000,
fsync_options = [],
- options = []
+ options = [],
+ compression
}).
@@ -271,5 +274,6 @@
extract_kv = fun({_Key, _Value} = KV) -> KV end,
assemble_kv = fun(Key, Value) -> {Key, Value} end,
less = fun(A, B) -> A < B end,
- reduce = nil
+ reduce = nil,
+ compression = ?DEFAULT_COMPRESSION
}).
Modified: couchdb/trunk/src/couchdb/couch_db_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db_updater.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db_updater.erl Mon May 2 13:28:20 2011
@@ -14,6 +14,7 @@
-behaviour(gen_server).
-export([btree_by_id_reduce/2,btree_by_seq_reduce/2]).
+-export([make_doc_summary/2]).
-export([init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3,handle_info/2]).
-include("couch_db.hrl").
@@ -66,8 +67,9 @@ handle_call(increment_update_seq, _From,
couch_db_update_notifier:notify({updated, Db#db.name}),
{reply, {ok, Db2#db.update_seq}, Db2};
-handle_call({set_security, NewSec}, _From, Db) ->
- {ok, Ptr, _} = couch_file:append_term(Db#db.updater_fd, NewSec),
+handle_call({set_security, NewSec}, _From, #db{compression = Comp} = Db) ->
+ {ok, Ptr, _} = couch_file:append_term(
+ Db#db.updater_fd, NewSec, [{compression, Comp}]),
Db2 = commit_data(Db#db{security=NewSec, security_ptr=Ptr,
update_seq=Db#db.update_seq+1}),
ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}),
@@ -88,7 +90,8 @@ handle_call({purge_docs, IdRevs}, _From,
fulldocinfo_by_id_btree = DocInfoByIdBTree,
docinfo_by_seq_btree = DocInfoBySeqBTree,
update_seq = LastSeq,
- header = Header = #db_header{purge_seq=PurgeSeq}
+ header = Header = #db_header{purge_seq=PurgeSeq},
+ compression = Comp
} = Db,
DocLookups = couch_btree:lookup(DocInfoByIdBTree,
[Id || {Id, _Revs} <- IdRevs]),
@@ -135,7 +138,8 @@ handle_call({purge_docs, IdRevs}, _From,
DocInfoToUpdate, SeqsToRemove),
{ok, DocInfoByIdBTree2} = couch_btree:add_remove(DocInfoByIdBTree,
FullDocInfoToUpdate, IdsToRemove),
- {ok, Pointer, _} = couch_file:append_term(Fd, IdRevsPurged),
+ {ok, Pointer, _} = couch_file:append_term(
+ Fd, IdRevsPurged, [{compression, Comp}]),
Db2 = commit_data(
Db#db{
@@ -419,15 +423,20 @@ init_db(DbName, Filepath, Fd, ReaderFd,
_ -> ok
end,
+ Compression = couch_compress:get_compression_method(),
+
{ok, IdBtree} = couch_btree:open(Header#db_header.fulldocinfo_by_id_btree_state, Fd,
[{split, fun(X) -> btree_by_id_split(X) end},
{join, fun(X,Y) -> btree_by_id_join(X,Y) end},
- {reduce, fun(X,Y) -> btree_by_id_reduce(X,Y) end}]),
+ {reduce, fun(X,Y) -> btree_by_id_reduce(X,Y) end},
+ {compression, Compression}]),
{ok, SeqBtree} = couch_btree:open(Header#db_header.docinfo_by_seq_btree_state, Fd,
[{split, fun(X) -> btree_by_seq_split(X) end},
{join, fun(X,Y) -> btree_by_seq_join(X,Y) end},
- {reduce, fun(X,Y) -> btree_by_seq_reduce(X,Y) end}]),
- {ok, LocalDocsBtree} = couch_btree:open(Header#db_header.local_docs_btree_state, Fd),
+ {reduce, fun(X,Y) -> btree_by_seq_reduce(X,Y) end},
+ {compression, Compression}]),
+ {ok, LocalDocsBtree} = couch_btree:open(Header#db_header.local_docs_btree_state, Fd,
+ [{compression, Compression}]),
case Header#db_header.security_ptr of
nil ->
Security = [],
@@ -458,7 +467,8 @@ init_db(DbName, Filepath, Fd, ReaderFd,
instance_start_time = StartTime,
revs_limit = Header#db_header.revs_limit,
fsync_options = FsyncOptions,
- options = Options
+ options = Options,
+ compression = Compression
}.
open_reader_fd(Filepath, Options) ->
@@ -496,31 +506,30 @@ flush_trees(#db{updater_fd = Fd} = Db,
{Flushed, LeafsSize} = couch_key_tree:mapfold(
fun(_Rev, Value, Type, Acc) ->
case Value of
- #doc{atts=Atts,deleted=IsDeleted}=Doc ->
+ #doc{deleted = IsDeleted, body = {summary, Summary, AttsFd}} ->
% this node value is actually an unwritten document summary,
% write to disk.
% make sure the Fd in the written bins is the same Fd we are
% and convert bins, removing the FD.
% All bins should have been written to disk already.
- DiskAtts =
- case Atts of
- [] -> [];
- [#att{data={BinFd, _Sp}} | _ ] when BinFd == Fd ->
- [{N,T,P,AL,DL,R,M,E}
- || #att{name=N,type=T,data={_,P},md5=M,revpos=R,
- att_len=AL,disk_len=DL,encoding=E}
- <- Atts];
+ case {AttsFd, Fd} of
+ {nil, _} ->
+ ok;
+ {SameFd, SameFd} ->
+ ok;
_ ->
- % BinFd must not equal our Fd. This can happen when a database
- % is being switched out during a compaction
+ % Fd where the attachments were written to is not the same
+ % as our Fd. This can happen when a database is being
+ % switched out during a compaction.
?LOG_DEBUG("File where the attachments are written has"
" changed. Possibly retrying.", []),
throw(retry)
end,
{ok, NewSummaryPointer, SummarySize} =
- couch_file:append_term_md5(Fd, {Doc#doc.body, DiskAtts}),
+ couch_file:append_raw_chunk(Fd, Summary),
TotalSize = lists:foldl(
- fun(#att{att_len = L}, A) -> A + L end, SummarySize, Atts),
+ fun(#att{att_len = L}, A) -> A + L end,
+ SummarySize, Value#doc.atts),
NewValue = {IsDeleted, NewSummaryPointer, UpdateSeq, TotalSize},
case Type of
leaf ->
@@ -783,7 +792,14 @@ commit_data(Db, _) ->
copy_doc_attachments(#db{updater_fd = SrcFd} = SrcDb, SrcSp, DestFd) ->
- {ok, {BodyData, BinInfos}} = couch_db:read_doc(SrcDb, SrcSp),
+ {ok, {BodyData, BinInfos0}} = couch_db:read_doc(SrcDb, SrcSp),
+ BinInfos = case BinInfos0 of
+ _ when is_binary(BinInfos0) ->
+ couch_compress:decompress(BinInfos0);
+ _ when is_list(BinInfos0) ->
+ % pre 1.2 file format
+ BinInfos0
+ end,
% copy the bin values
NewBinInfos = lists:map(
fun({Name, Type, BinSp, AttLen, RevPos, Md5}) ->
@@ -826,8 +842,9 @@ copy_docs(Db, #db{updater_fd = DestFd} =
Seq = element(3, LeafVal),
{_Body, AttsInfo} = Summary = copy_doc_attachments(
Db, Sp, DestFd),
- {ok, Pos, SummarySize} =
- couch_file:append_term_md5(DestFd, Summary),
+ SummaryChunk = make_doc_summary(NewDb, Summary),
+ {ok, Pos, SummarySize} = couch_file:append_raw_chunk(
+ DestFd, SummaryChunk),
TotalLeafSize = lists:foldl(
fun({_, _, _, AttLen, _, _, _, _}, S) -> S + AttLen end,
SummarySize, AttsInfo),
@@ -890,7 +907,9 @@ copy_compact(Db, NewDb0, Retry) ->
% copy misc header values
if NewDb3#db.security /= Db#db.security ->
- {ok, Ptr, _} = couch_file:append_term(NewDb3#db.updater_fd, Db#db.security),
+ {ok, Ptr, _} = couch_file:append_term(
+ NewDb3#db.updater_fd, Db#db.security,
+ [{compression, NewDb3#db.compression}]),
NewDb4 = NewDb3#db{security=Db#db.security, security_ptr=Ptr};
true ->
NewDb4 = NewDb3
@@ -921,7 +940,8 @@ start_copy_compact(#db{name=Name,filepat
NewDb = init_db(Name, CompactFile, Fd, ReaderFd, Header, Db#db.options),
NewDb2 = if PurgeSeq > 0 ->
{ok, PurgedIdsRevs} = couch_db:get_last_purged(Db),
- {ok, Pointer, _} = couch_file:append_term(Fd, PurgedIdsRevs),
+ {ok, Pointer, _} = couch_file:append_term(
+ Fd, PurgedIdsRevs, [{compression, NewDb#db.compression}]),
NewDb#db{header=Header#db_header{purge_seq=PurgeSeq, purged_docs=Pointer}};
true ->
NewDb
@@ -932,3 +952,19 @@ start_copy_compact(#db{name=Name,filepat
close_db(NewDb3),
gen_server:cast(Db#db.update_pid, {compact_done, CompactFile}).
+make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
+ Body = case couch_compress:is_compressed(Body0) of
+ true ->
+ Body0;
+ false ->
+ % pre 1.2 database file format
+ couch_compress:compress(Body0, Comp)
+ end,
+ Atts = case couch_compress:is_compressed(Atts0) of
+ true ->
+ Atts0;
+ false ->
+ couch_compress:compress(Atts0, Comp)
+ end,
+ SummaryBin = ?term_to_bin({Body, Atts}),
+ couch_file:assemble_file_chunk(SummaryBin, couch_util:md5(SummaryBin)).
Modified: couchdb/trunk/src/couchdb/couch_doc.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_doc.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_doc.erl (original)
+++ couchdb/trunk/src/couchdb/couch_doc.erl Mon May 2 13:28:20 2011
@@ -20,6 +20,7 @@
-export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
-export([to_path/1]).
-export([mp_parse_doc/2]).
+-export([with_ejson_body/1, with_bin_body/1]).
-include("couch_db.hrl").
@@ -132,7 +133,10 @@ to_json_attachments(Atts, OutputData, Da
end, Atts),
[{<<"_attachments">>, {AttProps}}].
-to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds},
+to_json_obj(Doc, Options) ->
+ doc_to_json_obj(with_ejson_body(Doc), Options).
+
+doc_to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds},
meta=Meta}=Doc,Options)->
{[{<<"_id">>, Id}]
++ to_json_rev(Start, RevIds)
@@ -530,3 +534,15 @@ mp_parse_atts({body, Bytes}) ->
fun mp_parse_atts/1;
mp_parse_atts(body_end) ->
fun mp_parse_atts/1.
+
+
+with_bin_body(#doc{body = Json} = Doc) when is_binary(Json) ->
+ Doc;
+with_bin_body(#doc{body = EJson} = Doc) ->
+ Doc#doc{body = couch_compress:compress(EJson)}.
+
+
+with_ejson_body(#doc{body = Body} = Doc) when is_binary(Body) ->
+ Doc#doc{body = couch_compress:decompress(Body)};
+with_ejson_body(#doc{body = {_}} = Doc) ->
+ Doc.
Modified: couchdb/trunk/src/couchdb/couch_file.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_file.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_file.erl (original)
+++ couchdb/trunk/src/couchdb/couch_file.erl Mon May 2 13:28:20 2011
@@ -26,7 +26,8 @@
-export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2]).
-export([pread_term/2, pread_iolist/2, pread_binary/2]).
-export([append_binary/2, append_binary_md5/2]).
--export([append_term/2, append_term_md5/2]).
+-export([append_raw_chunk/2, assemble_file_chunk/1, assemble_file_chunk/2]).
+-export([append_term/2, append_term/3, append_term_md5/2, append_term_md5/3]).
-export([write_header/2, read_header/1]).
-export([delete/2, delete/3, init_delete_dir/1]).
@@ -77,11 +78,18 @@ open(Filepath, Options) ->
%%----------------------------------------------------------------------
append_term(Fd, Term) ->
- append_binary(Fd, term_to_binary(Term)).
-
+ append_term(Fd, Term, []).
+
+append_term(Fd, Term, Options) ->
+ Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION),
+ append_binary(Fd, couch_compress:compress(Term, Comp)).
+
append_term_md5(Fd, Term) ->
- append_binary_md5(Fd, term_to_binary(Term)).
+ append_term_md5(Fd, Term, []).
+append_term_md5(Fd, Term, Options) ->
+ Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION),
+ append_binary_md5(Fd, couch_compress:compress(Term, Comp)).
%%----------------------------------------------------------------------
%% Purpose: To append an Erlang binary to the end of the file.
@@ -92,15 +100,21 @@ append_term_md5(Fd, Term) ->
%%----------------------------------------------------------------------
append_binary(Fd, Bin) ->
- Size = iolist_size(Bin),
- gen_server:call(Fd, {append_bin,
- [<<0:1/integer,Size:31/integer>>, Bin]}, infinity).
+ gen_server:call(Fd, {append_bin, assemble_file_chunk(Bin)}, infinity).
append_binary_md5(Fd, Bin) ->
- Size = iolist_size(Bin),
- gen_server:call(Fd, {append_bin,
- [<<1:1/integer,Size:31/integer>>, couch_util:md5(Bin), Bin]}, infinity).
+ gen_server:call(Fd,
+ {append_bin, assemble_file_chunk(Bin, couch_util:md5(Bin))}, infinity).
+
+append_raw_chunk(Fd, Chunk) ->
+ gen_server:call(Fd, {append_bin, Chunk}, infinity).
+
+
+assemble_file_chunk(Bin) ->
+ [<<0:1/integer, (iolist_size(Bin)):31/integer>>, Bin].
+assemble_file_chunk(Bin, Md5) ->
+ [<<1:1/integer, (iolist_size(Bin)):31/integer>>, Md5, Bin].
%%----------------------------------------------------------------------
%% Purpose: Reads a term from a file that was written with append_term
@@ -112,7 +126,7 @@ append_binary_md5(Fd, Bin) ->
pread_term(Fd, Pos) ->
{ok, Bin} = pread_binary(Fd, Pos),
- {ok, binary_to_term(Bin)}.
+ {ok, couch_compress:decompress(Bin)}.
%%----------------------------------------------------------------------
Modified: couchdb/trunk/src/couchdb/couch_httpd.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd.erl Mon May 2 13:28:20 2011
@@ -279,6 +279,11 @@ handle_request_int(MochiReq, DefaultFun,
send_error(HttpReq, bad_request);
exit:normal ->
exit(normal);
+ exit:snappy_nif_not_loaded ->
+ ErrorReason = "To access the database or view index, Apache CouchDB"
+ " must be built with Erlang OTP R13B04 or higher.",
+ ?LOG_ERROR("~s", [ErrorReason]),
+ send_error(HttpReq, {bad_otp_release, ErrorReason});
throw:Error ->
?LOG_DEBUG("Minor error in HTTP request: ~p",[Error]),
?LOG_DEBUG("Stacktrace: ~p",[erlang:get_stacktrace()]),
Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Mon May 2 13:28:20 2011
@@ -152,7 +152,7 @@ handle_design_req(#httpd{
}=Req, Db) ->
% load ddoc
DesignId = <<"_design/", DesignName/binary>>,
- DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]),
Handler = couch_util:dict_find(Action, DesignUrlHandlers, fun(_, _, _) ->
throw({not_found, <<"missing handler: ", Action/binary>>})
end),
Modified: couchdb/trunk/src/couchdb/couch_query_servers.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_query_servers.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_query_servers.erl (original)
+++ couchdb/trunk/src/couchdb/couch_query_servers.erl Mon May 2 13:28:20 2011
@@ -309,7 +309,8 @@ terminate(_Reason, #qserver{pid_procs=Pi
[couch_util:shutdown_sync(P) || {P,_} <- ets:tab2list(PidProcs)],
ok.
-handle_call({get_proc, #doc{body={Props}}=DDoc, DDocKey}, From, Server) ->
+handle_call({get_proc, DDoc1, DDocKey}, From, Server) ->
+ #doc{body = {Props}} = DDoc = couch_doc:with_ejson_body(DDoc1),
Lang = couch_util:get_value(<<"language">>, Props, <<"javascript">>),
case lang_proc(Lang, Server, fun(Procs) ->
% find a proc in the set that has the DDoc
Modified: couchdb/trunk/src/couchdb/couch_replication_manager.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_replication_manager.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_replication_manager.erl (original)
+++ couchdb/trunk/src/couchdb/couch_replication_manager.erl Mon May 2 13:28:20 2011
@@ -445,7 +445,7 @@ stop_all_replications() ->
update_rep_doc(RepDocId, KVs) ->
{ok, RepDb} = ensure_rep_db_exists(),
try
- case couch_db:open_doc(RepDb, RepDocId, []) of
+ case couch_db:open_doc(RepDb, RepDocId, [ejson_body]) of
{ok, LatestRepDoc} ->
update_rep_doc(RepDb, LatestRepDoc, KVs);
_ ->
Modified: couchdb/trunk/src/couchdb/couch_replicator.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_replicator.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_replicator.erl (original)
+++ couchdb/trunk/src/couchdb/couch_replicator.erl Mon May 2 13:28:20 2011
@@ -559,7 +559,7 @@ fold_replication_logs([], _Vsn, _LogId,
lists:reverse(Acc);
fold_replication_logs([Db | Rest] = Dbs, Vsn, LogId, NewId, Rep, Acc) ->
- case couch_api_wrap:open_doc(Db, LogId, []) of
+ case couch_api_wrap:open_doc(Db, LogId, [ejson_body]) of
{error, <<"not_found">>} when Vsn > 1 ->
OldRepId = couch_replicator_utils:replication_id(Rep, Vsn - 1),
fold_replication_logs(Dbs, Vsn - 1,
Modified: couchdb/trunk/src/couchdb/couch_replicator_utils.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_replicator_utils.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_replicator_utils.erl (original)
+++ couchdb/trunk/src/couchdb/couch_replicator_utils.erl Mon May 2 13:28:20 2011
@@ -89,8 +89,8 @@ filter_code(Filter, Source, UserCtx) ->
re:run(Filter, "(.*?)/(.*)", [{capture, [1, 2], binary}]),
{ok, Db} = couch_api_wrap:db_open(Source, [{user_ctx, UserCtx}]),
try
- {ok, #doc{body = Body}} =
- couch_api_wrap:open_doc(Db, <<"_design/", DDocName/binary>>, []),
+ {ok, #doc{body = Body}} = couch_api_wrap:open_doc(
+ Db, <<"_design/", DDocName/binary>>, [ejson_body]),
Code = couch_util:get_nested_json_value(
Body, [<<"filters">>, FilterName]),
re:replace(Code, [$^, "\s*(.*?)\s*", $$], "\\1", [{return, binary}])
Modified: couchdb/trunk/src/couchdb/couch_server.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_server.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_server.erl (original)
+++ couchdb/trunk/src/couchdb/couch_server.erl Mon May 2 13:28:20 2011
@@ -380,6 +380,29 @@ code_change(_OldVsn, State, _Extra) ->
handle_info({'EXIT', _Pid, config_change}, Server) ->
{noreply, shutdown, Server};
+handle_info({'EXIT', Pid, snappy_nif_not_loaded}, Server) ->
+ Server2 = case ets:lookup(couch_dbs_by_pid, Pid) of
+ [{Pid, Db}] ->
+ [{Db, {opening, Pid, Froms}}] = ets:lookup(couch_dbs_by_name, Db),
+ Msg = io_lib:format("To open the database `~s`, Apache CouchDB "
+ "must be built with Erlang OTP R13B04 or higher.", [Db]),
+ ?LOG_ERROR(Msg, []),
+ lists:foreach(
+ fun(F) -> gen_server:reply(F, {bad_otp_release, Msg}) end,
+ Froms),
+ true = ets:delete(couch_dbs_by_name, Db),
+ true = ets:delete(couch_dbs_by_pid, Pid),
+ case ets:lookup(couch_sys_dbs, Db) of
+ [{Db, _}] ->
+ true = ets:delete(couch_sys_dbs, Db),
+ Server;
+ [] ->
+ Server#server{dbs_open = Server#server.dbs_open - 1}
+ end;
+ _ ->
+ Server
+ end,
+ {noreply, Server2};
handle_info(Error, _Server) ->
?LOG_ERROR("Unexpected message, restarting couch_server: ~p", [Error]),
exit(kill).
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1098558&r1=1098557&r2=1098558&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Mon May 2 13:28:20 2011
@@ -497,7 +497,7 @@ sort_lib([{LName, LCode}|Rest], LAcc) ->
open_db_group(DbName, GroupId) ->
case couch_db:open_int(DbName, []) of
{ok, Db} ->
- case couch_db:open_doc(Db, GroupId) of
+ case couch_db:open_doc(Db, GroupId, [ejson_body]) of
{ok, Doc} ->
couch_db:close(Db),
{ok, design_doc_to_view_group(Doc)};
@@ -618,7 +618,8 @@ init_group(Db, Fd, #group{def_lang=Lang,
(State) -> {State, 0, 0}
end,
ViewStates2 = lists:map(StateUpdate, ViewStates),
- {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd),
+ {ok, IdBtree} = couch_btree:open(
+ IdBtreeState, Fd, [{compression, Db#db.compression}]),
Views2 = lists:zipwith(
fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
@@ -644,7 +645,8 @@ init_group(Db, Fd, #group{def_lang=Lang,
Less = fun(A,B) -> A < B end
end,
{ok, Btree} = couch_btree:open(BTState, Fd,
- [{less, Less}, {reduce, ReduceFun}]
+ [{less, Less}, {reduce, ReduceFun},
+ {compression, Db#db.compression}]
),
View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq}
end,
Added: couchdb/trunk/src/snappy/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/Makefile.am?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/Makefile.am (added)
+++ couchdb/trunk/src/snappy/Makefile.am Mon May 2 13:28:20 2011
@@ -0,0 +1,74 @@
+## Licensed under the Apache License, Version 2.0 (the "License"); you may not
+## use this file except in compliance with the License. You may obtain a copy of
+## the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+## License for the specific language governing permissions and limitations under
+## the License.
+
+snappyebindir = $(localerlanglibdir)/snappy-1.0.1/ebin
+snappyprivdir = $(localerlanglibdir)/snappy-1.0.1/priv
+
+CLEANFILES = \
+ $(snappyebin_make_generated_file_list) \
+ $(snappypriv_make_generated_file_list) \
+ priv/snappy_nif.so
+
+if USE_OTP_NIFS
+snappypriv_LTLIBRARIES = snappy_nif.la
+endif
+
+SNAPPY_CXX_SRCS = \
+ snappy_nif.cc \
+ google-snappy/snappy.cc \
+ google-snappy/snappy-sinksource.cc \
+ google-snappy/snappy-stubs-internal.cc
+
+SNAPPY_CXX_HDRS = \
+ erl_nif_compat.h \
+ google-snappy/snappy.h \
+ google-snappy/snappy-internal.h \
+ google-snappy/snappy-stubs-internal.h \
+ google-snappy/snappy-stubs-public.h
+
+if USE_OTP_NIFS
+snappy_nif_la_SOURCES = $(SNAPPY_CXX_SRCS)
+snappy_nif_la_LDFLAGS = -module -avoid-version
+
+if WINDOWS
+snappy_nif_la_LDFLAGS += -no-undefined
+endif
+endif
+
+snappy_file_collection = \
+ snappy.app.in \
+ snappy.erl
+
+snappyebin_make_generated_file_list = \
+ snappy.app \
+ snappy.beam
+
+EXTRA_DIST = \
+ $(SNAPPY_CXX_HDRS)
+ $(snappy_file_collection)
+
+snappyebin_DATA = \
+ $(snappyebin_make_generated_file_list)
+
+if USE_OTP_NIFS
+priv/snappy_nif.so: snappy_nif.la
+ @mkdir -p ./priv
+ cp .libs/snappy_nif.so $@
+
+all: priv/snappy_nif.so
+endif
+
+%.app: %.app.in
+ cp $< $@
+
+%.beam: %.erl
+ $(ERLC) $(ERLC_FLAGS) $<
Added: couchdb/trunk/src/snappy/erl_nif_compat.h
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/erl_nif_compat.h?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/erl_nif_compat.h (added)
+++ couchdb/trunk/src/snappy/erl_nif_compat.h Mon May 2 13:28:20 2011
@@ -0,0 +1,121 @@
+/* Copyright (c) 2010-2011 Basho Technologies, Inc.
+ * With some minor modifications by Filipe David Manana
+ * <fd...@apache.org>
+ *
+ * This file is provided to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+#ifndef ERL_NIF_COMPAT_H_
+#define ERL_NIF_COMPAT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+#include "erl_nif.h"
+
+
+#if ERL_NIF_MAJOR_VERSION == 0 && ERL_NIF_MINOR_VERSION == 1
+#define OTP_R13B03
+#elif ERL_NIF_MAJOR_VERSION == 1 && ERL_NIF_MINOR_VERSION == 0
+#define OTP_R13B04
+#elif ERL_NIF_MAJOR_VERSION == 2 && ERL_NIF_MINOR_VERSION == 0
+#define OTP_R14A
+#define OTP_R14B
+#define OTP_R14B01
+#elif ERL_NIF_MAJOR_VERSION == 2 && ERL_NIF_MINOR_VERSION == 1
+#define OTP_R14B02
+#endif
+
+
+#ifdef OTP_R13B03
+
+#define enif_open_resource_type_compat enif_open_resource_type
+#define enif_alloc_resource_compat enif_alloc_resource
+#define enif_release_resource_compat enif_release_resource
+#define enif_alloc_binary_compat enif_alloc_binary
+#define enif_alloc_compat enif_alloc
+#define enif_release_binary_compat enif_release_binary
+#define enif_free_compat enif_free
+#define enif_get_atom_compat enif_get_atom
+#define enif_priv_data_compat enif_get_data
+#define enif_make_uint_compat enif_make_ulong
+
+#define enif_make_string_compat(E, B, Enc) \
+ enif_make_string(E, B)
+
+#endif /* R13B03 */
+
+
+#ifdef OTP_R13B04
+
+#define enif_open_resource_type_compat enif_open_resource_type
+#define enif_alloc_resource_compat enif_alloc_resource
+#define enif_release_resource_compat enif_release_resource
+#define enif_alloc_binary_compat enif_alloc_binary
+#define enif_realloc_binary_compat enif_realloc_binary
+#define enif_release_binary_compat enif_release_binary
+#define enif_alloc_compat enif_alloc
+#define enif_free_compat enif_free
+#define enif_get_atom_compat enif_get_atom
+#define enif_priv_data_compat enif_priv_data
+#define enif_make_string_compat enif_make_string
+#define enif_make_uint_compat enif_make_uint
+
+#endif /* R13B04 */
+
+
+/* OTP R14A and future releases */
+#if !defined(OTP_R13B03) && !defined(OTP_R13B04)
+
+#define enif_open_resource_type_compat(E, N, D, F, T) \
+ enif_open_resource_type(E, NULL, N, D, F, T)
+
+#define enif_alloc_resource_compat(E, T, S) \
+ enif_alloc_resource(T, S)
+
+#define enif_release_resource_compat(E, H) \
+ enif_release_resource(H)
+
+#define enif_alloc_binary_compat(E, S, B) \
+ enif_alloc_binary(S, B)
+
+#define enif_realloc_binary_compat(E, S, B) \
+ enif_realloc_binary(S, B)
+
+#define enif_release_binary_compat(E, B) \
+ enif_release_binary(B)
+
+#define enif_alloc_compat(E, S) \
+ enif_alloc(S)
+
+#define enif_free_compat(E, P) \
+ enif_free(P)
+
+#define enif_get_atom_compat(E, T, B, S) \
+ enif_get_atom(E, T, B, S, ERL_NIF_LATIN1)
+
+#define enif_priv_data_compat enif_priv_data
+#define enif_make_string_compat enif_make_string
+#define enif_make_uint_compat enif_make_uint
+
+#endif /* R14 and future releases */
+
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* ERL_NIF_COMPAT_H_ */
Added: couchdb/trunk/src/snappy/google-snappy/AUTHORS
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/AUTHORS?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/AUTHORS (added)
+++ couchdb/trunk/src/snappy/google-snappy/AUTHORS Mon May 2 13:28:20 2011
@@ -0,0 +1 @@
+opensource@google.com
Added: couchdb/trunk/src/snappy/google-snappy/COPYING
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/COPYING?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/COPYING (added)
+++ couchdb/trunk/src/snappy/google-snappy/COPYING Mon May 2 13:28:20 2011
@@ -0,0 +1,28 @@
+Copyright 2011, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Added: couchdb/trunk/src/snappy/google-snappy/config.h.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/config.h.in?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/config.h.in (added)
+++ couchdb/trunk/src/snappy/google-snappy/config.h.in Mon May 2 13:28:20 2011
@@ -0,0 +1,130 @@
+/* src/snappy/google-snappy/config.h.in. Generated from configure.ac by autoheader. */
+
+/* Define if building universal (internal helper macro) */
+#undef AC_APPLE_UNIVERSAL_BUILD
+
+/* Define to 1 if the compiler supports __builtin_ctz and friends. */
+#undef HAVE_BUILTIN_CTZ
+
+/* Define to 1 if the compiler supports __builtin_expect. */
+#undef HAVE_BUILTIN_EXPECT
+
+/* "Provide HTTP support to couchjs" */
+#undef HAVE_CURL
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the <stddef.h> header file. */
+#undef HAVE_STDDEF_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#undef HAVE_SYS_MMAN_H
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#undef HAVE_SYS_RESOURCE_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#undef LT_OBJDIR
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Use new JS_SetOperationCallback */
+#undef USE_JS_SETOPCB
+
+/* Enable extensions on AIX 3, Interix. */
+#ifndef _ALL_SOURCE
+# undef _ALL_SOURCE
+#endif
+/* Enable GNU extensions on systems that have them. */
+#ifndef _GNU_SOURCE
+# undef _GNU_SOURCE
+#endif
+/* Enable threading extensions on Solaris. */
+#ifndef _POSIX_PTHREAD_SEMANTICS
+# undef _POSIX_PTHREAD_SEMANTICS
+#endif
+/* Enable extensions on HP NonStop. */
+#ifndef _TANDEM_SOURCE
+# undef _TANDEM_SOURCE
+#endif
+/* Enable general extensions on Solaris. */
+#ifndef __EXTENSIONS__
+# undef __EXTENSIONS__
+#endif
+
+
+/* Version number of package */
+#undef VERSION
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+# define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+# undef WORDS_BIGENDIAN
+# endif
+#endif
+
+/* Define to 1 if on MINIX. */
+#undef _MINIX
+
+/* Define to 2 if the system does not provide POSIX.1 features except with
+ this defined. */
+#undef _POSIX_1_SOURCE
+
+/* Define to 1 if you need to in order for `stat' and other things to work. */
+#undef _POSIX_SOURCE
Added: couchdb/trunk/src/snappy/google-snappy/snappy-internal.h
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/snappy-internal.h?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/snappy-internal.h (added)
+++ couchdb/trunk/src/snappy/google-snappy/snappy-internal.h Mon May 2 13:28:20 2011
@@ -0,0 +1,150 @@
+// Copyright 2008 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Internals shared between the Snappy implementation and its unittest.
+
+#ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_
+#define UTIL_SNAPPY_SNAPPY_INTERNAL_H_
+
+#include "snappy-stubs-internal.h"
+
+namespace snappy {
+namespace internal {
+
+class WorkingMemory {
+ public:
+ WorkingMemory() : large_table_(NULL) { }
+ ~WorkingMemory() { delete[] large_table_; }
+
+ // Allocates and clears a hash table using memory in "*this",
+ // stores the number of buckets in "*table_size" and returns a pointer to
+ // the base of the hash table.
+ uint16* GetHashTable(size_t input_size, int* table_size);
+
+ private:
+ uint16 small_table_[1<<10]; // 2KB
+ uint16* large_table_; // Allocated only when needed
+
+ DISALLOW_COPY_AND_ASSIGN(WorkingMemory);
+};
+
+// Flat array compression that does not emit the "uncompressed length"
+// prefix. Compresses "input" string to the "*op" buffer.
+//
+// REQUIRES: "input_length <= kBlockSize"
+// REQUIRES: "op" points to an array of memory that is at least
+// "MaxCompressedLength(input_length)" in size.
+// REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
+// REQUIRES: "table_size" is a power of two
+//
+// Returns an "end" pointer into "op" buffer.
+// "end - op" is the compressed size of "input".
+char* CompressFragment(const char* input,
+ size_t input_length,
+ char* op,
+ uint16* table,
+ const int table_size);
+
+// Return the largest n such that
+//
+// s1[0,n-1] == s2[0,n-1]
+// and n <= (s2_limit - s2).
+//
+// Does not read *s2_limit or beyond.
+// Does not read *(s1 + (s2_limit - s2)) or beyond.
+// Requires that s2_limit >= s2.
+//
+// Separate implementation for x86_64, for speed. Uses the fact that
+// x86_64 is little endian.
+#if defined(ARCH_K8)
+static inline int FindMatchLength(const char* s1,
+ const char* s2,
+ const char* s2_limit) {
+ DCHECK_GE(s2_limit, s2);
+ int matched = 0;
+
+ // Find out how long the match is. We loop over the data 64 bits at a
+ // time until we find a 64-bit block that doesn't match; then we find
+ // the first non-matching bit and use that to calculate the total
+ // length of the match.
+ while (PREDICT_TRUE(s2 <= s2_limit - 8)) {
+ if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) {
+ s2 += 8;
+ matched += 8;
+ } else {
+ // On current (mid-2008) Opteron models there is a 3% more
+ // efficient code sequence to find the first non-matching byte.
+ // However, what follows is ~10% better on Intel Core 2 and newer,
+ // and we expect AMD's bsf instruction to improve.
+ uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched);
+ int matching_bits = Bits::FindLSBSetNonZero64(x);
+ matched += matching_bits >> 3;
+ return matched;
+ }
+ }
+ while (PREDICT_TRUE(s2 < s2_limit)) {
+ if (PREDICT_TRUE(s1[matched] == *s2)) {
+ ++s2;
+ ++matched;
+ } else {
+ return matched;
+ }
+ }
+ return matched;
+}
+#else
+static inline int FindMatchLength(const char* s1,
+ const char* s2,
+ const char* s2_limit) {
+ // Implementation based on the x86-64 version, above.
+ DCHECK_GE(s2_limit, s2);
+ int matched = 0;
+
+ while (s2 <= s2_limit - 4 &&
+ UNALIGNED_LOAD32(s2) == UNALIGNED_LOAD32(s1 + matched)) {
+ s2 += 4;
+ matched += 4;
+ }
+ if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 4) {
+ uint32 x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched);
+ int matching_bits = Bits::FindLSBSetNonZero(x);
+ matched += matching_bits >> 3;
+ } else {
+ while ((s2 < s2_limit) && (s1[matched] == *s2)) {
+ ++s2;
+ ++matched;
+ }
+ }
+ return matched;
+}
+#endif
+
+} // end namespace internal
+} // end namespace snappy
+
+#endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_
Added: couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.cc
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.cc?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.cc (added)
+++ couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.cc Mon May 2 13:28:20 2011
@@ -0,0 +1,72 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string.h>
+
+#include "snappy-sinksource.h"
+
+namespace snappy {
+
+Source::~Source() { }
+
+Sink::~Sink() { }
+
+char* Sink::GetAppendBuffer(size_t length, char* scratch) {
+ return scratch;
+}
+
+ByteArraySource::~ByteArraySource() { }
+
+size_t ByteArraySource::Available() const { return left_; }
+
+const char* ByteArraySource::Peek(size_t* len) {
+ *len = left_;
+ return ptr_;
+}
+
+void ByteArraySource::Skip(size_t n) {
+ left_ -= n;
+ ptr_ += n;
+}
+
+UncheckedByteArraySink::~UncheckedByteArraySink() { }
+
+void UncheckedByteArraySink::Append(const char* data, size_t n) {
+ // Do no copying if the caller filled in the result of GetAppendBuffer()
+ if (data != dest_) {
+ memcpy(dest_, data, n);
+ }
+ dest_ += n;
+}
+
+char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) {
+ return dest_;
+}
+
+
+}
Added: couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.h
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.h?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.h (added)
+++ couchdb/trunk/src/snappy/google-snappy/snappy-sinksource.h Mon May 2 13:28:20 2011
@@ -0,0 +1,136 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
+#define UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
+
+#include <stddef.h>
+
+
+namespace snappy {
+
+// A Sink is an interface that consumes a sequence of bytes.
+class Sink {
+ public:
+ Sink() { }
+ virtual ~Sink();
+
+ // Append "bytes[0,n-1]" to this.
+ virtual void Append(const char* bytes, size_t n) = 0;
+
+ // Returns a writable buffer of the specified length for appending.
+ // May return a pointer to the caller-owned scratch buffer which
+ // must have at least the indicated length. The returned buffer is
+ // only valid until the next operation on this Sink.
+ //
+ // After writing at most "length" bytes, call Append() with the
+ // pointer returned from this function and the number of bytes
+ // written. Many Append() implementations will avoid copying
+ // bytes if this function returned an internal buffer.
+ //
+ // If a non-scratch buffer is returned, the caller may only pass a
+ // prefix of it to Append(). That is, it is not correct to pass an
+ // interior pointer of the returned array to Append().
+ //
+ // The default implementation always returns the scratch buffer.
+ virtual char* GetAppendBuffer(size_t length, char* scratch);
+
+ private:
+ // No copying
+ Sink(const Sink&);
+ void operator=(const Sink&);
+};
+
+// A Source is an interface that yields a sequence of bytes
+class Source {
+ public:
+ Source() { }
+ virtual ~Source();
+
+ // Return the number of bytes left to read from the source
+ virtual size_t Available() const = 0;
+
+ // Peek at the next flat region of the source. Does not reposition
+ // the source. The returned region is empty iff Available()==0.
+ //
+ // Returns a pointer to the beginning of the region and store its
+ // length in *len.
+ //
+ // The returned region is valid until the next call to Skip() or
+ // until this object is destroyed, whichever occurs first.
+ //
+ // The returned region may be larger than Available() (for example
+ // if this ByteSource is a view on a substring of a larger source).
+ // The caller is responsible for ensuring that it only reads the
+ // Available() bytes.
+ virtual const char* Peek(size_t* len) = 0;
+
+ // Skip the next n bytes. Invalidates any buffer returned by
+ // a previous call to Peek().
+ // REQUIRES: Available() >= n
+ virtual void Skip(size_t n) = 0;
+
+ private:
+ // No copying
+ Source(const Source&);
+ void operator=(const Source&);
+};
+
+// A Source implementation that yields the contents of a flat array
+class ByteArraySource : public Source {
+ public:
+ ByteArraySource(const char* p, size_t n) : ptr_(p), left_(n) { }
+ virtual ~ByteArraySource();
+ virtual size_t Available() const;
+ virtual const char* Peek(size_t* len);
+ virtual void Skip(size_t n);
+ private:
+ const char* ptr_;
+ size_t left_;
+};
+
+// A Sink implementation that writes to a flat array without any bound checks.
+class UncheckedByteArraySink : public Sink {
+ public:
+ explicit UncheckedByteArraySink(char* dest) : dest_(dest) { }
+ virtual ~UncheckedByteArraySink();
+ virtual void Append(const char* data, size_t n);
+ virtual char* GetAppendBuffer(size_t len, char* scratch);
+
+ // Return the current output pointer so that a caller can see how
+ // many bytes were produced.
+ // Note: this is not a Sink method.
+ char* CurrentDestination() const { return dest_; }
+ private:
+ char* dest_;
+};
+
+
+}
+
+#endif // UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_
Added: couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.cc
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.cc?rev=1098558&view=auto
==============================================================================
--- couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.cc (added)
+++ couchdb/trunk/src/snappy/google-snappy/snappy-stubs-internal.cc Mon May 2 13:28:20 2011
@@ -0,0 +1,42 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <algorithm>
+#include <string>
+
+#include "snappy-stubs-internal.h"
+
+namespace snappy {
+
+void Varint::Append32(string* s, uint32 value) {
+ char buf[Varint::kMax32];
+ const char* p = Varint::Encode32(buf, value);
+ s->append(buf, p - buf);
+}
+
+} // namespace snappy