You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/01/18 00:00:20 UTC
[47/50] [abbrv] git commit: upgrade to 1.0.3
upgrade to 1.0.3
Project: http://git-wip-us.apache.org/repos/asf/couchdb-snappy/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-snappy/commit/b840218d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-snappy/tree/b840218d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-snappy/diff/b840218d
Branch: refs/heads/import
Commit: b840218d0e4cdd8aec6d5cb741da5cf406295a89
Parents: 87d0742
Author: benoitc <bc...@gmail.com>
Authored: Sun Jun 19 13:22:52 2011 +0200
Committer: benoitc <bc...@gmail.com>
Committed: Sun Jun 19 13:22:52 2011 +0200
----------------------------------------------------------------------
c_src/erl_nif_compat.h | 8 +
c_src/snappy/NEWS | 29 ++++
c_src/snappy/snappy.cc | 78 +++++-----
c_src/snappy_nif.cc | 352 ++++++++++++++++++++++++--------------------
src/snappy.erl | 7 +-
test/snappy_tests.erl | 12 +-
6 files changed, 284 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/c_src/erl_nif_compat.h
----------------------------------------------------------------------
diff --git a/c_src/erl_nif_compat.h b/c_src/erl_nif_compat.h
index b6e4453..b8eb9b0 100644
--- a/c_src/erl_nif_compat.h
+++ b/c_src/erl_nif_compat.h
@@ -53,6 +53,9 @@ extern "C" {
#define enif_priv_data_compat enif_get_data
#define enif_make_uint_compat enif_make_ulong
+#define enif_make_existing_atom_compat(E, N, R, Enc) \
+ enif_make_existing_atom(E, N, R)
+
#define enif_make_string_compat(E, B, Enc) \
enif_make_string(E, B)
@@ -74,6 +77,10 @@ extern "C" {
#define enif_make_string_compat enif_make_string
#define enif_make_uint_compat enif_make_uint
+#define enif_make_existing_atom_compat(E, N, R, Enc) \
+ enif_make_existing_atom(E, N, R)
+
+
#endif /* R13B04 */
@@ -109,6 +116,7 @@ extern "C" {
#define enif_priv_data_compat enif_priv_data
#define enif_make_string_compat enif_make_string
+#define enif_make_existing_atom_compat enif_make_existing_atom
#define enif_make_uint_compat enif_make_uint
#endif /* R14 and future releases */
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/c_src/snappy/NEWS
----------------------------------------------------------------------
diff --git a/c_src/snappy/NEWS b/c_src/snappy/NEWS
index 611abb5..d514787 100644
--- a/c_src/snappy/NEWS
+++ b/c_src/snappy/NEWS
@@ -1,3 +1,32 @@
+Snappy v1.0.3, June 2nd 2011:
+
+ * Speeded up the decompressor somewhat; about 3-6% for Core 2,
+ 6-13% for Core i7, and 5-12% for Opteron (all in 64-bit mode).
+
+ * Added compressed format documentation. This text is new,
+ but an earlier version from Zeev Tarantov was used as reference.
+
+ * Only link snappy_unittest against -lz and other autodetected
+ libraries, not libsnappy.so (which doesn't need any such dependency).
+
+ * Fixed some display issues in the microbenchmarks, one of which would
+ frequently make the test crash on GNU/Hurd.
+
+
+Snappy v1.0.2, April 29th 2011:
+
+ * Relicense to a BSD-type license.
+
+ * Added C bindings, contributed by Martin Gieseking.
+
+ * More Win32 fixes, in particular for MSVC.
+
+ * Replace geo.protodata with a newer version.
+
+ * Fix timing inaccuracies in the unit test when comparing Snappy
+ to other algorithms.
+
+
Snappy v1.0.1, March 25th 2011:
This is a maintenance release, mostly containing minor fixes.
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/c_src/snappy/snappy.cc
----------------------------------------------------------------------
diff --git a/c_src/snappy/snappy.cc b/c_src/snappy/snappy.cc
index 625029c..a591aba 100644
--- a/c_src/snappy/snappy.cc
+++ b/c_src/snappy/snappy.cc
@@ -653,44 +653,50 @@ class SnappyDecompressor {
// Process the next item found in the input.
// Returns true if successful, false on error or end of input.
template <class Writer>
- bool Step(Writer* writer) {
+ void DecompressAllTags(Writer* writer) {
const char* ip = ip_;
- if (ip_limit_ - ip < 5) {
- if (!RefillTag()) return false;
- ip = ip_;
- }
-
- const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip++));
- const uint32 entry = char_table[c];
- const uint32 trailer = LittleEndian::Load32(ip) & wordmask[entry >> 11];
- ip += entry >> 11;
- const uint32 length = entry & 0xff;
+ for ( ;; ) {
+ if (ip_limit_ - ip < 5) {
+ ip_ = ip;
+ if (!RefillTag()) return;
+ ip = ip_;
+ }
- if ((c & 0x3) == LITERAL) {
- uint32 literal_length = length + trailer;
- uint32 avail = ip_limit_ - ip;
- while (avail < literal_length) {
+ const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip++));
+ const uint32 entry = char_table[c];
+ const uint32 trailer = LittleEndian::Load32(ip) & wordmask[entry >> 11];
+ ip += entry >> 11;
+ const uint32 length = entry & 0xff;
+
+ if ((c & 0x3) == LITERAL) {
+ uint32 literal_length = length + trailer;
+ uint32 avail = ip_limit_ - ip;
+ while (avail < literal_length) {
+ bool allow_fast_path = (avail >= 16);
+ if (!writer->Append(ip, avail, allow_fast_path)) return;
+ literal_length -= avail;
+ reader_->Skip(peeked_);
+ size_t n;
+ ip = reader_->Peek(&n);
+ avail = n;
+ peeked_ = avail;
+ if (avail == 0) return; // Premature end of input
+ ip_limit_ = ip + avail;
+ }
bool allow_fast_path = (avail >= 16);
- if (!writer->Append(ip, avail, allow_fast_path)) return false;
- literal_length -= avail;
- reader_->Skip(peeked_);
- size_t n;
- ip = reader_->Peek(&n);
- avail = n;
- peeked_ = avail;
- if (avail == 0) return false; // Premature end of input
- ip_limit_ = ip + avail;
+ if (!writer->Append(ip, literal_length, allow_fast_path)) {
+ return;
+ }
+ ip += literal_length;
+ } else {
+ // copy_offset/256 is encoded in bits 8..10. By just fetching
+ // those bits, we get copy_offset (since the bit-field starts at
+ // bit 8).
+ const uint32 copy_offset = entry & 0x700;
+ if (!writer->AppendFromSelf(copy_offset + trailer, length)) {
+ return;
+ }
}
- ip_ = ip + literal_length;
- bool allow_fast_path = (avail >= 16);
- return writer->Append(ip, literal_length, allow_fast_path);
- } else {
- ip_ = ip;
- // copy_offset/256 is encoded in bits 8..10. By just fetching
- // those bits, we get copy_offset (since the bit-field starts at
- // bit 8).
- const uint32 copy_offset = entry & 0x700;
- return writer->AppendFromSelf(copy_offset + trailer, length);
}
}
};
@@ -770,7 +776,7 @@ static bool InternalUncompress(Source* r,
writer->SetExpectedLength(uncompressed_len);
// Process the entire input
- while (decompressor.Step(writer)) { }
+ decompressor.DecompressAllTags(writer);
return (decompressor.eof() && writer->CheckLength());
}
@@ -866,7 +872,7 @@ size_t Compress(Source* reader, Sink* writer) {
// A type that writes to a flat array.
// Note that this is not a "ByteSink", but a type that matches the
-// Writer template argument to SnappyDecompressor::Step().
+// Writer template argument to SnappyDecompressor::DecompressAllTags().
class SnappyArrayWriter {
private:
char* base_;
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/c_src/snappy_nif.cc
----------------------------------------------------------------------
diff --git a/c_src/snappy_nif.cc b/c_src/snappy_nif.cc
index 91844d7..93c1859 100644
--- a/c_src/snappy_nif.cc
+++ b/c_src/snappy_nif.cc
@@ -22,204 +22,244 @@
#include "snappy/snappy.h"
#include "snappy/snappy-sinksource.h"
-class SnappyNifSink : public snappy::Sink {
-public:
- SnappyNifSink(ErlNifEnv* e) : env(e), length(0) {
- if (!enif_alloc_binary_compat(env, 4096, &bin)) {
- throw std::bad_alloc();
- }
+#ifdef OTP_R13B03
+#error OTP R13B03 not supported. Upgrade to R13B04 or later.
+#endif
+
+#ifdef __cplusplus
+#define BEGIN_C extern "C" {
+#define END_C }
+#else
+#define BEGIN_C
+#define END_C
+#endif
+
+#define SC_PTR(c) reinterpret_cast<char *>(c)
+
+class SnappyNifSink : public snappy::Sink
+{
+ public:
+ SnappyNifSink(ErlNifEnv* e);
+ ~SnappyNifSink();
+
+ void Append(const char* data, size_t n);
+ char* GetAppendBuffer(size_t len, char* scratch);
+ ErlNifBinary& getBin();
+
+ private:
+ ErlNifEnv* env;
+ ErlNifBinary bin;
+ size_t length;
+};
+
+SnappyNifSink::SnappyNifSink(ErlNifEnv* e) : env(e), length(0)
+{
+ if(!enif_alloc_binary_compat(env, 0, &bin)) {
+ env = NULL;
+ throw std::bad_alloc();
}
+}
- void Append(const char *data, size_t n) {
- if (data != reinterpret_cast<const char *>(bin.data + length)) {
- memcpy(bin.data + length, data, n);
- }
- length += n;
+SnappyNifSink::~SnappyNifSink()
+{
+ if(env != NULL) {
+ enif_release_binary_compat(env, &bin);
}
+}
- char* GetAppendBuffer(size_t len, char* scratch) {
- if ((length + len) > bin.size) {
- size_t sz = (len * 4) < 8192 ? 8192 : (len * 4);
+void
+SnappyNifSink::Append(const char *data, size_t n)
+{
+ if(data != (SC_PTR(bin.data) + length)) {
+ memcpy(bin.data + length, data, n);
+ }
+ length += n;
+}
- if (!enif_realloc_binary_compat(env, &bin, bin.size + sz)) {
- enif_release_binary_compat(env, &bin);
- throw std::bad_alloc();
- }
- }
+char*
+SnappyNifSink::GetAppendBuffer(size_t len, char* scratch)
+{
+ size_t sz;
+
+ if((length + len) > bin.size) {
+ sz = (len * 4) < 8192 ? 8192 : (len * 4);
- return reinterpret_cast<char *>(bin.data + length);
+ if(!enif_realloc_binary_compat(env, &bin, bin.size + sz)) {
+ throw std::bad_alloc();
+ }
}
- ErlNifBinary& getBin() {
- if (bin.size > length) {
- if (!enif_realloc_binary_compat(env, &bin, length)) {
- // shouldn't happen
- enif_release_binary_compat(env, &bin);
- throw std::bad_alloc();
- }
+ return SC_PTR(bin.data) + length;
+}
+
+ErlNifBinary&
+SnappyNifSink::getBin()
+{
+ if(bin.size > length) {
+ if(!enif_realloc_binary_compat(env, &bin, length)) {
+ throw std::bad_alloc();
}
- return bin;
}
+ return bin;
+}
-private:
- ErlNifEnv* env;
- ErlNifBinary bin;
- size_t length;
-};
+static inline ERL_NIF_TERM
+make_atom(ErlNifEnv* env, const char* name)
+{
+ ERL_NIF_TERM ret;
+ if(enif_make_existing_atom_compat(env, name, &ret, ERL_NIF_LATIN1)) {
+ return ret;
+ }
+ return enif_make_atom(env, name);
+}
-extern "C" {
- ERL_NIF_TERM snappy_compress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
- ErlNifBinary input;
+static inline ERL_NIF_TERM
+make_ok(ErlNifEnv* env, ERL_NIF_TERM mesg)
+{
+ ERL_NIF_TERM ok = make_atom(env, "ok");
+ return enif_make_tuple2(env, ok, mesg);
+}
- if (!enif_inspect_iolist_as_binary(env, argv[0], &input)) {
- return enif_make_badarg(env);
- }
- try {
- snappy::ByteArraySource source(reinterpret_cast<const char *>(input.data),
- input.size);
- SnappyNifSink sink(env);
-
- snappy::Compress(&source, &sink);
-
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "ok"),
- enif_make_binary(env, &sink.getBin()));
- } catch(std::bad_alloc e) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "insufficient_memory"));
- } catch(...) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "unknown"));
- }
- }
+static inline ERL_NIF_TERM
+make_error(ErlNifEnv* env, const char* mesg)
+{
+ ERL_NIF_TERM error = make_atom(env, "error");
+ return enif_make_tuple2(env, error, make_atom(env, mesg));
+}
- ERL_NIF_TERM snappy_decompress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
- ErlNifBinary input;
+BEGIN_C
- if (!enif_inspect_iolist_as_binary(env, argv[0], &input)) {
- return enif_make_badarg(env);
- }
- try {
- size_t len = -1;
- bool isCompressed = snappy::GetUncompressedLength(
- reinterpret_cast<const char *>(input.data), input.size, &len);
-
- if (!isCompressed) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "not_compressed_data"));
- }
-
- ErlNifBinary retBin;
-
- if (!enif_alloc_binary_compat(env, len, &retBin)) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "insufficient_memory"));
- }
-
- bool valid = snappy::RawUncompress(reinterpret_cast<const char *>(input.data),
- input.size,
- reinterpret_cast<char *>(retBin.data));
-
- if (!valid) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "corrupted_data"));
- }
-
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "ok"),
- enif_make_binary(env, &retBin));
- } catch(...) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "unknown"));
- }
+ERL_NIF_TERM
+snappy_compress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ ErlNifBinary input;
+
+ if(!enif_inspect_iolist_as_binary(env, argv[0], &input)) {
+ return enif_make_badarg(env);
+ }
+
+ try {
+ snappy::ByteArraySource source(SC_PTR(input.data), input.size);
+ SnappyNifSink sink(env);
+ snappy::Compress(&source, &sink);
+ return make_ok(env, enif_make_binary(env, &sink.getBin()));
+ } catch(std::bad_alloc e) {
+ return make_error(env, "insufficient_memory");
+ } catch(...) {
+ return make_error(env, "unknown");
}
+}
+
+
+ERL_NIF_TERM
+snappy_decompress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ ErlNifBinary bin;
+ ErlNifBinary ret;
+ size_t len;
+ if(!enif_inspect_iolist_as_binary(env, argv[0], &bin)) {
+ return enif_make_badarg(env);
+ }
- ERL_NIF_TERM snappy_get_uncompressed_length(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
- ErlNifBinary input;
+ try {
+ if(!snappy::GetUncompressedLength(SC_PTR(bin.data), bin.size, &len)) {
+ return make_error(env, "data_not_compressed");
+ }
- if (!enif_inspect_iolist_as_binary(env, argv[0], &input)) {
- return enif_make_badarg(env);
+ if(!enif_alloc_binary_compat(env, len, &ret)) {
+ return make_error(env, "insufficient_memory");
}
- try {
- size_t len = -1;
- bool isCompressed = snappy::GetUncompressedLength(
- reinterpret_cast<const char *>(input.data), input.size, &len);
-
- if (isCompressed) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "ok"),
- enif_make_ulong(env, len));
- } else {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "not_compressed_data"));
- }
- } catch(...) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "unknown"));
+ if(!snappy::RawUncompress(SC_PTR(bin.data), bin.size,
+ SC_PTR(ret.data))) {
+ return make_error(env, "corrupted_data");
}
+
+ return make_ok(env, enif_make_binary(env, &ret));
+ } catch(...) {
+ return make_error(env, "unknown");
}
+}
- ERL_NIF_TERM snappy_is_valid_compressed_buffer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
- ErlNifBinary input;
+ERL_NIF_TERM
+snappy_uncompressed_length(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ ErlNifBinary bin;
+ size_t len;
- if (!enif_inspect_iolist_as_binary(env, argv[0], &input)) {
- return enif_make_badarg(env);
- }
+ if(!enif_inspect_iolist_as_binary(env, argv[0], &bin)) {
+ return enif_make_badarg(env);
+ }
- try {
- bool valid = snappy::IsValidCompressedBuffer(
- reinterpret_cast<const char *>(input.data), input.size);
-
- if (valid) {
- return enif_make_atom(env, "true");
- } else {
- return enif_make_atom(env, "false");
- }
- } catch(...) {
- return enif_make_tuple(env, 2,
- enif_make_atom(env, "error"),
- enif_make_atom(env, "unknown"));
+ try {
+ if(!snappy::GetUncompressedLength(SC_PTR(bin.data), bin.size, &len)) {
+ return make_error(env, "data_not_compressed");
}
+ return make_ok(env, enif_make_ulong(env, len));
+ } catch(...) {
+ return make_error(env, "unknown");
}
+}
+ERL_NIF_TERM
+snappy_is_valid(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ ErlNifBinary bin;
- int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM info) {
- return 0;
+ if (!enif_inspect_iolist_as_binary(env, argv[0], &bin)) {
+ return enif_make_badarg(env);
}
-
- int on_reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM info) {
- return 0;
+ try {
+ if(snappy::IsValidCompressedBuffer(SC_PTR(bin.data), bin.size)) {
+ return make_atom(env, "true");
+ } else {
+ return make_atom(env, "false");
+ }
+ } catch(...) {
+ return make_error(env, "unknown");
}
+}
- int on_upgrade(ErlNifEnv* env, void** priv_data, void** old_data, ERL_NIF_TERM info) {
- return 0;
- }
+int
+on_load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info)
+{
+ return 0;
+}
+
+
+int
+on_reload(ErlNifEnv* env, void** priv, ERL_NIF_TERM info)
+{
+ return 0;
+}
- static ErlNifFunc nif_functions[] = {
- {"compress", 1, snappy_compress},
- {"decompress", 1, snappy_decompress},
- {"get_uncompressed_length", 1, snappy_get_uncompressed_length},
- {"is_valid_compressed_buffer", 1, snappy_is_valid_compressed_buffer}
- };
- ERL_NIF_INIT(snappy, nif_functions, &on_load, &on_reload, &on_upgrade, NULL);
+int
+on_upgrade(ErlNifEnv* env, void** priv, void** old_priv, ERL_NIF_TERM info)
+{
+ return 0;
}
+
+
+static ErlNifFunc nif_functions[] = {
+ {"compress", 1, snappy_compress},
+ {"decompress", 1, snappy_decompress},
+ {"uncompressed_length", 1, snappy_uncompressed_length},
+ {"is_valid", 1, snappy_is_valid}
+};
+
+
+ERL_NIF_INIT(snappy, nif_functions, &on_load, &on_reload, &on_upgrade, NULL);
+
+
+END_C
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/src/snappy.erl
----------------------------------------------------------------------
diff --git a/src/snappy.erl b/src/snappy.erl
index a38592f..7d3d36a 100644
--- a/src/snappy.erl
+++ b/src/snappy.erl
@@ -16,8 +16,7 @@
-module(snappy).
-export([compress/1, decompress/1]).
--export([get_uncompressed_length/1]).
--export([is_valid_compressed_buffer/1]).
+-export([uncompressed_length/1, is_valid/1]).
-on_load(init/0).
@@ -49,9 +48,9 @@ decompress(_IoList) ->
exit(snappy_nif_not_loaded).
-get_uncompressed_length(_IoList) ->
+uncompressed_length(_IoList) ->
exit(snappy_nif_not_loaded).
-is_valid_compressed_buffer(_IoList) ->
+is_valid(_IoList) ->
exit(snappy_nif_not_loaded).
http://git-wip-us.apache.org/repos/asf/couchdb-snappy/blob/b840218d/test/snappy_tests.erl
----------------------------------------------------------------------
diff --git a/test/snappy_tests.erl b/test/snappy_tests.erl
index 749c899..ac39c58 100644
--- a/test/snappy_tests.erl
+++ b/test/snappy_tests.erl
@@ -33,18 +33,18 @@ compression() ->
?assertEqual(true, byte_size(Compressed) < byte_size(Data)),
- ?assertEqual(true, snappy:is_valid_compressed_buffer(Compressed)),
- ?assertEqual(false, snappy:is_valid_compressed_buffer(Data)),
- ?assertEqual(false, snappy:is_valid_compressed_buffer(<<"foobar123">>)),
- ?assertEqual({ok, byte_size(Data)}, snappy:get_uncompressed_length(Compressed)),
+ ?assertEqual(true, snappy:is_valid(Compressed)),
+ ?assertEqual(false, snappy:is_valid(Data)),
+ ?assertEqual(false, snappy:is_valid(<<"foobar123">>)),
+ ?assertEqual({ok, byte_size(Data)}, snappy:uncompressed_length(Compressed)),
Result2 = snappy:compress(DataIoList),
?assertMatch({ok, _}, Result2),
{ok, Compressed2} = Result2,
?assertEqual(byte_size(Compressed2), byte_size(Compressed)),
- ?assertEqual(true, snappy:is_valid_compressed_buffer(Compressed2)),
- ?assertEqual({ok, byte_size(Data)}, snappy:get_uncompressed_length(Compressed2)),
+ ?assertEqual(true, snappy:is_valid(Compressed2)),
+ ?assertEqual({ok, byte_size(Data)}, snappy:uncompressed_length(Compressed2)),
ok.