You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2012/10/27 01:41:15 UTC
svn commit: r1402720 - in /incubator/mesos/trunk: src/tests/stout_tests.cpp
third_party/libprocess/Makefile.am
third_party/libprocess/include/stout/gzip.hpp
Author: benh
Date: Fri Oct 26 23:41:15 2012
New Revision: 1402720
URL: http://svn.apache.org/viewvc?rev=1402720&view=rev
Log:
Adding gzip compression in stout.
From: Ben Mahler <be...@gmail.com>
Review: https://reviews.apache.org/r/7675
Added:
incubator/mesos/trunk/third_party/libprocess/include/stout/gzip.hpp
Modified:
incubator/mesos/trunk/src/tests/stout_tests.cpp
incubator/mesos/trunk/third_party/libprocess/Makefile.am
Modified: incubator/mesos/trunk/src/tests/stout_tests.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/tests/stout_tests.cpp?rev=1402720&r1=1402719&r2=1402720&view=diff
==============================================================================
--- incubator/mesos/trunk/src/tests/stout_tests.cpp (original)
+++ incubator/mesos/trunk/src/tests/stout_tests.cpp Fri Oct 26 23:41:15 2012
@@ -1,5 +1,6 @@
#include <gmock/gmock.h>
+#include <cstdlib> // For rand.
#include <map>
#include <string>
#include <vector>
@@ -9,6 +10,7 @@
#include <stout/fatal.hpp>
#include <stout/foreach.hpp>
#include <stout/format.hpp>
+#include <stout/gzip.hpp>
#include <stout/hashmap.hpp>
#include <stout/hashset.hpp>
#include <stout/json.hpp>
@@ -26,6 +28,8 @@
#include "tests/utils.hpp"
+#include "tests/utils.hpp"
+
using namespace std;
@@ -621,3 +625,46 @@ TEST(StoutMultihashmapTest, Foreach)
}
}
}
+
+
+#ifdef HAVE_LIBZ
+TEST(StoutCompressionTest, Gzip)
+{
+ // Test bad compression levels, outside of [-1, Z_BEST_COMPRESSION].
+ ASSERT_ERROR(gzip::compress("", -2));
+ ASSERT_ERROR(gzip::compress("", Z_BEST_COMPRESSION + 1));
+
+ string s =
+ "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do "
+ "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad "
+ "minim veniam, quis nostrud exercitation ullamco laboris nisi ut "
+ "aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit "
+ "in voluptate velit esse cillum dolore eu fugiat nulla pariatur. "
+ "Excepteur sint occaecat cupidatat non proident, sunt in culpa qui "
+ "officia deserunt mollit anim id est laborum.";
+
+ Try<string> compressed = gzip::compress(s);
+ ASSERT_SOME(compressed);
+ Try<string> decompressed = gzip::decompress(compressed.get());
+ ASSERT_SOME(decompressed);
+ ASSERT_EQ(s, decompressed.get());
+
+ // Test with a 1MB random string!
+ s = "";
+ while (s.length() < (1024 * 1024)) {
+ s.append(1, ' ' + (rand() % ('~' - ' ')));
+ }
+ compressed = gzip::compress(s);
+ ASSERT_SOME(compressed);
+ decompressed = gzip::decompress(compressed.get());
+ ASSERT_SOME(decompressed);
+ ASSERT_EQ(s, decompressed.get());
+
+ s = "";
+ compressed = gzip::compress(s);
+ ASSERT_SOME(compressed);
+ decompressed = gzip::decompress(compressed.get());
+ ASSERT_SOME(decompressed);
+ ASSERT_EQ(s, decompressed.get());
+}
+#endif // HAVE_LIBZ
Modified: incubator/mesos/trunk/third_party/libprocess/Makefile.am
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/Makefile.am?rev=1402720&r1=1402719&r2=1402720&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/Makefile.am (original)
+++ incubator/mesos/trunk/third_party/libprocess/Makefile.am Fri Oct 26 23:41:15 2012
@@ -71,6 +71,7 @@ libprocess_la_SOURCES += $(top_srcdir)/i
$(top_srcdir)/include/stout/foreach.hpp \
$(top_srcdir)/include/stout/format.hpp \
$(top_srcdir)/include/stout/fs.hpp \
+ $(top_srcdir)/include/stout/gzip.hpp \
$(top_srcdir)/include/stout/hashmap.hpp \
$(top_srcdir)/include/stout/hashset.hpp \
$(top_srcdir)/include/stout/json.hpp \
Added: incubator/mesos/trunk/third_party/libprocess/include/stout/gzip.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/include/stout/gzip.hpp?rev=1402720&view=auto
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/include/stout/gzip.hpp (added)
+++ incubator/mesos/trunk/third_party/libprocess/include/stout/gzip.hpp Fri Oct 26 23:41:15 2012
@@ -0,0 +1,154 @@
+#ifndef __STOUT_GZIP_HPP__
+#define __STOUT_GZIP_HPP__
+
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
+#include <string>
+
+#include "try.hpp"
+
+// Compression utilities.
+// TODO(bmahler): Provide streaming compression / decompression as well.
+namespace gzip {
+
+// We use a 16KB buffer with zlib compression / decompression.
+#define GZIP_BUFFER_SIZE 16384
+
+// Returns a gzip compressed version of the provided string.
+// The compression level should be within the range [-1, 9].
+// See zlib.h:
+// #define Z_NO_COMPRESSION 0
+// #define Z_BEST_SPEED 1
+// #define Z_BEST_COMPRESSION 9
+// #define Z_DEFAULT_COMPRESSION (-1)
+inline Try<std::string> compress(
+ const std::string& decompressed,
+#ifdef HAVE_LIBZ
+ int level = Z_DEFAULT_COMPRESSION)
+#else
+ int level = -1)
+#endif
+{
+#ifndef HAVE_LIBZ
+ return Try<std::string>::error(
+ "Failed to compress because libz is not available");
+#else
+ // Verify the level is within range.
+ if (!(level == Z_DEFAULT_COMPRESSION ||
+ (level >= Z_NO_COMPRESSION && level <= Z_BEST_COMPRESSION))) {
+ return Try<std::string>::error("Invalid compression level: " + level);
+ }
+
+ z_stream_s stream;
+ stream.next_in =
+ const_cast<Bytef*>(reinterpret_cast<const Bytef*>(decompressed.data()));
+ stream.avail_in = decompressed.length();
+ stream.zalloc = Z_NULL;
+ stream.zfree = Z_NULL;
+ stream.opaque = Z_NULL;
+
+ int code = deflateInit2(
+ &stream,
+ level, // Compression level.
+ Z_DEFLATED, // Compression method.
+ MAX_WBITS + 16, // Zlib magic for gzip compression / decompression.
+ 8, // Default memLevel value.
+ Z_DEFAULT_STRATEGY);
+
+ if (code != Z_OK) {
+ return Try<std::string>::error(
+ "Error initializing zlib: " + std::string(stream.msg));
+ }
+
+ // Build up the compressed result.
+ Bytef buffer[GZIP_BUFFER_SIZE];
+ std::string result = "";
+ do {
+ stream.next_out = buffer;
+ stream.avail_out = GZIP_BUFFER_SIZE;
+ code = deflate(&stream, stream.avail_in > 0 ? Z_NO_FLUSH : Z_FINISH);
+
+ if (code != Z_OK && code != Z_STREAM_END) {
+ std::string error(stream.msg);
+ deflateEnd(&stream);
+ return Try<std::string>::error("Error during compression: " + error);
+ }
+
+ // Consume output and reset the buffer.
+ result.append(
+ reinterpret_cast<char*>(buffer),
+ GZIP_BUFFER_SIZE - stream.avail_out);
+ stream.next_out = buffer;
+ stream.avail_out = GZIP_BUFFER_SIZE;
+ } while (code != Z_STREAM_END);
+
+ code = deflateEnd(&stream);
+ if (code != Z_OK) {
+ return Try<std::string>::error(
+ "Error cleaning up zlib: " + std::string(stream.msg));
+ }
+ return result;
+#endif // HAVE_LIBZ
+}
+
+
+// Returns a gzip decompressed version of the provided string.
+inline Try<std::string> decompress(const std::string& compressed)
+{
+#ifndef HAVE_LIBZ
+ return Try<std::string>::error(
+ "Failed to decompress because libz is not available");
+#else
+ z_stream_s stream;
+ stream.next_in =
+ const_cast<Bytef*>(reinterpret_cast<const Bytef*>(compressed.data()));
+ stream.avail_in = compressed.length();
+ stream.zalloc = Z_NULL;
+ stream.zfree = Z_NULL;
+ stream.opaque = Z_NULL;
+
+ int code = inflateInit2(
+ &stream,
+ MAX_WBITS + 16); // Zlib magic for gzip compression / decompression.
+
+ if (code != Z_OK) {
+ return Try<std::string>::error(
+ "Error initializing zlib: " + std::string(stream.msg));
+ }
+
+ // Build up the decompressed result.
+ Bytef buffer[GZIP_BUFFER_SIZE];
+ std::string result = "";
+ do {
+ stream.next_out = buffer;
+ stream.avail_out = GZIP_BUFFER_SIZE;
+ code = inflate(&stream, stream.avail_in > 0 ? Z_NO_FLUSH : Z_FINISH);
+
+ if (code != Z_OK && code != Z_STREAM_END) {
+ std::string error(stream.msg);
+ inflateEnd(&stream);
+ return Try<std::string>::error("Error during decompression: " + error);
+ }
+
+ // Consume output and reset the buffer.
+ result.append(
+ reinterpret_cast<char*>(buffer),
+ GZIP_BUFFER_SIZE - stream.avail_out);
+ stream.next_out = buffer;
+ stream.avail_out = GZIP_BUFFER_SIZE;
+ } while (code != Z_STREAM_END);
+
+ code = inflateEnd(&stream);
+ if (code != Z_OK) {
+ return Try<std::string>::error(
+ "Error cleaning up zlib: " + std::string(stream.msg));
+ }
+ return result;
+#endif // HAVE_LIBZ
+}
+
+} // namespace gzip {
+
+#endif // __STOUT_GZIP_HPP__