You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by to...@apache.org on 2015/07/16 10:24:45 UTC
svn commit: r1691330 - in /avro/trunk: CHANGES.txt lang/ruby/Gemfile
lang/ruby/lib/avro/data_file.rb lang/ruby/test/test_datafile.rb
share/docker/Dockerfile
Author: tomwhite
Date: Thu Jul 16 08:24:45 2015
New Revision: 1691330
URL: http://svn.apache.org/r1691330
Log:
AVRO-1697. Ruby: Add support for the Snappy codec to the Ruby library. Contributed by Daniel Schierbeck.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/ruby/Gemfile
avro/trunk/lang/ruby/lib/avro/data_file.rb
avro/trunk/lang/ruby/test/test_datafile.rb
avro/trunk/share/docker/Dockerfile
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1691330&r1=1691329&r2=1691330&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jul 16 08:24:45 2015
@@ -99,6 +99,9 @@ Trunk (not yet released)
AVRO-1692. Allow more than one logical type for a Java class. (blue via
tomwhite)
+ AVRO-1697. Ruby: Add support for the Snappy codec to the Ruby library.
+ (Daniel Schierbeck via tomwhite)
+
BUG FIXES
AVRO-1553. Java: MapReduce never uses MapOutputValueSchema (tomwhite)
Modified: avro/trunk/lang/ruby/Gemfile
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/Gemfile?rev=1691330&r1=1691329&r2=1691330&view=diff
==============================================================================
--- avro/trunk/lang/ruby/Gemfile (original)
+++ avro/trunk/lang/ruby/Gemfile Thu Jul 16 08:24:45 2015
@@ -17,3 +17,4 @@ source 'https://rubygems.org'
gem 'rake'
gem 'echoe'
gem 'multi_json'
+gem 'snappy'
Modified: avro/trunk/lang/ruby/lib/avro/data_file.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/lib/avro/data_file.rb?rev=1691330&r1=1691329&r2=1691330&view=diff
==============================================================================
--- avro/trunk/lang/ruby/lib/avro/data_file.rb (original)
+++ avro/trunk/lang/ruby/lib/avro/data_file.rb Thu Jul 16 08:24:45 2015
@@ -333,8 +333,31 @@ module Avro
end
end
+ class SnappyCodec
+ def codec_name; 'snappy'; end
+
+ def decompress(data)
+ load_snappy!
+ Snappy.inflate(data)
+ end
+
+ def compress(data)
+ load_snappy!
+ Snappy.deflate(data)
+ end
+
+ private
+
+ def load_snappy!
+ require 'snappy' unless defined?(Snappy)
+ rescue LoadError
+ raise LoadError, "Snappy compression is not available, please install the `snappy` gem."
+ end
+ end
+
DataFile.register_codec NullCodec
DataFile.register_codec DeflateCodec
+ DataFile.register_codec SnappyCodec
# TODO this constant won't be updated if you register another codec.
# Deprecated in favor of Avro::DataFile::codecs
Modified: avro/trunk/lang/ruby/test/test_datafile.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/test/test_datafile.rb?rev=1691330&r1=1691329&r2=1691330&view=diff
==============================================================================
--- avro/trunk/lang/ruby/test/test_datafile.rb (original)
+++ avro/trunk/lang/ruby/test/test_datafile.rb Thu Jul 16 08:24:45 2015
@@ -167,6 +167,19 @@ JSON
assert_equal records, ['a' * 10_000]
end
+ def test_snappy
+ Avro::DataFile.open('data.avr', 'w', '"string"', :snappy) do |writer|
+ writer << 'a' * 10_000
+ end
+ assert(File.size('data.avr') < 600)
+
+ records = []
+ Avro::DataFile.open('data.avr') do |reader|
+ reader.each {|record| records << record }
+ end
+ assert_equal records, ['a' * 10_000]
+ end
+
def test_append_to_deflated_file
schema = Avro::Schema.parse('"string"')
writer = Avro::IO::DatumWriter.new(schema)
Modified: avro/trunk/share/docker/Dockerfile
URL: http://svn.apache.org/viewvc/avro/trunk/share/docker/Dockerfile?rev=1691330&r1=1691329&r2=1691330&view=diff
==============================================================================
--- avro/trunk/share/docker/Dockerfile (original)
+++ avro/trunk/share/docker/Dockerfile Thu Jul 16 08:24:45 2015
@@ -31,7 +31,8 @@ RUN apt-get update && apt-get install --
perl \
php5 phpunit php5-gmp \
python python-setuptools python3-setuptools \
- ruby ruby-dev rake
+ ruby ruby-dev rake \
+ libsnappy1 libsnappy-dev
# Install Forrest
RUN mkdir -p /usr/local/apache-forrest
@@ -48,7 +49,7 @@ RUN cpanm install Module::Install Module
Test::Exception Test::Pod
# Install Ruby modules
-RUN gem install echoe yajl-ruby multi_json
+RUN gem install echoe yajl-ruby multi_json snappy
# Install global Node modules
RUN npm install -g grunt-cli