[avro] branch master updated: AVRO-2464: Add zstd codec support to the Ruby bindings

[dk...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git


The following commit(s) were added to refs/heads/master by this push:
     new a150907  AVRO-2464: Add zstd codec support to the Ruby bindings
a150907 is described below

commit a15090758a09ef55196c1dd77cda0124e8278a1b
Author: Kengo Seki <se...@apache.org>
AuthorDate: Fri Jul 5 16:31:33 2019 +0900

    AVRO-2464: Add zstd codec support to the Ruby bindings
---
 BUILD.md                        |  3 ++-
 lang/ruby/Gemfile               |  1 +
 lang/ruby/lib/avro/data_file.rb | 23 +++++++++++++++++++++++
 lang/ruby/test/test_datafile.rb | 13 +++++++++++++
 share/docker/Dockerfile         |  2 +-
 5 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/BUILD.md b/BUILD.md
index 163a075..504b95c 100644
--- a/BUILD.md
+++ b/BUILD.md
@@ -12,7 +12,8 @@ The following packages must be installed before Avro can be built:
  - C++: cmake 3.7.2 or greater, g++, flex, bison, libboost-dev
  - C#: .NET Core 2.2 SDK
  - JavaScript: Node 6.x+, nodejs, npm
- - Ruby: Ruby 2.3.3 or greater, ruby-dev, gem, rake, echoe, yajl-ruby
+ - Ruby: Ruby 2.3.3 or greater, ruby-dev, gem, rake, echoe, yajl-ruby,
+   snappy, zstd-ruby
  - Perl: Perl 5.24.1 or greater, gmake, Module::Install,
    Module::Install::ReadmeFromPod, Module::Install::Repository,
    Math::BigInt, JSON::XS, Try::Tiny, Regexp::Common, Encode,
diff --git a/lang/ruby/Gemfile b/lang/ruby/Gemfile
index 24e66dd..59872d3 100644
--- a/lang/ruby/Gemfile
+++ b/lang/ruby/Gemfile
@@ -18,4 +18,5 @@ gem 'rake'
 gem 'echoe'
 gem 'multi_json'
 gem 'snappy'
+gem 'zstd-ruby'
 gem 'test-unit'
diff --git a/lang/ruby/lib/avro/data_file.rb b/lang/ruby/lib/avro/data_file.rb
index ee6cbe9..610c408 100644
--- a/lang/ruby/lib/avro/data_file.rb
+++ b/lang/ruby/lib/avro/data_file.rb
@@ -372,9 +372,32 @@ module Avro
       end
     end
 
+    class ZstandardCodec
+      def codec_name; 'zstandard'; end
+
+      def decompress(data)
+        load_zstandard!
+        Zstd.decompress(data)
+      end
+
+      def compress(data)
+        load_zstandard!
+        Zstd.compress(data)
+      end
+
+      private
+
+      def load_zstandard!
+        require 'zstd-ruby' unless defined?(Zstd)
+      rescue LoadError
+        raise LoadError, "Zstandard compression is not available, please install the `zstd-ruby` gem."
+      end
+    end
+
     DataFile.register_codec NullCodec
     DataFile.register_codec DeflateCodec
     DataFile.register_codec SnappyCodec
+    DataFile.register_codec ZstandardCodec
 
     # TODO this constant won't be updated if you register another codec.
     # Deprecated in favor of Avro::DataFile::codecs
diff --git a/lang/ruby/test/test_datafile.rb b/lang/ruby/test/test_datafile.rb
index 2877aac..ab4c48b 100644
--- a/lang/ruby/test/test_datafile.rb
+++ b/lang/ruby/test/test_datafile.rb
@@ -180,6 +180,19 @@ JSON
     assert_equal records, ['a' * 10_000]
   end
 
+  def test_zstandard
+    Avro::DataFile.open('data.avr', 'w', '"string"', :zstandard) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
   def test_append_to_deflated_file
     schema = Avro::Schema.parse('"string"')
     writer = Avro::IO::DatumWriter.new(schema)
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 89886ea..f0e9f18 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -97,7 +97,7 @@ RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.6.phar && c
 RUN pip3 install zstandard
 
 # Install Ruby modules
-RUN gem install echoe yajl-ruby multi_json snappy
+RUN gem install echoe yajl-ruby multi_json snappy zstd-ruby
 
 # Install global Node modules
 RUN npm install -g grunt-cli