You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by fo...@apache.org on 2019/07/12 17:55:48 UTC
[avro] branch master updated: AVRO-2456: Add interop test for the
snappy and zstd codec (#573)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new fbfb135 AVRO-2456: Add interop test for the snappy and zstd codec (#573)
fbfb135 is described below
commit fbfb135dd50fc39835c5323057561144eff2bc41
Author: Kengo Seki <se...@apache.org>
AuthorDate: Sat Jul 13 02:55:42 2019 +0900
AVRO-2456: Add interop test for the snappy and zstd codec (#573)
---
lang/java/avro/pom.xml | 32 ++++++++++++++++++++++++++++++--
lang/java/ipc/pom.xml | 8 ++++++++
lang/java/pom.xml | 1 -
lang/py/test/gen_interop_data.py | 32 +++++++++++++++++++++++++-------
lang/ruby/interop/test_interop.rb | 10 +++++++++-
share/docker/Dockerfile | 7 ++++++-
6 files changed, 78 insertions(+), 12 deletions(-)
diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml
index 6151b62..b1c313e 100644
--- a/lang/java/avro/pom.xml
+++ b/lang/java/avro/pom.xml
@@ -126,6 +126,36 @@
</configuration>
<goals><goal>java</goal></goals>
</execution>
+ <!-- Generate random data for interop tests, using snappy codec -->
+ <execution>
+ <id>interop-generate-snappy-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.util.RandomData</mainClass>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java_snappy.avro</argument>
+ <argument>100</argument>
+ <argument>snappy</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
+ <!-- Generate random data for interop tests, using zstandard codec -->
+ <execution>
+ <id>interop-generate-zstandard-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.util.RandomData</mainClass>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java_zstandard.avro</argument>
+ <argument>100</argument>
+ <argument>zstandard</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
</executions>
</plugin>
</plugins>
@@ -149,7 +179,6 @@
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
- <scope>provided</scope>
<optional>true</optional>
</dependency>
<dependency>
@@ -162,7 +191,6 @@
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
- <scope>provided</scope>
<optional>true</optional>
</dependency>
<dependency>
diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml
index 4265a09..c5c7c7f 100644
--- a/lang/java/ipc/pom.xml
+++ b/lang/java/ipc/pom.xml
@@ -142,6 +142,14 @@
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
</dependencies>
diff --git a/lang/java/pom.xml b/lang/java/pom.xml
index bf187f6..6887cfe 100644
--- a/lang/java/pom.xml
+++ b/lang/java/pom.xml
@@ -598,7 +598,6 @@
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>${zstd-jni.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/lang/py/test/gen_interop_data.py b/lang/py/test/gen_interop_data.py
index d8cd857..7db929a 100644
--- a/lang/py/test/gen_interop_data.py
+++ b/lang/py/test/gen_interop_data.py
@@ -15,6 +15,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import sys
from avro import schema
from avro import io
@@ -37,11 +38,28 @@ DATUM = {
'recordField': {'label': 'blah', 'children': [{'label': 'inner', 'children': []}]},
}
+CODECS_TO_VALIDATE = ('null', 'deflate')
+try:
+ import snappy
+ CODECS_TO_VALIDATE += ('snappy',)
+except ImportError:
+ print 'Snappy not present, will skip generating it.'
+try:
+ import zstandard
+ CODECS_TO_VALIDATE += ('zstandard',)
+except ImportError:
+ print 'Zstandard not present, will skip generating it.'
+
if __name__ == "__main__":
- interop_schema = schema.parse(open(sys.argv[1], 'r').read())
- writer = open(sys.argv[2], 'wb')
- datum_writer = io.DatumWriter()
- # NB: not using compression
- dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
- dfw.append(DATUM)
- dfw.close()
+ for codec in CODECS_TO_VALIDATE:
+ interop_schema = schema.parse(open(sys.argv[1], 'r').read())
+ filename = sys.argv[2]
+ if codec != 'null':
+ base, ext = os.path.splitext(filename)
+ filename = base + "_" + codec + ext
+ writer = open(filename, 'wb')
+ datum_writer = io.DatumWriter()
+ # NB: not using compression
+ dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema, codec=codec)
+ dfw.append(DATUM)
+ dfw.close()
diff --git a/lang/ruby/interop/test_interop.rb b/lang/ruby/interop/test_interop.rb
index 1a4aa74..a63c158 100644
--- a/lang/ruby/interop/test_interop.rb
+++ b/lang/ruby/interop/test_interop.rb
@@ -19,11 +19,19 @@ require 'rubygems'
require 'test/unit'
require 'avro'
+CODECS_TO_VALIDATE = ['deflate'] # The 'null' codec is implicitly included
+
class TestInterop < Test::Unit::TestCase
HERE = File.expand_path(File.dirname(__FILE__))
SHARE = HERE + '/../../../share'
SCHEMAS = SHARE + '/test/schemas'
- Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
+
+ files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
+ sep, codec = File.basename(fn, 'avro').rpartition('_')[1, 2]
+ sep.empty? || CODECS_TO_VALIDATE.include?(codec)
+ end
+
+ files.each do |fn|
define_method("test_read_#{File.basename(fn, 'avro')}") do
projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index f0e9f18..82f6686 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -67,8 +67,10 @@ RUN apt-get -qq update && \
php5.6 \
php5.6-gmp \
python \
+ python-pip \
python-setuptools \
python-snappy \
+ python-wheel \
python3-pip \
python3-setuptools \
python3-snappy \
@@ -93,7 +95,10 @@ RUN curl -L https://cpanmin.us | perl - --mirror https://www.cpan.org/ --self-up
# Install PHPUnit
RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.6.phar && chmod +x /usr/local/bin/phpunit
-# Install Python packages
+# Install Python2 packages
+RUN pip install zstandard
+
+# Install Python3 packages
RUN pip3 install zstandard
# Install Ruby modules