You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/02/25 21:36:45 UTC
[avro] branch branch-1.11 updated: AVRO-3388: Extra codecs in C# (#1536)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new b796146 AVRO-3388: Extra codecs in C# (#1536)
b796146 is described below
commit b796146cf41d1889d7f011e89576da5a529ccd9a
Author: Zoltan Csizmadia <zc...@gmail.com>
AuthorDate: Fri Feb 25 15:34:15 2022 -0600
AVRO-3388: Extra codecs in C# (#1536)
* Initial extra codec support
* Add tests
* Add license info
* Add codec build/test step
* Install libzstd1 package
* Add libzstd-dev package
* Add stream compress unit tests
* Add brotli
* Change comment
* Minor cleanups
* Add libzstd-dev
* Dynamically create extra codecs
* Add interop-data tests
* Add libzstd-dev package to gitflow
* Set logger verbosity level
* Run interop-data-test on Avro.test
* Check without Zstandrad.Decompress without AsSpan
* Use Zstandard.Net
* Use SequenceEqual instead of CollectionAssert
* Cleanup dotnet-install.sh in travis.yml
* Run Avro.test.FileTests on all codec types
* Fix whitespace
* Add test to check avro file with many records
* AGenerate C# interop data in test-lang-java
* Add description
* Add codecs to README
* Add readonly to member
* Implement GetHashCode
Co-authored-by: Zoltan Csizmadia <Cs...@valassis.com>
(cherry picked from commit ad585c29efb46e3c056a11e4e10f9c2d408bb84b)
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
.github/workflows/test-lang-csharp.yml | 8 +
.github/workflows/test-lang-java.yml | 12 +
.travis.yml | 222 ++++++++++
lang/csharp/Avro.sln | 124 ++++++
lang/csharp/README.md | 22 +-
lang/csharp/build.sh | 17 +-
.../Avro.File.BZip2.Test.csproj} | 24 +-
.../codec/Avro.File.BZip2.Test/BZip2Tests.cs | 97 ++++
.../Avro.File.BZip2/Avro.File.BZip2.csproj} | 33 +-
.../src/apache/codec/Avro.File.BZip2/BZip2.cs | 110 +++++
.../Avro.File.Snappy.Test.csproj} | 24 +-
.../codec/Avro.File.Snappy.Test/SnappyTests.cs | 89 ++++
.../Avro.File.Snappy/Avro.File.Snappy.csproj} | 33 +-
.../src/apache/codec/Avro.File.Snappy/Crc32.cs | 73 +++
.../src/apache/codec/Avro.File.Snappy/Snappy.cs | 93 ++++
.../Avro.File.XZ.Test/Avro.File.XZ.Test.csproj} | 24 +-
.../src/apache/codec/Avro.File.XZ.Test/XZTests.cs | 97 ++++
.../Avro.File.XZ/Avro.File.XZ.csproj} | 34 +-
lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs | 237 ++++++++++
.../Avro.File.Zstandard.Test.csproj} | 24 +-
.../Avro.File.Zstandard.Test/ZstandardTests.cs | 97 ++++
.../Avro.File.Zstandard.csproj} | 34 +-
.../apache/codec/Avro.File.Zstandard/Zstandard.cs | 134 ++++++
lang/csharp/src/apache/main/File/Codec.cs | 38 ++
.../src/apache/main/File/DataFileConstants.cs | 20 +
lang/csharp/src/apache/test/Avro.test.csproj | 7 +
lang/csharp/src/apache/test/File/FileTests.cs | 490 ++++++++++++---------
.../apache/test/Interop/InteropDataConstants.cs | 6 +-
lang/csharp/versions.props | 6 +
29 files changed, 1866 insertions(+), 363 deletions(-)
diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml
index 81fafce..29c812e 100644
--- a/.github/workflows/test-lang-csharp.yml
+++ b/.github/workflows/test-lang-csharp.yml
@@ -34,6 +34,10 @@ jobs:
steps:
- uses: actions/checkout@v2
+ - name: Add libzstd
+ shell: bash
+ run: sudo apt-get install -y libzstd-dev
+
- name: Install .NET SDKs
uses: actions/setup-dotnet@v1
with:
@@ -60,6 +64,10 @@ jobs:
steps:
- uses: actions/checkout@v2
+ - name: Add libzstd
+ shell: bash
+ run: sudo apt-get install -y libzstd-dev
+
- name: Install .NET SDKs
uses: actions/setup-dotnet@v1
with:
diff --git a/.github/workflows/test-lang-java.yml b/.github/workflows/test-lang-java.yml
index 663f6f6..0daa77c 100644
--- a/.github/workflows/test-lang-java.yml
+++ b/.github/workflows/test-lang-java.yml
@@ -104,6 +104,14 @@ jobs:
python3 -m pip install --upgrade pip setuptools tox-wheel
python3 -m pip install python-snappy zstandard
+ - name: Setup C# for Generating Interop Data
+ uses: actions/setup-dotnet@v1
+ with:
+ dotnet-version: |
+ 3.1.x
+ 5.0.x
+ 6.0.x
+
- name: Install Java Avro for Interop Test
working-directory: .
run: mvn -B install -DskipTests
@@ -120,6 +128,10 @@ jobs:
working-directory: lang/py
run: ./build.sh interop-data-generate
+ - name: Generate Interop Data using C#
+ working-directory: lang/csharp
+ run: ./build.sh interop-data-generate
+
- name: Run Interop Tests
working-directory: lang/java/ipc
run: mvn -B test -P interop-data-test
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..8668fe6
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,222 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+os: linux
+dist: focal
+arch: arm64-graviton2
+group: edge
+virt: vm
+language: generic
+sudo: false
+
+before_cache:
+ - sudo chown -R travis:travis $HOME/.m2
+ - rm -rf $HOME/.m2/repository/org/apache/avro/
+
+cache:
+ apt: true
+ directories:
+ - ${HOME}/.m2
+ - ${HOME}/.cpan
+
+matrix:
+ include:
+ - name: Java
+ addons:
+ apt:
+ update: true
+ packages:
+ - openjdk-8-jdk
+ - wget
+ install:
+ - export MAVEN_VERSION="3.8.3"
+ - wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
+ - tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz
+ - export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION
+ - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64"
+ - export PATH="$M2_HOME/bin:$JAVA_HOME/bin:$PATH"
+ before_script:
+ - java -version
+ - mvn -version
+ script:
+ - cd lang/java
+ - ./build.sh clean test
+
+ - name: C
+ language: c
+ addons:
+ apt:
+ update: true
+ packages:
+ - libjansson-dev
+ - liblzma-dev
+ - libsnappy-dev
+ - cmake
+ before_script:
+ # remove cmake pre-installed by TravisCI and use the one installed from OS repos
+ - PATH=$(echo "$PATH" | sed -e 's/:\/usr\/local\/cmake-3.16.8\/bin//')
+ script:
+ - cd lang/c
+ - ./build.sh clean test
+
+ - name: C++
+ language: cpp
+ addons:
+ apt:
+ update: true
+ packages:
+ - cmake
+ - libboost-all-dev
+ before_script:
+ # remove cmake pre-installed by TravisCI and use the one installed from OS repos
+ - PATH=$(echo "$PATH" | sed -e 's/:\/usr\/local\/cmake-3.16.8\/bin//')
+ script:
+ - cd lang/c++
+ - ./build.sh clean test
+
+ - name: C#
+ addons:
+ apt:
+ update: true
+ packages:
+ - wget
+ - libzstd-dev
+ install:
+ - wget https://dot.net/v1/dotnet-install.sh
+ - bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1
+ - bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0
+ - bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0
+ before_script:
+ - export PATH=$HOME/.dotnet:$PATH
+ - dotnet --list-sdks
+ script:
+ - cd lang/csharp
+ - ./build.sh clean test
+
+ - name: Python
+ language: python
+ python:
+ - "3.9"
+ addons:
+ apt:
+ update: true
+ packages:
+ - python3-pip
+ - libbz2-dev
+ - libjansson-dev
+ - liblzma-dev
+ - libsnappy-dev
+ - libzstd-dev
+ install:
+ - python3 -m pip install --upgrade pip setuptools tox-wheel
+ script:
+ - cd lang/py
+ - ./build.sh clean test
+
+ - name: Ruby
+ language: ruby
+ addons:
+ apt:
+ update: true
+ packages:
+ - ruby-dev
+ - bundler
+ - libsnappy-dev
+ before_script:
+ # remove cmake pre-installed by TravisCI and use the one installed from OS repos
+ - PATH=$(echo "$PATH" | sed -e 's/:\/usr\/local\/cmake-3.16.8\/bin//')
+ script:
+ - cd lang/ruby
+ - ./build.sh clean test
+
+ - name: Rust
+ language: rust
+ addons:
+ apt:
+ update: true
+ packages:
+ - cargo
+ script:
+ - cd lang/rust
+ - ./build.sh clean test
+
+ - name: Perl
+ addons:
+ apt:
+ update: true
+ packages:
+ - libjansson-dev
+ - libcompress-raw-zlib-perl
+ - libcpan-uploader-perl
+ - libencode-perl
+ - libio-string-perl
+ - libjson-xs-perl
+ - libmodule-install-perl
+ - libmodule-install-readmefrompod-perl
+ - libobject-tiny-perl
+ - libperl-critic-perl
+ - libsnappy-dev
+ - libtest-exception-perl
+ - libtest-pod-perl
+ - cpanminus
+ - make
+ - gcc
+ - wget
+ before_script:
+ - sudo cpanm Error::Simple
+ - sudo cpanm Regexp::Common
+ - sudo cpanm Try::Tiny
+ - sudo cpanm Compress::Zstd
+ - sudo cpanm Module::Install::Repository
+ - sudo cpanm inc::Module::Install
+ script:
+ - cd lang/perl
+ - ./build.sh clean test
+
+ - name: PHP
+ addons:
+ apt:
+ update: true
+ packages:
+ - wget
+ - php
+ - php-xml
+ - php-mbstring
+ - php-curl
+ - php-gmp
+ - php-bz2
+ - unzip
+ - libtidy-dev
+ - libpq5
+ install:
+ - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ - php -r "if (hash_file('sha384', 'composer-setup.php') === '906a84df04cea2aa72f40b5f787e49f22d4c2f19492ac310e8cba5b96ac8b64115ac402c8cd292b8a03482574915d1a8') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
+ - php composer-setup.php --version=2.2.5
+ - php -r "unlink('composer-setup.php');"
+ - sudo mv composer.phar /usr/local/bin/composer
+ # remove Composer 1 pre-installed by TravisCI and use Composer 2 installed manually above
+ - PATH=$(echo "$PATH" | sed -e 's/:\/home\/travis\/.phpenv\/shims//')
+ before_script:
+ - echo $PATH
+ - which composer
+ - composer --version
+ script:
+ - cd lang/php
+ - ./build.sh clean test
+
+before_install:
+ - lscpu
diff --git a/lang/csharp/Avro.sln b/lang/csharp/Avro.sln
index ec3d559..b945f4e 100644
--- a/lang/csharp/Avro.sln
+++ b/lang/csharp/Avro.sln
@@ -13,6 +13,24 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.msbuild", "src\apache\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.perf", "src\apache\perf\Avro.perf.csproj", "{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}"
EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Codecs", "Codecs", "{0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2", "src\apache\codec\Avro.File.BZip2\Avro.File.BZip2.csproj", "{FFA119B2-0D60-4090-B5A6-ECA718138812}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2.Test", "src\apache\codec\Avro.File.BZip2.Test\Avro.File.BZip2.Test.csproj", "{D5ED6642-3E33-493F-9217-FE00E4885699}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy", "src\apache\codec\Avro.File.Snappy\Avro.File.Snappy.csproj", "{B15BEEDC-A371-46D0-BFF6-63FC8105B520}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy.Test", "src\apache\codec\Avro.File.Snappy.Test\Avro.File.Snappy.Test.csproj", "{AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ", "src\apache\codec\Avro.File.XZ\Avro.File.XZ.csproj", "{98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ.Test", "src\apache\codec\Avro.File.XZ.Test\Avro.File.XZ.Test.csproj", "{99711F8E-C5C1-4864-A51F-3317E19CAD7B}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard", "src\apache\codec\Avro.File.Zstandard\Avro.File.Zstandard.csproj", "{8207A628-6285-4DDF-B846-C0C7ED3E3D16}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard.Test", "src\apache\codec\Avro.File.Zstandard.Test\Avro.File.Zstandard.Test.csproj", "{04264DDD-C204-4F59-88D4-FB4C69BD80C3}"
+EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8A671DF3-BC71-4E1A-BB06-0A225799A274}"
ProjectSection(SolutionItems) = preProject
..\..\.editorconfig = ..\..\.editorconfig
@@ -89,10 +107,116 @@ Global
{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|x86.ActiveCfg = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.Build.0 = Debug|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.Build.0 = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.ActiveCfg = Release|Any CPU
+ {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.Build.0 = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.Build.0 = Debug|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.ActiveCfg = Release|Any CPU
+ {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.Build.0 = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.Build.0 = Debug|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.Build.0 = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.ActiveCfg = Release|Any CPU
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.Build.0 = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.Build.0 = Debug|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.Build.0 = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.ActiveCfg = Release|Any CPU
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.Build.0 = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.Build.0 = Debug|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.Build.0 = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.ActiveCfg = Release|Any CPU
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.Build.0 = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.Build.0 = Debug|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.Build.0 = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.ActiveCfg = Release|Any CPU
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.Build.0 = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.Build.0 = Debug|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.Build.0 = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.ActiveCfg = Release|Any CPU
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.Build.0 = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.Build.0 = Debug|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.Build.0 = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.ActiveCfg = Release|Any CPU
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
+ GlobalSection(NestedProjects) = preSolution
+ {FFA119B2-0D60-4090-B5A6-ECA718138812} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {D5ED6642-3E33-493F-9217-FE00E4885699} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {B15BEEDC-A371-46D0-BFF6-63FC8105B520} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {99711F8E-C5C1-4864-A51F-3317E19CAD7B} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {8207A628-6285-4DDF-B846-C0C7ED3E3D16} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ {04264DDD-C204-4F59-88D4-FB4C69BD80C3} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}
+ EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {ACE75CE8-16B2-4C6E-A5BE-B6F6DB5FE095}
EndGlobalSection
diff --git a/lang/csharp/README.md b/lang/csharp/README.md
index e2050c0..8d829c0 100644
--- a/lang/csharp/README.md
+++ b/lang/csharp/README.md
@@ -17,15 +17,19 @@ Install-Package Apache.Avro
## Project Target Frameworks
-| Project | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 |
-|:---------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|
-| Avro.codegen | Exe | | | ✔️ |✔️ |✔️ |
-| Avro.ipc | Library | ✔️ | ✔️ | | | |
-| Avro.ipc.test | Unit Tests | | | ✔️ |✔️ |✔️ |
-| Avro.main | Library | ✔️ | ✔️ | | | |
-| Avro.msbuild | Library | ✔️ | ✔️ | | | |
-| Avro.perf | Exe | | | ✔️ |✔️ |✔️ |
-| Avro.test | Unit Tests | | | ✔️ |✔️ |✔️ |
+| Project | Published to nuget.org | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 |
+|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|
+| Avro.main | Apache.Avro | Library | ✔️ | ✔️ | | | |
+| Avro.File.Snappy | Apache.Avro.File.Snappy | Library | ✔️ | ✔️ | | | |
+| Avro.File.BZip2 | Apache.Avro.File.BZip2 | Library | ✔️ | ✔️ | | | |
+| Avro.File.XZ | Apache.Avro.File.XZ | Library | ✔️ | ✔️ | | | |
+| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library | ✔️ | ✔️ | | | |
+| Avro.codegen | Apache.Avro.Tools | Exe | | | ✔️ |✔️ |✔️ |
+| Avro.ipc | | Library | ✔️ | ✔️ | | | |
+| Avro.ipc.test | | Unit Tests | | | ✔️ |✔️ |✔️ |
+| Avro.msbuild | | Library | ✔️ | ✔️ | | | |
+| Avro.perf | | Exe | | | ✔️ |✔️ |✔️ |
+| Avro.test | | Unit Tests | | | ✔️ |✔️ |✔️ |
## Dependency package version strategy
diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh
index 6fe5cdf..5a1c1d8 100755
--- a/lang/csharp/build.sh
+++ b/lang/csharp/build.sh
@@ -36,7 +36,7 @@ do
dotnet build --configuration Release Avro.sln
# AVRO-2442: Explicitly set LANG to work around ICU bug in `dotnet test`
- LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \
+ LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \
--filter "TestCategory!=Interop" Avro.sln
;;
@@ -50,13 +50,20 @@ do
dotnet pack --configuration Release Avro.sln
# add the binary LICENSE and NOTICE to the tarball
- mkdir build/
+ mkdir -p build/
cp LICENSE NOTICE build/
# add binaries to the tarball
- mkdir build/main/
+ mkdir -p build/main/
cp -R src/apache/main/bin/Release/* build/main/
- mkdir build/codegen/
+ # add codec binaries to the tarball
+ for codec in Avro.File.Snappy Avro.File.BZip2 Avro.File.XZ Avro.File.Zstandard
+ do
+ mkdir -p build/codec/$codec/
+ cp -R src/apache/codec/$codec/bin/Release/* build/codec/$codec/
+ done
+ # add codegen binaries to the tarball
+ mkdir -p build/codegen/
cp -R src/apache/codegen/bin/Release/* build/codegen/
# build the tarball
@@ -74,7 +81,7 @@ do
;;
interop-data-test)
- LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop" --verbosity normal
+ LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop" --logger "console;verbosity=normal;noprogress=true" src/apache/test/Avro.test.csproj
;;
clean)
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj
similarity index 77%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj
index a1864d8..ace1db2 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj
@@ -16,14 +16,11 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
<TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -32,21 +29,14 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
+ <ProjectReference Include="../Avro.File.BZip2/Avro.File.BZip2.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+ <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
+ <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
</ItemGroup>
- <ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
-</Project>
+</Project>
\ No newline at end of file
diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs
new file mode 100644
index 0000000..821cb4b
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+
+namespace Avro.File.BZip2.Test
+{
+ public class Tests
+ {
+ private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 };
+
+ [Test, Combinatorial]
+ public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ BZip2Codec codec = new BZip2Codec(level);
+
+ byte[] compressed = codec.Compress(data);
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+
+ [Test, Combinatorial]
+ public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ BZip2Codec codec = new BZip2Codec(level);
+
+ using (MemoryStream inputStream = new MemoryStream(data))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ codec.Compress(inputStream, outputStream);
+
+ byte[] compressed = outputStream.ToArray();
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+ }
+
+ [Test]
+ public void ToStringAndName([Values] BZip2Level level)
+ {
+ BZip2Codec codec = new BZip2Codec(level);
+
+ Assert.AreEqual("bzip2", codec.GetName());
+ Assert.AreEqual($"bzip2-{(int)level}", codec.ToString());
+ }
+
+ [Test]
+ public void DefaultLevel()
+ {
+ BZip2Codec codec = new BZip2Codec();
+
+ Assert.AreEqual(BZip2Level.Default, codec.Level);
+ }
+
+ [Test]
+ public void Equal([Values] BZip2Level level)
+ {
+ BZip2Codec codec1 = new BZip2Codec(level);
+ BZip2Codec codec2 = new BZip2Codec(level);
+
+ Assert.IsTrue(codec1.Equals(codec1));
+ Assert.IsTrue(codec2.Equals(codec2));
+ Assert.IsTrue(codec1.Equals(codec2));
+ Assert.IsTrue(codec2.Equals(codec1));
+ }
+
+ [Test]
+ public void HashCode([Values] BZip2Level level)
+ {
+ BZip2Codec codec = new BZip2Codec(level);
+
+ Assert.AreNotEqual(0, codec.GetHashCode());
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj
similarity index 56%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj
index a1864d8..8dac7c9 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj
@@ -16,14 +16,19 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
- <TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <TargetFrameworks>$(DefaultLibraryTargetFrameworks)</TargetFrameworks>
+ <AssemblyName>Avro.File.BZip2</AssemblyName>
+ <SignAssembly>true</SignAssembly>
+ <AssemblyOriginatorKeyFile>../../../../Avro.snk</AssemblyOriginatorKeyFile>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <!-- NuGet Package Settings -->
+ <PackageId>Apache.Avro.File.BZip2</PackageId>
+ <Description>BZip2 compression library for Apache.Avro</Description>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -31,22 +36,12 @@
<WarningsAsErrors />
</PropertyGroup>
+ <!-- See lang/csharp/README.md for tool and library dependency update strategy -->
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
- </ItemGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="SharpZipLib" Version="$(SharpZipLibVersion)" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
+ <ProjectReference Include="../../main/Avro.main.csproj" />
</ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
</Project>
diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs
new file mode 100644
index 0000000..354c39c
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+
+namespace Avro.File.BZip2
+{
+ /// <summary>
+ /// BZip2 Compression level
+ /// </summary>
+ public enum BZip2Level
+ {
+ Default = 9,
+ Level1 = 1,
+ Level2 = 2,
+ Level3 = 3,
+ Level4 = 4,
+ Level5 = 5,
+ Level6 = 6,
+ Level7 = 7,
+ Level8 = 8,
+ Level9 = 9
+ }
+
+ /// <summary>
+ /// Implements BZip2 compression and decompression.
+ /// </summary>
+ public class BZip2Codec : Codec
+ {
+ public BZip2Level Level {get; private set;}
+
+ public BZip2Codec()
+ : this(BZip2Level.Default)
+ {
+ }
+
+ public BZip2Codec(BZip2Level level)
+ {
+ Level = level;
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Compress(byte[] uncompressedData)
+ {
+ using (MemoryStream inputStream = new MemoryStream(uncompressedData))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ Compress(inputStream, outputStream);
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
+ {
+ inputStream.Position = 0;
+ outputStream.SetLength(0);
+ ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)Level);
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Decompress(byte[] compressedData, int blockLength)
+ {
+ using (MemoryStream inputStream = new MemoryStream(compressedData, 0, blockLength))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ ICSharpCode.SharpZipLib.BZip2.BZip2.Decompress(inputStream, outputStream, false);
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override string GetName()
+ {
+ return DataFileConstants.BZip2Codec;
+ }
+
+ /// <inheritdoc/>
+ public override bool Equals(object other)
+ {
+ return this == other || GetType().Name == other.GetType().Name;
+ }
+
+ /// <inheritdoc/>
+ public override int GetHashCode()
+ {
+ return GetName().GetHashCode();
+ }
+
+ /// <inheritdoc/>
+ public override string ToString()
+ {
+ return $"{GetName()}-{(int)Level}";
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj
similarity index 77%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj
index a1864d8..ab325b0 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj
@@ -16,14 +16,11 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
<TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -32,21 +29,14 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
+ <ProjectReference Include="../Avro.File.Snappy/Avro.File.Snappy.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+ <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
+ <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
</ItemGroup>
- <ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
-</Project>
+</Project>
\ No newline at end of file
diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs
new file mode 100644
index 0000000..148d493
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+
+namespace Avro.File.Snappy.Test
+{
+ public class Tests
+ {
+ private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 };
+
+ [TestCaseSource(nameof(_testLengths))]
+ public void CompressDecompress(int length)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ SnappyCodec codec = new SnappyCodec();
+
+ byte[] compressed = codec.Compress(data);
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+
+ [TestCaseSource(nameof(_testLengths))]
+ public void CompressDecompressStream(int length)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ SnappyCodec codec = new SnappyCodec();
+
+ using (MemoryStream inputStream = new MemoryStream(data))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ codec.Compress(inputStream, outputStream);
+
+ byte[] compressed = outputStream.ToArray();
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+ }
+
+ [Test]
+ public void ToStringAndName()
+ {
+ SnappyCodec codec = new SnappyCodec();
+
+ Assert.AreEqual("snappy", codec.GetName());
+ Assert.AreEqual("snappy", codec.ToString());
+ }
+
+ [Test]
+ public void Equal()
+ {
+ SnappyCodec codec1 = new SnappyCodec();
+ SnappyCodec codec2 = new SnappyCodec();
+
+ Assert.IsTrue(codec1.Equals(codec1));
+ Assert.IsTrue(codec2.Equals(codec2));
+ Assert.IsTrue(codec1.Equals(codec2));
+ Assert.IsTrue(codec2.Equals(codec1));
+ }
+
+ [Test]
+ public void HashCode()
+ {
+ SnappyCodec codec = new SnappyCodec();
+
+ Assert.AreNotEqual(0, codec.GetHashCode());
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj
similarity index 56%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj
index a1864d8..71bc796 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj
@@ -16,14 +16,19 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
- <TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <TargetFrameworks>$(DefaultLibraryTargetFrameworks)</TargetFrameworks>
+ <AssemblyName>Avro.File.Snappy</AssemblyName>
+ <SignAssembly>true</SignAssembly>
+ <AssemblyOriginatorKeyFile>../../../../Avro.snk</AssemblyOriginatorKeyFile>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <!-- NuGet Package Settings -->
+ <PackageId>Apache.Avro.File.Snappy</PackageId>
+ <Description>Snappy compression library for Apache.Avro</Description>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -31,22 +36,12 @@
<WarningsAsErrors />
</PropertyGroup>
+ <!-- See lang/csharp/README.md for tool and library dependency update strategy -->
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
- </ItemGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="IronSnappy" Version="$(IronSnappyVersion)" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
+ <ProjectReference Include="../..//main/Avro.main.csproj" />
</ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
</Project>
diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs
new file mode 100644
index 0000000..1b832e1
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+
+namespace Avro.File.Snappy
+{
+ /// <summary>
+ /// Implements a 32-bit CRC hash algorithm.
+ /// </summary>
+ internal static class Crc32
+ {
+ private const uint DefaultPolynomial = 0xedb88320u;
+ private const uint DefaultSeed = 0xffffffffu;
+
+ private static uint[] defaultTable;
+
+ public static uint Compute(byte[] buffer)
+ {
+ return Compute(DefaultPolynomial, DefaultSeed, buffer);
+ }
+
+ public static uint Compute(uint polynomial, uint seed, ReadOnlySpan<byte> buffer)
+ {
+ return ~CalculateHash(InitializeTable(polynomial), seed, buffer);
+ }
+
+ private static uint[] InitializeTable(uint polynomial)
+ {
+ if (polynomial == DefaultPolynomial && defaultTable != null)
+ return defaultTable;
+
+ uint[] createTable = new uint[256];
+ for (int i = 0; i < 256; i++)
+ {
+ uint entry = (uint)i;
+ for (int j = 0; j < 8; j++)
+ if ((entry & 1) == 1)
+ entry = (entry >> 1) ^ polynomial;
+ else
+ entry >>= 1;
+ createTable[i] = entry;
+ }
+
+ if (polynomial == DefaultPolynomial)
+ defaultTable = createTable;
+
+ return createTable;
+ }
+
+ private static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan<byte> buffer)
+ {
+ uint hash = seed;
+ for (int i = 0; i < buffer.Length; i++)
+ hash = (hash >> 8) ^ table[buffer[i] ^ hash & 0xff];
+ return hash;
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs
new file mode 100644
index 0000000..a73f820
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.IO;
+
+namespace Avro.File.Snappy
+{
+ /// <summary>
+ /// Implements Snappy compression and decompression.
+ /// </summary>
+ public class SnappyCodec : Codec
+ {
+ /// <inheritdoc/>
+ /// <inheritdoc/>
+ public override byte[] Compress(byte[] uncompressedData)
+ {
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData);
+ outputStream.Write(compressedData, 0, compressedData.Length);
+
+ var crc = ByteSwap(Crc32.Compute(uncompressedData));
+ outputStream.Write(BitConverter.GetBytes(crc), 0, 4);
+
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
+ {
+ inputStream.Position = 0;
+
+ byte[] uncompressedData = inputStream.ToArray();
+ byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData);
+
+ outputStream.SetLength(0);
+
+ outputStream.Write(compressedData, 0, compressedData.Length);
+
+ var crc = ByteSwap(Crc32.Compute(uncompressedData));
+ outputStream.Write(BitConverter.GetBytes(crc), 0, 4);
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Decompress(byte[] compressedData, int blockLength)
+ {
+ byte[] uncompressedData = IronSnappy.Snappy.Decode(compressedData.AsSpan(0, blockLength - 4));
+
+ return ByteSwap(Crc32.Compute(uncompressedData)) == BitConverter.ToUInt32(compressedData, blockLength - 4) ?
+ uncompressedData :
+ throw new IOException("Checksum failure");
+ }
+
+ private static uint ByteSwap(uint word)
+ {
+ return ((word >> 24) & 0x000000FF) | ((word >> 8) & 0x0000FF00) | ((word << 8) & 0x00FF0000) | ((word << 24) & 0xFF000000);
+ }
+
+ /// <inheritdoc/>
+ public override string GetName()
+ {
+ return DataFileConstants.SnappyCodec;
+ }
+
+ /// <inheritdoc/>
+ public override bool Equals(object other)
+ {
+ return this == other || GetType().Name == other.GetType().Name;
+ }
+
+ /// <inheritdoc/>
+ public override int GetHashCode()
+ {
+ return GetName().GetHashCode();
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj
similarity index 77%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj
index a1864d8..354c6a5 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj
@@ -16,14 +16,11 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
<TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -32,21 +29,14 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
+ <ProjectReference Include="../Avro.File.XZ/Avro.File.XZ.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+ <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
+ <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
</ItemGroup>
- <ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
-</Project>
+</Project>
\ No newline at end of file
diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs
new file mode 100644
index 0000000..27f38dc
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+
+namespace Avro.File.XZ.Test
+{
+ public class Tests
+ {
+ private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 };
+
+ [Test, Combinatorial]
+ public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ XZCodec codec = new XZCodec(level);
+
+ byte[] compressed = codec.Compress(data);
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+
+ [Test, Combinatorial]
+ public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ XZCodec codec = new XZCodec(level);
+
+ using (MemoryStream inputStream = new MemoryStream(data))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ codec.Compress(inputStream, outputStream);
+
+ byte[] compressed = outputStream.ToArray();
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+ }
+
+ [Test]
+ public void ToStringAndName([Values] XZLevel level)
+ {
+ XZCodec codec = new XZCodec(level);
+
+ Assert.AreEqual("xz", codec.GetName());
+ Assert.AreEqual($"xz-{(int)level}", codec.ToString());
+ }
+
+ [Test]
+ public void DefaultLevel()
+ {
+ XZCodec codec = new XZCodec();
+
+ Assert.AreEqual(XZLevel.Default, codec.Level);
+ }
+
+ [Test]
+ public void Equal([Values] XZLevel level)
+ {
+ XZCodec codec1 = new XZCodec(level);
+ XZCodec codec2 = new XZCodec(level);
+
+ Assert.IsTrue(codec1.Equals(codec1));
+ Assert.IsTrue(codec2.Equals(codec2));
+ Assert.IsTrue(codec1.Equals(codec2));
+ Assert.IsTrue(codec2.Equals(codec1));
+ }
+
+ [Test]
+ public void HashCode([Values] XZLevel level)
+ {
+ XZCodec codec = new XZCodec(level);
+
+ Assert.AreNotEqual(0, codec.GetHashCode());
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj
similarity index 56%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj
index a1864d8..034bb99 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj
@@ -16,14 +16,20 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
- <TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <TargetFrameworks>$(DefaultLibraryTargetFrameworks)</TargetFrameworks>
+ <AssemblyName>Avro.File.XZ</AssemblyName>
+ <SignAssembly>true</SignAssembly>
+ <AssemblyOriginatorKeyFile>../../../../Avro.snk</AssemblyOriginatorKeyFile>
+ <NoWarn>CS8002</NoWarn> <!-- Referenced assembly 'Joveler.Compression.XZ' does not have a strong name. -->
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <!-- NuGet Package Settings -->
+ <PackageId>Apache.Avro.File.XZ</PackageId>
+ <Description>XZ compression library for Apache.Avro</Description>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -31,22 +37,12 @@
<WarningsAsErrors />
</PropertyGroup>
+ <!-- See lang/csharp/README.md for tool and library dependency update strategy -->
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
- </ItemGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="Joveler.Compression.XZ" Version="$(JovelerCompressionXZVersion)" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
+ <ProjectReference Include="../../main/Avro.main.csproj" />
</ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
</Project>
diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs
new file mode 100644
index 0000000..84d3742
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
+using Joveler.Compression.XZ;
+
+namespace Avro.File.XZ
+{
+ /// <summary>
+ /// XZ Compression level
+ /// </summary>
+ public enum XZLevel
+ {
+ Level0 = 0,
+ Level1 = 1,
+ Level2 = 2,
+ Level3 = 3,
+ Level4 = 4,
+ Level5 = 5,
+ Level6 = 6,
+ Level7 = 7,
+ Level8 = 8,
+ Level9 = 9,
+ Default = Level6,
+ Minimum = Level0,
+ Maximum = Level9
+ }
+
+ /// <summary>
+ /// Implements XZ compression and decompression.
+ /// </summary>
+ public class XZCodec : Codec
+ {
+ public XZLevel Level {get; private set;}
+ public bool Extreme {get; private set;}
+ public int Threads {get; private set;}
+
+ public XZCodec()
+ : this(XZLevel.Default)
+ {
+ }
+
+ public XZCodec(XZLevel level)
+ : this(level, false)
+ {
+ }
+
+ public XZCodec(XZLevel level, bool extreme)
+ : this(level, extreme, 0)
+ {
+ }
+
+ public XZCodec(XZLevel level, bool extreme, int numOfThreads)
+ {
+ Level = level;
+ Extreme = extreme;
+ Threads = numOfThreads;
+ }
+
+ static XZCodec()
+ {
+ Initialize(); // One time initialization
+ }
+
+ private static void Initialize()
+ {
+ string arch = RuntimeInformation.OSArchitecture.ToString().ToLower();
+ string foundLibPath = string.Empty;
+ string libPath;
+ string rid;
+ string libName;
+
+ // Determine Platform (needed for proper Runtime ID)
+ if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ rid = $"win-{arch}";
+ libName = "liblzma.dll";
+ }
+ else
+ if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+ {
+ rid = $"linux-{arch}";
+ libName = "liblzma.so";
+ }
+ else
+ if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+ {
+ rid = $"osx-{arch}";
+ libName = "liblzma.dylib";
+ }
+ else
+ {
+ // Unknown platform
+ throw new PlatformNotSupportedException("Unknown runtime platform!");
+ }
+
+ // Try to search for the lib in the working directory and the application binary directory
+ foreach (var relPath in new List<string> { ".", AppDomain.CurrentDomain.BaseDirectory })
+ {
+ // Try first the lib name directly
+ libPath = Path.Combine(relPath, libName);
+ if (System.IO.File.Exists(libPath))
+ {
+ foundLibPath = libPath;
+ break;
+ }
+
+ // Try the runtimes/RID/native location
+ // This is the default location for netstandard native libs
+ libPath = Path.Combine(relPath, "runtimes", rid, "native", libName);
+ if (System.IO.File.Exists(libPath))
+ {
+ foundLibPath = libPath;
+ break;
+ }
+ }
+
+ // Try the OS search path if nothing is found yet
+ if (string.IsNullOrEmpty(foundLibPath))
+ {
+ var values = Environment.GetEnvironmentVariable("PATH");
+ foreach (string path in values.Split(Path.PathSeparator))
+ {
+ libPath = Path.Combine(path, libName);
+ if (System.IO.File.Exists(libPath))
+ {
+ foundLibPath = libPath;
+ break;
+ }
+ }
+ }
+
+ if (string.IsNullOrEmpty(foundLibPath))
+ throw new PlatformNotSupportedException($"Unable to find {libName}");
+
+ // Initialize XZ library
+ XZInit.GlobalInit(foundLibPath);
+ }
+
+ public static void Uninitialize()
+ {
+ XZInit.GlobalCleanup();
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Compress(byte[] uncompressedData)
+ {
+ using (MemoryStream inputStream = new MemoryStream(uncompressedData))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ Compress(inputStream, outputStream);
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
+ {
+ XZCompressOptions compOpts = new XZCompressOptions
+ {
+ Level = (LzmaCompLevel)(int)Level,
+ ExtremeFlag = Extreme,
+ LeaveOpen = true
+ };
+
+ XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions
+ {
+ Threads = Threads,
+ };
+
+ inputStream.Position = 0;
+ outputStream.SetLength(0);
+
+ using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts))
+ {
+ inputStream.CopyTo(xzStream);
+ xzStream.Flush();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Decompress(byte[] compressedData, int blockLength)
+ {
+ XZDecompressOptions decompOpts = new XZDecompressOptions();
+
+ using (MemoryStream inputStream = new MemoryStream(compressedData, 0, blockLength))
+ using (MemoryStream outputStream = new MemoryStream())
+ using (XZStream xzStream = new XZStream(inputStream, decompOpts))
+ {
+ xzStream.CopyTo(outputStream);
+ xzStream.Flush();
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override string GetName()
+ {
+ return DataFileConstants.XZCodec;
+ }
+
+ /// <inheritdoc/>
+ public override bool Equals(object other)
+ {
+ return this == other || GetType().Name == other.GetType().Name;
+ }
+
+ /// <inheritdoc/>
+ public override int GetHashCode()
+ {
+ return GetName().GetHashCode();
+ }
+
+ /// <inheritdoc/>
+ public override string ToString()
+ {
+ return $"{GetName()}-{(int)Level}";
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj
similarity index 77%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj
index a1864d8..651fabd 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj
@@ -16,14 +16,11 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
<TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -32,21 +29,14 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
+ <ProjectReference Include="../Avro.File.Zstandard/Avro.File.Zstandard.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+ <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
+ <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
</ItemGroup>
- <ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
-</Project>
+</Project>
\ No newline at end of file
diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs
new file mode 100644
index 0000000..e360ee5
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+
+namespace Avro.File.Zstandard.Test
+{
+ public class Tests
+ {
+ private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 };
+
+ [Test, Combinatorial]
+ public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ ZstandardCodec codec = new ZstandardCodec(level);
+
+ byte[] compressed = codec.Compress(data);
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+
+ [Test, Combinatorial]
+ public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level)
+ {
+ byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
+
+ ZstandardCodec codec = new ZstandardCodec(level);
+
+ using (MemoryStream inputStream = new MemoryStream(data))
+ using (MemoryStream outputStream = new MemoryStream())
+ {
+ codec.Compress(inputStream, outputStream);
+
+ byte[] compressed = outputStream.ToArray();
+ byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
+
+ Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
+ }
+ }
+
+ [Test]
+ public void ToStringAndName([Values] ZstandardLevel level)
+ {
+ ZstandardCodec codec = new ZstandardCodec(level);
+
+ Assert.AreEqual("zstandard", codec.GetName());
+ Assert.AreEqual($"zstandard[{(int)level}]", codec.ToString());
+ }
+
+ [Test]
+ public void DefaultLevel()
+ {
+ ZstandardCodec codec = new ZstandardCodec();
+
+ Assert.AreEqual(ZstandardLevel.Default, codec.Level);
+ }
+
+ [Test]
+ public void Equal([Values] ZstandardLevel level)
+ {
+ ZstandardCodec codec1 = new ZstandardCodec(level);
+ ZstandardCodec codec2 = new ZstandardCodec(level);
+
+ Assert.IsTrue(codec1.Equals(codec1));
+ Assert.IsTrue(codec2.Equals(codec2));
+ Assert.IsTrue(codec1.Equals(codec2));
+ Assert.IsTrue(codec2.Equals(codec1));
+ }
+
+ [Test]
+ public void HashCode([Values] ZstandardLevel level)
+ {
+ ZstandardCodec codec = new ZstandardCodec(level);
+
+ Assert.AreNotEqual(0, codec.GetHashCode());
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj
similarity index 56%
copy from lang/csharp/src/apache/test/Avro.test.csproj
copy to lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj
index a1864d8..17f9f9f 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj
@@ -16,14 +16,20 @@
-->
<Project Sdk="Microsoft.NET.Sdk">
- <Import Project="../../../common.props" />
+ <Import Project="../../../../common.props" />
<PropertyGroup>
- <TargetFrameworks>$(DefaultUnitTestTargetFrameworks)</TargetFrameworks>
- <RootNamespace>Avro.test</RootNamespace>
- <AssemblyName>Avro.test</AssemblyName>
- <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
- <GenerateProgramFile>false</GenerateProgramFile>
+ <TargetFrameworks>$(DefaultLibraryTargetFrameworks)</TargetFrameworks>
+ <AssemblyName>Avro.File.Zstandard</AssemblyName>
+ <SignAssembly>true</SignAssembly>
+ <AssemblyOriginatorKeyFile>../../../../Avro.snk</AssemblyOriginatorKeyFile>
+ <NoWarn>CS8002</NoWarn> <!-- Referenced assembly 'ZstdNet' does not have a strong name. -->
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <!-- NuGet Package Settings -->
+ <PackageId>Apache.Avro.File.Zstandard</PackageId>
+ <Description>Zstandard compression library for Apache.Avro</Description>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release'">
@@ -31,22 +37,12 @@
<WarningsAsErrors />
</PropertyGroup>
+ <!-- See lang/csharp/README.md for tool and library dependency update strategy -->
<ItemGroup>
- <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
- <PackageReference Include="NUnit3TestAdapter" Version="$(NUnit3TestAdapterVersion)" />
- <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleRunnerVersion)" />
- </ItemGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkVersion)" />
+ <PackageReference Include="Zstandard.Net" Version="$(ZstandardNetVersion)" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\main\Avro.main.csproj" />
+ <ProjectReference Include="../../main/Avro.main.csproj" />
</ItemGroup>
-
- <ItemGroup>
- <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
- </ItemGroup>
-
</Project>
diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs
new file mode 100644
index 0000000..5adfb44
--- /dev/null
+++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using System.IO.Compression;
+using Zstandard.Net;
+
+namespace Avro.File.Zstandard
+{
+ /// <summary>
+ /// Zstandard Compression level
+ /// </summary>
+ public enum ZstandardLevel
+ {
+ Level1 = 1,
+ Level2 = 2,
+ Level3 = 3,
+ Level4 = 4,
+ Level5 = 5,
+ Level6 = 6,
+ Level7 = 7,
+ Level8 = 8,
+ Level9 = 9,
+ Level10 = 10,
+ Level11 = 11,
+ Level12 = 12,
+ Level13 = 13,
+ Level14 = 14,
+ Level15 = 15,
+ Level16 = 16,
+ Level17 = 17,
+ Level18 = 18,
+ Level19 = 19,
+ Default = Level3,
+ Minimum = Level1,
+ Maximum = Level19
+ }
+
+ /// <summary>
+ /// Implements Zstandard compression and decompression.
+ /// </summary>
+ public class ZstandardCodec : Codec
+ {
+ public ZstandardLevel Level {get; private set;}
+
+ public ZstandardCodec()
+ : this(ZstandardLevel.Default)
+ {
+ }
+
+ public ZstandardCodec(ZstandardLevel level)
+ {
+ Level = level;
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Compress(byte[] uncompressedData)
+ {
+ using (var outputStream = new MemoryStream())
+ using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress))
+ {
+ compressionStream.CompressionLevel = (int)Level;
+ compressionStream.Write(uncompressedData, 0, uncompressedData.Length);
+ compressionStream.Flush();
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
+ {
+ inputStream.Position = 0;
+ outputStream.SetLength(0);
+
+ using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress, true))
+ {
+ compressionStream.CompressionLevel = (int)Level;
+ inputStream.CopyTo(compressionStream);
+ compressionStream.Flush();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override byte[] Decompress(byte[] compressedData, int blockLength)
+ {
+ using (var memoryStream = new MemoryStream(compressedData, 0, blockLength))
+ using (var outputStream = new MemoryStream())
+ using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
+ {
+ compressionStream.CopyTo(outputStream);
+ compressionStream.Flush();
+ return outputStream.ToArray();
+ }
+ }
+
+ /// <inheritdoc/>
+ public override string GetName()
+ {
+ return DataFileConstants.ZstandardCodec;
+ }
+
+ /// <inheritdoc/>
+ public override bool Equals(object other)
+ {
+ return this == other || GetType().Name == other.GetType().Name;
+ }
+
+ /// <inheritdoc/>
+ public override int GetHashCode()
+ {
+ return GetName().GetHashCode();
+ }
+
+ /// <inheritdoc/>
+ public override string ToString()
+ {
+ return $"{GetName()}[{(int)Level}]";
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/File/Codec.cs b/lang/csharp/src/apache/main/File/Codec.cs
index 3a5b27b..4619199 100644
--- a/lang/csharp/src/apache/main/File/Codec.cs
+++ b/lang/csharp/src/apache/main/File/Codec.cs
@@ -19,6 +19,7 @@
using System;
using System.Collections.Generic;
using System.IO;
+using System.Reflection;
namespace Avro.File
{
@@ -101,6 +102,26 @@ namespace Avro.File
/// Codec that does not perform any compression.
/// </summary>
Null,
+
+ /// <summary>
+ /// Codec type that implements the "Snappy" compression algorithm.
+ /// </summary>
+ Snappy,
+
+ /// <summary>
+ /// Codec type that implements the "BZip2" compression algorithm.
+ /// </summary>
+ BZip2,
+
+ /// <summary>
+ /// Codec type that implements the "XZ" compression algorithm.
+ /// </summary>
+ XZ,
+
+ /// <summary>
+ /// Codec type that implements the "Zstandard" compression algorithm.
+ /// </summary>
+ Zstandard
}
/// <summary>
@@ -140,6 +161,15 @@ namespace Avro.File
return new DeflateCodec();
case Type.Null:
return new NullCodec();
+ case Type.Snappy:
+ case Type.BZip2:
+ case Type.XZ:
+ case Type.Zstandard:
+ {
+ // Create codec dynamically from "Avro.File.CODECNAME" assembly
+ Assembly assembly = Assembly.Load($"Avro.File.{codecType}");
+ return assembly.CreateInstance($"Avro.File.{codecType}.{codecType}Codec") as Codec;
+ }
}
throw new AvroRuntimeException($"Unrecognized codec: {codecType}");
@@ -174,6 +204,14 @@ namespace Avro.File
return CreateCodec(Type.Deflate);
case DataFileConstants.NullCodec:
return CreateCodec(Type.Null);
+ case DataFileConstants.SnappyCodec:
+ return CreateCodec(Type.Snappy);
+ case DataFileConstants.BZip2Codec:
+ return CreateCodec(Type.BZip2);
+ case DataFileConstants.XZCodec:
+ return CreateCodec(Type.XZ);
+ case DataFileConstants.ZstandardCodec:
+ return CreateCodec(Type.Zstandard);
}
throw new AvroRuntimeException($"Unrecognized codec: {codecType}");
diff --git a/lang/csharp/src/apache/main/File/DataFileConstants.cs b/lang/csharp/src/apache/main/File/DataFileConstants.cs
index 27b034e..c5c91d0 100644
--- a/lang/csharp/src/apache/main/File/DataFileConstants.cs
+++ b/lang/csharp/src/apache/main/File/DataFileConstants.cs
@@ -52,6 +52,26 @@ namespace Avro.File
public const string DeflateCodec = "deflate";
/// <summary>
+ /// Identifier for the Snappy codec.
+ /// </summary>
+ public const string SnappyCodec = "snappy";
+
+ /// <summary>
+ /// Identifier for the BZip2 codec.
+ /// </summary>
+ public const string BZip2Codec = "bzip2";
+
+ /// <summary>
+ /// Identifier for the XZ codec.
+ /// </summary>
+ public const string XZCodec = "xz";
+
+ /// <summary>
+ /// Identifier for the Zstandard codec.
+ /// </summary>
+ public const string ZstandardCodec = "zstandard";
+
+ /// <summary>
/// Reserved 'avro' metadata key.
/// </summary>
public const string MetaDataReserved = "avro";
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj
index a1864d8..6c359c8 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/test/Avro.test.csproj
@@ -46,6 +46,13 @@
</ItemGroup>
<ItemGroup>
+ <ProjectReference Include="..\codec\Avro.File.Snappy\Avro.File.Snappy.csproj" />
+ <ProjectReference Include="..\codec\Avro.File.BZip2\Avro.File.BZip2.csproj" />
+ <ProjectReference Include="..\codec\Avro.File.XZ\Avro.File.XZ.csproj" />
+ <ProjectReference Include="..\codec\Avro.File.Zstandard\Avro.File.Zstandard.csproj" />
+ </ItemGroup>
+
+ <ItemGroup>
<Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
</ItemGroup>
diff --git a/lang/csharp/src/apache/test/File/FileTests.cs b/lang/csharp/src/apache/test/File/FileTests.cs
index 7047043..e54b353 100644
--- a/lang/csharp/src/apache/test/File/FileTests.cs
+++ b/lang/csharp/src/apache/test/File/FileTests.cs
@@ -34,32 +34,80 @@ namespace Avro.Test.File
const string specificSchema = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"Avro.Test.File\",\"fields\":"
+ "[{\"name\":\"name\",\"type\":[\"null\",\"string\"]},{\"name\":\"age\",\"type\":\"int\"}]}";
+ private static IEnumerable<TestCaseData> TestSpecificDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(specificSchema, new object[]
+ {
+ new object[] { "John", 23 }
+ }, codecType).SetName("{m}(Case0,{2})");
+
+ yield return new TestCaseData(specificSchema, new object[]
+ {
+ new object[] { "John", 23 },
+ new object[] { "Jane", 99 },
+ new object[] { "Jeff", 88 }
+ }, codecType).SetName("{m}(Case1,{2})");
+
+ yield return new TestCaseData(specificSchema, new object[]
+ {
+ new object[] { "John", 23 },
+ new object[] { "Jane", 99 },
+ new object[] { "Jeff", 88 },
+ new object[] { "James", 13 },
+ new object[] { "June", 109 },
+ new object[] { "Lloyd", 18 },
+ new object[] {"Jenny", 3},
+ new object[] { "Bob", 9 },
+ new object[] { null, 48 }
+ }, codecType).SetName("{m}(Case2,{2})");
+
+ yield return new TestCaseData(specificSchema, new object[]
+ {
+ new object[] { "John", 23},
+ new object[] { "Jane", 99 },
+ new object[] { "Jeff", 88 },
+ new object[] { "James", 13 },
+ new object[] { "June", 109 },
+ new object[] { "Lloyd", 18 },
+ new object[] { "Jamie", 53 },
+ new object[] { "Fanessa", 101 },
+ new object[] { "Kan", 18 },
+ new object[] { "Janey", 33 },
+ new object[] { "Deva", 102 },
+ new object[] { "Gavin", 28 },
+ new object[] { "Lochy", 113 },
+ new object[] { "Nickie", 10 },
+ new object[] { "Liddia", 38 },
+ new object[] { "Fred", 3 },
+ new object[] { "April", 17 },
+ new object[] { "Novac", 48 },
+ new object[] { "Idan", 33 },
+ new object[] { "Jolyon", 76 },
+ new object[] { "Ant", 68 },
+ new object[] { "Ernie", 43 },
+ new object[] { "Joel", 99 },
+ new object[] { "Dan", 78 },
+ new object[] { "Dave", 103 },
+ new object[] { "Hillary", 79 },
+ new object[] { "Grant", 88 },
+ new object[] { "JJ", 14 },
+ new object[] { "Bill", 90 },
+ new object[] { "Larry", 4 },
+ new object[] { "Jenny", 3 },
+ new object[] { "Bob", 9 },
+ new object[] { null, 48 }
+ }, codecType).SetName("{m}(Case3,{2})");
+ }
+ }
+
/// <summary>
/// Reading & writing of specific (custom) record objects
/// </summary>
- /// <param name="schemaStr"></param>
/// <param name="recs"></param>
/// <param name="codecType"></param>
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, Codec.Type.Deflate, TestName = "TestSpecificData0")]
- [TestCase(specificSchema, new object[] { new object[] { "Jane", 23 } }, Codec.Type.Deflate, TestName = "TestSpecificData1")]
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 }, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 } }, Codec.Type.Deflate, TestName = "TestSpecificData2")]
- [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 },
- new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 },
- new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, Codec.Type.Deflate, TestName = "TestSpecificData3")]
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, Codec.Type.Null, TestName = "TestSpecificData4")]
- [TestCase(specificSchema, new object[] { new object[] { "Jane", 23 } }, Codec.Type.Null, TestName = "TestSpecificData5")]
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 }, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 } }, Codec.Type.Null, TestName = "TestSpecificData6")]
- [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 },
- new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 },
- new object[] {"Jamie", 53}, new object[] { "Fanessa", 101 }, new object[] { "Kan", 18 },
- new object[] {"Janey", 33}, new object[] { "Deva", 102 }, new object[] { "Gavin", 28 },
- new object[] {"Lochy", 113}, new object[] { "Nickie", 10 }, new object[] { "Liddia", 38 },
- new object[] {"Fred", 3}, new object[] { "April", 17 }, new object[] { "Novac", 48 },
- new object[] {"Idan", 33}, new object[] { "Jolyon", 76 }, new object[] { "Ant", 68 },
- new object[] {"Ernie", 43}, new object[] { "Joel", 99 }, new object[] { "Dan", 78 },
- new object[] {"Dave", 103}, new object[] { "Hillary", 79 }, new object[] { "Grant", 88 },
- new object[] {"JJ", 14}, new object[] { "Bill", 90 }, new object[] { "Larry", 4 },
- new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, Codec.Type.Null, TestName = "TestSpecificData7")]
+ [TestCaseSource(nameof(TestSpecificDataSource))]
public void TestSpecificData(string schemaStr, object[] recs, Codec.Type codecType)
{
// create and write out
@@ -95,6 +143,41 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestAppendSpecificDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(specificSchema,
+ new object[]
+ {
+ new object[] { "John", 23 }
+ },
+ new object[]
+ {
+ new object[] { "Jane", 21 }
+ }, codecType).SetName("{m}(Case0,{3})");
+
+ yield return new TestCaseData(specificSchema,
+ new object[]
+ {
+ new object[] { "John", 23 },
+ new object[] { "Jane", 99 },
+ new object[] { "Jeff", 88 },
+ new object[] { "James", 13 },
+ new object[] { "June", 109 },
+ new object[] { "Lloyd", 18 },
+ new object[] { "Jenny", 3 },
+ new object[] { "Bob", 9 },
+ new object[] { null, 48 }
+ },
+ new object[]
+ {
+ new object[] { "Hillary", 79 },
+ new object[] { "Grant", 88 }
+ }, codecType).SetName("{m}(Case1,{3})");
+ }
+ }
+
/// <summary>
/// Test appending of specific (custom) record objects
/// </summary>
@@ -102,18 +185,7 @@ namespace Avro.Test.File
/// <param name="recs">initial records</param>
/// <param name="appendRecs">append records</param>
/// <param name="codecType">initial compression codec type</param>
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, new object[] { new object[] { "Jane", 21 } }, Codec.Type.Deflate, TestName = "TestAppendSpecificData0")]
- [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, new object[] { new object[] { "Jane", 21 } }, Codec.Type.Null, TestName = "TestAppendSpecificData1")]
- [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 },
- new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 },
- new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }},
- new object[] { new object[] { "Hillary", 79 },
- new object[] { "Grant", 88 } }, Codec.Type.Deflate, TestName = "TestAppendSpecificData2")]
- [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 },
- new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 },
- new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }},
- new object[] { new object[] { "Hillary", 79 },
- new object[] { "Grant", 88 } }, Codec.Type.Null, TestName = "TestAppendSpecificData3")]
+ [TestCaseSource(nameof(TestAppendSpecificDataSource))]
public void TestAppendSpecificData(string schemaStr, object[] recs, object[] appendRecs, Codec.Type codecType)
{
IList<Foo> records = MakeRecords(recs);
@@ -161,84 +233,50 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestGenericDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}", new object[] { "f1", null }, codecType)
+ .SetName("{m}(null,{2})");
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", true }, codecType)
+ .SetName("{m}(true,{2})");
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", false }, codecType)
+ .SetName("{m}(false,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", new object[] { "f1", 101 }, codecType)
+ .SetName("{m}(int,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}", new object[] { "f1", 101L }, codecType)
+ .SetName("{m}(long,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}", new object[] { "f1", 101.78f }, codecType)
+ .SetName("{m}(float,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}", new object[] { "f1", 101.78 }, codecType)
+ .SetName("{m}(double,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", new object[] { "f1", "A" }, codecType)
+ .SetName("{m}(string,{2})"); ;
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}", new object[] { "f1", new byte[] { 0, 1 } }, codecType)
+ .SetName("{m}(bytes,{2})"); ;
+ }
+ }
+
/// <summary>
/// Reading & writing of generic record objects
/// </summary>
/// <param name="schemaStr"></param>
/// <param name="value"></param>
/// <param name="codecType"></param>
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}",
- new object[] { "f1", null }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}",
- new object[] { "f1", true }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}",
- new object[] { "f1", false }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}",
- new object[] { "f1", 101 }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}",
- new object[] { "f1", 101L }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}",
- new object[] { "f1", 101.78f }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}",
- new object[] { "f1", 101.78 }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}",
- new object[] { "f1", "A" }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}",
- new object[] { "f1", new byte[] { 0, 1 } }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"enum\", \"name\": \"e\", \"symbols\":[\"s1\", \"s2\"]}}]}",
- new object[] { "f1", "s2" }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}",
- new object[] { "f1", new object[] { 0, 1, 101 } }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}",
- new object[] { "f1", new int[] { 0, 1, 101 } }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}",
- new object[] { "f1", 100 }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}",
- new object[] { "f1", 100L }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"fixed\", \"name\": \"f\", \"size\": 2}}]}",
- new object[] { "f1", new byte[] { 1, 2 } }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}",
- new object[] { "f1", null }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}",
- new object[] { "f1", true }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}",
- new object[] { "f1", false }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}",
- new object[] { "f1", 101 }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}",
- new object[] { "f1", 101L }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}",
- new object[] { "f1", 101.78f }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}",
- new object[] { "f1", 101.78 }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}",
- new object[] { "f1", "A" }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}",
- new object[] { "f1", new byte[] { 0, 1 } }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"enum\", \"name\": \"e\", \"symbols\":[\"s1\", \"s2\"]}}]}",
- new object[] { "f1", "s2" }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}",
- new object[] { "f1", new object[] { 0, 1, 101 } }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}",
- new object[] { "f1", new int[] { 0, 1, 101 } }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}",
- new object[] { "f1", 100 }, Codec.Type.Null)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}",
- new object[] { "f1", 100L }, Codec.Type.Null)]
+ [TestCaseSource(nameof(TestGenericDataSource))]
public void TestGenericData(string schemaStr, object[] value, Codec.Type codecType)
{
- foreach(var rwFactory in GenericOptions<GenericRecord>())
+ foreach (var rwFactory in GenericOptions<GenericRecord>())
{
// Create and write out
MemoryStream dataFileOutputStream = new MemoryStream();
@@ -264,6 +302,22 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestAppendGenericDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", true }, new object[] { "f1", false }, codecType)
+ .SetName("{m}(bool,{3})");
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", new object[] { "f1", 1 }, new object[] { "f1", 2 }, codecType)
+ .SetName("{m}(int,{3})");
+ yield return new TestCaseData(
+ "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", new object[] { "f1", "A" }, new object[] { "f1", "B" }, codecType)
+ .SetName("{m}(string,{3})");
+ }
+ }
+
/// <summary>
/// Test appending of generic record objects
/// </summary>
@@ -271,10 +325,7 @@ namespace Avro.Test.File
/// <param name="recs">initial records</param>
/// <param name="appendRecs">append records</param>
/// <param name="codecType">innitial compression codec type</param>
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}",
- new object[] { "f1", true }, new object[] { "f1", false }, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}",
- new object[] { "f1", 1 }, new object[] { "f1", 2 }, Codec.Type.Null)]
+ [TestCaseSource(nameof(TestAppendGenericDataSource))]
public void TestAppendGenericData(string schemaStr, object[] recs, object[] appendRecs, Codec.Type codecType)
{
foreach (var rwFactory in GenericOptions<GenericRecord>())
@@ -346,9 +397,7 @@ namespace Avro.Test.File
/// DeflateStream as it is a standard non-seekable Stream that has the same behavior as the
/// NetworkStream, which we should handle.
/// </summary>
- [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" +
- "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}",
- new object[] { "f1", 100L }, Codec.Type.Null)]
+ [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", new object[] { "f1", 100L }, Codec.Type.Null)]
public void TestNonSeekableStream(string schemaStr, object[] value, Codec.Type codecType)
{
foreach (var rwFactory in GenericOptions<GenericRecord>())
@@ -387,57 +436,41 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestPrimitiveDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData("[\"boolean\", \"null\"]", null, codecType);
+ yield return new TestCaseData("[\"boolean\", \"null\"]", true, codecType);
+ yield return new TestCaseData("[\"int\", \"long\"]", 100, codecType);
+ yield return new TestCaseData("[\"int\", \"long\"]", 100L, codecType);
+ yield return new TestCaseData("[\"float\", \"double\"]", 100.75, codecType);
+ yield return new TestCaseData("[\"float\", \"double\"]", 23.67f, codecType);
+ yield return new TestCaseData("{\"type\": \"boolean\"}", true, codecType);
+ yield return new TestCaseData("{\"type\": \"boolean\"}", false, codecType);
+ yield return new TestCaseData("{\"type\": \"string\"}", "John", codecType);
+ yield return new TestCaseData("{\"type\": [\"null\",\"string\"]}", null, codecType);
+ yield return new TestCaseData("{\"type\": \"int\"}", 1, codecType);
+ yield return new TestCaseData("{\"type\": \"long\"}", 12312313123L, codecType);
+ yield return new TestCaseData("{\"type\": \"float\"}", 0.0f, codecType);
+ yield return new TestCaseData("{\"type\": \"double\"}", 0.0, codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, codecType);
+ yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, codecType);
+ yield return new TestCaseData("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", codecType);
+ }
+ }
+
/// <summary>
/// Reading & writing of primitive objects
/// </summary>
/// <param name="schemaStr"></param>
/// <param name="value"></param>
/// <param name="codecType"></param>
- [TestCase("{\"type\": \"boolean\"}", true, Codec.Type.Deflate)]
- [TestCase("{\"type\": \"boolean\"}", false, Codec.Type.Deflate)]
- [TestCase("{\"type\": \"boolean\"}", true, Codec.Type.Null)]
- [TestCase("{\"type\": \"boolean\"}", false, Codec.Type.Null)]
- [TestCase("[\"boolean\", \"null\"]", null, Codec.Type.Deflate)]
- [TestCase("[\"boolean\", \"null\"]", true, Codec.Type.Deflate)]
- [TestCase("[\"int\", \"long\"]", 100, Codec.Type.Deflate)]
- [TestCase("[\"int\", \"long\"]", 100L, Codec.Type.Deflate)]
- [TestCase("[\"float\", \"double\"]", 100.75, Codec.Type.Deflate)]
- [TestCase("[\"float\", \"double\"]", 23.67f, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, Codec.Type.Deflate)]
- [TestCase("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", Codec.Type.Deflate)]
- [TestCase("{\"type\":\"string\"}", "John", Codec.Type.Deflate)]
- [TestCase("{\"type\":[\"null\",\"string\"]}", null, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"int\"}", 1, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"boolean\"}", false, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"long\"}", 12312313123L, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"float\"}", 0.0f, Codec.Type.Deflate)]
- [TestCase("{\"type\":\"double\"}", 0.0, Codec.Type.Deflate)]
- [TestCase("[\"boolean\", \"null\"]", null, Codec.Type.Null)]
- [TestCase("[\"boolean\", \"null\"]", true, Codec.Type.Null)]
- [TestCase("[\"int\", \"long\"]", 100, Codec.Type.Null)]
- [TestCase("[\"int\", \"long\"]", 100L, Codec.Type.Null)]
- [TestCase("[\"float\", \"double\"]", 100.75, Codec.Type.Null)]
- [TestCase("[\"float\", \"double\"]", 23.67f, Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, Codec.Type.Null)]
- [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, Codec.Type.Null)]
- [TestCase("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", Codec.Type.Null)]
- [TestCase("{\"type\":\"string\"}", "John", Codec.Type.Null)]
- [TestCase("{\"type\":[\"null\",\"string\"]}", null, Codec.Type.Null)]
- [TestCase("{\"type\":\"int\"}", 1, Codec.Type.Null)]
- [TestCase("{\"type\":\"boolean\"}", false, Codec.Type.Null)]
- [TestCase("{\"type\":\"long\"}", 12312313123L, Codec.Type.Null)]
- [TestCase("{\"type\":\"float\"}", 0.0f, Codec.Type.Null)]
- [TestCase("{\"type\":\"double\"}", 0.0, Codec.Type.Null)]
- [TestCase("{\"type\":\"string\"}", "test", Codec.Type.Null)]
+ [TestCaseSource(nameof(TestPrimitiveDataSource))]
public void TestPrimitiveData(string schemaStr, object value, Codec.Type codecType)
{
foreach(var rwFactory in GenericOptions<object>())
@@ -455,32 +488,28 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestMetaDataSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ foreach (bool useTypeGetter in new bool[] { true, false })
+ {
+ yield return new TestCaseData("bytesTest", new byte[] { 1, 2, 3 }, codecType, useTypeGetter);
+ yield return new TestCaseData("stringTest", "testVal", codecType, useTypeGetter);
+ yield return new TestCaseData("longTest", 12312313123L, codecType, useTypeGetter);
+ yield return new TestCaseData("bytesTest", new byte[] { 1 }, codecType, useTypeGetter);
+ yield return new TestCaseData("longTest", -1211212L, codecType, useTypeGetter);
+ }
+ }
+ }
+
/// <summary>
/// Reading & writing of header meta data
/// </summary>
- /// <param name="schemaStr"></param>
/// <param name="value"></param>
/// <param name="codecType"></param>
- [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Null, true)]
- [TestCase("stringTest", "testVal", Codec.Type.Null, true)]
- [TestCase("longTest", 12312313123L, Codec.Type.Null, true)]
- [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Null, true)]
- [TestCase("longTest", -1211212L, Codec.Type.Null, true)]
- [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Deflate, true)]
- [TestCase("stringTest", "testVal", Codec.Type.Deflate, true)]
- [TestCase("longTest", 12312313123L, Codec.Type.Deflate, true)]
- [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Deflate, true)]
- [TestCase("longTest", -21211212L, Codec.Type.Deflate, true)]
- [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Null, false)]
- [TestCase("stringTest", "testVal", Codec.Type.Null, false)]
- [TestCase("longTest", 12312313123L, Codec.Type.Null, false)]
- [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Null, false)]
- [TestCase("longTest", -1211212L, Codec.Type.Null, false)]
- [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Deflate, false)]
- [TestCase("stringTest", "testVal", Codec.Type.Deflate, false)]
- [TestCase("longTest", 12312313123L, Codec.Type.Deflate, false)]
- [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Deflate, false)]
- [TestCase("longTest", -21211212L, Codec.Type.Deflate, false)]
+ /// <param name="useTypeGetter"></param>
+ [TestCaseSource(nameof(TestMetaDataSource))]
public void TestMetaData(string key, object value, Codec.Type codecType, bool useTypeGetter)
{
// create and write out
@@ -507,6 +536,20 @@ namespace Avro.Test.File
}
}
+ private static IEnumerable<TestCaseData> TestPartialReadSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(specificSchema, codecType, 0, 330).SetName("{m}({1},{2},{3})");
+ yield return new TestCaseData(specificSchema, codecType, 1, 330).SetName("{m}({1},{2},{3})");
+ yield return new TestCaseData(specificSchema, codecType, 135, 330).SetName("{m}({1},{2},{3})");
+ yield return new TestCaseData(specificSchema, codecType, 194, 264).SetName("{m}({1},{2},{3})");
+ }
+
+ // This is only for Null codec
+ yield return new TestCaseData(specificSchema, Codec.Type.Null, 888, 165).SetName("{m}({1},{2},{3})");
+ }
+
/// <summary>
/// Partial reading of file / stream from
/// position in stream
@@ -514,12 +557,7 @@ namespace Avro.Test.File
/// <param name="schemaStr"></param>
/// <param name="value"></param>
/// <param name="codecType"></param>
- [TestCase(specificSchema, Codec.Type.Null, 1, 330)] // 330
- [TestCase(specificSchema, Codec.Type.Null, 135, 330)] // 330
- [TestCase(specificSchema, Codec.Type.Null, 194, 264)] // 264
- [TestCase(specificSchema, Codec.Type.Null, 235, 264)] // 264
- [TestCase(specificSchema, Codec.Type.Null, 888, 165)] // 165
- [TestCase(specificSchema, Codec.Type.Null, 0, 330)] // 330
+ [TestCaseSource(nameof(TestPartialReadSource))]
public void TestPartialRead(string schemaStr, Codec.Type codecType, int position, int expectedRecords)
{
// create and write out
@@ -569,11 +607,9 @@ namespace Avro.Test.File
/// Tests reading from sync boundaries.
/// </summary>
/// <param name="schemaStr"></param>
- /// <param name="value"></param>
/// <param name="codecType"></param>
- [TestCase(specificSchema, Codec.Type.Null)]
- [TestCase(specificSchema, Codec.Type.Deflate)]
- public void TestPartialReadAll(string schemaStr, Codec.Type codecType)
+ [Test]
+ public void TestPartialReadAll([Values(specificSchema)] string schemaStr, [Values] Codec.Type codecType)
{
// create and write out
IList<Foo> records = MakeRecords(GetTestFooObject());
@@ -636,14 +672,11 @@ namespace Avro.Test.File
/// Test leaveOpen flag
/// </summary>
/// <param name="schemaStr"></param>
- /// <param name="value"></param>
/// <param name="codecType"></param>
- /// <param name="leaveOpen"></param>
- [TestCase(specificSchema, Codec.Type.Null, true, false)]
- [TestCase(specificSchema, Codec.Type.Null, true, true)]
- [TestCase(specificSchema, Codec.Type.Null, false, false)]
- [TestCase(specificSchema, Codec.Type.Null, false, true)]
- public void TestLeaveOpen(string schemaStr, Codec.Type codecType, bool leaveWriteOpen, bool leaveReadOpen)
+ /// <param name="leaveWriteOpen"></param>
+ /// <param name="leaveReadOpen"></param>
+ [Test]
+ public void TestLeaveOpen([Values(specificSchema)] string schemaStr, [Values] Codec.Type codecType, [Values] bool leaveWriteOpen, [Values] bool leaveReadOpen)
{
// create and write out
IList<Foo> records = MakeRecords(GetTestFooObject());
@@ -723,19 +756,23 @@ namespace Avro.Test.File
Assert.AreEqual( expectedRecords, readRecords, "didn't read expected records from position " + position );
}
+ private static IEnumerable<TestCaseData> TestSyncAndSeekPositionsSource()
+ {
+ foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type)))
+ {
+ yield return new TestCaseData(specificSchema, codecType, 2, 0, 1).SetName("{m}({1},{2},{3},{4})");
+ yield return new TestCaseData(specificSchema, codecType, 10, 1, 4).SetName("{m}({1},{2},{3},{4})");
+ yield return new TestCaseData(specificSchema, codecType, 200, 111, 15).SetName("{m}({1},{2},{3},{4})");
+ yield return new TestCaseData(specificSchema, codecType, 1000, 588, 998).SetName("{m}({1},{2},{3},{4})");
+ }
+ }
+
/// <summary>
/// Reading all sync positions and
/// verifying them with subsequent seek
/// positions
/// </summary>
- [TestCase(specificSchema, Codec.Type.Null, 2, 0, 1)]
- [TestCase(specificSchema, Codec.Type.Null, 10, 1, 4)]
- [TestCase(specificSchema, Codec.Type.Null, 200, 111, 15)]
- [TestCase(specificSchema, Codec.Type.Null, 1000, 588, 998)]
- [TestCase(specificSchema, Codec.Type.Deflate, 2, 0, 1)]
- [TestCase(specificSchema, Codec.Type.Deflate, 10, 1, 4)]
- [TestCase(specificSchema, Codec.Type.Deflate, 200, 111, 15)]
- [TestCase(specificSchema, Codec.Type.Deflate, 1000, 588, 998)]
+ [TestCaseSource(nameof(TestSyncAndSeekPositionsSource))]
public void TestSyncAndSeekPositions(string schemaStr, Codec.Type codecType, int iterations, int firstSyncPosition, int secondSyncPosition)
{
// create and write out
@@ -820,6 +857,44 @@ namespace Avro.Test.File
}
/// <summary>
+ /// Reading & writing many specific record objects
+ /// </summary>
+ /// <param name="codecType"></param>
+ /// <param name="numOfRecords"></param>
+ [Test]
+ public void TestLargeSpecificData([Values] Codec.Type codecType, [Values(0, 1000, 100000)] int numOfRecords)
+ {
+ foreach (var rwFactory in SpecificOptions<Foo>())
+ {
+ MemoryStream dataFileOutputStream = new MemoryStream();
+ Schema schema = Schema.Parse(specificSchema);
+ using (IFileWriter<Foo> dataFileWriter = rwFactory.CreateWriter(dataFileOutputStream, schema, Codec.CreateCodec(codecType)))
+ {
+ for (int index = 0; index < numOfRecords; index++)
+ {
+ dataFileWriter.Append(new Foo() { name = $"Name-{index}", age = index });
+ }
+ }
+
+ MemoryStream dataFileInputStream = new MemoryStream(dataFileOutputStream.ToArray());
+
+ // Read back and verify
+ using (IFileReader<Foo> reader = rwFactory.CreateReader(dataFileInputStream, null))
+ {
+ int index = 0;
+ foreach (Foo record in reader.NextEntries)
+ {
+ Assert.AreEqual($"Name-{index}", record.name);
+ Assert.AreEqual(index, record.age);
+ index++;
+ }
+
+ Assert.AreEqual(numOfRecords, index);
+ }
+ }
+ }
+
+ /// <summary>
/// Reading and writing using optional codecs
/// </summary>
/// <param name="schemaStr"></param>
@@ -827,6 +902,10 @@ namespace Avro.Test.File
[TestCase("zstd", true)]
[TestCase("deflate", false)]
[TestCase("null", false)]
+ [TestCase("snappy", false)]
+ [TestCase("bzip2", false)]
+ [TestCase("xz", false)]
+ [TestCase("zstandard", false)]
public void TestOptionalCodecs(string codecToUse, bool expectResolverProvidedCodec)
{
var resolverProvidedCodec = false;
@@ -886,14 +965,17 @@ namespace Avro.Test.File
[TestCase("deflate")]
[TestCase("null")]
[TestCase(null)] // If codec is absent, it is assumed to be "null"
+ [TestCase("snappy")]
+ [TestCase("bzip2")]
+ [TestCase("xz")]
+ [TestCase("zstandard")]
public void KnownCodecFromString(string codec)
{
Assert.NotNull(Codec.CreateCodecFromString(codec));
}
- [TestCase(Codec.Type.Deflate)]
- [TestCase(Codec.Type.Null)]
- public void KnownCodecFromType(Codec.Type codec)
+ [Test]
+ public void KnownCodecFromType([Values] Codec.Type codec)
{
Assert.NotNull(Codec.CreateCodec(codec));
}
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
index 94bfb40..170e28e 100644
--- a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
+++ b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
@@ -25,7 +25,11 @@ namespace Avro.Test.Interop
public static readonly HashSet<string> SupportedCodecNames = new HashSet<string>
{
DataFileConstants.NullCodec,
- DataFileConstants.DeflateCodec
+ DataFileConstants.DeflateCodec,
+ DataFileConstants.SnappyCodec,
+ DataFileConstants.BZip2Codec,
+ DataFileConstants.XZCodec,
+ DataFileConstants.ZstandardCodec
};
}
}
\ No newline at end of file
diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props
index 2ec4d2e..6b767c6 100644
--- a/lang/csharp/versions.props
+++ b/lang/csharp/versions.props
@@ -31,6 +31,12 @@
<SystemReflectionVersion>4.3.0</SystemReflectionVersion>
<SystemReflectionEmitILGenerationVersion>4.7.0</SystemReflectionEmitILGenerationVersion>
<SystemReflectionEmitLightweightVersion>4.7.0</SystemReflectionEmitLightweightVersion>
+
+ <!-- The following pacakges are required for the extra codec libraries. These are not direct dependencies of the Avro.main library. -->
+ <SharpZipLibVersion>1.3.3</SharpZipLibVersion>
+ <IronSnappyVersion>1.3.0</IronSnappyVersion>
+ <JovelerCompressionXZVersion>4.1.0</JovelerCompressionXZVersion>
+ <ZstandardNetVersion>1.1.7</ZstandardNetVersion>
</PropertyGroup>
<!--