You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by fo...@apache.org on 2020/05/21 10:18:13 UTC
[avro] branch master updated: AVRO-2839: PHP Add zstd and snappy
codec support. (#886)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new c536a35 AVRO-2839: PHP Add zstd and snappy codec support. (#886)
c536a35 is described below
commit c536a358e19644e3dc6b646b3ee541f531d6cc6c
Author: Siad Ardroumli <si...@gmail.com>
AuthorDate: Thu May 21 12:18:02 2020 +0200
AVRO-2839: PHP Add zstd and snappy codec support. (#886)
* AVRO-2839: Add zstd and snappy codec support.
* Fixed wrong sprintf calls
* Added zstd and snappy ext.
* Added zstd and snappy compiled ext.
* Added dev package.
* Added libzstd-dev
* Fixed install
* Fixed install 2
* Added debug modules output
* Added debug info output
* Added so to ini
* Added so to ini 2
* Path fix
* Added all codec fixture files to the test execution.
* Added crc32 to snappy compression
* Removed crc from compressed string.
* More work on crc32
---
lang/php/build.sh | 7 ------
lang/php/lib/avro/data_file.php | 48 +++++++++++++++++++++++++++++++++++------
lang/php/test/DataFileTest.php | 4 ++--
lang/php/test/InterOpTest.php | 4 ++--
share/docker/Dockerfile | 22 ++++++++++++++++++-
5 files changed, 67 insertions(+), 18 deletions(-)
diff --git a/lang/php/build.sh b/lang/php/build.sh
index 0d41cfc..54f7546 100755
--- a/lang/php/build.sh
+++ b/lang/php/build.sh
@@ -63,13 +63,6 @@ do
test)
composer install -d "../.."
vendor/bin/phpunit -v
-
- # Check backward compatibility with PHP 5.x if both PHP 5.6 and PHPUnit 5.7 are installed.
- # TODO: remove this check when we drop PHP 5.x support in the future
- if command -v php5.6 > /dev/null && phpunit --version | grep -q 'PHPUnit 5.7'; then
- echo 'Checking backward compatibility with PHP 5.x'
- php5.6 $(which phpunit) -v test/AllTests.php
- fi
;;
dist)
diff --git a/lang/php/lib/avro/data_file.php b/lang/php/lib/avro/data_file.php
index 420f079..3acd5f7 100644
--- a/lang/php/lib/avro/data_file.php
+++ b/lang/php/lib/avro/data_file.php
@@ -73,10 +73,14 @@ class AvroDataIO
*/
const DEFLATE_CODEC = 'deflate';
+ const SNAPPY_CODEC = 'snappy';
+
+ const ZSTANDARD_CODEC = 'zstandard';
+
/**
* @var array array of valid codec names
*/
- private static $valid_codecs = array(self::NULL_CODEC, self::DEFLATE_CODEC);
+ private static $valid_codecs = [self::NULL_CODEC, self::DEFLATE_CODEC, self::SNAPPY_CODEC, self::ZSTANDARD_CODEC];
/**
* @var AvroSchema cached version of metadata schema object
@@ -308,9 +312,28 @@ class AvroDataIOReader
$compressed = $decoder->read($length);
$datum = gzinflate($compressed);
$decoder = new AvroIOBinaryDecoder(new AvroStringIO($datum));
+ } elseif ($this->codec === AvroDataIO::ZSTANDARD_CODEC) {
+ if (!extension_loaded('zstd')) {
+ throw new AvroException('Please install ext-zstd to use zstandard compression.');
+ }
+ $compressed = $decoder->read($length);
+ $datum = zstd_uncompress($compressed);
+ $decoder = new AvroIOBinaryDecoder(new AvroStringIO($datum));
+ } elseif ($this->codec === AvroDataIO::SNAPPY_CODEC) {
+ if (!extension_loaded('snappy')) {
+ throw new AvroException('Please install ext-snappy to use snappy compression.');
+ }
+ $compressed = $decoder->read($length);
+ $crc32 = unpack('N', substr($compressed, -4))[1];
+ $datum = snappy_uncompress(substr($compressed, 0, -4));
+ if ($crc32 === crc32($datum)) {
+ $decoder = new AvroIOBinaryDecoder(new AvroStringIO($datum));
+ } else {
+ $decoder = new AvroIOBinaryDecoder(new AvroStringIO(snappy_uncompress($datum)));
+ }
}
}
- $data []= $this->datum_reader->read($decoder);
+ $data[] = $this->datum_reader->read($decoder);
$this->block_count -= 1;
}
return $data;
@@ -509,10 +532,23 @@ class AvroDataIOWriter
if ($this->block_count > 0)
{
$this->encoder->write_long($this->block_count);
- $to_write = strval($this->buffer);
-
- if ($this->codec == AvroDataIO::DEFLATE_CODEC)
- $to_write = gzdeflate($to_write);
+ $to_write = (string) $this->buffer;
+
+ if ($this->codec === AvroDataIO::DEFLATE_CODEC) {
+ $to_write = gzdeflate($to_write);
+ } elseif ($this->codec === AvroDataIO::ZSTANDARD_CODEC) {
+ if (!extension_loaded('zstd')) {
+ throw new AvroException('Please install ext-zstd to use zstandard compression.');
+ }
+ $to_write = zstd_compress($to_write);
+ } elseif ($this->codec === AvroDataIO::SNAPPY_CODEC) {
+ if (!extension_loaded('snappy')) {
+ throw new AvroException('Please install ext-snappy to use snappy compression.');
+ }
+ $crc32 = crc32($to_write);
+ $compressed = snappy_compress($to_write);
+ $to_write = pack('a*N', $compressed, $crc32);
+ }
$this->encoder->write_long(strlen($to_write));
$this->write($to_write);
diff --git a/lang/php/test/DataFileTest.php b/lang/php/test/DataFileTest.php
index f7befd4..f36bdf6 100644
--- a/lang/php/test/DataFileTest.php
+++ b/lang/php/test/DataFileTest.php
@@ -192,7 +192,7 @@ class DataFileTest extends PHPUnit\Framework\TestCase
public function test_differing_schemas_with_primitives()
{
foreach (AvroDataIO::valid_codecs() as $codec) {
- $data_file = $this->add_data_file('data-prim-%s.avr');
+ $data_file = $this->add_data_file(sprintf('data-prim-%s.avr', $codec));
$writer_schema = <<<JSON
{ "type": "record",
@@ -229,7 +229,7 @@ JSON;
public function test_differing_schemas_with_complex_objects()
{
foreach (AvroDataIO::valid_codecs() as $codec) {
- $data_file = $this->add_data_file('data-complex-%s.avr', $codec);
+ $data_file = $this->add_data_file(sprintf('data-complex-%s.avr', $codec));
$writers_schema = <<<JSON
{ "type": "record",
diff --git a/lang/php/test/InterOpTest.php b/lang/php/test/InterOpTest.php
index 95ad732..0fc2d2c 100644
--- a/lang/php/test/InterOpTest.php
+++ b/lang/php/test/InterOpTest.php
@@ -39,8 +39,8 @@ class InterOpTest extends PHPUnit\Framework\TestCase
die("Could not open data dir '$data_dir'\n");
while ($file = readdir($dh))
- if (0 < preg_match('/^[a-z]+(_deflate)?\.avro$/', $file))
- $data_files []= join(DIRECTORY_SEPARATOR, array($data_dir, $file));
+ if (0 < preg_match('/^[a-z]+(_deflate|_snappy|_zstandard)?\.avro$/', $file))
+ $data_files []= implode(DIRECTORY_SEPARATOR, array($data_dir, $file));
closedir($dh);
$ary = array();
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 42b9b98..3d142f5 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -86,10 +86,30 @@ RUN curl -sSL https://packages.sury.org/php/apt.gpg \
| apt-key add --no-tty - \
&& echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list \
&& apt-get -qqy update \
- && apt-get -qqy install --no-install-recommends php${PHP7_VERSION} \
+ && apt-get -qqy install --no-install-recommends libzstd-dev \
+ php${PHP7_VERSION} \
php${PHP7_VERSION}-gmp \
php${PHP7_VERSION}-xml \
php${PHP7_VERSION}-mbstring \
+ php${PHP7_VERSION}-dev \
+ && mkdir tmp && cd tmp \
+ && git clone --recursive --depth=1 https://github.com/kjdev/php-ext-zstd.git \
+ && cd php-ext-zstd \
+ && phpize \
+ && ./configure \
+ && make \
+ && make install \
+ && echo "extension=zstd.so" > /etc/php/7.3/cli/conf.d/10-zstd.ini \
+ && cd .. && rm -rf php-ext-zstd \
+ && git clone --recursive --depth=1 https://github.com/kjdev/php-ext-snappy.git \
+ && cd php-ext-snappy \
+ && phpize \
+ && ./configure \
+ && make \
+ && make install \
+ && echo "extension=snappy.so" > /etc/php/7.3/cli/conf.d/10-snappy.ini \
+ && cd .. && rm -rf php-ext-snappy \
+ && php -m \
&& apt-get -qqy clean \
&& rm -rf /var/lib/apt/lists