You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/10/17 18:23:24 UTC
[1/2] commons-compress git commit: COMPRESS-423 - Add ZStandard
decompression support using Zstd-JNI
Repository: commons-compress
Updated Branches:
refs/heads/master 89bc17055 -> 1c382914c
COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/7984387a
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/7984387a
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/7984387a
Branch: refs/heads/master
Commit: 7984387af004fcfe1d1ee12e2c8e6b68f23be001
Parents: 89bc170
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Sat Oct 14 17:57:19 2017 +1100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Oct 17 20:17:01 2017 +0200
----------------------------------------------------------------------
pom.xml | 6 +
.../compressors/CompressorStreamFactory.java | 25 ++-
.../zstandard/ZstdCompressorInputStream.java | 95 +++++++++++
.../compressors/zstandard/ZstdUtils.java | 88 +++++++++++
.../ZstdCompressorInputStreamTest.java | 157 +++++++++++++++++++
src/test/resources/bla.tar.zst | Bin 0 -> 473 bytes
src/test/resources/zstandard.testdata | 3 +
src/test/resources/zstandard.testdata.zst | Bin 0 -> 94 bytes
8 files changed, 372 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 31fc4cd..6a33b38 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,12 @@ jar, tar, zip, dump, 7z, arj.
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <version>1.3.1-1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
<groupId>org.brotli</groupId>
<artifactId>dec</artifactId>
<version>0.1.2</version>
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 4bd22aa..b446963 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -55,6 +55,8 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.apache.commons.compress.compressors.xz.XZUtils;
import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdUtils;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.compress.utils.ServiceLoaderIterator;
@@ -191,6 +193,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
public static final String LZ4_FRAMED = "lz4-framed";
/**
+ * Constant (value {@value}) used to identify the ZStandard compression
+ * algorithm. Not supported as an output stream type.
+ *
+ * @since 1.15
+ */
+ public static final String ZSTANDARD = "zst";
+
+ /**
* Constructs a new sorted map from input stream provider names to provider
* objects.
*
@@ -279,7 +289,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
public static String getBrotli() {
return BROTLI;
}
-
+
public static String getBzip2() {
return BZIP2;
}
@@ -328,6 +338,10 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return LZ4_BLOCK;
}
+ public static String getZstandard() {
+ return ZSTANDARD;
+ }
+
static void putAll(final Set<String> names, final CompressorStreamProvider provider,
final TreeMap<String, CompressorStreamProvider> map) {
for (final String name : names) {
@@ -555,6 +569,13 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
}
+ if (ZSTANDARD.equalsIgnoreCase(name)) {
+ if (!ZstdUtils.isZstdCompressionAvailable()) {
+ throw new CompressorException("XZ compression is not available.");
+ }
+ return new ZstdCompressorInputStream(in);
+ }
+
if (LZMA.equalsIgnoreCase(name)) {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
@@ -701,7 +722,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
@Override
public Set<String> getInputStreamCompressorNames() {
return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK,
- LZ4_FRAMED);
+ LZ4_FRAMED, ZSTANDARD);
}
@Override
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
new file mode 100644
index 0000000..1e5dd8d
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.compress.compressors.zstandard;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.github.luben.zstd.ZstdInputStream;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+
+/**
+ * {@link CompressorInputStream} implementation to decode Zstandard encoded stream.
+ * Library relies on <a href="https://github.com/luben/zstd-jni/">Zstandard JNI</a>
+ *
+ * @since 1.15
+ */
+public class ZstdCompressorInputStream extends CompressorInputStream {
+
+ private final com.github.luben.zstd.ZstdInputStream decIS;
+
+ public ZstdCompressorInputStream(final InputStream in) throws IOException {
+ this.decIS = new ZstdInputStream(in);
+ }
+
+ @Override
+ public int available() throws IOException {
+ return decIS.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ decIS.close();
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return decIS.read(b);
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ return decIS.skip(n);
+ }
+
+ @Override
+ public void mark(final int readlimit) {
+ decIS.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return decIS.markSupported();
+ }
+
+ @Override
+ public int read() throws IOException {
+ final int ret = decIS.read();
+ count(ret == -1 ? 0 : 1);
+ return ret;
+ }
+
+ @Override
+ public int read(final byte[] buf, final int off, final int len) throws IOException {
+ final int ret = decIS.read(buf, off, len);
+ count(ret);
+ return ret;
+ }
+
+ @Override
+ public String toString() {
+ return decIS.toString();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ decIS.reset();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
new file mode 100644
index 0000000..0eb8fa1
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.zstandard;
+
+/**
+ * Utility code for the Zstandard compression format.
+ * @ThreadSafe
+ * @since 1.14
+ */
+public class ZstdUtils {
+
+ static enum CachedAvailability {
+ DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE
+ }
+
+ private static volatile CachedAvailability cachedZstdAvailability;
+
+ static {
+ cachedZstdAvailability = CachedAvailability.DONT_CACHE;
+ try {
+ Class.forName("org.osgi.framework.BundleEvent");
+ } catch (final Exception ex) { // NOSONAR
+ setCacheZstdAvailablity(true);
+ }
+ }
+
+ /** Private constructor to prevent instantiation of this utility class. */
+ private ZstdUtils() {
+ }
+
+ /**
+ * Are the classes required to support Zstandard compression available?
+ * @return true if the classes required to support Zstandard compression are available
+ */
+ public static boolean isZstdCompressionAvailable() {
+ final CachedAvailability cachedResult = cachedZstdAvailability;
+ if (cachedResult != CachedAvailability.DONT_CACHE) {
+ return cachedResult == CachedAvailability.CACHED_AVAILABLE;
+ }
+ return internalIsZstdCompressionAvailable();
+ }
+
+ private static boolean internalIsZstdCompressionAvailable() {
+ try {
+ Class.forName("com.github.luben.zstd.ZstdInputStream");
+ return true;
+ } catch (NoClassDefFoundError | Exception error) {
+ return false;
+ }
+ }
+
+ /**
+ * Whether to cache the result of the Zstandard for Java check.
+ *
+ * <p>This defaults to {@code false} in an OSGi environment and {@code true} otherwise.</p>
+ * @param doCache whether to cache the result
+ */
+ public static void setCacheZstdAvailablity(final boolean doCache) {
+ if (!doCache) {
+ cachedZstdAvailability = CachedAvailability.DONT_CACHE;
+ } else if (cachedZstdAvailability == CachedAvailability.DONT_CACHE) {
+ final boolean hasZstd = internalIsZstdCompressionAvailable();
+ cachedZstdAvailability = hasZstd ? CachedAvailability.CACHED_AVAILABLE
+ : CachedAvailability.CACHED_UNAVAILABLE;
+ }
+ }
+
+ // only exists to support unit tests
+ static CachedAvailability getCachedZstdAvailability() {
+ return cachedZstdAvailability;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java
new file mode 100644
index 0000000..5ed276c
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.compress.compressors.zstandard;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.compress.AbstractTestCase;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ZstdCompressorInputStreamTest extends AbstractTestCase {
+
+ /**
+ * Test bridge works fine
+ * @throws {@link IOException}
+ */
+ @Test
+ public void testZstdDecode() throws IOException {
+ final File input = getFile("zstandard.testdata.zst");
+ final File expected = getFile("zstandard.testdata");
+ try (InputStream inputStream = new FileInputStream(input);
+ InputStream expectedStream = new FileInputStream(expected);
+ ZstdCompressorInputStream zstdInputStream = new ZstdCompressorInputStream(inputStream)) {
+ final byte[] b = new byte[97];
+ IOUtils.readFully(expectedStream, b);
+ final ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ int readByte = -1;
+ while((readByte = zstdInputStream.read()) != -1) {
+ bos.write(readByte);
+ }
+ Assert.assertArrayEquals(b, bos.toByteArray());
+ }
+ }
+
+ @Test
+ public void testCachingIsEnabledByDefaultAndZstdUtilsPresent() {
+ assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability());
+ assertTrue(ZstdUtils.isZstdCompressionAvailable());
+ }
+
+ @Test
+ public void testCanTurnOffCaching() {
+ try {
+ ZstdUtils.setCacheZstdAvailablity(false);
+ assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, ZstdUtils.getCachedZstdAvailability());
+ assertTrue(ZstdUtils.isZstdCompressionAvailable());
+ } finally {
+ ZstdUtils.setCacheZstdAvailablity(true);
+ }
+ }
+
+ @Test
+ public void testTurningOnCachingReEvaluatesAvailability() {
+ try {
+ ZstdUtils.setCacheZstdAvailablity(false);
+ assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, ZstdUtils.getCachedZstdAvailability());
+ ZstdUtils.setCacheZstdAvailablity(true);
+ assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability());
+ } finally {
+ ZstdUtils.setCacheZstdAvailablity(true);
+ }
+ }
+
+ @Test
+ public void shouldBeAbleToSkipAByte() throws IOException {
+ final File input = getFile("zstandard.testdata.zst");
+ try (InputStream is = new FileInputStream(input)) {
+ final ZstdCompressorInputStream in =
+ new ZstdCompressorInputStream(is);
+ Assert.assertEquals(1, in.skip(1));
+ in.close();
+ }
+ }
+
+ @Test
+ public void singleByteReadWorksAsExpected() throws IOException {
+
+ final File input = getFile("zstandard.testdata.zst");
+
+ final File original = getFile("zstandard.testdata");
+ final long originalFileLength = original.length();
+
+ byte[] originalFileContent = new byte[((int) originalFileLength)];
+
+ try (InputStream ois = new FileInputStream(original)) {
+ ois.read(originalFileContent);
+ }
+
+ try (InputStream is = new FileInputStream(input)) {
+ final ZstdCompressorInputStream in =
+ new ZstdCompressorInputStream(is);
+
+ Assert.assertEquals(originalFileContent[0], in.read());
+ in.close();
+ }
+ }
+
+ @Test
+ public void singleByteReadReturnsMinusOneAtEof() throws IOException {
+ final File input = getFile("zstandard.testdata.zst");
+ try (InputStream is = new FileInputStream(input)) {
+ final ZstdCompressorInputStream in =
+ new ZstdCompressorInputStream(is);
+ IOUtils.toByteArray(in);
+ Assert.assertEquals(-1, in.read());
+ in.close();
+ }
+ }
+
+ @Test
+ public void testZstandardUnarchive() throws Exception {
+ final File input = getFile("bla.tar.zst");
+ final File output = new File(dir, "bla.tar");
+ try (InputStream is = new FileInputStream(input)) {
+ final CompressorInputStream in = new CompressorStreamFactory()
+ .createCompressorInputStream("zst", is);
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(output);
+ IOUtils.copy(in, out);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ in.close();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/test/resources/bla.tar.zst
----------------------------------------------------------------------
diff --git a/src/test/resources/bla.tar.zst b/src/test/resources/bla.tar.zst
new file mode 100644
index 0000000..d5fd6e0
Binary files /dev/null and b/src/test/resources/bla.tar.zst differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/test/resources/zstandard.testdata
----------------------------------------------------------------------
diff --git a/src/test/resources/zstandard.testdata b/src/test/resources/zstandard.testdata
new file mode 100644
index 0000000..e51bfd4
--- /dev/null
+++ b/src/test/resources/zstandard.testdata
@@ -0,0 +1,3 @@
+And as usual, instead of ipsum lorem we shall just state very clearly:
+
+Test test test chocolate
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7984387a/src/test/resources/zstandard.testdata.zst
----------------------------------------------------------------------
diff --git a/src/test/resources/zstandard.testdata.zst b/src/test/resources/zstandard.testdata.zst
new file mode 100644
index 0000000..86c03fe
Binary files /dev/null and b/src/test/resources/zstandard.testdata.zst differ
Re: [1/2] commons-compress git commit: COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI
Posted by Stefan Bodewig <bo...@apache.org>.
On 2017-10-17, Gary Gregory wrote:
> Should it be "Z_STANDARD" instead of "ZSTANDARD" since "standard" is a word?
Zstandard seems to be the official captitalization: http://www.zstd.net/
But I should check whether it's been used consistently over the last few
commits, I doubt it.
Stefan
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@commons.apache.org
For additional commands, e-mail: dev-help@commons.apache.org
Re: [1/2] commons-compress git commit: COMPRESS-423 - Add ZStandard
decompression support using Zstd-JNI
Posted by Gary Gregory <ga...@gmail.com>.
Should it be "Z_STANDARD" instead of "ZSTANDARD" since "standard" is a word?
Gary
On Tue, Oct 17, 2017 at 12:23 PM, <bo...@apache.org> wrote:
> Repository: commons-compress
> Updated Branches:
> refs/heads/master 89bc17055 -> 1c382914c
>
>
> COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI
>
>
> Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
> Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/
> commit/7984387a
> Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/
> tree/7984387a
> Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/
> diff/7984387a
>
> Branch: refs/heads/master
> Commit: 7984387af004fcfe1d1ee12e2c8e6b68f23be001
> Parents: 89bc170
> Author: Andre F de Miranda <tr...@users.noreply.github.com>
> Authored: Sat Oct 14 17:57:19 2017 +1100
> Committer: Stefan Bodewig <bo...@apache.org>
> Committed: Tue Oct 17 20:17:01 2017 +0200
>
> ----------------------------------------------------------------------
> pom.xml | 6 +
> .../compressors/CompressorStreamFactory.java | 25 ++-
> .../zstandard/ZstdCompressorInputStream.java | 95 +++++++++++
> .../compressors/zstandard/ZstdUtils.java | 88 +++++++++++
> .../ZstdCompressorInputStreamTest.java | 157 +++++++++++++++++++
> src/test/resources/bla.tar.zst | Bin 0 -> 473 bytes
> src/test/resources/zstandard.testdata | 3 +
> src/test/resources/zstandard.testdata.zst | Bin 0 -> 94 bytes
> 8 files changed, 372 insertions(+), 2 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/pom.xml
> ----------------------------------------------------------------------
> diff --git a/pom.xml b/pom.xml
> index 31fc4cd..6a33b38 100644
> --- a/pom.xml
> +++ b/pom.xml
> @@ -74,6 +74,12 @@ jar, tar, zip, dump, 7z, arj.
> <scope>test</scope>
> </dependency>
> <dependency>
> + <groupId>com.github.luben</groupId>
> + <artifactId>zstd-jni</artifactId>
> + <version>1.3.1-1</version>
> + <optional>true</optional>
> + </dependency>
> + <dependency>
> <groupId>org.brotli</groupId>
> <artifactId>dec</artifactId>
> <version>0.1.2</version>
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
> b/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> index 4bd22aa..b446963 100644
> --- a/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> +++ b/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> @@ -55,6 +55,8 @@ import org.apache.commons.compress.compressors.xz.
> XZCompressorInputStream;
> import org.apache.commons.compress.compressors.xz.
> XZCompressorOutputStream;
> import org.apache.commons.compress.compressors.xz.XZUtils;
> import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
> +import org.apache.commons.compress.compressors.zstandard.
> ZstdCompressorInputStream;
> +import org.apache.commons.compress.compressors.zstandard.ZstdUtils;
> import org.apache.commons.compress.utils.IOUtils;
> import org.apache.commons.compress.utils.Lists;
> import org.apache.commons.compress.utils.ServiceLoaderIterator;
> @@ -191,6 +193,14 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
> public static final String LZ4_FRAMED = "lz4-framed";
>
> /**
> + * Constant (value {@value}) used to identify the ZStandard
> compression
> + * algorithm. Not supported as an output stream type.
> + *
> + * @since 1.15
> + */
> + public static final String ZSTANDARD = "zst";
> +
> + /**
> * Constructs a new sorted map from input stream provider names to
> provider
> * objects.
> *
> @@ -279,7 +289,7 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
> public static String getBrotli() {
> return BROTLI;
> }
> -
> +
> public static String getBzip2() {
> return BZIP2;
> }
> @@ -328,6 +338,10 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
> return LZ4_BLOCK;
> }
>
> + public static String getZstandard() {
> + return ZSTANDARD;
> + }
> +
> static void putAll(final Set<String> names, final
> CompressorStreamProvider provider,
> final TreeMap<String, CompressorStreamProvider> map) {
> for (final String name : names) {
> @@ -555,6 +569,13 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
> return new XZCompressorInputStream(in,
> actualDecompressConcatenated, memoryLimitInKb);
> }
>
> + if (ZSTANDARD.equalsIgnoreCase(name)) {
> + if (!ZstdUtils.isZstdCompressionAvailable()) {
> + throw new CompressorException("XZ compression is not
> available.");
> + }
> + return new ZstdCompressorInputStream(in);
> + }
> +
> if (LZMA.equalsIgnoreCase(name)) {
> if (!LZMAUtils.isLZMACompressionAvailable()) {
> throw new CompressorException("LZMA compression is
> not available");
> @@ -701,7 +722,7 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
> @Override
> public Set<String> getInputStreamCompressorNames() {
> return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200,
> DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK,
> - LZ4_FRAMED);
> + LZ4_FRAMED, ZSTANDARD);
> }
>
> @Override
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/
> compressors/zstandard/ZstdCompressorInputStream.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/
> commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
> new file mode 100644
> index 0000000..1e5dd8d
> --- /dev/null
> +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/
> ZstdCompressorInputStream.java
> @@ -0,0 +1,95 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +
> +import java.io.IOException;
> +import java.io.InputStream;
> +
> +import com.github.luben.zstd.ZstdInputStream;
> +import org.apache.commons.compress.compressors.CompressorInputStream;
> +
> +/**
> + * {@link CompressorInputStream} implementation to decode Zstandard
> encoded stream.
> + * Library relies on <a href="https://github.com/luben/zstd-jni/">Zstandard
> JNI</a>
> + *
> + * @since 1.15
> + */
> +public class ZstdCompressorInputStream extends CompressorInputStream {
> +
> + private final com.github.luben.zstd.ZstdInputStream decIS;
> +
> + public ZstdCompressorInputStream(final InputStream in) throws
> IOException {
> + this.decIS = new ZstdInputStream(in);
> + }
> +
> + @Override
> + public int available() throws IOException {
> + return decIS.available();
> + }
> +
> + @Override
> + public void close() throws IOException {
> + decIS.close();
> + }
> +
> + @Override
> + public int read(final byte[] b) throws IOException {
> + return decIS.read(b);
> + }
> +
> + @Override
> + public long skip(final long n) throws IOException {
> + return decIS.skip(n);
> + }
> +
> + @Override
> + public void mark(final int readlimit) {
> + decIS.mark(readlimit);
> + }
> +
> + @Override
> + public boolean markSupported() {
> + return decIS.markSupported();
> + }
> +
> + @Override
> + public int read() throws IOException {
> + final int ret = decIS.read();
> + count(ret == -1 ? 0 : 1);
> + return ret;
> + }
> +
> + @Override
> + public int read(final byte[] buf, final int off, final int len)
> throws IOException {
> + final int ret = decIS.read(buf, off, len);
> + count(ret);
> + return ret;
> + }
> +
> + @Override
> + public String toString() {
> + return decIS.toString();
> + }
> +
> + @Override
> + public void reset() throws IOException {
> + decIS.reset();
> + }
> +
> +}
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/
> compressors/zstandard/ZstdUtils.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
> b/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdUtils.java
> new file mode 100644
> index 0000000..0eb8fa1
> --- /dev/null
> +++ b/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdUtils.java
> @@ -0,0 +1,88 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied. See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +/**
> + * Utility code for the Zstandard compression format.
> + * @ThreadSafe
> + * @since 1.14
> + */
> +public class ZstdUtils {
> +
> + static enum CachedAvailability {
> + DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE
> + }
> +
> + private static volatile CachedAvailability cachedZstdAvailability;
> +
> + static {
> + cachedZstdAvailability = CachedAvailability.DONT_CACHE;
> + try {
> + Class.forName("org.osgi.framework.BundleEvent");
> + } catch (final Exception ex) { // NOSONAR
> + setCacheZstdAvailablity(true);
> + }
> + }
> +
> + /** Private constructor to prevent instantiation of this utility
> class. */
> + private ZstdUtils() {
> + }
> +
> + /**
> + * Are the classes required to support Zstandard compression
> available?
> + * @return true if the classes required to support Zstandard
> compression are available
> + */
> + public static boolean isZstdCompressionAvailable() {
> + final CachedAvailability cachedResult = cachedZstdAvailability;
> + if (cachedResult != CachedAvailability.DONT_CACHE) {
> + return cachedResult == CachedAvailability.CACHED_AVAILABLE;
> + }
> + return internalIsZstdCompressionAvailable();
> + }
> +
> + private static boolean internalIsZstdCompressionAvailable() {
> + try {
> + Class.forName("com.github.luben.zstd.ZstdInputStream");
> + return true;
> + } catch (NoClassDefFoundError | Exception error) {
> + return false;
> + }
> + }
> +
> + /**
> + * Whether to cache the result of the Zstandard for Java check.
> + *
> + * <p>This defaults to {@code false} in an OSGi environment and
> {@code true} otherwise.</p>
> + * @param doCache whether to cache the result
> + */
> + public static void setCacheZstdAvailablity(final boolean doCache) {
> + if (!doCache) {
> + cachedZstdAvailability = CachedAvailability.DONT_CACHE;
> + } else if (cachedZstdAvailability ==
> CachedAvailability.DONT_CACHE) {
> + final boolean hasZstd = internalIsZstdCompressionAvailable();
> + cachedZstdAvailability = hasZstd ? CachedAvailability.CACHED_
> AVAILABLE
> + : CachedAvailability.CACHED_UNAVAILABLE;
> + }
> + }
> +
> + // only exists to support unit tests
> + static CachedAvailability getCachedZstdAvailability() {
> + return cachedZstdAvailability;
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/java/org/apache/commons/compress/
> compressors/zstandard/ZstdCompressorInputStreamTest.java
> ----------------------------------------------------------------------
> diff --git a/src/test/java/org/apache/commons/compress/compressors/
> zstandard/ZstdCompressorInputStreamTest.java b/src/test/java/org/apache/
> commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java
> new file mode 100644
> index 0000000..5ed276c
> --- /dev/null
> +++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/
> ZstdCompressorInputStreamTest.java
> @@ -0,0 +1,157 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +import static org.junit.Assert.assertEquals;
> +import static org.junit.Assert.assertTrue;
> +
> +import java.io.ByteArrayOutputStream;
> +import java.io.File;
> +import java.io.FileInputStream;
> +import java.io.FileOutputStream;
> +import java.io.IOException;
> +import java.io.InputStream;
> +
> +import org.apache.commons.compress.AbstractTestCase;
> +import org.apache.commons.compress.compressors.CompressorInputStream;
> +import org.apache.commons.compress.compressors.CompressorStreamFactory;
> +import org.apache.commons.compress.compressors.zstandard.
> ZstdCompressorInputStream;
> +import org.apache.commons.compress.utils.IOUtils;
> +import org.junit.Assert;
> +import org.junit.Test;
> +
> +public class ZstdCompressorInputStreamTest extends AbstractTestCase {
> +
> + /**
> + * Test bridge works fine
> + * @throws {@link IOException}
> + */
> + @Test
> + public void testZstdDecode() throws IOException {
> + final File input = getFile("zstandard.testdata.zst");
> + final File expected = getFile("zstandard.testdata");
> + try (InputStream inputStream = new FileInputStream(input);
> + InputStream expectedStream = new FileInputStream(expected);
> + ZstdCompressorInputStream zstdInputStream = new
> ZstdCompressorInputStream(inputStream)) {
> + final byte[] b = new byte[97];
> + IOUtils.readFully(expectedStream, b);
> + final ByteArrayOutputStream bos = new ByteArrayOutputStream();
> + int readByte = -1;
> + while((readByte = zstdInputStream.read()) != -1) {
> + bos.write(readByte);
> + }
> + Assert.assertArrayEquals(b, bos.toByteArray());
> + }
> + }
> +
> + @Test
> + public void testCachingIsEnabledByDefaultAndZstdUtilsPresent() {
> + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE,
> ZstdUtils.getCachedZstdAvailability());
> + assertTrue(ZstdUtils.isZstdCompressionAvailable());
> + }
> +
> + @Test
> + public void testCanTurnOffCaching() {
> + try {
> + ZstdUtils.setCacheZstdAvailablity(false);
> + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE,
> ZstdUtils.getCachedZstdAvailability());
> + assertTrue(ZstdUtils.isZstdCompressionAvailable());
> + } finally {
> + ZstdUtils.setCacheZstdAvailablity(true);
> + }
> + }
> +
> + @Test
> + public void testTurningOnCachingReEvaluatesAvailability() {
> + try {
> + ZstdUtils.setCacheZstdAvailablity(false);
> + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE,
> ZstdUtils.getCachedZstdAvailability());
> + ZstdUtils.setCacheZstdAvailablity(true);
> + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE,
> ZstdUtils.getCachedZstdAvailability());
> + } finally {
> + ZstdUtils.setCacheZstdAvailablity(true);
> + }
> + }
> +
> + @Test
> + public void shouldBeAbleToSkipAByte() throws IOException {
> + final File input = getFile("zstandard.testdata.zst");
> + try (InputStream is = new FileInputStream(input)) {
> + final ZstdCompressorInputStream in =
> + new ZstdCompressorInputStream(is);
> + Assert.assertEquals(1, in.skip(1));
> + in.close();
> + }
> + }
> +
> + @Test
> + public void singleByteReadWorksAsExpected() throws IOException {
> +
> + final File input = getFile("zstandard.testdata.zst");
> +
> + final File original = getFile("zstandard.testdata");
> + final long originalFileLength = original.length();
> +
> + byte[] originalFileContent = new byte[((int) originalFileLength)];
> +
> + try (InputStream ois = new FileInputStream(original)) {
> + ois.read(originalFileContent);
> + }
> +
> + try (InputStream is = new FileInputStream(input)) {
> + final ZstdCompressorInputStream in =
> + new ZstdCompressorInputStream(is);
> +
> + Assert.assertEquals(originalFileContent[0], in.read());
> + in.close();
> + }
> + }
> +
> + @Test
> + public void singleByteReadReturnsMinusOneAtEof() throws IOException {
> + final File input = getFile("zstandard.testdata.zst");
> + try (InputStream is = new FileInputStream(input)) {
> + final ZstdCompressorInputStream in =
> + new ZstdCompressorInputStream(is);
> + IOUtils.toByteArray(in);
> + Assert.assertEquals(-1, in.read());
> + in.close();
> + }
> + }
> +
> + @Test
> + public void testZstandardUnarchive() throws Exception {
> + final File input = getFile("bla.tar.zst");
> + final File output = new File(dir, "bla.tar");
> + try (InputStream is = new FileInputStream(input)) {
> + final CompressorInputStream in = new CompressorStreamFactory()
> + .createCompressorInputStream("zst", is);
> + FileOutputStream out = null;
> + try {
> + out = new FileOutputStream(output);
> + IOUtils.copy(in, out);
> + } finally {
> + if (out != null) {
> + out.close();
> + }
> + in.close();
> + }
> + }
> + }
> +
> +}
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/bla.tar.zst
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/bla.tar.zst b/src/test/resources/bla.tar.
> zst
> new file mode 100644
> index 0000000..d5fd6e0
> Binary files /dev/null and b/src/test/resources/bla.tar.zst differ
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/zstandard.testdata
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/zstandard.testdata b/src/test/resources/
> zstandard.testdata
> new file mode 100644
> index 0000000..e51bfd4
> --- /dev/null
> +++ b/src/test/resources/zstandard.testdata
> @@ -0,0 +1,3 @@
> +And as usual, instead of ipsum lorem we shall just state very clearly:
> +
> +Test test test chocolate
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/zstandard.testdata.zst
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/zstandard.testdata.zst
> b/src/test/resources/zstandard.testdata.zst
> new file mode 100644
> index 0000000..86c03fe
> Binary files /dev/null and b/src/test/resources/zstandard.testdata.zst
> differ
>
>
[2/2] commons-compress git commit: COMPRESS-423 small fixes
Posted by bo...@apache.org.
COMPRESS-423 small fixes
closes #55
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/1c382914
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/1c382914
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/1c382914
Branch: refs/heads/master
Commit: 1c382914c0ca52e786937e222d8e7aad17348cfa
Parents: 7984387
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Oct 17 20:22:47 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Oct 17 20:22:47 2017 +0200
----------------------------------------------------------------------
.../commons/compress/compressors/CompressorStreamFactory.java | 4 ++--
.../compressors/zstandard/ZstdCompressorInputStream.java | 4 ++--
.../apache/commons/compress/compressors/zstandard/ZstdUtils.java | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/1c382914/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index b446963..8b25611 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -196,7 +196,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
* Constant (value {@value}) used to identify the ZStandard compression
* algorithm. Not supported as an output stream type.
*
- * @since 1.15
+ * @since 1.16
*/
public static final String ZSTANDARD = "zst";
@@ -571,7 +571,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (ZSTANDARD.equalsIgnoreCase(name)) {
if (!ZstdUtils.isZstdCompressionAvailable()) {
- throw new CompressorException("XZ compression is not available.");
+ throw new CompressorException("ZStandard compression is not available.");
}
return new ZstdCompressorInputStream(in);
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/1c382914/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
index 1e5dd8d..c85a444 100644
--- a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
@@ -28,7 +28,7 @@ import org.apache.commons.compress.compressors.CompressorInputStream;
* {@link CompressorInputStream} implementation to decode Zstandard encoded stream.
* Library relies on <a href="https://github.com/luben/zstd-jni/">Zstandard JNI</a>
*
- * @since 1.15
+ * @since 1.16
*/
public class ZstdCompressorInputStream extends CompressorInputStream {
@@ -92,4 +92,4 @@ public class ZstdCompressorInputStream extends CompressorInputStream {
decIS.reset();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/1c382914/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
index 0eb8fa1..a12492e 100644
--- a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
+++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
@@ -21,7 +21,7 @@ package org.apache.commons.compress.compressors.zstandard;
/**
* Utility code for the Zstandard compression format.
* @ThreadSafe
- * @since 1.14
+ * @since 1.16
*/
public class ZstdUtils {