[GitHub] gianm closed pull request #6224: Zstandard decompression support

[GitBox <gi...@apache.org>]

gianm closed pull request #6224: Zstandard decompression support
URL: https://github.com/apache/incubator-druid/pull/6224
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/java-util/pom.xml b/java-util/pom.xml
index 696e579a73d..569aa4fa8d3 100644
--- a/java-util/pom.xml
+++ b/java-util/pom.xml
@@ -93,6 +93,10 @@
             <groupId>org.tukaani</groupId>
             <artifactId>xz</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path</artifactId>
diff --git a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
index 99c6367b095..77ef23a358c 100644
--- a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
+++ b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
@@ -31,6 +31,7 @@
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
@@ -58,6 +59,7 @@
   private static final String XZ_SUFFIX = ".xz";
   private static final String ZIP_SUFFIX = ".zip";
   private static final String SNAPPY_SUFFIX = ".sz";
+  private static final String ZSTD_SUFFIX = ".zst";
 
   /**
    * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped
@@ -567,6 +569,8 @@ public static InputStream decompress(final InputStream in, final String fileName
       return new XZCompressorInputStream(in, true);
     } else if (fileName.endsWith(SNAPPY_SUFFIX)) {
       return new FramedSnappyCompressorInputStream(in);
+    } else if (fileName.endsWith(ZSTD_SUFFIX)) {
+      return new ZstdCompressorInputStream(in);
     } else if (fileName.endsWith(ZIP_SUFFIX)) {
       // This reads the first file in the archive.
       final ZipInputStream zipIn = new ZipInputStream(in, StandardCharsets.UTF_8);
diff --git a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
index 26d3d7914b6..2e02c770ea8 100644
--- a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
+++ b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
@@ -28,6 +28,7 @@
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -290,6 +291,20 @@ public void testDecompressSnappy() throws IOException
     }
   }
 
+  @Test
+  public void testDecompressZstd() throws IOException
+  {
+    final File tmpDir = temporaryFolder.newFolder("testDecompressZstd");
+    final File zstdFile = new File(tmpDir, testFile.getName() + ".zst");
+    Assert.assertFalse(zstdFile.exists());
+    try (final OutputStream out = new ZstdCompressorOutputStream(new FileOutputStream(zstdFile))) {
+      ByteStreams.copy(new FileInputStream(testFile), out);
+    }
+    try (final InputStream inputStream = CompressionUtils.decompress(new FileInputStream(zstdFile), zstdFile.getName())) {
+      assertGoodDataStream(inputStream);
+    }
+  }
+
   @Test
   public void testDecompressZip() throws IOException
   {
diff --git a/pom.xml b/pom.xml
index a7bc4faac46..9774ac8b33e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -340,6 +340,11 @@
                 <artifactId>xz</artifactId>
                 <version>1.8</version>
             </dependency>
+            <dependency>
+                <groupId>com.github.luben</groupId>
+                <artifactId>zstd-jni</artifactId>
+                <version>1.3.3-1</version>
+            </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-annotations</artifactId>


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org