You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/29 17:55:05 UTC
[2/2] commons-compress git commit: add DEFLATE as compression format
to play with
add DEFLATE as compression format to play with
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b74e5386
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b74e5386
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b74e5386
Branch: refs/heads/compress-2.0
Commit: b74e53866767f2323af1ac09681bf80318fe2c49
Parents: 27686ef
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sat Apr 29 19:54:35 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sat Apr 29 19:54:35 2017 +0200
----------------------------------------------------------------------
.../formats/deflate/DeflateCompressedInput.java | 92 +++++++++
.../deflate/DeflateCompressedOutput.java | 87 +++++++++
.../deflate/DeflateCompressionFormat.java | 89 +++++++++
.../deflate/DeflateCompressionFormatTest.java | 53 ++++++
.../formats/deflate/RoundTripTest.java | 185 +++++++++++++++++++
.../test-archives/default.tar.deflatez | Bin 0 -> 468 bytes
6 files changed, 506 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
new file mode 100644
index 0000000..a7d9977
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressedInput;
+
+/**
+ * Input for DEFLATE compressed channels.
+ */
+public class DeflateCompressedInput extends AbstractCompressedInput {
+ private final InputStream in;
+ private final Inflater inflater;
+ private boolean nextCalled = false;
+
+ public DeflateCompressedInput(ReadableByteChannel channel) {
+ in = Channels.newInputStream(channel);
+ inflater = new Inflater(false);
+ }
+
+ @Override
+ public ReadableByteChannel next() throws IOException {
+ if (nextCalled) {
+ return null;
+ }
+ nextCalled = true;
+ return Channels.newChannel(new InflaterInputStream(new WrappedStream(in), inflater));
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ in.close();
+ } finally {
+ inflater.end();
+ }
+ }
+
+ private class WrappedStream extends FilterInputStream {
+ private WrappedStream(InputStream i) {
+ super(i);
+ }
+
+ @Override
+ public void close() {
+ inflater.reset();
+ }
+
+ @Override
+ public int read() throws IOException {
+ int r = super.read();
+ count(r < 0 ? 0 : 1);
+ return r;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int r = super.read(b, off, len);
+ count(r);
+ return r;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ long r = super.skip(n);
+ count(r);
+ return r;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
new file mode 100644
index 0000000..f563213
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressedOutput;
+
+/**
+ * Output for DEFLATE compressed channels.
+ */
+public class DeflateCompressedOutput extends AbstractCompressedOutput {
+ private WritableByteChannel channel;
+ private final DeflaterOutputStream out;
+ private final Deflater deflater;
+
+ public DeflateCompressedOutput(WritableByteChannel channel) {
+ this.channel = channel;
+ deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
+ out = new DeflaterOutputStream(new WrappedStream(Channels.newOutputStream(channel)), deflater);
+ }
+
+ @Override
+ public WritableByteChannel startCompressing() {
+ return Channels.newChannel(out);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ out.finish();
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ out.close();
+ } finally {
+ try {
+ deflater.end();
+ } finally {
+ channel.close();
+ }
+ }
+ }
+
+ private class WrappedStream extends FilterOutputStream {
+ private WrappedStream(OutputStream o) {
+ super(o);
+ }
+
+ @Override
+ public void close() { }
+
+ @Override
+ public void write(int b) throws IOException {
+ super.write(b);
+ count(1);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ super.write(b, off, len);
+ count(len);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
new file mode 100644
index 0000000..ee3bed3
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressionFormat;
+
+/**
+ * Format descriptor for the GZIP format.
+ */
+public class DeflateCompressionFormat extends AbstractCompressionFormat {
+ private static final int MAGIC_1 = 0x78;
+ private static final int MAGIC_2a = 0x01;
+ private static final int MAGIC_2b = 0x5e;
+ private static final int MAGIC_2c = 0x9c;
+ private static final int MAGIC_2d = 0xda;
+
+ /**
+ * "DEFLATE"
+ */
+ public static final String DEFLATE_FORMAT_NAME = "DEFLATE";
+
+ /**
+ * "DEFLATE"
+ */
+ @Override
+ public String getName() {
+ return DEFLATE_FORMAT_NAME;
+ }
+
+ /**
+ * Yes.
+ */
+ @Override
+ public boolean supportsWriting() { return true; }
+
+ /**
+ * Yes.
+ */
+ @Override
+ public boolean supportsAutoDetection() { return true; }
+
+ /**
+ * @return 2
+ */
+ @Override
+ public int getNumberOfBytesRequiredForAutodetection() {
+ return 2;
+ }
+
+ @Override
+ public boolean matches(ByteBuffer probe) {
+ byte[] sig = new byte[2];
+ probe.get(sig);
+ return sig[0] == MAGIC_1 && (
+ sig[1] == (byte) MAGIC_2a ||
+ sig[1] == (byte) MAGIC_2b ||
+ sig[1] == (byte) MAGIC_2c ||
+ sig[1] == (byte) MAGIC_2d);
+ }
+
+ @Override
+ public DeflateCompressedInput readFrom(ReadableByteChannel channel) {
+ return new DeflateCompressedInput(channel);
+ }
+
+ @Override
+ public DeflateCompressedOutput writeTo(WritableByteChannel channel) {
+ return new DeflateCompressedOutput(channel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
new file mode 100644
index 0000000..837f31d
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.commons.compress2.util.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DeflateCompressionFormatTest {
+
+ @Test
+ public void shouldDetectFormat() throws IOException {
+ Assert.assertTrue(isAr("test-archives/default.tar.deflatez"));
+ }
+
+ @Test
+ public void shouldRejectXMLFile() throws IOException {
+ Assert.assertFalse(isAr("test1.xml"));
+ }
+
+
+ private boolean isAr(String file) throws IOException {
+ File f = RoundTripTest.getFile(file);
+ FileInputStream c = new FileInputStream(f);
+ try {
+ byte[] b = new byte[10];
+ IOUtils.readFully(c, b);
+ return new DeflateCompressionFormat().matches(ByteBuffer.wrap(b));
+ } finally {
+ c.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
new file mode 100644
index 0000000..72f36d4
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.Locale;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.commons.compress2.compressors.CompressedInput;
+import org.apache.commons.compress2.compressors.CompressedOutput;
+import org.apache.commons.compress2.util.IOUtils;
+
+public class RoundTripTest {
+
+ private File dir;
+
+ @Before
+ public void createTempDir() throws Exception {
+ dir = mkdir("dir");
+ }
+
+ @After
+ public void removeTempDir() throws Exception {
+ rmdir(dir);
+ }
+
+ @Test
+ public void testRoundtripUsingConstructors() throws Exception {
+ final File output = new File(dir, "constructors.def");
+ final File file1 = getFile("test1.xml");
+ try (WritableByteChannel out = new FileOutputStream(output).getChannel();
+ DeflateCompressedOutput os = new DeflateCompressedOutput(out);
+ WritableByteChannel c = os.startCompressing();
+ ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+ IOUtils.copy(in, c);
+ }
+
+ final File input = output;
+ final File target = new File(dir, "test1.xml");
+ try (ReadableByteChannel is = new FileInputStream(input).getChannel();
+ DeflateCompressedInput in = new DeflateCompressedInput(is);
+ ReadableByteChannel r = in.next();
+ WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+ IOUtils.copy(r, out);
+ }
+ }
+
+ @Test
+ public void testRoundtripUsingFormatInstanceAndChannels() throws Exception {
+ DeflateCompressionFormat format = new DeflateCompressionFormat();
+ final File output = new File(dir, "format-channels.def");
+ final File file1 = getFile("test1.xml");
+ try (WritableByteChannel out = new FileOutputStream(output).getChannel();
+ DeflateCompressedOutput os = format.writeTo(out);
+ WritableByteChannel c = os.startCompressing();
+ ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+ IOUtils.copy(in, c);
+ }
+
+ final File input = output;
+ final File target = new File(dir, "test1.xml");
+ try (ReadableByteChannel is = new FileInputStream(input).getChannel();
+ DeflateCompressedInput in = format.readFrom(is);
+ ReadableByteChannel r = in.next();
+ WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+ IOUtils.copy(r, out);
+ }
+ }
+
+ @Test
+ public void testRoundtripUsingFormatInstanceAndPaths() throws Exception {
+ DeflateCompressionFormat format = new DeflateCompressionFormat();
+ final File output = new File(dir, "format-files.def");
+ final File file1 = getFile("test1.xml");
+ try (CompressedOutput os = format.writeTo(output.toPath());
+ WritableByteChannel c = os.startCompressing();
+ ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+ IOUtils.copy(in, c);
+ }
+
+ final File input = output;
+ final File target = new File(dir, "test1.xml");
+ try (CompressedInput in = format.readFrom(input.toPath());
+ ReadableByteChannel r = in.next();
+ WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+ IOUtils.copy(r, out);
+ }
+ }
+
+ public static File mkdir(String name) throws IOException {
+ File f = File.createTempFile(name, "");
+ f.delete();
+ f.mkdir();
+ return f;
+ }
+
+ public static File getFile(String path) throws IOException {
+ URL url = RoundTripTest.class.getClassLoader().getResource(path);
+ if (url == null) {
+ throw new FileNotFoundException("couldn't find " + path);
+ }
+ URI uri = null;
+ try {
+ uri = url.toURI();
+ } catch (java.net.URISyntaxException ex) {
+ throw new IOException(ex);
+ }
+ return new File(uri);
+ }
+
+ public static void rmdir(File f) {
+ String[] s = f.list();
+ if (s != null) {
+ for (String element : s) {
+ final File file = new File(f, element);
+ if (file.isDirectory()){
+ rmdir(file);
+ }
+ boolean ok = tryHardToDelete(file);
+ if (!ok && file.exists()){
+ System.out.println("Failed to delete "+element+" in "+f.getPath());
+ }
+ }
+ }
+ tryHardToDelete(f); // safer to delete and check
+ if (f.exists()){
+ throw new Error("Failed to delete "+f.getPath());
+ }
+ }
+
+ private static final boolean ON_WINDOWS =
+ System.getProperty("os.name").toLowerCase(Locale.ENGLISH)
+ .indexOf("windows") > -1;
+
+ /**
+ * Accommodate Windows bug encountered in both Sun and IBM JDKs.
+ * Others possible. If the delete does not work, call System.gc(),
+ * wait a little and try again.
+ *
+ * @return whether deletion was successful
+ * @since Stolen from FileUtils in Ant 1.8.0
+ */
+ public static boolean tryHardToDelete(File f) {
+ if (f != null && f.exists() && !f.delete()) {
+ if (ON_WINDOWS) {
+ System.gc();
+ }
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException ex) {
+ // Ignore Exception
+ }
+ return f.delete();
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/resources/test-archives/default.tar.deflatez
----------------------------------------------------------------------
diff --git a/src/test/resources/test-archives/default.tar.deflatez b/src/test/resources/test-archives/default.tar.deflatez
new file mode 100644
index 0000000..32ebe77
Binary files /dev/null and b/src/test/resources/test-archives/default.tar.deflatez differ