You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2019/12/15 11:10:53 UTC
[commons-compress] 03/05: Merge branch 'master' of
https://github.com/apache/commons-compress into COMPRESS-477-constructing
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit 95140bd9675df0284e97706c1b48d34e339738c9
Merge: e61d22e dfa9ed3
Author: Lee <pe...@gmail.com>
AuthorDate: Mon Dec 9 10:02:20 2019 +0800
Merge branch 'master' of https://github.com/apache/commons-compress into COMPRESS-477-constructing
.../compress/archivers/zip/ZipArchiveEntry.java | 12 +
.../archivers/zip/ZipArchiveInputStream.java | 35 +-
.../commons/compress/archivers/zip/ZipFile.java | 167 +-
.../archivers/zip/ZipSplitOutputStream.java | 12 +-
.../zip/ZipSplitReadOnlySeekableByteChannel.java | 250 ++
.../commons/compress/compressors/FileNameUtil.java | 15 -
.../commons/compress/utils/FileNameUtils.java | 78 +
.../utils/MultiReadOnlySeekableByteChannel.java | 17 +
.../archivers/zip/ZipArchiveInputStreamTest.java | 72 +
.../compress/archivers/zip/ZipFileTest.java | 83 +
.../commons/compress/utils/FileNameUtilsTest.java | 53 +
.../MultiReadOnlySeekableByteChannelTest.java | 8 +-
.../ZipSplitReadOnlySeekableByteChannelTest.java | 181 ++
.../split_zip_created_by_winrar/file_to_compare_1} | 2594 ++++++++++----------
.../split_zip_created_by_winrar.z01 | Bin 0 -> 262144 bytes
.../split_zip_created_by_winrar.z02 | Bin 0 -> 262144 bytes
.../split_zip_created_by_winrar.zip | Bin 0 -> 50536 bytes
.../zip_to_compare_created_by_winrar.zip | Bin 0 -> 574820 bytes
.../split_zip_created_by_zip/file_to_compare_1 | 38 +
.../split_zip_created_by_zip/file_to_compare_2 | 79 +
.../split_zip_created_by_zip.z01 | Bin 0 -> 262144 bytes
.../split_zip_created_by_zip.z02 | Bin 0 -> 262144 bytes
.../split_zip_created_by_zip.zip | Bin 0 -> 57763 bytes
.../split_zip_created_by_zip_zip64.z01 | Bin 0 -> 262144 bytes
.../split_zip_created_by_zip_zip64.z02 | Bin 0 -> 262144 bytes
.../split_zip_created_by_zip_zip64.zip | Bin 0 -> 69177 bytes
.../zip_to_compare_created_by_zip_zip64.zip | Bin 0 -> 584681 bytes
27 files changed, 2347 insertions(+), 1347 deletions(-)
diff --cc src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
index bcf3459,53af93a..47a3edd
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
@@@ -1080,11 -1081,23 +1080,23 @@@ public class ZipArchiveEntry extends ja
this.commentSource = commentSource;
}
+ /**
+ * The number of the split segment this entry starts at.
+ *
+ * @return the number of the split segment this entry starts at.
+ * @since 1.20
+ */
- public long getDiskNumberStart() {
+ public int getDiskNumberStart() {
return diskNumberStart;
}
+ /**
+ * The number of the split segment this entry starts at.
+ *
+ * @param diskNumberStart the number of the split segment this entry starts at.
+ * @since 1.20
+ */
- public void setDiskNumberStart(long diskNumberStart) {
+ public void setDiskNumberStart(int diskNumberStart) {
this.diskNumberStart = diskNumberStart;
}
diff --cc src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index 152272b,2319456..6f2a16c
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@@ -852,6 -860,10 +860,10 @@@ public class ZipFile implements Closeab
if (hasRelativeHeaderOffset) {
ze.setLocalHeaderOffset(z64.getRelativeHeaderOffset().getLongValue());
}
+
+ if (hasDiskStart) {
- ze.setDiskNumberStart(z64.getDiskStartNumber().getValue());
++ ze.setDiskNumberStart(z64.getDiskStartNumber().getIntValue());
+ }
}
}
diff --cc src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
index 20ed3c1,0000000..d622604
mode 100644,000000..100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
@@@ -1,236 -1,0 +1,238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.archivers.zip;
+
- import org.apache.commons.compress.compressors.FileNameUtil;
++import org.apache.commons.compress.utils.FileNameUtils;
+
- import java.io.*;
- import java.nio.ByteBuffer;
++import java.io.File;
++import java.io.FileOutputStream;
++import java.io.IOException;
++import java.io.OutputStream;
+
+public class ZipSplitOutputStream extends OutputStream {
+ private OutputStream outputStream;
+ private File zipFile;
+ private final long splitSize;
+ private int currentSplitSegmentIndex = 0;
+ private long currentSplitSegmentBytesWritten = 0;
+ private boolean finished = false;
+ private final byte[] singleByte = new byte[1];
+
+ /**
+ * 8.5.1 Capacities for split archives are as follows:
+ * <p>
+ * Maximum number of segments = 4,294,967,295 - 1
+ * Maximum .ZIP segment size = 4,294,967,295 bytes (refer to section 8.5.6)
+ * Minimum segment size = 64K
+ * Maximum PKSFX segment size = 2,147,483,647 bytes
+ */
+ private final long ZIP_SEGMENT_MIN_SIZE = 64 * 1024L;
+ private final long ZIP_SEGMENT_MAX_SIZE = 4294967295L;
+
+ /**
+ * Create a split zip. If the zip file is smaller than the split size,
+ * then there will only be one split zip, and its suffix is .zip,
+ * otherwise the split segments should be like .z01, .z02, ... .z(N-1), .zip
+ *
+ * @param zipFile the zip file to write to
+ * @param splitSize the split size
+ */
+ public ZipSplitOutputStream(final File zipFile, final long splitSize) throws IllegalArgumentException, IOException {
+ if (splitSize < ZIP_SEGMENT_MIN_SIZE || splitSize > ZIP_SEGMENT_MAX_SIZE) {
+ throw new IllegalArgumentException("zip split segment size should between 64K and 4,294,967,295");
+ }
+
+ this.zipFile = zipFile;
+ this.splitSize = splitSize;
+
+ this.outputStream = new FileOutputStream(zipFile);
+ // write the zip split signature 0x08074B50 to the zip file
+ writeZipSplitSignature();
+ }
+
+ /**
+ * Some data can not be written to different split segments, for example:
+ * <p>
+ * 4.4.1.5 The end of central directory record and the Zip64 end
+ * of central directory locator record MUST reside on the same
+ * disk when splitting or spanning an archive.
+ *
+ * @param unsplittableContentSize
+ * @throws IllegalArgumentException
+ * @throws IOException
+ */
+ public void prepareToWriteUnsplittableContent(long unsplittableContentSize) throws IllegalArgumentException, IOException {
+ if (unsplittableContentSize > this.splitSize) {
+ throw new IllegalArgumentException("The unsplittable content size is bigger than the split segment size");
+ }
+
+ long bytesRemainingInThisSegment = this.splitSize - this.currentSplitSegmentBytesWritten;
+ if (bytesRemainingInThisSegment < unsplittableContentSize) {
+ openNewSplitSegment();
+ }
+ }
+
+ @Override
+ public void write(int i) throws IOException {
+ singleByte[0] = (byte)(i & 0xff);
+ write(singleByte);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ /**
+ * Write the data to zip split segments, if the remaining space of current split segment
+ * is not enough, then a new split segment should be created
+ *
+ * @param b data to write
+ * @param off offset of the start of data in param b
+ * @param len the length of data to write
+ * @throws IOException
+ */
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ if (len <= 0) {
+ return;
+ }
+
+ if (currentSplitSegmentBytesWritten >= splitSize) {
+ openNewSplitSegment();
+ write(b, off, len);
+ } else if (currentSplitSegmentBytesWritten + len > splitSize) {
+ int bytesToWriteForThisSegment = (int) splitSize - (int) currentSplitSegmentBytesWritten;
+ write(b, off, bytesToWriteForThisSegment);
+ openNewSplitSegment();
+ write(b, off + bytesToWriteForThisSegment, len - bytesToWriteForThisSegment);
+ } else {
+ outputStream.write(b, off, len);
+ currentSplitSegmentBytesWritten += len;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!finished) {
+ finish();
+ }
+ }
+
+ /**
+ * The last zip split segment's suffix should be .zip
+ *
+ * @throws IOException
+ */
+ private void finish() throws IOException {
+ if (finished) {
+ throw new IOException("This archive has already been finished");
+ }
+
- String zipFileBaseName = FileNameUtil.getBaseName(zipFile.getName());
++ String zipFileBaseName = FileNameUtils.getBaseName(zipFile.getName());
+ File lastZipSplitSegmentFile = new File(zipFile.getParentFile(), zipFileBaseName + ".zip");
+ outputStream.close();
+ zipFile.renameTo(lastZipSplitSegmentFile);
+ finished = true;
+ }
+
+ /**
+ * Create a new zip split segment and prepare to write to the new segment
+ *
+ * @return
+ * @throws IOException
+ */
+ private OutputStream openNewSplitSegment() throws IOException {
+ File newFile;
+ if (currentSplitSegmentIndex == 0) {
+ outputStream.close();
+ newFile = createNewSplitSegmentFile(1);
+ zipFile.renameTo(newFile);
+ }
+
+ newFile = createNewSplitSegmentFile(null);
+
+
+ OutputStream newFileOutputStream = new FileOutputStream(newFile);
+ outputStream.close();
+ outputStream = newFileOutputStream;
+ currentSplitSegmentBytesWritten = 0;
+ zipFile = newFile;
+ currentSplitSegmentIndex++;
+
+ return newFileOutputStream;
+ }
+
+ /**
+ * Write the zip split signature (0x08074B50) to the head of the first zip split segment
+ *
+ * @throws IOException
+ */
+ private void writeZipSplitSignature() throws IOException {
+ outputStream.write(ZipArchiveOutputStream.DD_SIG);
+ currentSplitSegmentBytesWritten += ZipArchiveOutputStream.DD_SIG.length;
+ }
+
+ /**
+ * Create the new zip split segment, the last zip segment should be .zip, and the zip split segments' suffix should be
+ * like .z01, .z02, .z03, ... .z99, .z100, ..., .z(N-1), .zip
+ * <p>
+ * 8.3.3 Split ZIP files are typically written to the same location
+ * and are subject to name collisions if the spanned name
+ * format is used since each segment will reside on the same
+ * drive. To avoid name collisions, split archives are named
+ * as follows.
+ * <p>
+ * Segment 1 = filename.z01
+ * Segment n-1 = filename.z(n-1)
+ * Segment n = filename.zip
+ * <p>
+ * NOTE:
+ * The zip split segment begin from 1,2,3,... , and we're creating a new segment,
+ * so the new segment suffix should be (currentSplitSegmentIndex + 2)
+ *
+ * @param zipSplitSegmentSuffixIndex
+ * @return
+ * @throws IOException
+ */
+ private File createNewSplitSegmentFile(Integer zipSplitSegmentSuffixIndex) throws IOException {
+ int newZipSplitSegmentSuffixIndex = zipSplitSegmentSuffixIndex == null ? (currentSplitSegmentIndex + 2) : zipSplitSegmentSuffixIndex;
- String baseName = FileNameUtil.getBaseName(zipFile.getName());
++ String baseName = FileNameUtils.getBaseName(zipFile.getName());
+ String extension = ".z";
+ if (newZipSplitSegmentSuffixIndex <= 9) {
+ extension += "0" + newZipSplitSegmentSuffixIndex;
+ } else {
+ extension += newZipSplitSegmentSuffixIndex;
+ }
+
+ File newFile = new File(zipFile.getParent(), baseName + extension);
+
+ if (newFile.exists()) {
+ throw new IOException("split zip segment " + baseName + extension + " already exists");
+ }
+ return newFile;
+ }
+
+ public int getCurrentSplitSegmentIndex() {
+ return currentSplitSegmentIndex;
+ }
+
+ public long getCurrentSplitSegmentBytesWritten() {
+ return currentSplitSegmentBytesWritten;
+ }
+}