You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2020/01/01 17:11:32 UTC
[commons-compress] branch master updated (9839a3e -> b465bfa)
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git.
from 9839a3e the sentence is incomplete
new f7f7c53 Happy new year
new b87cc99 COMPRESS-124 add support for sparse files in tar
new 50569e5 COMPRESS-124 : add testcases for extracting sparse
new 03b7d1e Merge branch 'COMPRESS-124' of https://github.com/PeterAlfreadLee/commons-compress into PeterAlfreadLee-COMPRESS-124
new 9bc32de Merge branch 'PeterAlfreadLee-COMPRESS-124'
new b465bfa COMPRESS-124 review cosmetics
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
NOTICE.txt | 2 +-
.../compress/archivers/tar/TarArchiveEntry.java | 56 ++-
.../archivers/tar/TarArchiveInputStream.java | 415 +++++++++++++++++++--
.../archivers/tar/TarArchiveSparseEntry.java | 22 ++
.../archivers/tar/TarArchiveStructSparse.java | 82 ++++
.../compress/archivers/tar/TarConstants.java | 24 ++
.../commons/compress/archivers/tar/TarUtils.java | 19 +-
.../commons/compress/utils/BoundedInputStream.java | 12 +
.../compress/archivers/tar/SparseFilesTest.java | 182 ++++++++-
.../archivers/tar/TarArchiveInputStreamTest.java | 10 +-
.../compress/archivers/tar/TarUtilsTest.java | 12 +
...ldgnu_sparse.tar => oldgnu_extended_sparse.tar} | Bin 10240 -> 10240 bytes
12 files changed, 798 insertions(+), 38 deletions(-)
create mode 100644 src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
copy src/test/resources/{oldgnu_sparse.tar => oldgnu_extended_sparse.tar} (80%)
[commons-compress] 01/06: COMPRESS-124 add support for sparse files
in tar
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit b87cc99c8312630427ade0319d100b40143dd9e7
Author: Lee <pe...@gmail.com>
AuthorDate: Sat Nov 16 18:49:35 2019 +0800
COMPRESS-124 add support for sparse files in tar
add support for extracting tar archives containing sparse files
---
.../compress/archivers/tar/TarArchiveEntry.java | 65 +++++-
.../archivers/tar/TarArchiveInputStream.java | 242 ++++++++++++++++++---
.../archivers/tar/TarArchiveSparseEntry.java | 17 ++
.../archivers/tar/TarArchiveSparseInputStream.java | 222 +++++++++++++++++++
...parseEntry.java => TarArchiveStructSparse.java} | 48 ++--
.../compress/archivers/tar/TarConstants.java | 24 ++
.../commons/compress/archivers/tar/TarUtils.java | 15 +-
.../commons/compress/utils/BoundedInputStream.java | 9 +
.../archivers/tar/TarArchiveInputStreamTest.java | 10 +-
9 files changed, 589 insertions(+), 63 deletions(-)
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
index fc187f4..9012a3c 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
@@ -20,11 +20,14 @@ package org.apache.commons.compress.archivers.tar;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
+
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipEncoding;
import org.apache.commons.compress.utils.ArchiveUtils;
@@ -200,6 +203,9 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/** The entry's minor device number. */
private int devMinor = 0;
+ /** The sparse headers in tar */
+ private List<TarArchiveStructSparse> sparseHeaders;
+
/** If an extension sparse header follows. */
private boolean isExtended;
@@ -209,6 +215,10 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/** is this entry a GNU sparse entry using one of the PAX formats? */
private boolean paxGNUSparse;
+ /** is this entry a GNU sparse entry using 1.X PAX formats?
+ * the sparse headers of 1.x PAX Format is stored in file data block */
+ private boolean paxGNU1XSparse = false;
+
/** is this entry a star sparse entry using the PAX header? */
private boolean starSparse;
@@ -742,6 +752,32 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
}
/**
+ * Set this entry's sparse headers
+ * @param sparseHeaders The new sparse headers
+ */
+ public void setSparseHeaders(List<TarArchiveStructSparse> sparseHeaders) {
+ this.sparseHeaders = sparseHeaders;
+ }
+
+ /**
+ * Get this entry's sparse headers
+ *
+ * @return This entry's sparse headers
+ */
+ public List<TarArchiveStructSparse> getSparseHeaders() {
+ return sparseHeaders;
+ }
+
+ /**
+ * Get if this entry is a sparse file with 1.X PAX Format or not
+ *
+ * @return True if this entry is a sparse file with 1.X PAX Format
+ */
+ public boolean isPaxGNU1XSparse() {
+ return paxGNU1XSparse;
+ }
+
+ /**
* Set this entry's file size.
*
* @param size This entry's new file size.
@@ -1041,13 +1077,16 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/**
* Update the entry using a map of pax headers.
* @param headers
+ * @param sparseHeaders for 0.0 PAX Format, the sparse headers may appear more than 1 time in headers map,
+* * this means it can not be read from a map, therefore the sparse headers have already
+* * been parsed to a list and was passed through parameter sparseHeaders
* @since 1.15
*/
- void updateEntryFromPaxHeaders(Map<String, String> headers) {
+ void updateEntryFromPaxHeaders(Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) {
for (final Map.Entry<String, String> ent : headers.entrySet()) {
final String key = ent.getKey();
final String val = ent.getValue();
- processPaxHeader(key, val, headers);
+ processPaxHeader(key, val, headers, sparseHeaders);
}
}
@@ -1062,6 +1101,10 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
processPaxHeader(key,val,extraPaxHeaders);
}
+ private void processPaxHeader(String key, String val, Map<String, String> headers) {
+ processPaxHeader(key, val, headers, null);
+ }
+
/**
* Process one pax header, using the supplied map as source for extra headers to be used when handling
* entries for sparse files
@@ -1069,9 +1112,13 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
* @param key the header name.
* @param val the header value.
* @param headers map of headers used for dealing with sparse file.
+ * @param sparseHeaders for 0.0 PAX Format, the sparse headers may appear more than 1 time in headers map,
+ * this means it can not be read from a map, therefore the sparse headers have already
+ * been parsed to a list and was passed through parameter sparseHeaders
* @since 1.15
*/
- private void processPaxHeader(String key, String val, Map<String, String> headers) {
+ private void processPaxHeader(String key, String val, Map<String, String> headers,
+ final List<TarArchiveStructSparse> sparseHeaders) {
/*
* The following headers are defined for Pax.
* atime, ctime, charset: cannot use these without changing TarArchiveEntry fields
@@ -1125,6 +1172,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
break;
case "GNU.sparse.size":
fillGNUSparse0xData(headers);
+ this.sparseHeaders = sparseHeaders;
break;
case "GNU.sparse.realsize":
fillGNUSparse1xData(headers);
@@ -1341,6 +1389,16 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
offset += OFFSETLEN_GNU;
offset += LONGNAMESLEN_GNU;
offset += PAD2LEN_GNU;
+ sparseHeaders = new ArrayList<>();
+ for (int i = 0; i < SPARSE_HEADERS_IN_OLDGNU_HEADER; i++) {
+ TarArchiveStructSparse sparseHeader = TarUtils.parseSparse(header,
+ offset + i * (SPARSE_OFFSET_LEN + SPARSE_NUMBYTES_LEN));
+
+ // some sparse headers are empty, we need to skip these sparse headers
+ if(sparseHeader.getOffset() > 0 || sparseHeader.getNumbytes() > 0) {
+ sparseHeaders.add(sparseHeader);
+ }
+ }
offset += SPARSELEN_GNU;
isExtended = TarUtils.parseBoolean(header, offset);
offset += ISEXTENDEDLEN_GNU;
@@ -1461,6 +1519,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
void fillGNUSparse1xData(final Map<String, String> headers) {
paxGNUSparse = true;
+ paxGNU1XSparse = true;
realSize = Integer.parseInt(headers.get("GNU.sparse.realsize"));
name = headers.get("GNU.sparse.name");
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
index 7ec8ed7..9194db8 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
@@ -26,7 +26,9 @@ package org.apache.commons.compress.archivers.tar;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.commons.compress.archivers.ArchiveEntry;
@@ -68,6 +70,9 @@ public class TarArchiveInputStream extends ArchiveInputStream {
/** An input stream to read from */
private final InputStream is;
+ /** An input stream to read sparse file */
+ private InputStream sparseInputStream;
+
/** The meta-data about the current entry */
private TarArchiveEntry currEntry;
@@ -80,6 +85,9 @@ public class TarArchiveInputStream extends ArchiveInputStream {
// the global PAX header
private Map<String, String> globalPaxHeaders = new HashMap<>();
+ // the global sparse headers, this is only used in PAX Format 0.X
+ private List<TarArchiveStructSparse> globalSparseHeaders = new ArrayList<>();
+
private final boolean lenient;
/**
@@ -214,10 +222,17 @@ public class TarArchiveInputStream extends ArchiveInputStream {
if (isDirectory()) {
return 0;
}
- if (entrySize - entryOffset > Integer.MAX_VALUE) {
+
+ // for sparse entries, there are actually currEntry.getRealSize() bytes to read
+ long entryActualSize = entrySize;
+ if (currEntry.isSparse()) {
+ entryActualSize = currEntry.getRealSize();
+ }
+
+ if (entryActualSize - entryOffset > Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
- return (int) (entrySize - entryOffset);
+ return (int) (entryActualSize - entryOffset);
}
@@ -243,8 +258,16 @@ public class TarArchiveInputStream extends ArchiveInputStream {
return 0;
}
- final long available = entrySize - entryOffset;
- final long skipped = IOUtils.skip(is, Math.min(n, available));
+ long available;
+ long skipped;
+ if(!currEntry.isSparse()) {
+ available = entrySize - entryOffset;
+ skipped = IOUtils.skip(is, Math.min(n, available));
+ } else {
+ // for sparse entries, there are actually currEntry.getRealSize() bytes to read
+ available = currEntry.getRealSize() - entryOffset;
+ skipped = IOUtils.skip(sparseInputStream, Math.min(n, available));
+ }
count(skipped);
entryOffset += skipped;
return skipped;
@@ -348,7 +371,7 @@ public class TarArchiveInputStream extends ArchiveInputStream {
if (currEntry.isPaxHeader()){ // Process Pax headers
paxHeaders();
} else if (!globalPaxHeaders.isEmpty()) {
- applyPaxHeadersToCurrentEntry(globalPaxHeaders);
+ applyPaxHeadersToCurrentEntry(globalPaxHeaders, globalSparseHeaders);
}
if (currEntry.isOldGNUSparse()){ // Process sparse files
@@ -466,22 +489,166 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
private void readGlobalPaxHeaders() throws IOException {
- globalPaxHeaders = parsePaxHeaders(this);
+ globalPaxHeaders = parsePaxHeaders(this, globalSparseHeaders);
getNextEntry(); // Get the actual file entry
}
+ /**
+ * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes)
+ * may appear multi times, and they look like:
+ *
+ * GNU.sparse.size=size
+ * GNU.sparse.numblocks=numblocks
+ * repeat numblocks times
+ * GNU.sparse.offset=offset
+ * GNU.sparse.numbytes=numbytes
+ * end repeat
+ *
+ *
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ *
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ *
+ * For PAX Format 1.X:
+ * The sparse map itself is stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary.
+ * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers
+ * giving the offset and size of the data block it describes.
+ * @throws IOException
+ */
private void paxHeaders() throws IOException{
- final Map<String, String> headers = parsePaxHeaders(this);
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ final Map<String, String> headers = parsePaxHeaders(this, sparseHeaders);
+
+ // for 0.1 PAX Headers
+ if (headers.containsKey("GNU.sparse.map")) {
+ sparseHeaders = parsePAX01SparseHeaders(headers.get("GNU.sparse.map"));
+ }
getNextEntry(); // Get the actual file entry
- applyPaxHeadersToCurrentEntry(headers);
+ applyPaxHeadersToCurrentEntry(headers, sparseHeaders);
+
+ // for 1.0 PAX Format, the sparse map is stored in the file data block
+ if(currEntry.isPaxGNU1XSparse()) {
+ sparseHeaders = parsePAX1XSparseHeaders();
+ currEntry.setSparseHeaders(sparseHeaders);
+ }
+
+ // sparse headers are all done reading, we need to build
+ // a new input stream using these sparse headers
+ sparseInputStream = new TarArchiveSparseInputStream(sparseHeaders, is);
+ }
+
+ /**
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ * @param sparseMap the sparse map string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ * @return sparse headers parsed from sparse map
+ * @throws IOException
+ */
+ private List<TarArchiveStructSparse> parsePAX01SparseHeaders(String sparseMap) throws IOException {
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ String[] sparseHeaderStrings = sparseMap.split(",");
+
+ for (int i = 0; i < sparseHeaderStrings.length;i += 2) {
+ long sparseOffset = Long.parseLong(sparseHeaderStrings[i]);
+ long sparseNumbytes = Long.parseLong(sparseHeaderStrings[i + 1]);
+ sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes));
+ }
+
+ return sparseHeaders;
+ }
+
+ /**
+ * For PAX Format 1.X:
+ * The sparse map itself is stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary.
+ * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers
+ * giving the offset and size of the data block it describes.
+ * @return sparse headers
+ * @throws IOException
+ */
+ private List<TarArchiveStructSparse> parsePAX1XSparseHeaders() throws IOException {
+ // for 1.X PAX Headers
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ long bytesRead = 0;
+ long[] readResult;
+ long sparseHeadersCount;
+
+ readResult = readLineOfNumberForPax1X(is);
+ sparseHeadersCount = readResult[0];
+ bytesRead += readResult[1];
+ while (sparseHeadersCount-- > 0) {
+ readResult = readLineOfNumberForPax1X(is);
+ long sparseOffset = readResult[0];
+ bytesRead += readResult[1];
+
+ readResult = readLineOfNumberForPax1X(is);
+ long sparseNumbytes = readResult[0];
+ bytesRead += readResult[1];
+ sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes));
+ }
+
+ // skip the rest of this record data
+ long bytesToSkip = recordSize - bytesRead % recordSize;
+ IOUtils.skip(is, bytesToSkip);
+ return sparseHeaders;
+ }
+
+ /**
+ * For 1.X PAX Format, the sparse headers are stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines.
+ *
+ * @param inputStream the input stream of the tar file
+ * @return the decimal number delimited by '\n', and the bytes read from input stream
+ * @throws IOException
+ */
+ private long[] readLineOfNumberForPax1X(InputStream inputStream) throws IOException {
+ int number;
+ long result = 0;
+ long bytesRead = 0;
+
+ while((number = inputStream.read()) != '\n') {
+ bytesRead += 1;
+ if(number == -1) {
+ throw new IOException("Unexpected EOF when reading parse information of 1.X PAX format");
+ }
+ result = result * 10 + (number - '0');
+ }
+ bytesRead += 1;
+
+ return new long[] {result, bytesRead};
}
- // NOTE, using a Map here makes it impossible to ever support GNU
- // sparse files using the PAX Format 0.0, see
- // https://www.gnu.org/software/tar/manual/html_section/tar_92.html#SEC188
- Map<String, String> parsePaxHeaders(final InputStream i)
+ /**
+ * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes)
+ * may appear multi times, and they look like:
+ *
+ * GNU.sparse.size=size
+ * GNU.sparse.numblocks=numblocks
+ * repeat numblocks times
+ * GNU.sparse.offset=offset
+ * GNU.sparse.numbytes=numbytes
+ * end repeat
+ *
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ *
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ * @param i inputstream to read keys and values
+ * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times,
+ * the sparse headers need to be stored in an array, not a map
+ * @return
+ * @throws IOException
+ */
+ Map<String, String> parsePaxHeaders(final InputStream i, List<TarArchiveStructSparse> sparseHeaders)
throws IOException {
final Map<String, String> headers = new HashMap<>(globalPaxHeaders);
+ TarArchiveStructSparse sparseHeader = null;
// Format is "length keyword=value\n";
while(true){ // get length
int ch;
@@ -516,6 +683,22 @@ public class TarArchiveInputStream extends ArchiveInputStream {
final String value = new String(rest, 0,
restLen - 1, CharsetNames.UTF_8);
headers.put(keyword, value);
+
+ // for 0.0 PAX Headers
+ if(keyword.equals("GNU.sparse.offset")) {
+ sparseHeader = new TarArchiveStructSparse(Long.parseLong(value), 0);
+ sparseHeaders.add(sparseHeader);
+ }
+
+ // for 0.0 PAX Headers
+ if(keyword.equals("GNU.sparse.numbytes")) {
+ if(sparseHeader == null) {
+ throw new IOException("Failed to read Paxheader." +
+ "GNU.sparse.offset is expected before GNU.sparse.numbytes show up.");
+ }
+
+ sparseHeader.setNumbytes(Long.parseLong(value));
+ }
}
break;
}
@@ -533,8 +716,8 @@ public class TarArchiveInputStream extends ArchiveInputStream {
return headers;
}
- private void applyPaxHeadersToCurrentEntry(final Map<String, String> headers) {
- currEntry.updateEntryFromPaxHeaders(headers);
+ private void applyPaxHeadersToCurrentEntry(final Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) {
+ currEntry.updateEntryFromPaxHeaders(headers, sparseHeaders);
}
@@ -543,14 +726,8 @@ public class TarArchiveInputStream extends ArchiveInputStream {
* including any additional sparse entries following the current entry.
*
* @throws IOException on error
- *
- * @todo Sparse files get not yet really processed.
*/
private void readOldGNUSparse() throws IOException {
- /* we do not really process sparse files yet
- sparses = new ArrayList();
- sparses.addAll(currEntry.getSparses());
- */
if (currEntry.isExtended()) {
TarArchiveSparseEntry entry;
do {
@@ -560,11 +737,13 @@ public class TarArchiveInputStream extends ArchiveInputStream {
break;
}
entry = new TarArchiveSparseEntry(headerBuf);
- /* we do not really process sparse files yet
- sparses.addAll(entry.getSparses());
- */
+ currEntry.getSparseHeaders().addAll(entry.getSparseHeaders());
} while (entry.isExtended());
}
+
+ // sparse headers are all done reading, we need to build
+ // a new input stream using these sparse headers
+ sparseInputStream = new TarArchiveSparseInputStream(currEntry.getSparseHeaders(), is);
}
private boolean isDirectory() {
@@ -629,7 +808,7 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
int totalRead = 0;
- if (isAtEOF() || isDirectory() || entryOffset >= entrySize) {
+ if (isAtEOF() || isDirectory()) {
return -1;
}
@@ -637,9 +816,22 @@ public class TarArchiveInputStream extends ArchiveInputStream {
throw new IllegalStateException("No current tar entry");
}
+ if(!currEntry.isSparse() && entryOffset >= entrySize) {
+ return -1;
+ }
+
+ // for sparse entries, there are actually currEntry.getRealSize() bytes to read
+ if(currEntry.isSparse() && entryOffset >= currEntry.getRealSize()) {
+ return -1;
+ }
+
numToRead = Math.min(numToRead, available());
- totalRead = is.read(buf, offset, numToRead);
+ if(currEntry.isSparse()) {
+ totalRead = sparseInputStream.read(buf, offset, numToRead);
+ } else {
+ totalRead = is.read(buf, offset, numToRead);
+ }
if (totalRead == -1) {
if (numToRead > 0) {
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
index a49e718..7a8e2ee 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
@@ -19,6 +19,7 @@
package org.apache.commons.compress.archivers.tar;
import java.io.IOException;
+import java.util.List;
/**
* This class represents a sparse entry in a Tar archive.
@@ -44,6 +45,8 @@ public class TarArchiveSparseEntry implements TarConstants {
/** If an extension sparse header follows. */
private final boolean isExtended;
+ private List<TarArchiveStructSparse> sparseHeaders;
+
/**
* Construct an entry from an archive's header bytes. File is set
* to null.
@@ -53,6 +56,16 @@ public class TarArchiveSparseEntry implements TarConstants {
*/
public TarArchiveSparseEntry(final byte[] headerBuf) throws IOException {
int offset = 0;
+ for(int i = 0; i < SPARSE_HEADERS_IN_EXTENSION_HEADER;i++) {
+ TarArchiveStructSparse sparseHeader = TarUtils.parseSparse(headerBuf,
+ offset + i * (SPARSE_OFFSET_LEN + SPARSE_NUMBYTES_LEN));
+
+ // some sparse headers are empty, we need to skip these sparse headers
+ if(sparseHeader.getOffset() > 0 || sparseHeader.getNumbytes() > 0) {
+ sparseHeaders.add(sparseHeader);
+ }
+ }
+
offset += SPARSELEN_GNU_SPARSE;
isExtended = TarUtils.parseBoolean(headerBuf, offset);
}
@@ -60,4 +73,8 @@ public class TarArchiveSparseEntry implements TarConstants {
public boolean isExtended() {
return isExtended;
}
+
+ public List<TarArchiveStructSparse> getSparseHeaders() {
+ return sparseHeaders;
+ }
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java
new file mode 100644
index 0000000..7dd609b
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.archivers.tar;
+
+import org.apache.commons.compress.utils.BoundedInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * For sparse tar entries, there are many "holes"(consisting of all 0) in the file. Only the non-zero data is
+ * stored in tar files, and they are stored separately. The structure of non-zero data is introduced by the
+ * sparse headers using the offset, where a block of non-zero data starts, and numbytes, the length of the
+ * non-zero data block.
+ * This class is used to construct an input stream that combines the "holes" and the non-zero data together using
+ * the sparse headers. When reading from this input stream, the actual data is read out with "holes" and non-zero
+ * data combined together according to the sparse headers.
+ */
+public class TarArchiveSparseInputStream extends InputStream {
+ /** the sparse headers describing the sparse information */
+ private List<TarArchiveStructSparse> sparseHeaders;
+
+ /** the input stream of the tar file */
+ private InputStream inputStream;
+
+ /** the input streams consisting of all-zero input streams and non-zero streams */
+ private List<InputStream> inputStreams;
+
+ /** the index of current input stream being read */
+ private int currentInputStreamIndex = -1;
+
+ public TarArchiveSparseInputStream(List<TarArchiveStructSparse> sparseHeaders, InputStream inputStream) {
+ this.sparseHeaders = sparseHeaders;
+ this.inputStream = inputStream;
+ buildInputStreams();
+
+ if (inputStreams.size() > 0) {
+ currentInputStreamIndex = 0;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ // if there are no actual input streams, just read from the original input stream
+ if (inputStreams.size() == 0) {
+ return inputStream.read();
+ }
+
+ int value = inputStreams.get(currentInputStreamIndex).read();
+ if (value != -1) {
+ return value;
+ }
+
+ if (currentInputStreamIndex == inputStreams.size() - 1) {
+ return -1;
+ }
+
+ currentInputStreamIndex++;
+ return inputStreams.get(currentInputStreamIndex).read();
+ }
+
+ @Override
+ public int read(byte[] buf) throws IOException {
+ return read(buf, 0, buf.length);
+ }
+
+ @Override
+ public int read(byte[] buf, int offset, int len) throws IOException {
+ // if there are no actual input streams, just read from the original input stream
+ if (inputStreams.size() == 0) {
+ return inputStream.read(buf, offset, len);
+ }
+
+ InputStream currentInputStream = inputStreams.get(currentInputStreamIndex);
+ int readLen = currentInputStream.read(buf, offset, len);
+
+ // if the current input stream is the last input stream,
+ // just return the number of bytes read from current input stream
+ if (currentInputStreamIndex == inputStreams.size() - 1) {
+ return readLen;
+ }
+
+ // if EOF of current input stream is meet, open a new input stream and recursively call read
+ if (readLen == -1) {
+ currentInputStreamIndex++;
+ return read(buf, offset, len);
+ }
+
+ // if the rest data of current input stream is not long enough, open a new input stream
+ // and recursively call read
+ if (readLen < len) {
+ currentInputStreamIndex++;
+ int readLenOfNext = read(buf, offset + readLen, len - readLen);
+ if (readLenOfNext == -1) {
+ return readLen;
+ }
+
+ return readLen + readLenOfNext;
+ }
+
+ // if the rest data of current input stream is enough(which means readLen == len), just return readLen
+ return readLen;
+ }
+
+ /**
+ * Skip n bytes from current input stream, if the current input stream doesn't have enough data to skip,
+ * jump to the next input stream and skip the rest bytes, keep doing this until total n bytes are skipped
+ * or the input streams are all skipped
+ *
+ * @param n bytes of data to skip
+ * @return actual bytes of data skipped
+ * @throws IOException
+ */
+ @Override
+ public long skip(final long n) throws IOException {
+ if (inputStreams.size() == 0) {
+ return inputStream.skip(n);
+ }
+
+ long bytesSkipped = 0;
+ InputStream currentInputStream;
+
+ while (bytesSkipped < n && currentInputStreamIndex < inputStreams.size()) {
+ currentInputStream = inputStreams.get(currentInputStreamIndex);
+ bytesSkipped += currentInputStream.skip(n - bytesSkipped);
+
+ if (bytesSkipped < n) {
+ currentInputStreamIndex++;
+ }
+ }
+
+ return bytesSkipped;
+ }
+
+ /**
+ * Close all the input streams in inputStreams
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ for (InputStream inputStream : inputStreams) {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Build the input streams consisting of all-zero input streams and non-zero input streams.
+ * When reading from the non-zero input streams, the data is actually read from the original input stream.
+ * The size of each input stream is introduced by the sparse headers.
+ *
+ * NOTE : Some all-zero input streams and non-zero input streams have the size of 0. We DO NOT store the
+ * 0 size input streams because they are meaningless.
+ */
+ private void buildInputStreams() {
+ inputStreams = new ArrayList<>();
+ InputStream zeroInputStream = new TarArchiveSparseZeroInputStream();
+
+ long offset = 0;
+ for (TarArchiveStructSparse sparseHeader : sparseHeaders) {
+ if (sparseHeader.getOffset() == 0 && sparseHeader.getNumbytes() == 0) {
+ break;
+ }
+
+ // only store the input streams with non-zero size
+ if ((sparseHeader.getOffset() - offset) > 0) {
+ inputStreams.add(new BoundedInputStream(zeroInputStream, sparseHeader.getOffset() - offset));
+ }
+
+ // only store the input streams with non-zero size
+ if (sparseHeader.getNumbytes() > 0) {
+ inputStreams.add(new BoundedInputStream(inputStream, sparseHeader.getNumbytes()));
+ }
+
+ offset = sparseHeader.getOffset() + sparseHeader.getNumbytes();
+ }
+ }
+
+ /**
+ * This is an inputstream that always return 0,
+ * this is used when writing the holes of a sparse file
+ */
+ public class TarArchiveSparseZeroInputStream extends InputStream {
+ /**
+ * Just return 0
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public int read() throws IOException {
+ return 0;
+ }
+
+ /**
+ * these's nothing need to do when skipping
+ *
+ * @param n bytes to skip
+ * @return bytes actually skipped
+ */
+ @Override
+ public long skip(final long n) {
+ return n;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
similarity index 51%
copy from src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
copy to src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
index a49e718..ac08e68 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
@@ -18,19 +18,9 @@
*/
package org.apache.commons.compress.archivers.tar;
-import java.io.IOException;
-
/**
- * This class represents a sparse entry in a Tar archive.
- *
+ * This class represents struct sparse in a Tar archive.
* <p>
- * The C structure for a sparse entry is:
- * <pre>
- * struct posix_header {
- * struct sparse sp[21]; // TarConstants.SPARSELEN_GNU_SPARSE - offset 0
- * char isextended; // TarConstants.ISEXTENDEDLEN_GNU_SPARSE - offset 504
- * };
- * </pre>
* Whereas, "struct sparse" is:
* <pre>
* struct sparse {
@@ -39,25 +29,29 @@ import java.io.IOException;
* };
* </pre>
*/
+public class TarArchiveStructSparse {
+ private long offset;
+
+ private long numbytes;
-public class TarArchiveSparseEntry implements TarConstants {
- /** If an extension sparse header follows. */
- private final boolean isExtended;
+ public TarArchiveStructSparse(long offset, long numbytes) {
+ this.offset = offset;
+ this.numbytes = numbytes;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public void setOffset(long offset) {
+ this.offset = offset;
+ }
- /**
- * Construct an entry from an archive's header bytes. File is set
- * to null.
- *
- * @param headerBuf The header bytes from a tar archive entry.
- * @throws IOException on unknown format
- */
- public TarArchiveSparseEntry(final byte[] headerBuf) throws IOException {
- int offset = 0;
- offset += SPARSELEN_GNU_SPARSE;
- isExtended = TarUtils.parseBoolean(headerBuf, offset);
+ public long getNumbytes() {
+ return numbytes;
}
- public boolean isExtended() {
- return isExtended;
+ public void setNumbytes(long numbytes) {
+ this.numbytes = numbytes;
}
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
index 751840d..6fd959d 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
@@ -186,6 +186,30 @@ public interface TarConstants {
int REALSIZELEN_GNU = 12;
/**
+ * The length of offset in struct sparse
+ *
+ */
+ int SPARSE_OFFSET_LEN = 12;
+
+ /**
+ * The length of numbytes in struct sparse
+ *
+ */
+ int SPARSE_NUMBYTES_LEN = 12;
+
+ /**
+ * The number of sparse headers in an old GNU header
+ *
+ */
+ int SPARSE_HEADERS_IN_OLDGNU_HEADER = 4;
+
+ /**
+ * The number of sparse headers in an extension header
+ *
+ */
+ int SPARSE_HEADERS_IN_EXTENSION_HEADER = 21;
+
+ /**
* The sum of the length of all sparse headers in a sparse header buffer.
*
*/
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
index de30c89..1ecb57e 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
@@ -18,15 +18,17 @@
*/
package org.apache.commons.compress.archivers.tar;
-import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUMLEN;
-import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUM_OFFSET;
-
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import org.apache.commons.compress.archivers.zip.ZipEncoding;
import org.apache.commons.compress.archivers.zip.ZipEncodingHelper;
+import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUMLEN;
+import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUM_OFFSET;
+import static org.apache.commons.compress.archivers.tar.TarConstants.SPARSE_NUMBYTES_LEN;
+import static org.apache.commons.compress.archivers.tar.TarConstants.SPARSE_OFFSET_LEN;
+
/**
* This class provides static utility methods to work with byte streams.
*
@@ -301,6 +303,13 @@ public class TarUtils {
return "";
}
+ public static TarArchiveStructSparse parseSparse(final byte[] buffer, final int offset) {
+ long sparseOffset = parseOctalOrBinary(buffer, offset, SPARSE_OFFSET_LEN);
+ long sparseNumbytes = parseOctalOrBinary(buffer, offset + SPARSE_OFFSET_LEN, SPARSE_NUMBYTES_LEN);
+
+ return new TarArchiveStructSparse(sparseOffset, sparseNumbytes);
+ }
+
/**
* Copy a name into a buffer.
* Copies characters from the name into the buffer
diff --git a/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java b/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
index e089631..6d33445 100644
--- a/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
@@ -73,4 +73,13 @@ public class BoundedInputStream extends InputStream {
// there isn't anything to close in this stream and the nested
// stream is controlled externally
}
+
+ @Override
+ public long skip(final long n) throws IOException {
+ long bytesToSkip = Math.min(bytesRemaining, n);
+ long bytesSkipped = in.skip(bytesToSkip);
+ bytesRemaining -= bytesSkipped;
+
+ return bytesSkipped;
+ }
}
diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java
index b7dcb06..661e66b 100644
--- a/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java
@@ -54,7 +54,7 @@ public class TarArchiveInputStreamTest {
final TarArchiveInputStream tais = new TarArchiveInputStream(is);
final Map<String, String> headers = tais
.parsePaxHeaders(new ByteArrayInputStream("30 atime=1321711775.972059463\n"
- .getBytes(CharsetNames.UTF_8)));
+ .getBytes(CharsetNames.UTF_8)), null);
assertEquals(1, headers.size());
assertEquals("1321711775.972059463", headers.get("atime"));
tais.close();
@@ -66,7 +66,7 @@ public class TarArchiveInputStreamTest {
final TarArchiveInputStream tais = new TarArchiveInputStream(is);
final Map<String, String> headers = tais
.parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n11 foo=baz\n"
- .getBytes(CharsetNames.UTF_8)));
+ .getBytes(CharsetNames.UTF_8)), null);
assertEquals(1, headers.size());
assertEquals("baz", headers.get("foo"));
tais.close();
@@ -78,7 +78,7 @@ public class TarArchiveInputStreamTest {
final TarArchiveInputStream tais = new TarArchiveInputStream(is);
final Map<String, String> headers = tais
.parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n7 foo=\n"
- .getBytes(CharsetNames.UTF_8)));
+ .getBytes(CharsetNames.UTF_8)), null);
assertEquals(0, headers.size());
tais.close();
}
@@ -89,7 +89,7 @@ public class TarArchiveInputStreamTest {
final TarArchiveInputStream tais = new TarArchiveInputStream(is);
final Map<String, String> headers = tais
.parsePaxHeaders(new ByteArrayInputStream("28 comment=line1\nline2\nand3\n"
- .getBytes(CharsetNames.UTF_8)));
+ .getBytes(CharsetNames.UTF_8)), null);
assertEquals(1, headers.size());
assertEquals("line1\nline2\nand3", headers.get("comment"));
tais.close();
@@ -103,7 +103,7 @@ public class TarArchiveInputStreamTest {
final InputStream is = new ByteArrayInputStream(new byte[1]);
final TarArchiveInputStream tais = new TarArchiveInputStream(is);
final Map<String, String> headers = tais
- .parsePaxHeaders(new ByteArrayInputStream(line.getBytes(CharsetNames.UTF_8)));
+ .parsePaxHeaders(new ByteArrayInputStream(line.getBytes(CharsetNames.UTF_8)), null);
assertEquals(1, headers.size());
assertEquals(ae, headers.get("path"));
tais.close();
[commons-compress] 02/06: COMPRESS-124 : add testcases for
extracting sparse
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit 50569e5bfb1526c54acf0abe5a6e4d5463c5a4bd
Author: Lee <pe...@gmail.com>
AuthorDate: Fri Nov 22 17:24:10 2019 +0800
COMPRESS-124 : add testcases for extracting sparse
---
.../compress/archivers/tar/TarArchiveEntry.java | 24 +--
.../archivers/tar/TarArchiveInputStream.java | 237 ++++++++++++++++++---
.../archivers/tar/TarArchiveSparseEntry.java | 2 +
.../archivers/tar/TarArchiveSparseInputStream.java | 222 -------------------
.../archivers/tar/TarArchiveStructSparse.java | 24 +++
.../compress/archivers/tar/SparseFilesTest.java | 205 +++++++++++++++++-
.../compress/archivers/tar/TarUtilsTest.java | 12 ++
src/test/resources/oldgnu_extended_sparse.tar | Bin 0 -> 10240 bytes
8 files changed, 452 insertions(+), 274 deletions(-)
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
index 9012a3c..2aac73e 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
@@ -852,10 +852,14 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/**
* Get this entry's real file size in case of a sparse file.
+ * If the file is not a sparse file, return size instead of realSize.
*
- * @return This entry's real file size.
+ * @return This entry's real file size, if the file is not a sparse file, return size instead of realSize.
*/
public long getRealSize() {
+ if (!isSparse()) {
+ return size;
+ }
return realSize;
}
@@ -1077,16 +1081,13 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/**
* Update the entry using a map of pax headers.
* @param headers
- * @param sparseHeaders for 0.0 PAX Format, the sparse headers may appear more than 1 time in headers map,
-* * this means it can not be read from a map, therefore the sparse headers have already
-* * been parsed to a list and was passed through parameter sparseHeaders
* @since 1.15
*/
- void updateEntryFromPaxHeaders(Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) {
+ void updateEntryFromPaxHeaders(Map<String, String> headers) {
for (final Map.Entry<String, String> ent : headers.entrySet()) {
final String key = ent.getKey();
final String val = ent.getValue();
- processPaxHeader(key, val, headers, sparseHeaders);
+ processPaxHeader(key, val, headers);
}
}
@@ -1101,10 +1102,6 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
processPaxHeader(key,val,extraPaxHeaders);
}
- private void processPaxHeader(String key, String val, Map<String, String> headers) {
- processPaxHeader(key, val, headers, null);
- }
-
/**
* Process one pax header, using the supplied map as source for extra headers to be used when handling
* entries for sparse files
@@ -1112,13 +1109,9 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
* @param key the header name.
* @param val the header value.
* @param headers map of headers used for dealing with sparse file.
- * @param sparseHeaders for 0.0 PAX Format, the sparse headers may appear more than 1 time in headers map,
- * this means it can not be read from a map, therefore the sparse headers have already
- * been parsed to a list and was passed through parameter sparseHeaders
* @since 1.15
*/
- private void processPaxHeader(String key, String val, Map<String, String> headers,
- final List<TarArchiveStructSparse> sparseHeaders) {
+ private void processPaxHeader(String key, String val, Map<String, String> headers) {
/*
* The following headers are defined for Pax.
* atime, ctime, charset: cannot use these without changing TarArchiveEntry fields
@@ -1172,7 +1165,6 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
break;
case "GNU.sparse.size":
fillGNUSparse0xData(headers);
- this.sparseHeaders = sparseHeaders;
break;
case "GNU.sparse.realsize":
fillGNUSparse1xData(headers);
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
index 9194db8..72b6653 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
@@ -26,16 +26,14 @@ package org.apache.commons.compress.archivers.tar;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipEncoding;
import org.apache.commons.compress.archivers.zip.ZipEncodingHelper;
import org.apache.commons.compress.utils.ArchiveUtils;
+import org.apache.commons.compress.utils.BoundedInputStream;
import org.apache.commons.compress.utils.CharsetNames;
import org.apache.commons.compress.utils.IOUtils;
@@ -70,7 +68,12 @@ public class TarArchiveInputStream extends ArchiveInputStream {
/** An input stream to read from */
private final InputStream is;
- /** An input stream to read sparse file */
+ /** Input streams for reading sparse entries **/
+ private List<InputStream> sparseInputStreams;
+
+ /** the index of current input stream being read when reading sparse entries */
+ private int currentSparseInputStreamIndex;
+
private InputStream sparseInputStream;
/** The meta-data about the current entry */
@@ -86,7 +89,7 @@ public class TarArchiveInputStream extends ArchiveInputStream {
private Map<String, String> globalPaxHeaders = new HashMap<>();
// the global sparse headers, this is only used in PAX Format 0.X
- private List<TarArchiveStructSparse> globalSparseHeaders = new ArrayList<>();
+ private final List<TarArchiveStructSparse> globalSparseHeaders = new ArrayList<>();
private final boolean lenient;
@@ -193,6 +196,13 @@ public class TarArchiveInputStream extends ArchiveInputStream {
*/
@Override
public void close() throws IOException {
+ // Close all the input streams in sparseInputStreams
+ if(sparseInputStreams != null) {
+ for (InputStream inputStream : sparseInputStreams) {
+ inputStream.close();
+ }
+ }
+
is.close();
}
@@ -223,16 +233,10 @@ public class TarArchiveInputStream extends ArchiveInputStream {
return 0;
}
- // for sparse entries, there are actually currEntry.getRealSize() bytes to read
- long entryActualSize = entrySize;
- if (currEntry.isSparse()) {
- entryActualSize = currEntry.getRealSize();
- }
-
- if (entryActualSize - entryOffset > Integer.MAX_VALUE) {
+ if (currEntry.getRealSize() - entryOffset > Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
- return (int) (entryActualSize - entryOffset);
+ return (int) (currEntry.getRealSize() - entryOffset);
}
@@ -258,15 +262,12 @@ public class TarArchiveInputStream extends ArchiveInputStream {
return 0;
}
- long available;
+ long available = currEntry.getRealSize() - entryOffset;
long skipped;
if(!currEntry.isSparse()) {
- available = entrySize - entryOffset;
skipped = IOUtils.skip(is, Math.min(n, available));
} else {
- // for sparse entries, there are actually currEntry.getRealSize() bytes to read
- available = currEntry.getRealSize() - entryOffset;
- skipped = IOUtils.skip(sparseInputStream, Math.min(n, available));
+ skipped = skipSparse(n);
}
count(skipped);
entryOffset += skipped;
@@ -274,6 +275,35 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
/**
+ * Skip n bytes from current input stream, if the current input stream doesn't have enough data to skip,
+ * jump to the next input stream and skip the rest bytes, keep doing this until total n bytes are skipped
+ * or the input streams are all skipped
+ *
+ * @param n bytes of data to skip
+ * @return actual bytes of data skipped
+ * @throws IOException
+ */
+ private long skipSparse(final long n) throws IOException {
+ if (sparseInputStreams.size() == 0) {
+ return is.skip(n);
+ }
+
+ long bytesSkipped = 0;
+ InputStream currentInputStream;
+
+ while (bytesSkipped < n && currentSparseInputStreamIndex < sparseInputStreams.size()) {
+ currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
+ bytesSkipped += currentInputStream.skip(n - bytesSkipped);
+
+ if (bytesSkipped < n) {
+ currentSparseInputStreamIndex++;
+ }
+ }
+
+ return bytesSkipped;
+ }
+
+ /**
* Since we do not support marking just yet, we return false.
*
* @return False.
@@ -536,8 +566,8 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
// sparse headers are all done reading, we need to build
- // a new input stream using these sparse headers
- sparseInputStream = new TarArchiveSparseInputStream(sparseHeaders, is);
+ // sparse input streams using these sparse headers
+ buildSparseInputStreams();
}
/**
@@ -717,8 +747,8 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
private void applyPaxHeadersToCurrentEntry(final Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) {
- currEntry.updateEntryFromPaxHeaders(headers, sparseHeaders);
-
+ currEntry.updateEntryFromPaxHeaders(headers);
+ currEntry.setSparseHeaders(sparseHeaders);
}
/**
@@ -742,8 +772,8 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
// sparse headers are all done reading, we need to build
- // a new input stream using these sparse headers
- sparseInputStream = new TarArchiveSparseInputStream(currEntry.getSparseHeaders(), is);
+ // sparse input streams using these sparse headers
+ buildSparseInputStreams();
}
private boolean isDirectory() {
@@ -816,19 +846,22 @@ public class TarArchiveInputStream extends ArchiveInputStream {
throw new IllegalStateException("No current tar entry");
}
- if(!currEntry.isSparse() && entryOffset >= entrySize) {
- return -1;
- }
-
- // for sparse entries, there are actually currEntry.getRealSize() bytes to read
- if(currEntry.isSparse() && entryOffset >= currEntry.getRealSize()) {
- return -1;
+ if (!currEntry.isSparse()) {
+ if (entryOffset >= entrySize) {
+ return -1;
+ }
+ } else {
+ // for sparse entries, there are actually currEntry.getRealSize() bytes to read
+ if (entryOffset >= currEntry.getRealSize()) {
+ return -1;
+ }
}
numToRead = Math.min(numToRead, available());
- if(currEntry.isSparse()) {
- totalRead = sparseInputStream.read(buf, offset, numToRead);
+ if (currEntry.isSparse()) {
+ // for sparse entries, we need to read them in another way
+ totalRead = readSparse(buf, offset, numToRead);
} else {
totalRead = is.read(buf, offset, numToRead);
}
@@ -847,6 +880,61 @@ public class TarArchiveInputStream extends ArchiveInputStream {
}
/**
+ * For sparse tar entries, there are many "holes"(consisting of all 0) in the file. Only the non-zero data is
+ * stored in tar files, and they are stored separately. The structure of non-zero data is introduced by the
+ * sparse headers using the offset, where a block of non-zero data starts, and numbytes, the length of the
+ * non-zero data block.
+ * When reading sparse entries, the actual data is read out with "holes" and non-zero data combined together
+ * according to the sparse headers.
+ *
+ * @param buf The buffer into which to place bytes read.
+ * @param offset The offset at which to place bytes read.
+ * @param numToRead The number of bytes to read.
+ * @return The number of bytes read, or -1 at EOF.
+ * @throws IOException on error
+ */
+ private int readSparse(final byte[] buf, final int offset, int numToRead) throws IOException {
+ // if there are no actual input streams, just read from the original input stream
+ if (sparseInputStreams.size() == 0) {
+ return is.read(buf, offset, numToRead);
+ }
+
+ if(currentSparseInputStreamIndex >= sparseInputStreams.size()) {
+ return -1;
+ }
+
+ InputStream currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
+ int readLen = currentInputStream.read(buf, offset, numToRead);
+
+ // if the current input stream is the last input stream,
+ // just return the number of bytes read from current input stream
+ if (currentSparseInputStreamIndex == sparseInputStreams.size() - 1) {
+ return readLen;
+ }
+
+ // if EOF of current input stream is meet, open a new input stream and recursively call read
+ if (readLen == -1) {
+ currentSparseInputStreamIndex++;
+ return readSparse(buf, offset, numToRead);
+ }
+
+ // if the rest data of current input stream is not long enough, open a new input stream
+ // and recursively call read
+ if (readLen < numToRead) {
+ currentSparseInputStreamIndex++;
+ int readLenOfNext = readSparse(buf, offset + readLen, numToRead - readLen);
+ if (readLenOfNext == -1) {
+ return readLen;
+ }
+
+ return readLen + readLenOfNext;
+ }
+
+ // if the rest data of current input stream is enough(which means readLen == len), just return readLen
+ return readLen;
+ }
+
+ /**
* Whether this class is able to read the given entry.
*
* <p>May return false if the current entry is a sparse file.</p>
@@ -937,4 +1025,85 @@ public class TarArchiveInputStream extends ArchiveInputStream {
signature, TarConstants.VERSION_OFFSET, TarConstants.VERSIONLEN);
}
+ /**
+ * Build the input streams consisting of all-zero input streams and non-zero input streams.
+ * When reading from the non-zero input streams, the data is actually read from the original input stream.
+ * The size of each input stream is introduced by the sparse headers.
+ *
+ * NOTE : Some all-zero input streams and non-zero input streams have the size of 0. We DO NOT store the
+ * 0 size input streams because they are meaningless.
+ */
+ private void buildSparseInputStreams() throws IOException {
+ currentSparseInputStreamIndex = -1;
+ sparseInputStreams = new ArrayList<>();
+ InputStream zeroInputStream = new TarArchiveSparseZeroInputStream();
+
+ long offset = 0;
+ List<TarArchiveStructSparse> sparseHeaders = currEntry.getSparseHeaders();
+ // sort the sparse headers in case they are written in wrong order
+ if (sparseHeaders != null && sparseHeaders.size() > 1) {
+ final Comparator<TarArchiveStructSparse> sparseHeaderComparator = new Comparator<TarArchiveStructSparse>() {
+ @Override
+ public int compare(final TarArchiveStructSparse p, final TarArchiveStructSparse q) {
+ Long pOffset = p.getOffset();
+ Long qOffset = q.getOffset();
+ return pOffset.compareTo(qOffset);
+ }
+ };
+ Collections.sort(sparseHeaders, sparseHeaderComparator);
+ }
+
+ for (TarArchiveStructSparse sparseHeader : sparseHeaders) {
+ if (sparseHeader.getOffset() == 0 && sparseHeader.getNumbytes() == 0) {
+ break;
+ }
+
+ if ((sparseHeader.getOffset() - offset) < 0) {
+ throw new IOException("Corrupted struct sparse detected");
+ }
+
+ // only store the input streams with non-zero size
+ if ((sparseHeader.getOffset() - offset) > 0) {
+ sparseInputStreams.add(new BoundedInputStream(zeroInputStream, sparseHeader.getOffset() - offset));
+ }
+
+ // only store the input streams with non-zero size
+ if (sparseHeader.getNumbytes() > 0) {
+ sparseInputStreams.add(new BoundedInputStream(is, sparseHeader.getNumbytes()));
+ }
+
+ offset = sparseHeader.getOffset() + sparseHeader.getNumbytes();
+ }
+
+ if (sparseInputStreams.size() > 0) {
+ currentSparseInputStreamIndex = 0;
+ }
+ }
+
+ /**
+ * This is an inputstream that always return 0,
+ * this is used when reading the "holes" of a sparse file
+ */
+ public class TarArchiveSparseZeroInputStream extends InputStream {
+ /**
+ * Just return 0
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public int read() throws IOException {
+ return 0;
+ }
+
+ /**
+ * these's nothing need to do when skipping
+ *
+ * @param n bytes to skip
+ * @return bytes actually skipped
+ */
+ @Override
+ public long skip(final long n) {
+ return n;
+ }
+ }
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
index 7a8e2ee..5599e06 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
@@ -19,6 +19,7 @@
package org.apache.commons.compress.archivers.tar;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -56,6 +57,7 @@ public class TarArchiveSparseEntry implements TarConstants {
*/
public TarArchiveSparseEntry(final byte[] headerBuf) throws IOException {
int offset = 0;
+ sparseHeaders = new ArrayList<>();
for(int i = 0; i < SPARSE_HEADERS_IN_EXTENSION_HEADER;i++) {
TarArchiveStructSparse sparseHeader = TarUtils.parseSparse(headerBuf,
offset + i * (SPARSE_OFFSET_LEN + SPARSE_NUMBYTES_LEN));
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java
deleted file mode 100644
index 7dd609b..0000000
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseInputStream.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.commons.compress.archivers.tar;
-
-import org.apache.commons.compress.utils.BoundedInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * For sparse tar entries, there are many "holes"(consisting of all 0) in the file. Only the non-zero data is
- * stored in tar files, and they are stored separately. The structure of non-zero data is introduced by the
- * sparse headers using the offset, where a block of non-zero data starts, and numbytes, the length of the
- * non-zero data block.
- * This class is used to construct an input stream that combines the "holes" and the non-zero data together using
- * the sparse headers. When reading from this input stream, the actual data is read out with "holes" and non-zero
- * data combined together according to the sparse headers.
- */
-public class TarArchiveSparseInputStream extends InputStream {
- /** the sparse headers describing the sparse information */
- private List<TarArchiveStructSparse> sparseHeaders;
-
- /** the input stream of the tar file */
- private InputStream inputStream;
-
- /** the input streams consisting of all-zero input streams and non-zero streams */
- private List<InputStream> inputStreams;
-
- /** the index of current input stream being read */
- private int currentInputStreamIndex = -1;
-
- public TarArchiveSparseInputStream(List<TarArchiveStructSparse> sparseHeaders, InputStream inputStream) {
- this.sparseHeaders = sparseHeaders;
- this.inputStream = inputStream;
- buildInputStreams();
-
- if (inputStreams.size() > 0) {
- currentInputStreamIndex = 0;
- }
- }
-
- @Override
- public int read() throws IOException {
- // if there are no actual input streams, just read from the original input stream
- if (inputStreams.size() == 0) {
- return inputStream.read();
- }
-
- int value = inputStreams.get(currentInputStreamIndex).read();
- if (value != -1) {
- return value;
- }
-
- if (currentInputStreamIndex == inputStreams.size() - 1) {
- return -1;
- }
-
- currentInputStreamIndex++;
- return inputStreams.get(currentInputStreamIndex).read();
- }
-
- @Override
- public int read(byte[] buf) throws IOException {
- return read(buf, 0, buf.length);
- }
-
- @Override
- public int read(byte[] buf, int offset, int len) throws IOException {
- // if there are no actual input streams, just read from the original input stream
- if (inputStreams.size() == 0) {
- return inputStream.read(buf, offset, len);
- }
-
- InputStream currentInputStream = inputStreams.get(currentInputStreamIndex);
- int readLen = currentInputStream.read(buf, offset, len);
-
- // if the current input stream is the last input stream,
- // just return the number of bytes read from current input stream
- if (currentInputStreamIndex == inputStreams.size() - 1) {
- return readLen;
- }
-
- // if EOF of current input stream is meet, open a new input stream and recursively call read
- if (readLen == -1) {
- currentInputStreamIndex++;
- return read(buf, offset, len);
- }
-
- // if the rest data of current input stream is not long enough, open a new input stream
- // and recursively call read
- if (readLen < len) {
- currentInputStreamIndex++;
- int readLenOfNext = read(buf, offset + readLen, len - readLen);
- if (readLenOfNext == -1) {
- return readLen;
- }
-
- return readLen + readLenOfNext;
- }
-
- // if the rest data of current input stream is enough(which means readLen == len), just return readLen
- return readLen;
- }
-
- /**
- * Skip n bytes from current input stream, if the current input stream doesn't have enough data to skip,
- * jump to the next input stream and skip the rest bytes, keep doing this until total n bytes are skipped
- * or the input streams are all skipped
- *
- * @param n bytes of data to skip
- * @return actual bytes of data skipped
- * @throws IOException
- */
- @Override
- public long skip(final long n) throws IOException {
- if (inputStreams.size() == 0) {
- return inputStream.skip(n);
- }
-
- long bytesSkipped = 0;
- InputStream currentInputStream;
-
- while (bytesSkipped < n && currentInputStreamIndex < inputStreams.size()) {
- currentInputStream = inputStreams.get(currentInputStreamIndex);
- bytesSkipped += currentInputStream.skip(n - bytesSkipped);
-
- if (bytesSkipped < n) {
- currentInputStreamIndex++;
- }
- }
-
- return bytesSkipped;
- }
-
- /**
- * Close all the input streams in inputStreams
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- for (InputStream inputStream : inputStreams) {
- inputStream.close();
- }
- }
-
- /**
- * Build the input streams consisting of all-zero input streams and non-zero input streams.
- * When reading from the non-zero input streams, the data is actually read from the original input stream.
- * The size of each input stream is introduced by the sparse headers.
- *
- * NOTE : Some all-zero input streams and non-zero input streams have the size of 0. We DO NOT store the
- * 0 size input streams because they are meaningless.
- */
- private void buildInputStreams() {
- inputStreams = new ArrayList<>();
- InputStream zeroInputStream = new TarArchiveSparseZeroInputStream();
-
- long offset = 0;
- for (TarArchiveStructSparse sparseHeader : sparseHeaders) {
- if (sparseHeader.getOffset() == 0 && sparseHeader.getNumbytes() == 0) {
- break;
- }
-
- // only store the input streams with non-zero size
- if ((sparseHeader.getOffset() - offset) > 0) {
- inputStreams.add(new BoundedInputStream(zeroInputStream, sparseHeader.getOffset() - offset));
- }
-
- // only store the input streams with non-zero size
- if (sparseHeader.getNumbytes() > 0) {
- inputStreams.add(new BoundedInputStream(inputStream, sparseHeader.getNumbytes()));
- }
-
- offset = sparseHeader.getOffset() + sparseHeader.getNumbytes();
- }
- }
-
- /**
- * This is an inputstream that always return 0,
- * this is used when writing the holes of a sparse file
- */
- public class TarArchiveSparseZeroInputStream extends InputStream {
- /**
- * Just return 0
- * @return
- * @throws IOException
- */
- @Override
- public int read() throws IOException {
- return 0;
- }
-
- /**
- * these's nothing need to do when skipping
- *
- * @param n bytes to skip
- * @return bytes actually skipped
- */
- @Override
- public long skip(final long n) {
- return n;
- }
- }
-}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
index ac08e68..8221a25 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
@@ -18,6 +18,8 @@
*/
package org.apache.commons.compress.archivers.tar;
+import java.util.Objects;
+
/**
* This class represents struct sparse in a Tar archive.
* <p>
@@ -39,6 +41,28 @@ public class TarArchiveStructSparse {
this.numbytes = numbytes;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TarArchiveStructSparse that = (TarArchiveStructSparse) o;
+ return offset == that.offset &&
+ numbytes == that.numbytes;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(offset, numbytes);
+ }
+
+ @Override
+ public String toString() {
+ return "TarArchiveStructSparse{" +
+ "offset=" + offset +
+ ", numbytes=" + numbytes +
+ '}';
+ }
+
public long getOffset() {
return offset;
}
diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
index 56ac227..33b0c91 100644
--- a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
@@ -18,15 +18,24 @@
package org.apache.commons.compress.archivers.tar;
-import static org.apache.commons.compress.AbstractTestCase.getFile;
import static org.junit.Assert.*;
+
+import org.apache.commons.compress.AbstractTestCase;
+import org.junit.Assert;
import org.junit.Test;
+import shaded.org.apache.commons.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Locale;
+
+public class SparseFilesTest extends AbstractTestCase {
-public class SparseFilesTest {
+ private final boolean isOnWindows = System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows");
@Test
public void testOldGNU() throws Throwable {
@@ -40,6 +49,18 @@ public class SparseFilesTest {
assertTrue(ae.isGNUSparse());
assertFalse(ae.isPaxGNUSparse());
assertFalse(tin.canReadEntryData(ae));
+
+ List<TarArchiveStructSparse> sparseHeaders = ae.getSparseHeaders();
+ assertEquals(3, sparseHeaders.size());
+
+ assertEquals(0, sparseHeaders.get(0).getOffset());
+ assertEquals(2048, sparseHeaders.get(0).getNumbytes());
+
+ assertEquals(1050624L, sparseHeaders.get(1).getOffset());
+ assertEquals(2560, sparseHeaders.get(1).getNumbytes());
+
+ assertEquals(3101184L, sparseHeaders.get(2).getOffset());
+ assertEquals(0, sparseHeaders.get(2).getNumbytes());
} finally {
if (tin != null) {
tin.close();
@@ -63,6 +84,159 @@ public class SparseFilesTest {
}
}
+ @Test
+ public void testExtractSparseTarsOnWindows() throws IOException {
+ if (!isOnWindows) {
+ return;
+ }
+
+ final File oldGNUSparseTar = getFile("oldgnu_sparse.tar");
+ final File paxGNUSparseTar = getFile("pax_gnu_sparse.tar");
+ TarArchiveInputStream oldGNUSparseInputStream = null;
+ TarArchiveInputStream paxGNUSparseInputStream = null;
+ try {
+ // compare between old GNU and PAX 0.0
+ oldGNUSparseInputStream = new TarArchiveInputStream(new FileInputStream(oldGNUSparseTar));
+ oldGNUSparseInputStream.getNextTarEntry();
+ paxGNUSparseInputStream = new TarArchiveInputStream(new FileInputStream(paxGNUSparseTar));
+ paxGNUSparseInputStream.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(oldGNUSparseInputStream, paxGNUSparseInputStream));
+
+ // compare between old GNU and PAX 0.1
+ oldGNUSparseInputStream.close();
+ oldGNUSparseInputStream = new TarArchiveInputStream(new FileInputStream(oldGNUSparseTar));
+ oldGNUSparseInputStream.getNextTarEntry();
+ paxGNUSparseInputStream.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(oldGNUSparseInputStream, paxGNUSparseInputStream));
+
+ // compare between old GNU and PAX 1.0
+ oldGNUSparseInputStream.close();
+ oldGNUSparseInputStream = new TarArchiveInputStream(new FileInputStream(oldGNUSparseTar));
+ oldGNUSparseInputStream.getNextTarEntry();
+ paxGNUSparseInputStream.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(oldGNUSparseInputStream, paxGNUSparseInputStream));
+ } finally {
+ if (oldGNUSparseInputStream != null) {
+ oldGNUSparseInputStream.close();
+ }
+
+ if (paxGNUSparseInputStream != null) {
+ paxGNUSparseInputStream.close();
+ }
+ }
+ }
+
+ @Test
+ public void testExtractOldGNU() throws IOException, InterruptedException {
+ if (isOnWindows) {
+ return;
+ }
+
+ final File file = getFile("oldgnu_sparse.tar");
+ InputStream sparseFileInputStream = null;
+ TarArchiveInputStream tin = null;
+ try {
+ sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile");
+ tin = new TarArchiveInputStream(new FileInputStream(file));
+ tin.getNextTarEntry();
+
+ Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
+ } finally {
+ if (sparseFileInputStream != null) {
+ sparseFileInputStream.close();
+ }
+
+ if (tin != null) {
+ tin.close();
+ }
+ }
+ }
+
+ @Test
+ public void testExtractExtendedOldGNU() throws IOException, InterruptedException {
+ if (isOnWindows) {
+ return;
+ }
+
+ final File file = getFile("oldgnu_extended_sparse.tar");
+ InputStream sparseFileInputStream = null;
+ TarArchiveInputStream tin = null;
+ try {
+ sparseFileInputStream = extractTarAndGetInputStream(file, "sparse6");
+ tin = new TarArchiveInputStream(new FileInputStream(file));
+ final TarArchiveEntry ae = tin.getNextTarEntry();
+
+ Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
+
+ List<TarArchiveStructSparse> sparseHeaders = ae.getSparseHeaders();
+ assertEquals(7, sparseHeaders.size());
+
+ assertEquals(0, sparseHeaders.get(0).getOffset());
+ assertEquals(1024, sparseHeaders.get(0).getNumbytes());
+
+ assertEquals(10240, sparseHeaders.get(1).getOffset());
+ assertEquals(1024, sparseHeaders.get(1).getNumbytes());
+
+ assertEquals(16384, sparseHeaders.get(2).getOffset());
+ assertEquals(1024, sparseHeaders.get(2).getNumbytes());
+
+ assertEquals(24576, sparseHeaders.get(3).getOffset());
+ assertEquals(1024, sparseHeaders.get(3).getNumbytes());
+
+ assertEquals(29696, sparseHeaders.get(4).getOffset());
+ assertEquals(1024, sparseHeaders.get(4).getNumbytes());
+
+ assertEquals(36864, sparseHeaders.get(5).getOffset());
+ assertEquals(1024, sparseHeaders.get(5).getNumbytes());
+
+ assertEquals(51200, sparseHeaders.get(6).getOffset());
+ assertEquals(0, sparseHeaders.get(6).getNumbytes());
+ } finally {
+ if (sparseFileInputStream != null) {
+ sparseFileInputStream.close();
+ }
+
+ if (tin != null) {
+ tin.close();
+ }
+ }
+ }
+
+ @Test
+ public void testExtractPaxGNU() throws IOException, InterruptedException {
+ if (isOnWindows) {
+ return;
+ }
+
+ final File file = getFile("pax_gnu_sparse.tar");
+ InputStream sparseFileInputStream = null;
+ TarArchiveInputStream tin = null;
+ try {
+ sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-0.0");
+ tin = new TarArchiveInputStream(new FileInputStream(file));
+ tin.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
+
+ // TODO : it's wired that I can only get a 0 size sparsefile-0.1 on my Ubuntu 16.04
+ // using "tar -xf pax_gnu_sparse.tar"
+ sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-0.0");
+ tin.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
+
+ sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-1.0");
+ tin.getNextTarEntry();
+ Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
+ } finally {
+ if (sparseFileInputStream != null) {
+ sparseFileInputStream.close();
+ }
+
+ if (tin != null) {
+ tin.close();
+ }
+ }
+ }
+
private void assertPaxGNUEntry(final TarArchiveInputStream tin, final String suffix) throws Throwable {
final TarArchiveEntry ae = tin.getNextTarEntry();
assertEquals("sparsefile-" + suffix, ae.getName());
@@ -70,6 +244,33 @@ public class SparseFilesTest {
assertTrue(ae.isPaxGNUSparse());
assertFalse(ae.isOldGNUSparse());
assertFalse(tin.canReadEntryData(ae));
+
+ List<TarArchiveStructSparse> sparseHeaders = ae.getSparseHeaders();
+ assertEquals(3, sparseHeaders.size());
+
+ assertEquals(0, sparseHeaders.get(0).getOffset());
+ assertEquals(2048, sparseHeaders.get(0).getNumbytes());
+
+ assertEquals(1050624L, sparseHeaders.get(1).getOffset());
+ assertEquals(2560, sparseHeaders.get(1).getNumbytes());
+
+ assertEquals(3101184L, sparseHeaders.get(2).getOffset());
+ assertEquals(0, sparseHeaders.get(2).getNumbytes());
+ }
+
+ private InputStream extractTarAndGetInputStream(File tarFile, String sparseFileName) throws IOException, InterruptedException {
+ Runtime runtime = Runtime.getRuntime();
+ Process process = runtime.exec("tar -xf " + tarFile.getPath() + " -C " + resultDir.getPath());
+ // wait until the extract finishes
+ process.waitFor();
+
+ for (File file : resultDir.listFiles()) {
+ if(file.getName().equals(sparseFileName)) {
+ return new FileInputStream(file);
+ }
+ }
+
+ return null;
}
}
diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java
index 393c0aa..91b0ef7 100644
--- a/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java
@@ -381,4 +381,16 @@ public class TarUtilsTest {
}
}
+ @Test
+ public void testParseSparse() {
+ final long expectedOffset = 0100000;
+ final long expectedNumbytes = 0111000;
+ final byte [] buffer = new byte[] {
+ ' ', ' ', ' ', ' ', ' ', '0', '1', '0', '0', '0', '0', '0', // sparseOffset
+ ' ', ' ', ' ', ' ', ' ', '0', '1', '1', '1', '0', '0', '0'};
+ TarArchiveStructSparse sparse = TarUtils.parseSparse(buffer, 0);
+ assertEquals(sparse.getOffset(), expectedOffset);
+ assertEquals(sparse.getNumbytes(), expectedNumbytes);
+ }
+
}
diff --git a/src/test/resources/oldgnu_extended_sparse.tar b/src/test/resources/oldgnu_extended_sparse.tar
new file mode 100644
index 0000000..cba3ebc
Binary files /dev/null and b/src/test/resources/oldgnu_extended_sparse.tar differ
[commons-compress] 04/06: Merge branch 'COMPRESS-124' of
https://github.com/PeterAlfreadLee/commons-compress into
PeterAlfreadLee-COMPRESS-124
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit 03b7d1ecec4811fe7e6ae63eacb1e63291eaaf6e
Merge: f7f7c53 50569e5
Author: Stefan Bodewig <bo...@apache.org>
AuthorDate: Wed Jan 1 17:26:07 2020 +0100
Merge branch 'COMPRESS-124' of https://github.com/PeterAlfreadLee/commons-compress into PeterAlfreadLee-COMPRESS-124
.../compress/archivers/tar/TarArchiveEntry.java | 53 ++-
.../archivers/tar/TarArchiveInputStream.java | 417 +++++++++++++++++++--
.../archivers/tar/TarArchiveSparseEntry.java | 19 +
.../archivers/tar/TarArchiveStructSparse.java | 81 ++++
.../compress/archivers/tar/TarConstants.java | 24 ++
.../commons/compress/archivers/tar/TarUtils.java | 15 +-
.../commons/compress/utils/BoundedInputStream.java | 9 +
.../compress/archivers/tar/SparseFilesTest.java | 205 +++++++++-
.../archivers/tar/TarArchiveInputStreamTest.java | 10 +-
.../compress/archivers/tar/TarUtilsTest.java | 12 +
src/test/resources/oldgnu_extended_sparse.tar | Bin 0 -> 10240 bytes
11 files changed, 806 insertions(+), 39 deletions(-)
diff --cc src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
index 65b7e32,72b6653..0f6b70f
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
@@@ -66,8 -66,16 +66,16 @@@ public class TarArchiveInputStream exte
private long entryOffset;
/** An input stream to read from */
- private final InputStream is;
+ private final InputStream inputStream;
+ /** Input streams for reading sparse entries **/
+ private List<InputStream> sparseInputStreams;
+
+ /** the index of current input stream being read when reading sparse entries */
+ private int currentSparseInputStreamIndex;
+
+ private InputStream sparseInputStream;
+
/** The meta-data about the current entry */
private TarArchiveEntry currEntry;
@@@ -185,7 -196,14 +196,14 @@@
*/
@Override
public void close() throws IOException {
+ // Close all the input streams in sparseInputStreams
+ if(sparseInputStreams != null) {
+ for (InputStream inputStream : sparseInputStreams) {
+ inputStream.close();
+ }
+ }
+
- is.close();
+ inputStream.close();
}
/**
@@@ -243,14 -262,48 +262,48 @@@
return 0;
}
- final long available = entrySize - entryOffset;
- final long skipped = IOUtils.skip(inputStream, Math.min(n, available));
- long available = currEntry.getRealSize() - entryOffset;
- long skipped;
++ final long available = currEntry.getRealSize() - entryOffset;
++ final long skipped;
+ if(!currEntry.isSparse()) {
- skipped = IOUtils.skip(is, Math.min(n, available));
++ skipped = IOUtils.skip(inputStream, Math.min(n, available));
+ } else {
+ skipped = skipSparse(n);
+ }
count(skipped);
entryOffset += skipped;
return skipped;
}
/**
+ * Skip n bytes from current input stream, if the current input stream doesn't have enough data to skip,
+ * jump to the next input stream and skip the rest bytes, keep doing this until total n bytes are skipped
+ * or the input streams are all skipped
+ *
+ * @param n bytes of data to skip
+ * @return actual bytes of data skipped
+ * @throws IOException
+ */
+ private long skipSparse(final long n) throws IOException {
+ if (sparseInputStreams.size() == 0) {
- return is.skip(n);
++ return inputStream.skip(n);
+ }
+
+ long bytesSkipped = 0;
+ InputStream currentInputStream;
+
+ while (bytesSkipped < n && currentSparseInputStreamIndex < sparseInputStreams.size()) {
+ currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
+ bytesSkipped += currentInputStream.skip(n - bytesSkipped);
+
+ if (bytesSkipped < n) {
+ currentSparseInputStreamIndex++;
+ }
+ }
+
+ return bytesSkipped;
+ }
+
+ /**
* Since we do not support marking just yet, we return false.
*
* @return False.
@@@ -470,20 -523,164 +523,164 @@@
getNextEntry(); // Get the actual file entry
}
+ /**
+ * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes)
+ * may appear multi times, and they look like:
+ *
+ * GNU.sparse.size=size
+ * GNU.sparse.numblocks=numblocks
+ * repeat numblocks times
+ * GNU.sparse.offset=offset
+ * GNU.sparse.numbytes=numbytes
+ * end repeat
+ *
+ *
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ *
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ *
+ * For PAX Format 1.X:
+ * The sparse map itself is stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary.
+ * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers
+ * giving the offset and size of the data block it describes.
+ * @throws IOException
+ */
private void paxHeaders() throws IOException{
- final Map<String, String> headers = parsePaxHeaders(this);
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ final Map<String, String> headers = parsePaxHeaders(this, sparseHeaders);
+
+ // for 0.1 PAX Headers
+ if (headers.containsKey("GNU.sparse.map")) {
+ sparseHeaders = parsePAX01SparseHeaders(headers.get("GNU.sparse.map"));
+ }
getNextEntry(); // Get the actual file entry
- applyPaxHeadersToCurrentEntry(headers);
+ applyPaxHeadersToCurrentEntry(headers, sparseHeaders);
+
+ // for 1.0 PAX Format, the sparse map is stored in the file data block
+ if(currEntry.isPaxGNU1XSparse()) {
+ sparseHeaders = parsePAX1XSparseHeaders();
+ currEntry.setSparseHeaders(sparseHeaders);
+ }
+
+ // sparse headers are all done reading, we need to build
+ // sparse input streams using these sparse headers
+ buildSparseInputStreams();
}
- // NOTE, using a Map here makes it impossible to ever support GNU
- // sparse files using the PAX Format 0.0, see
- // https://www.gnu.org/software/tar/manual/html_section/tar_92.html#SEC188
- Map<String, String> parsePaxHeaders(final InputStream inputStream)
+ /**
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ * @param sparseMap the sparse map string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ * @return sparse headers parsed from sparse map
+ * @throws IOException
+ */
+ private List<TarArchiveStructSparse> parsePAX01SparseHeaders(String sparseMap) throws IOException {
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ String[] sparseHeaderStrings = sparseMap.split(",");
+
+ for (int i = 0; i < sparseHeaderStrings.length;i += 2) {
+ long sparseOffset = Long.parseLong(sparseHeaderStrings[i]);
+ long sparseNumbytes = Long.parseLong(sparseHeaderStrings[i + 1]);
+ sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes));
+ }
+
+ return sparseHeaders;
+ }
+
+ /**
+ * For PAX Format 1.X:
+ * The sparse map itself is stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary.
+ * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers
+ * giving the offset and size of the data block it describes.
+ * @return sparse headers
+ * @throws IOException
+ */
+ private List<TarArchiveStructSparse> parsePAX1XSparseHeaders() throws IOException {
+ // for 1.X PAX Headers
+ List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
+ long bytesRead = 0;
+ long[] readResult;
+ long sparseHeadersCount;
+
- readResult = readLineOfNumberForPax1X(is);
++ readResult = readLineOfNumberForPax1X(inputStream);
+ sparseHeadersCount = readResult[0];
+ bytesRead += readResult[1];
+ while (sparseHeadersCount-- > 0) {
- readResult = readLineOfNumberForPax1X(is);
++ readResult = readLineOfNumberForPax1X(inputStream);
+ long sparseOffset = readResult[0];
+ bytesRead += readResult[1];
+
- readResult = readLineOfNumberForPax1X(is);
++ readResult = readLineOfNumberForPax1X(inputStream);
+ long sparseNumbytes = readResult[0];
+ bytesRead += readResult[1];
+ sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes));
+ }
+
+ // skip the rest of this record data
+ long bytesToSkip = recordSize - bytesRead % recordSize;
- IOUtils.skip(is, bytesToSkip);
++ IOUtils.skip(inputStream, bytesToSkip);
+ return sparseHeaders;
+ }
+
+ /**
+ * For 1.X PAX Format, the sparse headers are stored in the file data block, preceding the actual file data.
+ * It consists of a series of decimal numbers delimited by newlines.
+ *
+ * @param inputStream the input stream of the tar file
+ * @return the decimal number delimited by '\n', and the bytes read from input stream
+ * @throws IOException
+ */
+ private long[] readLineOfNumberForPax1X(InputStream inputStream) throws IOException {
+ int number;
+ long result = 0;
+ long bytesRead = 0;
+
+ while((number = inputStream.read()) != '\n') {
+ bytesRead += 1;
+ if(number == -1) {
+ throw new IOException("Unexpected EOF when reading parse information of 1.X PAX format");
+ }
+ result = result * 10 + (number - '0');
+ }
+ bytesRead += 1;
+
+ return new long[] {result, bytesRead};
+ }
+
+ /**
+ * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes)
+ * may appear multi times, and they look like:
+ *
+ * GNU.sparse.size=size
+ * GNU.sparse.numblocks=numblocks
+ * repeat numblocks times
+ * GNU.sparse.offset=offset
+ * GNU.sparse.numbytes=numbytes
+ * end repeat
+ *
+ * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map
+ *
+ * GNU.sparse.map
+ * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
+ *
+ * @param i inputstream to read keys and values
+ * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times,
+ * the sparse headers need to be stored in an array, not a map
+ * @return
+ * @throws IOException
+ */
- Map<String, String> parsePaxHeaders(final InputStream i, List<TarArchiveStructSparse> sparseHeaders)
++ Map<String, String> parsePaxHeaders(final InputStream inputStream, List<TarArchiveStructSparse> sparseHeaders)
throws IOException {
final Map<String, String> headers = new HashMap<>(globalPaxHeaders);
+ TarArchiveStructSparse sparseHeader = null;
// Format is "length keyword=value\n";
-- while(true){ // get length
++ while(true) { // get length
int ch;
int len = 0;
int read = 0;
@@@ -637,9 -846,25 +846,25 @@@
throw new IllegalStateException("No current tar entry");
}
+ if (!currEntry.isSparse()) {
+ if (entryOffset >= entrySize) {
+ return -1;
+ }
+ } else {
+ // for sparse entries, there are actually currEntry.getRealSize() bytes to read
+ if (entryOffset >= currEntry.getRealSize()) {
+ return -1;
+ }
+ }
+
numToRead = Math.min(numToRead, available());
- totalRead = inputStream.read(buf, offset, numToRead);
+ if (currEntry.isSparse()) {
+ // for sparse entries, we need to read them in another way
+ totalRead = readSparse(buf, offset, numToRead);
+ } else {
- totalRead = is.read(buf, offset, numToRead);
++ totalRead = inputStream.read(buf, offset, numToRead);
+ }
if (totalRead == -1) {
if (numToRead > 0) {
@@@ -655,6 -880,61 +880,61 @@@
}
/**
+ * For sparse tar entries, there are many "holes"(consisting of all 0) in the file. Only the non-zero data is
+ * stored in tar files, and they are stored separately. The structure of non-zero data is introduced by the
+ * sparse headers using the offset, where a block of non-zero data starts, and numbytes, the length of the
+ * non-zero data block.
+ * When reading sparse entries, the actual data is read out with "holes" and non-zero data combined together
+ * according to the sparse headers.
+ *
+ * @param buf The buffer into which to place bytes read.
+ * @param offset The offset at which to place bytes read.
+ * @param numToRead The number of bytes to read.
+ * @return The number of bytes read, or -1 at EOF.
+ * @throws IOException on error
+ */
+ private int readSparse(final byte[] buf, final int offset, int numToRead) throws IOException {
+ // if there are no actual input streams, just read from the original input stream
+ if (sparseInputStreams.size() == 0) {
- return is.read(buf, offset, numToRead);
++ return inputStream.read(buf, offset, numToRead);
+ }
+
+ if(currentSparseInputStreamIndex >= sparseInputStreams.size()) {
+ return -1;
+ }
+
+ InputStream currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
+ int readLen = currentInputStream.read(buf, offset, numToRead);
+
+ // if the current input stream is the last input stream,
+ // just return the number of bytes read from current input stream
+ if (currentSparseInputStreamIndex == sparseInputStreams.size() - 1) {
+ return readLen;
+ }
+
+ // if EOF of current input stream is meet, open a new input stream and recursively call read
+ if (readLen == -1) {
+ currentSparseInputStreamIndex++;
+ return readSparse(buf, offset, numToRead);
+ }
+
+ // if the rest data of current input stream is not long enough, open a new input stream
+ // and recursively call read
+ if (readLen < numToRead) {
+ currentSparseInputStreamIndex++;
+ int readLenOfNext = readSparse(buf, offset + readLen, numToRead - readLen);
+ if (readLenOfNext == -1) {
+ return readLen;
+ }
+
+ return readLen + readLenOfNext;
+ }
+
+ // if the rest data of current input stream is enough(which means readLen == len), just return readLen
+ return readLen;
+ }
+
+ /**
* Whether this class is able to read the given entry.
*
* <p>May return false if the current entry is a sparse file.</p>
@@@ -745,4 -1025,85 +1025,85 @@@
signature, TarConstants.VERSION_OFFSET, TarConstants.VERSIONLEN);
}
+ /**
+ * Build the input streams consisting of all-zero input streams and non-zero input streams.
+ * When reading from the non-zero input streams, the data is actually read from the original input stream.
+ * The size of each input stream is introduced by the sparse headers.
+ *
+ * NOTE : Some all-zero input streams and non-zero input streams have the size of 0. We DO NOT store the
+ * 0 size input streams because they are meaningless.
+ */
+ private void buildSparseInputStreams() throws IOException {
+ currentSparseInputStreamIndex = -1;
+ sparseInputStreams = new ArrayList<>();
+ InputStream zeroInputStream = new TarArchiveSparseZeroInputStream();
+
+ long offset = 0;
+ List<TarArchiveStructSparse> sparseHeaders = currEntry.getSparseHeaders();
+ // sort the sparse headers in case they are written in wrong order
+ if (sparseHeaders != null && sparseHeaders.size() > 1) {
+ final Comparator<TarArchiveStructSparse> sparseHeaderComparator = new Comparator<TarArchiveStructSparse>() {
+ @Override
+ public int compare(final TarArchiveStructSparse p, final TarArchiveStructSparse q) {
+ Long pOffset = p.getOffset();
+ Long qOffset = q.getOffset();
+ return pOffset.compareTo(qOffset);
+ }
+ };
+ Collections.sort(sparseHeaders, sparseHeaderComparator);
+ }
+
+ for (TarArchiveStructSparse sparseHeader : sparseHeaders) {
+ if (sparseHeader.getOffset() == 0 && sparseHeader.getNumbytes() == 0) {
+ break;
+ }
+
+ if ((sparseHeader.getOffset() - offset) < 0) {
+ throw new IOException("Corrupted struct sparse detected");
+ }
+
+ // only store the input streams with non-zero size
+ if ((sparseHeader.getOffset() - offset) > 0) {
+ sparseInputStreams.add(new BoundedInputStream(zeroInputStream, sparseHeader.getOffset() - offset));
+ }
+
+ // only store the input streams with non-zero size
+ if (sparseHeader.getNumbytes() > 0) {
- sparseInputStreams.add(new BoundedInputStream(is, sparseHeader.getNumbytes()));
++ sparseInputStreams.add(new BoundedInputStream(inputStream, sparseHeader.getNumbytes()));
+ }
+
+ offset = sparseHeader.getOffset() + sparseHeader.getNumbytes();
+ }
+
+ if (sparseInputStreams.size() > 0) {
+ currentSparseInputStreamIndex = 0;
+ }
+ }
+
+ /**
+ * This is an inputstream that always return 0,
+ * this is used when reading the "holes" of a sparse file
+ */
+ public class TarArchiveSparseZeroInputStream extends InputStream {
+ /**
+ * Just return 0
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public int read() throws IOException {
+ return 0;
+ }
+
+ /**
+ * these's nothing need to do when skipping
+ *
+ * @param n bytes to skip
+ * @return bytes actually skipped
+ */
+ @Override
+ public long skip(final long n) {
+ return n;
+ }
+ }
}
[commons-compress] 03/06: Happy new year
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit f7f7c53904e27e591d921caf4f17d95775fbbc0e
Author: Stefan Bodewig <bo...@apache.org>
AuthorDate: Wed Jan 1 17:00:32 2020 +0100
Happy new year
---
NOTICE.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/NOTICE.txt b/NOTICE.txt
index 63f8bd9..e3182cd 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
Apache Commons Compress
-Copyright 2002-2019 The Apache Software Foundation
+Copyright 2002-2020 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
[commons-compress] 06/06: COMPRESS-124 review cosmetics
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit b465bfa6a73cfdaf440d26e56966cf60d6761f83
Author: Stefan Bodewig <bo...@apache.org>
AuthorDate: Wed Jan 1 18:11:01 2020 +0100
COMPRESS-124 review cosmetics
see #87
---
.../compress/archivers/tar/TarArchiveEntry.java | 5 +++-
.../archivers/tar/TarArchiveInputStream.java | 26 +++++++++---------
.../archivers/tar/TarArchiveSparseEntry.java | 3 +++
.../archivers/tar/TarArchiveStructSparse.java | 1 +
.../compress/archivers/tar/TarConstants.java | 10 +++----
.../commons/compress/archivers/tar/TarUtils.java | 4 +++
.../commons/compress/utils/BoundedInputStream.java | 3 +++
.../compress/archivers/tar/SparseFilesTest.java | 31 +++-------------------
8 files changed, 38 insertions(+), 45 deletions(-)
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
index 2aac73e..b240157 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java
@@ -754,6 +754,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/**
* Set this entry's sparse headers
* @param sparseHeaders The new sparse headers
+ * @since 1.20
*/
public void setSparseHeaders(List<TarArchiveStructSparse> sparseHeaders) {
this.sparseHeaders = sparseHeaders;
@@ -763,6 +764,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
* Get this entry's sparse headers
*
* @return This entry's sparse headers
+ * @since 1.20
*/
public List<TarArchiveStructSparse> getSparseHeaders() {
return sparseHeaders;
@@ -772,6 +774,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
* Get if this entry is a sparse file with 1.X PAX Format or not
*
* @return True if this entry is a sparse file with 1.X PAX Format
+ * @since 1.20
*/
public boolean isPaxGNU1XSparse() {
return paxGNU1XSparse;
@@ -852,7 +855,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants {
/**
* Get this entry's real file size in case of a sparse file.
- * If the file is not a sparse file, return size instead of realSize.
+ * <p>If the file is not a sparse file, return size instead of realSize.</p>
*
* @return This entry's real file size, if the file is not a sparse file, return size instead of realSize.
*/
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
index 0f6b70f..9687ec7 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java
@@ -26,7 +26,12 @@ package org.apache.commons.compress.archivers.tar;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
@@ -264,10 +269,10 @@ public class TarArchiveInputStream extends ArchiveInputStream {
final long available = currEntry.getRealSize() - entryOffset;
final long skipped;
- if(!currEntry.isSparse()) {
+ if (!currEntry.isSparse()) {
skipped = IOUtils.skip(inputStream, Math.min(n, available));
} else {
- skipped = skipSparse(n);
+ skipped = skipSparse(Math.min(n, available));
}
count(skipped);
entryOffset += skipped;
@@ -284,15 +289,14 @@ public class TarArchiveInputStream extends ArchiveInputStream {
* @throws IOException
*/
private long skipSparse(final long n) throws IOException {
- if (sparseInputStreams.size() == 0) {
+ if (sparseInputStreams == null || sparseInputStreams.size() == 0) {
return inputStream.skip(n);
}
long bytesSkipped = 0;
- InputStream currentInputStream;
while (bytesSkipped < n && currentSparseInputStreamIndex < sparseInputStreams.size()) {
- currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
+ final InputStream currentInputStream = sparseInputStreams.get(currentSparseInputStreamIndex);
bytesSkipped += currentInputStream.skip(n - bytesSkipped);
if (bytesSkipped < n) {
@@ -605,11 +609,9 @@ public class TarArchiveInputStream extends ArchiveInputStream {
// for 1.X PAX Headers
List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>();
long bytesRead = 0;
- long[] readResult;
- long sparseHeadersCount;
- readResult = readLineOfNumberForPax1X(inputStream);
- sparseHeadersCount = readResult[0];
+ long[] readResult = readLineOfNumberForPax1X(inputStream);
+ long sparseHeadersCount = readResult[0];
bytesRead += readResult[1];
while (sparseHeadersCount-- > 0) {
readResult = readLineOfNumberForPax1X(inputStream);
@@ -670,7 +672,7 @@ public class TarArchiveInputStream extends ArchiveInputStream {
* Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]"
*
* @param i inputstream to read keys and values
- * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times,
+ * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times,
* the sparse headers need to be stored in an array, not a map
* @return
* @throws IOException
@@ -895,7 +897,7 @@ public class TarArchiveInputStream extends ArchiveInputStream {
*/
private int readSparse(final byte[] buf, final int offset, int numToRead) throws IOException {
// if there are no actual input streams, just read from the original input stream
- if (sparseInputStreams.size() == 0) {
+ if (sparseInputStreams == null || sparseInputStreams.size() == 0) {
return inputStream.read(buf, offset, numToRead);
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
index 5599e06..1d89fca 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseEntry.java
@@ -76,6 +76,9 @@ public class TarArchiveSparseEntry implements TarConstants {
return isExtended;
}
+ /**
+ * @since 1.20
+ */
public List<TarArchiveStructSparse> getSparseHeaders() {
return sparseHeaders;
}
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
index 8221a25..6bd4450 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveStructSparse.java
@@ -30,6 +30,7 @@ import java.util.Objects;
* char numbytes[12]; // offset 12
* };
* </pre>
+ * @since 1.20
*/
public class TarArchiveStructSparse {
private long offset;
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
index 6fd959d..ffd2aa5 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java
@@ -187,27 +187,27 @@ public interface TarConstants {
/**
* The length of offset in struct sparse
- *
+ * @since 1.20
*/
int SPARSE_OFFSET_LEN = 12;
/**
* The length of numbytes in struct sparse
- *
+ * @since 1.20
*/
int SPARSE_NUMBYTES_LEN = 12;
/**
* The number of sparse headers in an old GNU header
- *
+ * @since 1.20
*/
int SPARSE_HEADERS_IN_OLDGNU_HEADER = 4;
/**
* The number of sparse headers in an extension header
- *
+ * @since 1.20
*/
- int SPARSE_HEADERS_IN_EXTENSION_HEADER = 21;
+ int SPARSE_HEADERS_IN_EXTENSION_HEADER = 21;
/**
* The sum of the length of all sparse headers in a sparse header buffer.
diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
index 1ecb57e..860cd19 100644
--- a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
+++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java
@@ -303,6 +303,10 @@ public class TarUtils {
return "";
}
+ /**
+ * Parses the content of a PAX 1.0 sparse block.
+ * @since 1.20
+ */
public static TarArchiveStructSparse parseSparse(final byte[] buffer, final int offset) {
long sparseOffset = parseOctalOrBinary(buffer, offset, SPARSE_OFFSET_LEN);
long sparseNumbytes = parseOctalOrBinary(buffer, offset + SPARSE_OFFSET_LEN, SPARSE_NUMBYTES_LEN);
diff --git a/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java b/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
index 6d33445..b6b3622 100644
--- a/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java
@@ -74,6 +74,9 @@ public class BoundedInputStream extends InputStream {
// stream is controlled externally
}
+ /**
+ * @since 1.20
+ */
@Override
public long skip(final long n) throws IOException {
long bytesToSkip = Math.min(bytesRemaining, n);
diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
index 33b0c91..6056feb 100644
--- a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java
@@ -133,22 +133,10 @@ public class SparseFilesTest extends AbstractTestCase {
}
final File file = getFile("oldgnu_sparse.tar");
- InputStream sparseFileInputStream = null;
- TarArchiveInputStream tin = null;
- try {
- sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile");
- tin = new TarArchiveInputStream(new FileInputStream(file));
+ try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile");
+ TarArchiveInputStream tin = new TarArchiveInputStream(new FileInputStream(file))) {
tin.getNextTarEntry();
-
Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
- } finally {
- if (sparseFileInputStream != null) {
- sparseFileInputStream.close();
- }
-
- if (tin != null) {
- tin.close();
- }
}
}
@@ -159,11 +147,8 @@ public class SparseFilesTest extends AbstractTestCase {
}
final File file = getFile("oldgnu_extended_sparse.tar");
- InputStream sparseFileInputStream = null;
- TarArchiveInputStream tin = null;
- try {
- sparseFileInputStream = extractTarAndGetInputStream(file, "sparse6");
- tin = new TarArchiveInputStream(new FileInputStream(file));
+ try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparse6");
+ TarArchiveInputStream tin = new TarArchiveInputStream(new FileInputStream(file))) {
final TarArchiveEntry ae = tin.getNextTarEntry();
Assert.assertTrue(IOUtils.contentEquals(tin, sparseFileInputStream));
@@ -191,14 +176,6 @@ public class SparseFilesTest extends AbstractTestCase {
assertEquals(51200, sparseHeaders.get(6).getOffset());
assertEquals(0, sparseHeaders.get(6).getNumbytes());
- } finally {
- if (sparseFileInputStream != null) {
- sparseFileInputStream.close();
- }
-
- if (tin != null) {
- tin.close();
- }
}
}
[commons-compress] 05/06: Merge branch
'PeterAlfreadLee-COMPRESS-124'
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit 9bc32de43f5313696cbb90cf5998573a8288a6d0
Merge: f7f7c53 03b7d1e
Author: Stefan Bodewig <bo...@apache.org>
AuthorDate: Wed Jan 1 17:26:53 2020 +0100
Merge branch 'PeterAlfreadLee-COMPRESS-124'
.../compress/archivers/tar/TarArchiveEntry.java | 53 ++-
.../archivers/tar/TarArchiveInputStream.java | 417 +++++++++++++++++++--
.../archivers/tar/TarArchiveSparseEntry.java | 19 +
.../archivers/tar/TarArchiveStructSparse.java | 81 ++++
.../compress/archivers/tar/TarConstants.java | 24 ++
.../commons/compress/archivers/tar/TarUtils.java | 15 +-
.../commons/compress/utils/BoundedInputStream.java | 9 +
.../compress/archivers/tar/SparseFilesTest.java | 205 +++++++++-
.../archivers/tar/TarArchiveInputStreamTest.java | 10 +-
.../compress/archivers/tar/TarUtilsTest.java | 12 +
src/test/resources/oldgnu_extended_sparse.tar | Bin 0 -> 10240 bytes
11 files changed, 806 insertions(+), 39 deletions(-)