You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pdfbox.apache.org by le...@apache.org on 2023/07/22 08:40:51 UTC
svn commit: r1911194 - in /pdfbox/trunk: io/src/main/java/org/apache/pdfbox/io/ pdfbox/src/main/java/org/apache/pdfbox/cos/ pdfbox/src/main/java/org/apache/pdfbox/filter/ pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/
Author: lehmi
Date: Sat Jul 22 08:40:51 2023
New Revision: 1911194
URL: http://svn.apache.org/viewvc?rev=1911194&view=rev
Log:
PDFBOX-3712: use the new RandomAccessReadWriteBuffer when decoding streams to support streams bigger the 2GB and to reduce the memory foot print, DRY refactoring
Modified:
pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java
pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java
Modified: pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java (original)
+++ pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java Sat Jul 22 08:40:51 2023
@@ -29,7 +29,7 @@ import java.util.List;
public class RandomAccessReadBuffer implements RandomAccessRead
{
// default chunk size is 4kb
- protected static final int DEFAULT_CHUNK_SIZE_4KB = 1 << 12;
+ public static final int DEFAULT_CHUNK_SIZE_4KB = 1 << 12;
// use the default chunk size
protected int chunkSize = DEFAULT_CHUNK_SIZE_4KB;
// list containing all chunks
Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java Sat Jul 22 08:40:51 2023
@@ -17,20 +17,18 @@
package org.apache.pdfbox.cos;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.pdfbox.filter.DecodeOptions;
import org.apache.pdfbox.filter.DecodeResult;
import org.apache.pdfbox.filter.Filter;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
/**
* An InputStream which reads from an encoded COS stream.
@@ -71,26 +69,9 @@ public final class COSInputStream extend
{
return new COSInputStream(in, Collections.<DecodeResult>emptyList());
}
-
List<DecodeResult> results = new ArrayList<>(filters.size());
- InputStream input = in;
- if (filters.size() > 1)
- {
- Set<Filter> filterSet = new HashSet<>(filters);
- if (filterSet.size() != filters.size())
- {
- throw new IOException("Duplicate");
- }
- }
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- // apply filters
- for (int i = 0; i < filters.size(); i++)
- {
- output.reset();
- results.add(filters.get(i).decode(input, output, parameters, i, options));
- input = new ByteArrayInputStream(output.toByteArray());
- }
- return new COSInputStream(input, results);
+ RandomAccessRead decoded = Filter.decode(in, filters, parameters, options, results);
+ return new COSInputStream(new RandomAccessInputStream(decoded), results);
}
private final List<DecodeResult> decodeResults;
Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java Sat Jul 22 08:40:51 2023
@@ -16,17 +16,13 @@
*/
package org.apache.pdfbox.cos;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -197,37 +193,7 @@ public class COSStream extends COSDictio
return new RandomAccessReadBuffer(createRawInputStream());
}
}
- else
- {
- if (filterList.size() > 1)
- {
- Set<Filter> filterSet = new HashSet<>(filterList);
- if (filterSet.size() != filterList.size())
- {
- throw new IOException("Duplicate");
- }
- }
- InputStream input = createRawInputStream();
- ByteArrayOutputStream output = new ByteArrayOutputStream(input.available());
- // apply filters
- for (int i = 0; i < filterList.size(); i++)
- {
- if (i > 0)
- {
- input = new ByteArrayInputStream(output.toByteArray());
- output.reset();
- }
- try
- {
- filterList.get(i).decode(input, output, this, i, DecodeOptions.DEFAULT);
- }
- finally
- {
- IOUtils.closeQuietly(input);
- }
- }
- return new RandomAccessReadBuffer(output.toByteArray());
- }
+ return Filter.decode(createRawInputStream(), filterList, this, DecodeOptions.DEFAULT, null);
}
/**
Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java Sat Jul 22 08:40:51 2023
@@ -19,7 +19,10 @@ package org.apache.pdfbox.filter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
import java.util.zip.Deflater;
import javax.imageio.ImageIO;
@@ -31,6 +34,12 @@ import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.io.IOUtils;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessOutputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
+import org.apache.pdfbox.io.RandomAccessReadBuffer;
+import org.apache.pdfbox.io.RandomAccessReadWriteBuffer;
/**
* A filter for stream data.
@@ -178,4 +187,66 @@ public abstract class Filter
}
return Math.max(-1, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));
}
+
+ /**
+ * Decodes data, with optional DecodeOptions. Not all filters support all options, and so callers should check the
+ * options' <code>honored</code> flag to test if they were applied.
+ *
+ * @param encoded the input stream holding the encoded data
+ * @param filterList list of filters to be used for decoding
+ * @param parameters the parameters used for decoding
+ * @param options additional options for decoding
+ * @param results list of optional decoding results for each filter
+ * @return the decoded stream data
+ * @throws IOException if the stream cannot be decoded
+ */
+ public static RandomAccessRead decode(InputStream encoded, List<Filter> filterList,
+ COSDictionary parameters, DecodeOptions options, List<DecodeResult> results)
+ throws IOException
+ {
+ int length = parameters.getInt(COSName.LENGTH,
+ RandomAccessReadBuffer.DEFAULT_CHUNK_SIZE_4KB);
+ if (filterList.size() > 1)
+ {
+ Set<Filter> filterSet = new HashSet<>(filterList);
+ if (filterSet.size() != filterList.size())
+ {
+ throw new IOException("Duplicate");
+ }
+ }
+ InputStream input = encoded;
+ RandomAccessReadWriteBuffer randomAccessWriteBuffer = null;
+ OutputStream output = null;
+ // apply filters
+ for (int i = 0; i < filterList.size(); i++)
+ {
+ if (i > 0)
+ {
+ randomAccessWriteBuffer.seek(0);
+ input = new RandomAccessInputStream(randomAccessWriteBuffer);
+ length = (int) randomAccessWriteBuffer.length();
+ }
+ // we don't know the size of the decoded stream, just estimate a 4 times bigger size than the encoded stream
+ // use the estimated stream size as chunk size, use the default chunk size as limit to avoid to big values
+ randomAccessWriteBuffer = new RandomAccessReadWriteBuffer(
+ Math.min(length << 2, RandomAccessReadBuffer.DEFAULT_CHUNK_SIZE_4KB));
+ output = new RandomAccessOutputStream(randomAccessWriteBuffer);
+ try
+ {
+ DecodeResult result = filterList.get(i).decode(input, output, parameters, i,
+ options);
+ if (results != null)
+ {
+ results.add(result);
+ }
+ }
+ finally
+ {
+ IOUtils.closeQuietly(input);
+ }
+ }
+ randomAccessWriteBuffer.seek(0);
+ return randomAccessWriteBuffer;
+ }
+
}
Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java Sat Jul 22 08:40:51 2023
@@ -16,8 +16,6 @@
*/
package org.apache.pdfbox.pdmodel.common;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -40,6 +38,8 @@ import org.apache.pdfbox.filter.DecodeOp
import org.apache.pdfbox.filter.Filter;
import org.apache.pdfbox.filter.FilterFactory;
import org.apache.pdfbox.io.IOUtils;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.common.filespecification.PDFileSpecification;
@@ -205,31 +205,23 @@ public class PDStream implements COSObje
public InputStream createInputStream(List<String> stopFilters) throws IOException
{
InputStream is = stream.createRawInputStream();
- ByteArrayOutputStream os = new ByteArrayOutputStream();
+ List<Filter> someFilters = new ArrayList<>();
List<COSName> filters = getFilters();
- for (int i = 0; i < filters.size(); i++)
+ for (COSName nextFilter : filters)
{
- COSName nextFilter = filters.get(i);
- if ((stopFilters != null) && stopFilters.contains(nextFilter.getName()))
+ if (stopFilters != null && stopFilters.contains(nextFilter.getName()))
{
break;
}
- else
- {
- Filter filter = FilterFactory.INSTANCE.getFilter(nextFilter);
- try
- {
- filter.decode(is, os, stream, i);
- }
- finally
- {
- IOUtils.closeQuietly(is);
- }
- is = new ByteArrayInputStream(os.toByteArray());
- os.reset();
- }
+ someFilters.add(FilterFactory.INSTANCE.getFilter(nextFilter));
+ }
+ if (someFilters.isEmpty())
+ {
+ return is;
}
- return is;
+ RandomAccessRead decoded = Filter.decode(is, someFilters, getCOSObject(),
+ DecodeOptions.DEFAULT, null);
+ return new RandomAccessInputStream(decoded);
}
/**