You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pdfbox.apache.org by le...@apache.org on 2023/07/22 08:40:51 UTC

svn commit: r1911194 - in /pdfbox/trunk: io/src/main/java/org/apache/pdfbox/io/ pdfbox/src/main/java/org/apache/pdfbox/cos/ pdfbox/src/main/java/org/apache/pdfbox/filter/ pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/

Author: lehmi
Date: Sat Jul 22 08:40:51 2023
New Revision: 1911194

URL: http://svn.apache.org/viewvc?rev=1911194&view=rev
Log:
PDFBOX-3712: use the new RandomAccessReadWriteBuffer when decoding streams to support streams bigger the 2GB and to reduce the memory foot print, DRY refactoring

Modified:
    pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java
    pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java
    pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java
    pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java
    pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java

Modified: pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java (original)
+++ pdfbox/trunk/io/src/main/java/org/apache/pdfbox/io/RandomAccessReadBuffer.java Sat Jul 22 08:40:51 2023
@@ -29,7 +29,7 @@ import java.util.List;
 public class RandomAccessReadBuffer implements RandomAccessRead
 {
     // default chunk size is 4kb
-    protected static final int DEFAULT_CHUNK_SIZE_4KB = 1 << 12;
+    public static final int DEFAULT_CHUNK_SIZE_4KB = 1 << 12;
     // use the default chunk size
     protected int chunkSize = DEFAULT_CHUNK_SIZE_4KB;
     // list containing all chunks

Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSInputStream.java Sat Jul 22 08:40:51 2023
@@ -17,20 +17,18 @@
 
 package org.apache.pdfbox.cos;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.pdfbox.filter.DecodeOptions;
 import org.apache.pdfbox.filter.DecodeResult;
 import org.apache.pdfbox.filter.Filter;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
 
 /**
  * An InputStream which reads from an encoded COS stream.
@@ -71,26 +69,9 @@ public final class COSInputStream extend
         {
             return new COSInputStream(in, Collections.<DecodeResult>emptyList());
         }
-
         List<DecodeResult> results = new ArrayList<>(filters.size());
-        InputStream input = in;
-        if (filters.size() > 1)
-        {
-            Set<Filter> filterSet = new HashSet<>(filters);
-            if (filterSet.size() != filters.size())
-            {
-                throw new IOException("Duplicate");
-            }
-        }
-        ByteArrayOutputStream output = new ByteArrayOutputStream();
-        // apply filters
-        for (int i = 0; i < filters.size(); i++)
-        {
-            output.reset();
-            results.add(filters.get(i).decode(input, output, parameters, i, options));
-            input = new ByteArrayInputStream(output.toByteArray());
-        }
-        return new COSInputStream(input, results);
+        RandomAccessRead decoded = Filter.decode(in, filters, parameters, options, results);
+        return new COSInputStream(new RandomAccessInputStream(decoded), results);
     }
 
     private final List<DecodeResult> decodeResults;

Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/cos/COSStream.java Sat Jul 22 08:40:51 2023
@@ -16,17 +16,13 @@
  */
 package org.apache.pdfbox.cos;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.FilterOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -197,37 +193,7 @@ public class COSStream extends COSDictio
                 return new RandomAccessReadBuffer(createRawInputStream());
             }
         }
-        else
-        {
-            if (filterList.size() > 1)
-            {
-                Set<Filter> filterSet = new HashSet<>(filterList);
-                if (filterSet.size() != filterList.size())
-                {
-                    throw new IOException("Duplicate");
-                }
-            }
-            InputStream input = createRawInputStream();
-            ByteArrayOutputStream output = new ByteArrayOutputStream(input.available());
-            // apply filters
-            for (int i = 0; i < filterList.size(); i++)
-            {
-                if (i > 0)
-                {
-                    input = new ByteArrayInputStream(output.toByteArray());
-                    output.reset();
-                }
-                try
-                {
-                    filterList.get(i).decode(input, output, this, i, DecodeOptions.DEFAULT);
-                }
-                finally
-                {
-                    IOUtils.closeQuietly(input);
-                }
-            }
-            return new RandomAccessReadBuffer(output.toByteArray());
-        }
+        return Filter.decode(createRawInputStream(), filterList, this, DecodeOptions.DEFAULT, null);
     }
 
     /**

Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java Sat Jul 22 08:40:51 2023
@@ -19,7 +19,10 @@ package org.apache.pdfbox.filter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 import java.util.zip.Deflater;
 
 import javax.imageio.ImageIO;
@@ -31,6 +34,12 @@ import org.apache.pdfbox.cos.COSArray;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSDictionary;
 import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.io.IOUtils;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessOutputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
+import org.apache.pdfbox.io.RandomAccessReadBuffer;
+import org.apache.pdfbox.io.RandomAccessReadWriteBuffer;
 
 /**
  * A filter for stream data.
@@ -178,4 +187,66 @@ public abstract class Filter
         }
         return Math.max(-1, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));
     }
+
+    /**
+     * Decodes data, with optional DecodeOptions. Not all filters support all options, and so callers should check the
+     * options' <code>honored</code> flag to test if they were applied.
+     *
+     * @param encoded the input stream holding the encoded data
+     * @param filterList list of filters to be used for decoding
+     * @param parameters the parameters used for decoding
+     * @param options additional options for decoding
+     * @param results list of optional decoding results for each filter
+     * @return the decoded stream data
+     * @throws IOException if the stream cannot be decoded
+     */
+    public static RandomAccessRead decode(InputStream encoded, List<Filter> filterList,
+            COSDictionary parameters, DecodeOptions options, List<DecodeResult> results)
+            throws IOException
+    {
+        int length = parameters.getInt(COSName.LENGTH,
+                RandomAccessReadBuffer.DEFAULT_CHUNK_SIZE_4KB);
+        if (filterList.size() > 1)
+        {
+            Set<Filter> filterSet = new HashSet<>(filterList);
+            if (filterSet.size() != filterList.size())
+            {
+                throw new IOException("Duplicate");
+            }
+        }
+        InputStream input = encoded;
+        RandomAccessReadWriteBuffer randomAccessWriteBuffer = null;
+        OutputStream output = null;
+        // apply filters
+        for (int i = 0; i < filterList.size(); i++)
+        {
+            if (i > 0)
+            {
+                randomAccessWriteBuffer.seek(0);
+                input = new RandomAccessInputStream(randomAccessWriteBuffer);
+                length = (int) randomAccessWriteBuffer.length();
+            }
+            // we don't know the size of the decoded stream, just estimate a 4 times bigger size than the encoded stream
+            // use the estimated stream size as chunk size, use the default chunk size as limit to avoid to big values
+            randomAccessWriteBuffer = new RandomAccessReadWriteBuffer(
+                    Math.min(length << 2, RandomAccessReadBuffer.DEFAULT_CHUNK_SIZE_4KB));
+            output = new RandomAccessOutputStream(randomAccessWriteBuffer);
+            try
+            {
+                DecodeResult result = filterList.get(i).decode(input, output, parameters, i,
+                        options);
+                if (results != null)
+                {
+                    results.add(result);
+                }
+            }
+            finally
+            {
+                IOUtils.closeQuietly(input);
+            }
+        }
+        randomAccessWriteBuffer.seek(0);
+        return randomAccessWriteBuffer;
+    }
+    
 }

Modified: pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java
URL: http://svn.apache.org/viewvc/pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java?rev=1911194&r1=1911193&r2=1911194&view=diff
==============================================================================
--- pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java (original)
+++ pdfbox/trunk/pdfbox/src/main/java/org/apache/pdfbox/pdmodel/common/PDStream.java Sat Jul 22 08:40:51 2023
@@ -16,8 +16,6 @@
  */
 package org.apache.pdfbox.pdmodel.common;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -40,6 +38,8 @@ import org.apache.pdfbox.filter.DecodeOp
 import org.apache.pdfbox.filter.Filter;
 import org.apache.pdfbox.filter.FilterFactory;
 import org.apache.pdfbox.io.IOUtils;
+import org.apache.pdfbox.io.RandomAccessInputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.common.filespecification.PDFileSpecification;
 
@@ -205,31 +205,23 @@ public class PDStream implements COSObje
     public InputStream createInputStream(List<String> stopFilters) throws IOException
     {
         InputStream is = stream.createRawInputStream();
-        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        List<Filter> someFilters = new ArrayList<>();
         List<COSName> filters = getFilters();
-        for (int i = 0; i < filters.size(); i++)
+        for (COSName nextFilter : filters)
         {
-            COSName nextFilter = filters.get(i);
-            if ((stopFilters != null) && stopFilters.contains(nextFilter.getName()))
+            if (stopFilters != null && stopFilters.contains(nextFilter.getName()))
             {
                 break;
             }
-            else
-            {
-                Filter filter = FilterFactory.INSTANCE.getFilter(nextFilter);
-                try
-                {
-                    filter.decode(is, os, stream, i);
-                }
-                finally
-                {
-                    IOUtils.closeQuietly(is);
-                }
-                is = new ByteArrayInputStream(os.toByteArray());
-                os.reset();
-            }
+            someFilters.add(FilterFactory.INSTANCE.getFilter(nextFilter));
+        }
+        if (someFilters.isEmpty())
+        {
+            return is;
         }
-        return is;
+        RandomAccessRead decoded = Filter.decode(is, someFilters, getCOSObject(),
+                DecodeOptions.DEFAULT, null);
+        return new RandomAccessInputStream(decoded);
     }
 
     /**