You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@commons.apache.org by GitBox <gi...@apache.org> on 2019/05/05 21:10:55 UTC

[GitHub] [commons-compress] Arnaud-Nauwynck commented on a change in pull request #77: MNG-6276 reproducible builds : ensure ZipEntry order in ParallelScatterZipCreator

Arnaud-Nauwynck commented on a change in pull request #77: MNG-6276 reproducible builds : ensure ZipEntry order in ParallelScatterZipCreator
URL: https://github.com/apache/commons-compress/pull/77#discussion_r281038943
 
 

 ##########
 File path: src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
 ##########
 @@ -17,49 +17,145 @@
  */
 package org.apache.commons.compress.archivers.zip;
 
-import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
-import org.apache.commons.compress.parallel.InputStreamSupplier;
-import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
-import org.apache.commons.compress.parallel.ScatterGatherBackingStoreSupplier;
+import static java.util.Collections.synchronizedList;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.Deflater;
+import java.util.zip.ZipEntry;
 
-import static java.util.Collections.synchronizedList;
-import static org.apache.commons.compress.archivers.zip.ZipArchiveEntryRequest.createZipArchiveEntryRequest;
+import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
+import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
+import org.apache.commons.compress.parallel.ScatterGatherBackingStoreSupplier;
+import org.apache.commons.compress.utils.BoundedInputStream;
 
 /**
- * Creates a zip in parallel by using multiple threadlocal {@link ScatterZipOutputStream} instances.
- * <p>
- * Note that this class generally makes no guarantees about the order of things written to
- * the output file. Things that need to come in a specific order (manifests, directories)
- * must be handled by the client of this class, usually by writing these things to the
- * {@link ZipArchiveOutputStream} <em>before</em> calling {@link #writeTo writeTo} on this class.</p>
- * <p>
- * The client can supply an {@link java.util.concurrent.ExecutorService}, but for reasons of
- * memory model consistency, this will be shut down by this class prior to completion.
- * </p>
- * @since 1.10
+ * Creates a zip using internally parallel thread executor, but ensuring consistent zip entries order
+ *
+ * Public method of this class are expected to be called from a single client thread
+ *
+ * creating instances:
+ * either use externally defined ThreadPool executor (must have at least 2 threads, 1 for compressing + 1 for writing)
+ * <PRE>
+ * ExecutorService es = ..
+ * OrderedParallelScatterZipCreator zipCreator = new OrderedParallelScatterZipCreator(es, false);
+ * </PRE>
+ * or use auto-created (and shutdown) ThreadPool executor
+ * <PRE>
+ * OrderedParallelScatterZipCreator zipCreator = new OrderedParallelScatterZipCreator();
+ * <PRE>
+ *
+ * Adding entries, then writing result zip output
+ * <PRE>
+ * zipCreator.addArchiveEntry(..)
+ * zipCreator.addArchiveEntry(..)
+ *
+ * ZipArchiveOutputStream zipOutputStream = ...
+ * zipCreator.finishWrite(zipOutputStream);
+ * </PRE>
+ *
+ * or (better parallelization, writing compressed results on the fly)
+ *
+ * <PRE>
+ * zipCreator.addArchiveEntry(..)
+ * zipCreator.addArchiveEntry(..)
+ *
+ * ZipArchiveOutputStream zipOutputStream = ...
+ * zipCreator.startWriteTo(zipOutputStream);
+ *
+ * zipCreator.addArchiveEntry(..)
+ * zipCreator.addArchiveEntry(..)
+ *
+ * zipCreator.waitFinishWriteTo();
+ * </PRE>
  */
 public class ParallelScatterZipCreator {
-    private final List<ScatterZipOutputStream> streams = synchronizedList(new ArrayList<ScatterZipOutputStream>());
+
+    private static final ParallelScatterZipEntry EOF_MARKER = new ParallelScatterZipEntry(null, 0, null);
+
     private final ExecutorService es;
+    private final boolean closeExecutorService;
     private final ScatterGatherBackingStoreSupplier backingStoreSupplier;
-    private final List<Future<Object>> futures = new ArrayList<>();
 
-    private final long startedAt = System.currentTimeMillis();
-    private long compressionDoneAt = 0;
-    private long scatterDoneAt;
+    private final List<ThreadBackingStore> threadBackingStores = synchronizedList(new ArrayList<ThreadBackingStore>());
+
+    private BlockingQueue<Future<ParallelScatterZipEntry>> futureWriteQueue = new LinkedBlockingDeque<>();
+    private Future<?> mainWriter;
+    private Exception mainWriterEx;
+
+    private final AtomicLong compressionElapsed = new AtomicLong();
+    private final AtomicLong scatterElapsed = new AtomicLong();
+
+    public static class ParallelScatterZipEntry {
+        private final ZipArchiveEntry zipArchiveEntry;
+        private final long len;
+        private final ThreadBackingStore threadBackingStore;
+
+        ParallelScatterZipEntry(ZipArchiveEntry zipArchiveEntry,
+                long len, ThreadBackingStore threadBackingStore) {
+            this.zipArchiveEntry = zipArchiveEntry;
+            this.len = len;
+            this.threadBackingStore = threadBackingStore;
+        }
+
+        InputStream createInputStream() {
+            return new BoundedInputStream(threadBackingStore.inputStream, len);
+        }
+
+    }
+
+    /** inner input/output Streams backing store, per Thread */
+    private class ThreadBackingStore {
+        ScatterGatherBackingStore backingStore;
+        OutputStream ouputStream;
+        InputStream inputStream;
+
+        ThreadBackingStore(ScatterGatherBackingStore backingStore) throws IOException {
+            this.backingStore = backingStore;
+            this.ouputStream = createOutputStream();
+            this.inputStream = backingStore.getInputStream();
+        }
+
+        // ... adapter for non-existent method backingStore.createOutputStream();
+        // TODO move to class ScatterGatherBackingStore
+        OutputStream createOutputStream() {
+            return new OutputStream() {
+                @Override
+                public void write(int b) throws IOException {
+                    byte[] data = new byte[] { (byte) b };
 
 Review comment:
   Hi,
   This method is necessary to implements OutputStream methods, and delegate to 
      ScatterGatherBackingStore.writeOut(byte[] data, int offset, int length) throws IOException;
   Anyway, this method from this private adapter class is not used.
   
   Maybe, It would be better that ScatterGatherBackingStore expose directly an OutputStream (but it would change signatures), or expose a unitary "write(byte data)"
   
   Regards,

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services