You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/03/26 21:51:57 UTC

[tika] branch main updated: TIKA-3304 (#420)

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new e47c6cd  TIKA-3304 (#420)
e47c6cd is described below

commit e47c6cd62e587fdaae7e2e999f37122d09449754
Author: Tim Allison <ta...@apache.org>
AuthorDate: Fri Mar 26 17:51:49 2021 -0400

    TIKA-3304 (#420)
    
    * fix npe in when OCR is available and fix incorrect mark offset exception
    
    * fix indentation
    
    * Avoid reporting of temporary ocr-based mime type in xhtml output
    
    * TIKA-3304 -- experimental... WIP -- not to be merged
    
    * allow instance credentials in s3 components
    
    * WIP -- do not merge...allow instance credentials in s3 components
    
    * WIP -- do not merge...still a bunch to do
    
    * WIP -- do not merge...still a bunch to do
    
    * merge from main and required updates/conflict resolution
    
    * TIKA-3304 -- basically works...lots more remains
    
    * checkstyle
---
 .gitignore                                         |   1 +
 pom.xml                                            |   2 +-
 .../src/main/java/org/apache/tika/gui/TikaGUI.java | 188 +++----
 .../java/org/apache/tika/config/TikaConfig.java    |   2 +-
 .../TikaTimeoutException.java}                     |  24 +-
 .../apache/tika/pipes/emitter/AbstractEmitter.java |   4 +-
 .../org/apache/tika/pipes/emitter/EmitKey.java     |  10 +-
 .../org/apache/tika/pipes/emitter/Emitter.java     |   2 +-
 .../apache/tika/pipes/emitter/EmptyEmitter.java    |   2 +-
 .../org/apache/tika/pipes/fetcher/FetchKey.java    |  11 +-
 .../tika/pipes/fetchiterator/FetchEmitTuple.java   |   5 +-
 .../tika/pipes/fetchiterator/FetchIterator.java    | 129 +++--
 .../fetchiterator/FileSystemFetchIterator.java     |   4 +-
 .../fetchiterator/FileSystemFetchIteratorTest.java |  26 +-
 .../tika/parser/ocr/TesseractOCRParserTest.java    |   3 +-
 tika-pipes/pom.xml                                 |  38 +-
 tika-pipes/tika-emitters/tika-emitter-fs/pom.xml   |  13 +-
 .../tika/pipes/emitter/fs/FileSystemEmitter.java   |  44 +-
 tika-pipes/tika-emitters/tika-emitter-s3/pom.xml   |  22 +-
 .../apache/tika/pipes/emitter/s3/S3Emitter.java    | 162 +++---
 tika-pipes/tika-emitters/tika-emitter-solr/pom.xml |  13 +-
 .../tika/pipes/emitter/solr/SolrEmitter.java       | 133 +++--
 .../apache/tika/pipes/emitter/solr/TestBasic.java  |  34 +-
 .../src/test/resources/log4j.properties            |   6 +-
 .../pipes/fetchiterator/csv/CSVFetchIterator.java  |  82 +--
 .../src/test/java/TestCSVFetchIterator.java        |  56 +-
 .../tika-fetch-iterator-jdbc/pom.xml               |  17 +-
 .../fetchiterator/jdbc/JDBCFetchIterator.java      |  84 +--
 .../fetchiterator/jdbc/TestJDBCFetchIterator.java  |  83 +--
 .../src/test/resources/log4j.properties            |   4 +-
 .../tika-fetch-iterator-s3/pom.xml                 |  22 +-
 .../pipes/fetchiterator/s3/S3FetchIterator.java    |  80 ++-
 .../fetchiterator/s3/TestS3FetchIterator.java      |  26 +-
 .../src/test/resources/log4j.properties            |   4 +-
 tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml |  13 +-
 .../tika/pipes/fetcher/http/HttpFetcher.java       |  33 +-
 .../tika/pipes/fetcher/http/HttpFetcherTest.java   |  50 +-
 tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml   |   9 +
 .../apache/tika/pipes/fetcher/s3/S3Fetcher.java    | 157 +++---
 .../tika/pipes/fetcher/s3/TestS3Fetcher.java       |  23 +-
 .../src/test/resources/tika-config-s3.xml          |   1 +
 .../org/apache/tika/client/HttpClientFactory.java  | 147 +++--
 .../org/apache/tika/client/HttpClientUtil.java     |  26 +-
 tika-pipes/tika-pipes-async/pom.xml                | 101 ++++
 .../java/org/apache/tika/pipes/async/AsyncCli.java | 358 +++++++++++++
 .../org/apache/tika/pipes/async/AsyncConfig.java   |  76 +++
 .../org/apache/tika/pipes/async/AsyncData.java     |  57 ++
 .../org/apache/tika/pipes/async/AsyncEmitHook.java |  18 +-
 .../org/apache/tika/pipes/async/AsyncEmitter.java  | 125 +++++
 .../tika/pipes/async/AsyncEmitterProcess.java      | 379 +++++++++++++
 .../tika/pipes/async/AsyncPipesEmitHook.java       |  61 +++
 .../apache/tika/pipes/async/AsyncProcessor.java    | 594 +++++++++++++++++++++
 .../tika/pipes/async/AsyncRuntimeException.java    |  23 +-
 .../org/apache/tika/pipes/async/AsyncTask.java     |  54 ++
 .../org/apache/tika/pipes/async/AsyncWorker.java   | 195 +++++++
 .../tika/pipes/async/AsyncWorkerProcess.java       | 505 ++++++++++++++++++
 .../src/main}/resources/log4j.properties           |   6 +-
 .../tika/pipes/async/AsyncProcessorTest.java       | 105 ++++
 .../org/apache/tika/pipes/async/MockEmitter.java   | 103 ++++
 .../org/apache/tika/pipes/async/MockFetcher.java   |  27 +-
 .../apache/tika/pipes/async/SerializationTest.java |  49 ++
 .../apache/tika/pipes/PipeIntegrationTests.java    | 103 ++--
 .../src/test/resources/log4j.properties            |   4 +-
 tika-serialization/pom.xml                         |   5 +
 .../tika/metadata/serialization/JsonEmitData.java  |   2 +-
 .../metadata/serialization/JsonFetchEmitTuple.java |   6 +-
 .../serialization/JsonMetadataDeserializer.java    |  28 +-
 .../serialization/JsonMetadataSerializer.java      |  28 +-
 .../serialization/JsonFetchEmitTupleTest.java      |   6 +-
 .../org/apache/tika/server/client/TikaClient.java  |   2 +-
 .../apache/tika/server/client/TikaClientCLI.java   |  35 +-
 .../tika/server/client/TikaEmitterResult.java      |   7 +-
 .../apache/tika/server/client/TikaHttpClient.java  |   6 +-
 .../tika/server/core/resource/AsyncEmitter.java    |  11 +-
 .../tika/server/core/resource/AsyncParser.java     |  13 +-
 .../tika/server/core/resource/AsyncResource.java   |   2 +-
 .../tika/server/core/resource/EmitterResource.java |  30 +-
 .../core/TikaServerEmitterIntegrationTest.java     |   5 +-
 78 files changed, 3978 insertions(+), 946 deletions(-)

diff --git a/.gitignore b/.gitignore
index dda6180..c608dc2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 .svn
 target
 dependency-reduced-pom.xml
+.editorconfig
 .idea
 .classpath
 .project
diff --git a/pom.xml b/pom.xml
index 17bc12d..1ad3c85 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,11 +37,11 @@
   <modules>
     <module>tika-parent</module>
     <module>tika-core</module>
+    <module>tika-serialization</module>
     <module>tika-pipes</module>
     <module>tika-parsers</module>
     <module>tika-bundles</module>
     <module>tika-xmp</module>
-    <module>tika-serialization</module>
     <module>tika-batch</module>
     <module>tika-langdetect</module>
     <module>tika-app</module>
diff --git a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
index 50b3c22..3f9587b 100644
--- a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
+++ b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
@@ -72,6 +72,7 @@ import org.apache.tika.metadata.serialization.JsonMetadataList;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AbstractParser;
 import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.DelegatingParser;
 import org.apache.tika.parser.DigestingParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
@@ -99,7 +100,7 @@ public class TikaGUI extends JFrame
         implements ActionListener, HyperlinkListener {
 
     //maximum length to allow for mark for reparse to get JSON
-    private static final int MAX_MARK = 20*1024*1024;//20MB
+    private static final int MAX_MARK = 20 * 1024 * 1024;//20MB
 
     /**
      * Serial version UID.
@@ -129,7 +130,7 @@ public class TikaGUI extends JFrame
                         new CommonsDigester(MAX_MARK,
                                 CommonsDigester.DigestAlgorithm.MD5,
                                 CommonsDigester.DigestAlgorithm.SHA256)
-                        )).setVisible(true);
+                )).setVisible(true);
             }
         });
     }
@@ -143,7 +144,7 @@ public class TikaGUI extends JFrame
      * Configured parser instance.
      */
     private final Parser parser;
-    
+
     /**
      * Captures requested embedded images
      */
@@ -173,7 +174,7 @@ public class TikaGUI extends JFrame
      * Main content output.
      */
     private final JEditorPane textMain;
-    
+
     /**
      * Raw XHTML source.
      */
@@ -341,14 +342,13 @@ public class TikaGUI extends JFrame
 
         context.set(DocumentSelector.class, new ImageDocumentSelector());
 
-        input = TikaInputStream.get(new ProgressMonitorInputStream(
-                this, "Parsing stream", input));
+        input = TikaInputStream.get(input);
 
         if (input.markSupported()) {
             int mark = -1;
             if (input instanceof TikaInputStream) {
-                if (((TikaInputStream)input).hasFile()) {
-                    mark = (int)((TikaInputStream)input).getLength();
+                if (((TikaInputStream) input).hasFile()) {
+                    mark = (int) ((TikaInputStream) input).getLength();
                 }
             }
             if (mark == -1) {
@@ -391,8 +391,8 @@ public class TikaGUI extends JFrame
             input.reset();
             isReset = true;
         } catch (IOException e) {
-            setText(json, "Error during stream reset.\n"+
-                    "There's a limit of "+MAX_MARK + " bytes for this type of processing in the GUI.\n"+
+            setText(json, "Error during stream reset.\n" +
+                    "There's a limit of " + MAX_MARK + " bytes for this type of processing in the GUI.\n" +
                     "Try the app with command line argument of -J."
             );
         }
@@ -420,7 +420,7 @@ public class TikaGUI extends JFrame
         t.printStackTrace(new PrintWriter(writer));
 
         JEditorPane editor =
-            new JEditorPane("text/plain", writer.toString());
+                new JEditorPane("text/plain", writer.toString());
         editor.setEditable(false);
         editor.setBackground(Color.WHITE);
         editor.setCaretPosition(0);
@@ -435,7 +435,7 @@ public class TikaGUI extends JFrame
     private void addWelcomeCard(JPanel panel, String name) {
         try {
             JEditorPane editor =
-                new JEditorPane(TikaGUI.class.getResource("welcome.html"));
+                    new JEditorPane(TikaGUI.class.getResource("welcome.html"));
             editor.setContentType("text/html");
             editor.setEditable(false);
             editor.setBackground(Color.WHITE);
@@ -480,7 +480,7 @@ public class TikaGUI extends JFrame
                 URL url = e.getURL();
                 try (InputStream stream = url.openStream()) {
                     JEditorPane editor =
-                        new JEditorPane("text/plain", IOUtils.toString(stream, UTF_8));
+                            new JEditorPane("text/plain", IOUtils.toString(stream, UTF_8));
                     editor.setEditable(false);
                     editor.setBackground(Color.WHITE);
                     editor.setCaretPosition(0);
@@ -519,7 +519,7 @@ public class TikaGUI extends JFrame
      * {@link JEditorPane} fail thinking that the document character set
      * is inconsistent.
      * <p>
-     * Additionally, it will use ImageSavingParser to re-write embedded:(image) 
+     * Additionally, it will use ImageSavingParser to re-write embedded:(image)
      * image links to be file:///(temporary file) so that they can be loaded.
      *
      * @param writer output writer
@@ -529,7 +529,7 @@ public class TikaGUI extends JFrame
     private ContentHandler getHtmlHandler(Writer writer)
             throws TransformerConfigurationException {
         SAXTransformerFactory factory = (SAXTransformerFactory)
-            SAXTransformerFactory.newInstance();
+                SAXTransformerFactory.newInstance();
         TransformerHandler handler = factory.newTransformerHandler();
         handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
         handler.setResult(new StreamResult(writer));
@@ -542,36 +542,37 @@ public class TikaGUI extends JFrame
                     uri = null;
                 }
                 if (!"head".equals(localName)) {
-                    if("img".equals(localName)) {
-                       AttributesImpl newAttrs;
-                       if(atts instanceof AttributesImpl) {
-                          newAttrs = (AttributesImpl)atts;
-                       } else {
-                          newAttrs = new AttributesImpl(atts);
-                       }
-                       
-                       for(int i=0; i<newAttrs.getLength(); i++) {
-                          if("src".equals(newAttrs.getLocalName(i))) {
-                             String src = newAttrs.getValue(i);
-                             if(src.startsWith("embedded:")) {
-                                String filename = src.substring(src.indexOf(':')+1);
-                                try {
-                                   File img = imageParser.requestSave(filename);
-                                   String newSrc = img.toURI().toString();
-                                   newAttrs.setValue(i, newSrc);
-                                } catch(IOException e) {
-                                   System.err.println("Error creating temp image file " + filename);
-                                   // The html viewer will show a broken image too to alert them
+                    if ("img".equals(localName)) {
+                        AttributesImpl newAttrs;
+                        if (atts instanceof AttributesImpl) {
+                            newAttrs = (AttributesImpl) atts;
+                        } else {
+                            newAttrs = new AttributesImpl(atts);
+                        }
+
+                        for (int i = 0; i < newAttrs.getLength(); i++) {
+                            if ("src".equals(newAttrs.getLocalName(i))) {
+                                String src = newAttrs.getValue(i);
+                                if (src.startsWith("embedded:")) {
+                                    String filename = src.substring(src.indexOf(':') + 1);
+                                    try {
+                                        File img = imageParser.requestSave(filename);
+                                        String newSrc = img.toURI().toString();
+                                        newAttrs.setValue(i, newSrc);
+                                    } catch (IOException e) {
+                                        System.err.println("Error creating temp image file " + filename);
+                                        // The html viewer will show a broken image too to alert them
+                                    }
                                 }
-                             }
-                          }
-                       }
-                       super.startElement(uri, localName, name, newAttrs);
+                            }
+                        }
+                        super.startElement(uri, localName, name, newAttrs);
                     } else {
-                       super.startElement(uri, localName, name, atts);
+                        super.startElement(uri, localName, name, atts);
                     }
                 }
             }
+
             @Override
             public void endElement(String uri, String localName, String name)
                     throws SAXException {
@@ -582,9 +583,11 @@ public class TikaGUI extends JFrame
                     super.endElement(uri, localName, name);
                 }
             }
+
             @Override
             public void startPrefixMapping(String prefix, String uri) {
             }
+
             @Override
             public void endPrefixMapping(String prefix) {
             }
@@ -594,6 +597,7 @@ public class TikaGUI extends JFrame
     private ContentHandler getTextContentHandler(Writer writer) {
         return new BodyContentHandler(writer);
     }
+
     private ContentHandler getTextMainContentHandler(Writer writer) {
         return new BoilerpipeContentHandler(writer);
     }
@@ -601,7 +605,7 @@ public class TikaGUI extends JFrame
     private ContentHandler getXmlContentHandler(Writer writer)
             throws TransformerConfigurationException {
         SAXTransformerFactory factory = (SAXTransformerFactory)
-            SAXTransformerFactory.newInstance();
+                SAXTransformerFactory.newInstance();
         TransformerHandler handler = factory.newTransformerHandler();
         handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
         handler.setResult(new StreamResult(writer));
@@ -612,63 +616,63 @@ public class TikaGUI extends JFrame
      * A {@link DocumentSelector} that accepts only images.
      */
     private static class ImageDocumentSelector implements DocumentSelector {
-      public boolean select(Metadata metadata) {
-         String type = metadata.get(Metadata.CONTENT_TYPE);
-         return type != null && type.startsWith("image/");
-      }
+        public boolean select(Metadata metadata) {
+            String type = metadata.get(Metadata.CONTENT_TYPE);
+            return type != null && type.startsWith("image/");
+        }
     }
-    
+
     /**
      * A recursive parser that saves certain images into the temporary
-     *  directory, and delegates everything else to another downstream
-     *  parser.
+     * directory, and delegates everything else to another downstream
+     * parser.
      */
     private static class ImageSavingParser extends AbstractParser {
-      private Map<String,File> wanted = new HashMap<String,File>();
-      private Parser downstreamParser;
-      private File tmpDir;
-      
-      private ImageSavingParser(Parser downstreamParser) {
-         this.downstreamParser = downstreamParser;
-         
-         try {
-            File t = File.createTempFile("tika", ".test");
-            tmpDir = t.getParentFile();
-         } catch(IOException e) {}
-      }
-      
-      public File requestSave(String embeddedName) throws IOException {
-         String suffix = ".tika";
-         
-         int splitAt = embeddedName.lastIndexOf('.');
-         if (splitAt > 0) {
-            embeddedName.substring(splitAt);
-         }
-         
-         File tmp = File.createTempFile("tika-embedded-", suffix);
-         wanted.put(embeddedName, tmp);
-         return tmp;
-      }
-      
-      public Set<MediaType> getSupportedTypes(ParseContext context) {
-         // Never used in an auto setup
-         return null;
-      }
-
-      public void parse(InputStream stream, ContentHandler handler,
-            Metadata metadata, ParseContext context) throws IOException,
-            SAXException, TikaException {
-         String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
-         if(name != null && wanted.containsKey(name)) {
-            FileOutputStream out = new FileOutputStream(wanted.get(name));
-            IOUtils.copy(stream, out);
-            out.close();
-         } else {
-            if(downstreamParser != null) {
-               downstreamParser.parse(stream, handler, metadata, context);
+        private Map<String, File> wanted = new HashMap<String, File>();
+        private Parser downstreamParser;
+        private File tmpDir;
+
+        private ImageSavingParser(Parser downstreamParser) {
+            this.downstreamParser = downstreamParser;
+
+            try {
+                File t = File.createTempFile("tika", ".test");
+                tmpDir = t.getParentFile();
+            } catch (IOException e) {
             }
-         }
-      }
+        }
+
+        public File requestSave(String embeddedName) throws IOException {
+            String suffix = ".tika";
+
+            int splitAt = embeddedName.lastIndexOf('.');
+            if (splitAt > 0) {
+                embeddedName.substring(splitAt);
+            }
+
+            File tmp = File.createTempFile("tika-embedded-", suffix);
+            wanted.put(embeddedName, tmp);
+            return tmp;
+        }
+
+        public Set<MediaType> getSupportedTypes(ParseContext context) {
+            return downstreamParser.getSupportedTypes(context);
+        }
+
+        public void parse(InputStream stream, ContentHandler handler,
+                          Metadata metadata, ParseContext context) throws IOException,
+                SAXException, TikaException {
+            String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+            if (name != null && wanted.containsKey(name)) {
+                FileOutputStream out = new FileOutputStream(wanted.get(name));
+                IOUtils.copy(stream, out);
+                out.close();
+            } else {
+                if (downstreamParser != null) {
+                    downstreamParser.parse(stream, handler, metadata, context);
+                }
+            }
+        }
 
     }
 
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index b987b4e..8c3ad86 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -467,7 +467,7 @@ public class TikaConfig {
 
         if (serviceLoaderElement != null) {
             boolean dynamic = Boolean.parseBoolean(serviceLoaderElement.getAttribute("dynamic"));
-            LoadErrorHandler loadErrorHandler = LoadErrorHandler.IGNORE;
+            LoadErrorHandler loadErrorHandler = LoadErrorHandler.THROW;
             String loadErrorHandleConfig = serviceLoaderElement.getAttribute("loadErrorHandler");
             if (LoadErrorHandler.WARN.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
                 loadErrorHandler = LoadErrorHandler.WARN;
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
index bc9fa3b..a53dbd6 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
@@ -14,21 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
-    String getName();
-
-    void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
-    void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
-    //TODO -- add this later for xhtml?
-    //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+package org.apache.tika.exception;
 
+/**
+ * Runtime/unchecked version of {@link java.util.concurrent.TimeoutException}
+ */
+public class TikaTimeoutException extends RuntimeException {
+    public TikaTimeoutException(String message) {
+        super(message);
+    }
 }
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
index db6dd4c..dda0054 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
@@ -59,9 +59,9 @@ public abstract class AbstractEmitter implements Emitter {
      * @throws TikaEmitterException
      */
     @Override
-    public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+    public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
         for (EmitData d : emitData) {
-            emit(d.getEmitKey().getKey(), d.getMetadataList());
+            emit(d.getEmitKey().getEmitKey(), d.getMetadataList());
         }
     }
 }
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
index 65271e3..b565a43 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
@@ -18,9 +18,13 @@ package org.apache.tika.pipes.emitter;
 
 public class EmitKey {
 
-    private final String emitterName;
-    private final String emitKey;
+    private String emitterName;
+    private String emitKey;
 
+    //for serialization only...yuck.
+    public EmitKey() {
+
+    }
     public EmitKey(String emitterName, String emitKey) {
         this.emitterName = emitterName;
         this.emitKey = emitKey;
@@ -30,7 +34,7 @@ public class EmitKey {
         return emitterName;
     }
 
-    public String getKey() {
+    public String getEmitKey() {
         return emitKey;
     }
 
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
index bc9fa3b..f60ef3b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
@@ -27,7 +27,7 @@ public interface Emitter {
 
     void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
 
-    void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
+    void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException;
     //TODO -- add this later for xhtml?
     //void emit(String txt, Metadata metadata) throws IOException, TikaException;
 
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
index 95bb462..b77107b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
@@ -35,7 +35,7 @@ public class EmptyEmitter implements Emitter {
     }
 
     @Override
-    public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+    public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
 
     }
 }
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
index 2992bb1..2c0ea64 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
@@ -21,8 +21,13 @@ package org.apache.tika.pipes.fetcher;
  * to send to that fetcher to retrieve a specific file.
  */
 public class FetchKey {
-    private final String fetcherName;
-    private final String fetchKey;
+    private String fetcherName;
+    private String fetchKey;
+
+    //this is for serialization...yuck
+    public FetchKey(){
+
+    }
 
     public FetchKey(String fetcherName, String fetchKey) {
         this.fetcherName = fetcherName;
@@ -33,7 +38,7 @@ public class FetchKey {
         return fetcherName;
     }
 
-    public String getKey() {
+    public String getFetchKey() {
         return fetchKey;
     }
 
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
index 1f7d5b9..974aa3b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
@@ -24,7 +24,7 @@ public class FetchEmitTuple {
 
     public static final ON_PARSE_EXCEPTION DEFAULT_ON_PARSE_EXCEPTION = ON_PARSE_EXCEPTION.EMIT;
     private final FetchKey fetchKey;
-    private final EmitKey emitKey;
+    private EmitKey emitKey;
     private final Metadata metadata;
     private final ON_PARSE_EXCEPTION onParseException;
     public FetchEmitTuple(FetchKey fetchKey, EmitKey emitKey, Metadata metadata) {
@@ -55,6 +55,9 @@ public class FetchEmitTuple {
         return onParseException;
     }
 
+    public void setEmitKey(EmitKey emitKey) {
+        this.emitKey = emitKey;
+    }
     @Override
     public String toString() {
         return "FetchEmitTuple{" + "fetchKey=" + fetchKey + ", emitKey=" + emitKey + ", metadata=" +
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
index efc1723..2b9273d 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
@@ -17,32 +17,42 @@
 package org.apache.tika.pipes.fetchiterator;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
 import org.apache.tika.config.Param;
 import org.apache.tika.exception.TikaConfigException;
-
+import org.apache.tika.exception.TikaTimeoutException;
 /**
  * Abstract class that handles the testing for timeouts/thread safety
  * issues.  Concrete classes implement the blocking {@link #enqueue()}.
- * <p>
- * This must be "called", obviously...
+ * If there's an exception in the enqueuing thread, this will throw
+ * a RuntimeException.  It will throw an IllegalStateException if
+ * next() is called after hasNext() has returned false.
  */
-public abstract class FetchIterator implements Callable<Integer>, Initializable {
+public abstract class FetchIterator
+        implements Callable<Integer>, Iterable<FetchEmitTuple>, Initializable {
 
     public static final long DEFAULT_MAX_WAIT_MS = 300_000;
     public static final int DEFAULT_QUEUE_SIZE = 1000;
+
     public static final FetchEmitTuple COMPLETED_SEMAPHORE = new FetchEmitTuple(null, null, null);
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(FetchIterator.class);
+
     private long maxWaitMs = DEFAULT_MAX_WAIT_MS;
-    private int numConsumers = -1;
     private ArrayBlockingQueue<FetchEmitTuple> queue = null;
     private int queueSize = DEFAULT_QUEUE_SIZE;
     private String fetcherName;
@@ -50,25 +60,7 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
     private int added = 0;
     private FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
             FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
-
-    public FetchIterator() {
-
-    }
-
-    public FetchIterator(String fetcherName) {
-        this.fetcherName = fetcherName;
-    }
-
-    /**
-     * This must be called before 'calling' this object.
-     *
-     * @param numConsumers
-     */
-    public ArrayBlockingQueue<FetchEmitTuple> init(int numConsumers) {
-        this.queue = new ArrayBlockingQueue<>(queueSize);
-        this.numConsumers = numConsumers;
-        return queue;
-    }
+    private FutureTask<Integer> futureTask;
 
     public String getFetcherName() {
         return fetcherName;
@@ -117,29 +109,16 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
         this.onParseException = onParseException;
     }
 
-    @Override
     public Integer call() throws Exception {
-        if (queue == null || numConsumers < 0) {
-            throw new IllegalStateException("Must call 'init' before calling this object");
-        }
-
         enqueue();
-        for (int i = 0; i < numConsumers; i++) {
-            try {
-                tryToAdd(COMPLETED_SEMAPHORE);
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            }
-        }
+        tryToAdd(COMPLETED_SEMAPHORE);
         return added;
     }
 
     protected abstract void enqueue() throws IOException, TimeoutException, InterruptedException;
 
     protected void tryToAdd(FetchEmitTuple p) throws InterruptedException, TimeoutException {
-        if (p != COMPLETED_SEMAPHORE) {
-            added++;
-        }
+        added++;
         boolean offered = queue.offer(p, maxWaitMs, TimeUnit.MILLISECONDS);
         if (!offered) {
             throw new TimeoutException("timed out while offering");
@@ -157,4 +136,76 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
         //no-op
     }
 
+    @Override
+    public Iterator<FetchEmitTuple> iterator() {
+        if (futureTask != null) {
+            throw new IllegalStateException("Can't call iterator more than once!");
+        }
+        futureTask = new FutureTask<>(this);
+        queue = new ArrayBlockingQueue<>(queueSize);
+        new Thread(futureTask).start();
+        return new TupleIterator();
+    }
+
+    private class TupleIterator implements Iterator<FetchEmitTuple> {
+        FetchEmitTuple next = null;
+
+        @Override
+        public boolean hasNext() {
+            if (next == null) {
+                next = pollNext();
+            }
+            return next != COMPLETED_SEMAPHORE;
+        }
+
+        @Override
+        public FetchEmitTuple next() {
+            if (next == COMPLETED_SEMAPHORE) {
+                throw new IllegalStateException(
+                        "don't call next() after hasNext() has returned false!");
+            }
+            FetchEmitTuple ret = next;
+            next = pollNext();
+            return ret;
+        }
+
+        private FetchEmitTuple pollNext() throws TikaTimeoutException {
+
+            FetchEmitTuple t = null;
+            long start = System.currentTimeMillis();
+            try {
+                long elapsed = System.currentTimeMillis() - start;
+                while (t == null && elapsed < maxWaitMs) {
+                    checkThreadOk();
+                    t = queue.poll(100, TimeUnit.MILLISECONDS);
+                    elapsed = System.currentTimeMillis() - start;
+                }
+            } catch (InterruptedException e) {
+                LOGGER.warn("interrupted");
+                return COMPLETED_SEMAPHORE;
+            }
+            if (t == null) {
+                throw new TikaTimeoutException(
+                        "waited longer than " + maxWaitMs + "ms for the next tuple");
+            }
+            return t;
+        }
+
+        /**
+         * this checks to make sure that the thread hasn't terminated early.
+         * Will return true if the thread has successfully completed or if
+         * it has not completed.  Will return false if there has been a thread
+         * interrupt. Will throw a RuntimeException if there's been
+         * an execution exception in the thread.
+         */
+        private void checkThreadOk() throws InterruptedException {
+            if (futureTask.isDone()) {
+                try {
+                    futureTask.get();
+                } catch (ExecutionException e) {
+                    throw new RuntimeException(e.getCause());
+                }
+            }
+        }
+    }
 }
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
index 2d08090..3487b63 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
@@ -39,11 +39,9 @@ public class FileSystemFetchIterator extends FetchIterator implements Initializa
     private Path basePath;
 
     public FileSystemFetchIterator() {
-
     }
 
-    public FileSystemFetchIterator(String fetcherName, Path basePath) {
-        super(fetcherName);
+    public FileSystemFetchIterator(Path basePath) {
         this.basePath = basePath;
     }
 
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
index 8bfb432..4e314bc 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
@@ -25,11 +25,6 @@ import java.nio.file.Paths;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -60,24 +55,13 @@ public class FileSystemFetchIteratorTest {
         }
 
         String fetcherName = "fs";
-        ExecutorService es = Executors.newFixedThreadPool(1);
-        ExecutorCompletionService<Integer> cs = new ExecutorCompletionService<>(es);
-        FetchIterator it = new FileSystemFetchIterator(fetcherName, root);
-        it.setQueueSize(20000);
-        ArrayBlockingQueue<FetchEmitTuple> q = it.init(1);
-
-        cs.submit(it);
-
-
-        Future<Integer> f = cs.take();
-        f.get();
+        FetchIterator it = new FileSystemFetchIterator(root);
+        it.setFetcherName(fetcherName);
+        it.setQueueSize(2);
 
         Set<String> iteratorSet = new HashSet<>();
-        for (FetchEmitTuple p : q) {
-            if (p == FetchIterator.COMPLETED_SEMAPHORE) {
-                break;
-            }
-            iteratorSet.add(p.getFetchKey().getKey());
+        for (FetchEmitTuple p : it) {
+            iteratorSet.add(p.getFetchKey().getFetchKey());
         }
 
         assertEquals(truthSet, iteratorSet);
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index c75b430..187cc3e 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -169,7 +169,6 @@ public class TesseractOCRParserTest extends TikaTest {
     @Test
     public void testSingleImage() throws Exception {
         Assume.assumeTrue("can run OCR", canRun());
-
         String xml = getXML("testOCR.jpg").xml;
         assertContains("OCR Testing", xml);
         //test metadata extraction
@@ -182,6 +181,8 @@ public class TesseractOCRParserTest extends TikaTest {
         assertContainsCount("<body", xml, 1);
         assertContainsCount("</body", xml, 1);
         assertContainsCount("</html", xml, 1);
+
+        assertNotContained("<meta name=\"Content-Type\" content=\"image/ocr-jpeg\" />", xml);
     }
 
 
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index b90d3c3..21a5a49 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -14,7 +14,8 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
 
@@ -35,7 +36,42 @@
         <module>tika-fetchers</module>
         <module>tika-emitters</module>
         <module>tika-fetch-iterators</module>
+        <module>tika-pipes-async</module>
         <module>tika-pipes-integration-tests</module>
     </modules>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+                <version>${checkstyle.plugin.version}</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>com.puppycrawl.tools</groupId>
+                        <artifactId>checkstyle</artifactId>
+                        <version>8.41</version>
+                    </dependency>
+                </dependencies>
+                <executions>
+                    <execution>
+                        <id>validate</id>
+                        <phase>validate</phase>
+                        <configuration>
+                            <configLocation>checkstyle.xml</configLocation>
+                            <encoding>UTF-8</encoding>
+                            <consoleOutput>false</consoleOutput>
+                            <includeTestSourceDirectory>true</includeTestSourceDirectory>
+                            <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
+                            <violationSeverity>error</violationSeverity>
+                            <failOnViolation>true</failOnViolation>
+                        </configuration>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
index cf214da..df61b96 100644
--- a/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>tika-emitters</artifactId>
@@ -90,15 +90,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
index d4a0448..285142b 100644
--- a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
@@ -16,16 +16,6 @@
  */
 package org.apache.tika.pipes.emitter.fs;
 
-import org.apache.tika.config.Field;
-import org.apache.tika.pipes.emitter.AbstractEmitter;
-import org.apache.tika.pipes.emitter.Emitter;
-import org.apache.tika.pipes.emitter.StreamEmitter;
-import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.metadata.serialization.JsonMetadataList;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
@@ -35,9 +25,15 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.util.Collections;
 import java.util.List;
-import java.util.Set;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.StreamEmitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
 
 /**
  * Emitter to write to a file system.
@@ -56,7 +52,8 @@ import java.util.Set;
  *                  &lt;param name="basePath" type="string"&gt;/path/to/output&lt;/param&gt;
  *                  &lt;!-- optional; default is 'json' --&gt;
  *                  &lt;param name="fileExtension" type="string"&gt;json&lt;/param&gt;
- *                  &lt;!-- optional; if the file already exists, options ('skip', 'replace', 'exception')
+ *                  &lt;!-- optional; if the file already exists,
+ *                       options ('skip', 'replace', 'exception')
  *                  default is 'exception' --&gt;
  *                  &lt;param name="onExists" type="string"&gt;skip&lt;/param&gt;
  *              &lt;/params&gt;
@@ -66,17 +63,13 @@ import java.util.Set;
  */
 public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter {
 
-    enum ON_EXISTS {
-        SKIP, EXCEPTION, REPLACE
-    }
-
     private Path basePath = null;
     private String fileExtension = "json";
     private ON_EXISTS onExists = ON_EXISTS.EXCEPTION;
 
-
     @Override
-    public void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException {
+    public void emit(String emitKey, List<Metadata> metadataList)
+            throws IOException, TikaEmitterException {
         Path output;
         if (metadataList == null || metadataList.size() == 0) {
             throw new TikaEmitterException("metadata list must not be null or of size 0");
@@ -124,15 +117,14 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
         } else if (onExists.equals("exception")) {
             this.onExists = ON_EXISTS.EXCEPTION;
         } else {
-            throw new IllegalArgumentException(
-                    "Don't understand '" + onExists +
-                            "'; must be one of: 'skip', 'replace', 'exception'");
+            throw new IllegalArgumentException("Don't understand '" + onExists +
+                    "'; must be one of: 'skip', 'replace', 'exception'");
         }
     }
 
     @Override
-    public void emit(String path, InputStream inputStream, Metadata userMetadata) throws IOException,
-            TikaEmitterException {
+    public void emit(String path, InputStream inputStream, Metadata userMetadata)
+            throws IOException, TikaEmitterException {
         Path target = basePath.resolve(path);
 
         if (!Files.isDirectory(target.getParent())) {
@@ -152,4 +144,8 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
             }
         }
     }
+
+    enum ON_EXISTS {
+        SKIP, EXCEPTION, REPLACE
+    }
 }
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
index 62a79f0..773865c 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>tika-emitters</artifactId>
@@ -60,9 +60,18 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${commons.codec.version}</version>
+        </dependency>
+        <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <version>${jackson.version}</version>
@@ -128,15 +137,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
index cb97d20..6f9c745 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
@@ -16,11 +16,31 @@
  */
 package org.apache.tika.pipes.emitter.s3;
 
-import com.amazonaws.SdkClientException;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Map;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
@@ -29,31 +49,13 @@ import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.io.TemporaryResources;
 import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.metadata.serialization.JsonMetadataList;
 import org.apache.tika.pipes.emitter.AbstractEmitter;
 import org.apache.tika.pipes.emitter.StreamEmitter;
 import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.apache.tika.metadata.Metadata;
 import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedWriter;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
 
 /**
  * Emits to existing s3 bucket
@@ -67,15 +69,21 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
  *                  &lt;!-- required --&gt;
  *                  &lt;param name="region" type="string"&gt;us-east-1&lt;/param&gt;
  *                  &lt;!-- required --&gt;
+ *                  &lt;param name="credentialsProvider"
+ *                       type="string"&gt;(profile|instance)&lt;/param&gt;
+ *                  &lt;!-- required if credentialsProvider=profile--&gt;
  *                  &lt;param name="profile" type="string"&gt;my-profile&lt;/param&gt;
  *                  &lt;!-- required --&gt;
  *                  &lt;param name="bucket" type="string"&gt;my-bucket&lt;/param&gt;
- *                  &lt;!-- optional; prefix to add to the path before emitting; default is no prefix --&gt;
+ *                  &lt;!-- optional; prefix to add to the path before emitting;
+ *                       default is no prefix --&gt;
  *                  &lt;param name="prefix" type="string"&gt;my-prefix&lt;/param&gt;
  *                  &lt;!-- optional; default is 'json' this will be added to the SOURCE_PATH
- *                                    if no emitter key is specified --&gt;
+ *                                    if no emitter key is specified. Do not add a "."
+ *                                     before the extension --&gt;
  *                  &lt;param name="fileExtension" type="string"&gt;json&lt;/param&gt;
- *                  &lt;!-- optional; default is 'true'-- whether to copy the json to a local file before putting to s3 --&gt;
+ *                  &lt;!-- optional; default is 'true'-- whether to copy the
+ *                     json to a local file before putting to s3 --&gt;
  *                  &lt;param name="spoolToTemp" type="bool"&gt;true&lt;/param&gt;
  *              &lt;/params&gt;
  *          &lt;/emitter&gt;
@@ -88,6 +96,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
     private String region;
     private String profile;
     private String bucket;
+    private String credentialsProvider;
     private String fileExtension = "json";
     private boolean spoolToTemp = true;
     private String prefix = null;
@@ -101,15 +110,16 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
      * @throws TikaException
      */
     @Override
-    public void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException {
+    public void emit(String emitKey, List<Metadata> metadataList)
+            throws IOException, TikaEmitterException {
         if (metadataList == null || metadataList.size() == 0) {
             throw new TikaEmitterException("metadata list must not be null or of size 0");
         }
 
-        if (! spoolToTemp) {
+        if (!spoolToTemp) {
             ByteArrayOutputStream bos = new ByteArrayOutputStream();
-            try (Writer writer =
-                         new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
+            try (Writer writer = new BufferedWriter(
+                    new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
                 JsonMetadataList.toJson(metadataList, writer);
             } catch (IOException e) {
                 throw new TikaEmitterException("can't jsonify", e);
@@ -122,8 +132,8 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
             TemporaryResources tmp = new TemporaryResources();
             try {
                 Path tmpPath = tmp.createTempFile();
-                try (Writer writer = Files.newBufferedWriter(tmpPath,
-                        StandardCharsets.UTF_8, StandardOpenOption.CREATE)) {
+                try (Writer writer = Files.newBufferedWriter(tmpPath, StandardCharsets.UTF_8,
+                        StandardOpenOption.CREATE)) {
                     JsonMetadataList.toJson(metadataList, writer);
                 } catch (IOException e) {
                     throw new TikaEmitterException("can't jsonify", e);
@@ -138,28 +148,27 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
     }
 
     /**
-     *
-     * @param path -- object path, not including the bucket
-     * @param is inputStream to copy
+     * @param path         -- object path, not including the bucket
+     * @param is           inputStream to copy
      * @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
-     * @throws TikaEmitterException
+     * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
      */
     @Override
-    public void emit(String path, InputStream is, Metadata userMetadata) throws IOException, TikaEmitterException {
+    public void emit(String path, InputStream is, Metadata userMetadata)
+            throws IOException, TikaEmitterException {
 
         if (!StringUtils.isBlank(prefix)) {
             path = prefix + "/" + path;
         }
 
-        if (! StringUtils.isBlank(fileExtension)) {
+        if (!StringUtils.isBlank(fileExtension)) {
             path += "." + fileExtension;
         }
 
-        LOGGER.debug("about to emit to target bucket: ({}) path:({})",
-                bucket, path);
+        LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
         long length = -1;
         if (is instanceof TikaInputStream) {
-            if (((TikaInputStream)is).hasFile()) {
+            if (((TikaInputStream) is).hasFile()) {
                 try {
                     length = ((TikaInputStream) is).getLength();
                 } catch (IOException e) {
@@ -167,22 +176,23 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
                 }
             }
         }
-        ObjectMetadata objectMetadata = new ObjectMetadata();
-        if (length > 0) {
-            objectMetadata.setContentLength(length);
-        }
-        for (String n : userMetadata.names()) {
-            String[] vals = userMetadata.getValues(n);
-            if (vals.length > 1) {
-                LOGGER.warn("Can only write the first value for key {}. I see {} values.",
-                        n, vals.length);
-            }
-            objectMetadata.addUserMetadata(n, vals[0]);
-        }
         try {
+            ObjectMetadata objectMetadata = new ObjectMetadata();
+            if (length > 0) {
+                objectMetadata.setContentLength(length);
+            }
+            for (String n : userMetadata.names()) {
+                String[] vals = userMetadata.getValues(n);
+                if (vals.length > 1) {
+                    LOGGER.warn("Can only write the first value for key {}. I see {} values.",
+                            n,
+                            vals.length);
+                }
+                objectMetadata.addUserMetadata(n, vals[0]);
+            }
             s3Client.putObject(bucket, path, is, objectMetadata);
-        } catch (SdkClientException e) {
-            throw new TikaEmitterException("problem writing s3object", e);
+        } catch (AmazonClientException e) {
+            throw new IOException("problem writing s3object", e);
         }
     }
 
@@ -190,6 +200,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
      * Whether or not to spool the metadatalist to a tmp file before putting object.
      * Default: <code>true</code>.  If this is set to <code>false</code>,
      * this emitter writes the json object to memory and then puts that into s3.
+     *
      * @param spoolToTemp
      */
     @Field
@@ -216,15 +227,25 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
     public void setPrefix(String prefix) {
         //strip final "/" if it exists
         if (prefix.endsWith("/")) {
-            this.prefix = prefix.substring(0, prefix.length()-1);
+            this.prefix = prefix.substring(0, prefix.length() - 1);
         } else {
             this.prefix = prefix;
         }
     }
 
+    @Field
+    public void setCredentialsProvider(String credentialsProvider) {
+        if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+            throw new IllegalArgumentException(
+                    "credentialsProvider must be either 'profile' or instance'");
+        }
+        this.credentialsProvider = credentialsProvider;
+    }
+
     /**
      * If you want to customize the output file's file extension.
      * Do not include the "."
+     *
      * @param fileExtension
      */
     @Field
@@ -232,20 +253,39 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
         this.fileExtension = fileExtension;
     }
 
+
+    /**
+     * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+     * e.g. AmazonClientException in a TikaConfigException.
+     *
+     * @param params params to use for initialization
+     * @throws TikaConfigException
+     */
     @Override
     public void initialize(Map<String, Param> params) throws TikaConfigException {
-        //params have already been set
-        //ignore them
-        s3Client = AmazonS3ClientBuilder.standard()
-                .withRegion(region)
-                .withCredentials(new ProfileCredentialsProvider(profile))
-                .build();
+        //params have already been set...ignore them
+        AWSCredentialsProvider provider = null;
+        if ("instance".equals(credentialsProvider)) {
+            provider = InstanceProfileCredentialsProvider.getInstance();
+        } else if ("profile".equals(credentialsProvider)) {
+            provider = new ProfileCredentialsProvider(profile);
+        } else {
+            throw new TikaConfigException("credentialsProvider must be set and " +
+                    "must be either 'instance' or 'profile'");
+        }
+
+        try {
+            s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+                    .build();
+        } catch (AmazonClientException e) {
+            throw new TikaConfigException("can't initialize s3 emitter", e);
+        }
     }
 
     @Override
-    public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+    public void checkInitialization(InitializableProblemHandler problemHandler)
+            throws TikaConfigException {
         mustNotBeEmpty("bucket", this.bucket);
-        mustNotBeEmpty("profile", this.profile);
         mustNotBeEmpty("region", this.region);
     }
 
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
index e36d1c8..a831fa9 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>tika-emitters</artifactId>
@@ -102,15 +102,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
index 0e09e9c..f43963e 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
@@ -16,9 +16,23 @@
  */
 package org.apache.tika.pipes.emitter.solr;
 
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.zip.GZIPOutputStream;
+
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import org.apache.http.client.HttpClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.client.HttpClientFactory;
 import org.apache.tika.client.HttpClientUtil;
 import org.apache.tika.client.TikaClientException;
@@ -26,41 +40,19 @@ import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
 import org.apache.tika.config.Param;
-import org.apache.tika.pipes.emitter.AbstractEmitter;
-import org.apache.tika.pipes.emitter.EmitData;
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.EmitData;
 import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.zip.GZIPOutputStream;
 
 public class SolrEmitter extends AbstractEmitter implements Initializable {
 
-    enum AttachmentStrategy {
-        SKIP,
-        CONCATENATE_CONTENT,
-        PARENT_CHILD,
-        //anything else?
-    }
     private static final String ATTACHMENTS = "attachments";
     private static final String UPDATE_PATH = "/update";
     private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
     //one day this will be allowed or can be configured?
     private final boolean gzipJson = false;
-
     private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
     private String url;
     private String contentField = "content";
@@ -68,77 +60,71 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
     private int commitWithin = 100;
     private HttpClientFactory httpClientFactory;
     private HttpClient httpClient;
-
     public SolrEmitter() throws TikaConfigException {
         httpClientFactory = new HttpClientFactory();
     }
+
     @Override
-    public void emit(String emitKey, List<Metadata> metadataList) throws IOException,
-            TikaEmitterException {
+    public void emit(String emitKey, List<Metadata> metadataList)
+            throws IOException, TikaEmitterException {
 
         if (metadataList == null || metadataList.size() == 0) {
             LOG.warn("metadataList is null or empty");
             return;
         }
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
-        Writer writer = gzipJson ?
-                new BufferedWriter(
-                        new OutputStreamWriter(
-                                new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
-                new BufferedWriter(
-                        new OutputStreamWriter(bos, StandardCharsets.UTF_8));
-        try (
-            JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+        Writer writer = gzipJson ? new BufferedWriter(
+                new OutputStreamWriter(new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+                new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+        try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
             jsonGenerator.writeStartArray();
             jsonify(jsonGenerator, emitKey, metadataList);
             jsonGenerator.writeEndArray();
         }
-        LOG.debug("emitting json ({})",
-                new String(bos.toByteArray(), StandardCharsets.UTF_8));
+        LOG.debug("emitting json ({})", new String(bos.toByteArray(), StandardCharsets.UTF_8));
         try {
-            HttpClientUtil.postJson(httpClient,
-                    url+UPDATE_PATH+"?commitWithin="+getCommitWithin(), bos.toByteArray(), gzipJson);
+            HttpClientUtil
+                    .postJson(httpClient, url + UPDATE_PATH +
+                                    "?commitWithin=" + getCommitWithin(),
+                            bos.toByteArray(), gzipJson);
         } catch (TikaClientException e) {
             throw new TikaEmitterException("can't post", e);
         }
     }
 
     @Override
-    public void emit(List<EmitData> batch) throws IOException,
-            TikaEmitterException {
+    public void emit(List<? extends EmitData> batch) throws IOException, TikaEmitterException {
         if (batch == null || batch.size() == 0) {
             LOG.warn("batch is null or empty");
             return;
         }
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
-        Writer writer = gzipJson ?
-                new BufferedWriter(
-                        new OutputStreamWriter(
-                                new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
-            new BufferedWriter(
-                new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+        Writer writer = gzipJson ? new BufferedWriter(
+                new OutputStreamWriter(new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+                new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8));
         try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
             jsonGenerator.writeStartArray();
             for (EmitData d : batch) {
-                jsonify(jsonGenerator, d.getEmitKey().getKey(), d.getMetadataList());
+                jsonify(jsonGenerator, d.getEmitKey().getEmitKey(), d.getMetadataList());
             }
             jsonGenerator.writeEndArray();
         }
-        LOG.debug("emitting json ({})",
-                new String(bos.toByteArray(), StandardCharsets.UTF_8));
+        LOG.debug("emitting json ({})", new String(bos.toByteArray(), StandardCharsets.UTF_8));
         try {
-            HttpClientUtil.postJson(httpClient,
-                    url+UPDATE_PATH+"?commitWithin="+getCommitWithin(),
-                    bos.toByteArray(), gzipJson);
+            HttpClientUtil
+                    .postJson(httpClient, url + UPDATE_PATH +
+                                    "?commitWithin=" + getCommitWithin(),
+                            bos.toByteArray(), gzipJson);
         } catch (TikaClientException e) {
             throw new TikaEmitterException("can't post", e);
         }
     }
 
-    private void jsonify(JsonGenerator jsonGenerator, String emitKey, List<Metadata> metadataList) throws IOException {
+    private void jsonify(JsonGenerator jsonGenerator, String emitKey,
+                         List<Metadata> metadataList)
+            throws IOException {
         metadataList.get(0).set(idField, emitKey);
-        if (attachmentStrategy == AttachmentStrategy.SKIP ||
-            metadataList.size() == 1) {
+        if (attachmentStrategy == AttachmentStrategy.SKIP || metadataList.size() == 1) {
             jsonify(metadataList.get(0), jsonGenerator);
         } else if (attachmentStrategy == AttachmentStrategy.CONCATENATE_CONTENT) {
             //this only handles text for now, not xhtml
@@ -164,12 +150,13 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
             jsonGenerator.writeEndArray();
             jsonGenerator.writeEndObject();
         } else {
-            throw new IllegalArgumentException("I don't yet support this attachment strategy: "
-                    + attachmentStrategy);
+            throw new IllegalArgumentException(
+                    "I don't yet support this attachment strategy: " + attachmentStrategy);
         }
     }
 
-    private void jsonify(Metadata metadata, JsonGenerator jsonGenerator, boolean writeEndObject) throws IOException {
+    private void jsonify(Metadata metadata, JsonGenerator jsonGenerator, boolean writeEndObject)
+            throws IOException {
         jsonGenerator.writeStartObject();
         for (String n : metadata.names()) {
             String[] vals = metadata.getValues(n);
@@ -215,29 +202,35 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
         } else if (attachmentStrategy.equals("parent-child")) {
             this.attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
         } else {
-            throw new IllegalArgumentException("Expected 'skip', 'concatenate-content' or "+
+            throw new IllegalArgumentException("Expected 'skip', 'concatenate-content' or " +
                     "'parent-child'. I regret I do not recognize: " + attachmentStrategy);
         }
     }
 
     /**
      * Specify the url for Solr
+     *
      * @param url
      */
     @Field
     public void setUrl(String url) {
         if (url.endsWith("/")) {
-            url = url.substring(0, url.length()-1);
+            url = url.substring(0, url.length() - 1);
         }
         this.url = url;
     }
 
+    public String getContentField() {
+        return contentField;
+    }
+
     /**
      * This is the field _after_ metadata mappings have been applied
      * that contains the "content" for each metadata object.
-     *
+     * <p>
      * This is the field that is used if {@link #attachmentStrategy}
      * is {@link AttachmentStrategy#CONCATENATE_CONTENT}.
+     *
      * @param contentField
      */
     @Field
@@ -245,8 +238,8 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
         this.contentField = contentField;
     }
 
-    public String getContentField() {
-        return contentField;
+    public int getCommitWithin() {
+        return commitWithin;
     }
 
     @Field
@@ -254,10 +247,6 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
         this.commitWithin = commitWithin;
     }
 
-    public int getCommitWithin() {
-        return commitWithin;
-    }
-
     /**
      * Specify the field in the first Metadata that should be
      * used as the id field for the document.
@@ -302,8 +291,14 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
     }
 
     @Override
-    public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+    public void checkInitialization(InitializableProblemHandler problemHandler)
+            throws TikaConfigException {
 
     }
 
+    enum AttachmentStrategy {
+        SKIP, CONCATENATE_CONTENT, PARENT_CHILD,
+        //anything else?
+    }
+
 }
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
index 069e7a2..049fe96 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
@@ -17,19 +17,20 @@
 package org.apache.tika.pipes.emitter.solr;
 
 
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
 import org.apache.tika.config.TikaConfig;
 import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.emitter.EmitData;
-import org.apache.tika.pipes.emitter.EmitKey;
-import org.apache.tika.pipes.emitter.Emitter;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.metadata.filter.MetadataFilter;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.emitter.Emitter;
 
 @Ignore("requires solr to be up and running")
 public class TestBasic {
@@ -39,8 +40,7 @@ public class TestBasic {
         TikaConfig tikaConfig = new TikaConfig(
                 TestBasic.class.getResourceAsStream("/tika-config-simple-emitter.xml"));
         Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr1");
-        List<Metadata> metadataList = getParentChild(tikaConfig,
-                "id1", 2);
+        List<Metadata> metadataList = getParentChild(tikaConfig, "id1", 2);
 
         emitter.emit("1", metadataList);
     }
@@ -52,17 +52,15 @@ public class TestBasic {
         Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr2");
         List<EmitData> emitData = new ArrayList<>();
         for (int i = 0; i < 100; i++) {
-            List<Metadata> metadataList = getParentChild(tikaConfig,
-                    "batch_"+i, 4);
-            emitData.add(new EmitData(
-                    new EmitKey(emitter.getName(),  "batch_"+i),
-                    metadataList));
+            List<Metadata> metadataList = getParentChild(tikaConfig, "batch_" + i, 4);
+            emitData.add(new EmitData(new EmitKey(emitter.getName(),
+                    "batch_" + i), metadataList));
         }
         emitter.emit(emitData);
     }
 
-    private List<Metadata> getParentChild(TikaConfig tikaConfig,
-                                          String id, int numChildren) throws TikaException {
+    private List<Metadata> getParentChild(TikaConfig tikaConfig, String id, int numChildren)
+            throws TikaException {
         List<Metadata> metadataList = new ArrayList<>();
         MetadataFilter filter = tikaConfig.getMetadataFilter();
 
@@ -75,7 +73,7 @@ public class TestBasic {
         m1.add(TikaCoreProperties.CREATOR, "secondAuthor");
         filter.filter(m1);
         metadataList.add(m1);
-        for (int i = 1; i < numChildren; i++ ) {
+        for (int i = 1; i < numChildren; i++) {
             Metadata m2 = new Metadata();
             m2.set(TikaCoreProperties.EMBEDDED_RESOURCE_PATH, "/path_to_this.txt");
             m2.set(TikaCoreProperties.TIKA_CONTENT, "fox jumped over the lazy " + i);
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
index 92b6d56..11e5887 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #info,debug, error,fatal ...
 log4j.rootLogger=debug,stderr
-
 #console
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
 log4j.appender.stderr.Target=System.err
-
-log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
+log4j.appender.stderr.layout.ConversionPattern=%-5p %m%n
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
index 7cb0c02..3063124 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
@@ -16,8 +16,24 @@
  */
 package org.apache.tika.pipes.fetchiterator.csv;
 
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
@@ -29,21 +45,6 @@ import org.apache.tika.pipes.fetcher.FetchKey;
 import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
 import org.apache.tika.pipes.fetchiterator.FetchIterator;
 import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
 
 /**
  * Iterates through a UTF-8 CSV file. This adds all columns
@@ -51,17 +52,20 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
  * to the metadata object.
  * <p>
  *  <ul>
- *      <li>If a 'fetchKeyColumn' is specified, this will use that column's value as the fetchKey.</li>
- *      <li>If no 'fetchKeyColumn' is specified, this will send the metadata from the other columns.</li>
+ *      <li>If a 'fetchKeyColumn' is specified, this will use that
+ *      column's value as the fetchKey.</li>
+ *      <li>If no 'fetchKeyColumn' is specified, this will send the
+ *      metadata from the other columns.</li>
  *      <li>The 'fetchKeyColumn' value is not added to the metadata.</li>
  *  </ul>
  * <p>
  *  <ul>
- *      <li>If an 'emitKeyColumn' is specified, this will use that column's value as the emit key.</li>
- *      <li>If an 'emitKeyColumn' is not specified, this will use the value from the 'fetchKeyColumn'.</li>
+ *      <li>If an 'emitKeyColumn' is specified, this will use that
+ *      column's value as the emit key.</li>
+ *      <li>If an 'emitKeyColumn' is not specified, this will use
+ *      the value from the 'fetchKeyColumn'.</li>
  *      <li>The 'emitKeyColumn' value is not added to the metadata.</li>
  *  </ul>
- *
  */
 public class CSVFetchIterator extends FetchIterator implements Initializable {
 
@@ -110,24 +114,21 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
             for (CSVRecord record : records) {
                 String fetchKey = getFetchKey(fetchEmitKeyIndices, record);
                 String emitKey = getEmitKey(fetchEmitKeyIndices, record);
-                if (StringUtils.isBlank(fetchKey) && ! StringUtils.isBlank(fetcherName)) {
+                if (StringUtils.isBlank(fetchKey) && !StringUtils.isBlank(fetcherName)) {
                     LOGGER.debug("Fetcher specified ({}), but no fetchkey was found in ({})",
                             fetcherName, record);
                 }
                 if (StringUtils.isBlank(emitKey)) {
-                    throw new IOException("emitKey must not be blank in :"+record);
+                    throw new IOException("emitKey must not be blank in :" + record);
                 }
                 Metadata metadata = loadMetadata(fetchEmitKeyIndices, headers, record);
-                tryToAdd(new FetchEmitTuple(
-                        new FetchKey(fetcherName, fetchKey),
-                        new EmitKey(emitterName, emitKey), metadata,
-                        getOnParseException()));
+                tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, fetchKey),
+                        new EmitKey(emitterName, emitKey), metadata, getOnParseException()));
             }
         }
     }
 
-    private void checkFetchEmitValidity(String fetcherName,
-                                        String emitterName,
+    private void checkFetchEmitValidity(String fetcherName, String emitterName,
                                         FetchEmitKeyIndices fetchEmitKeyIndices,
                                         List<String> headers) throws IOException {
 
@@ -135,7 +136,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
             throw new IOException(new TikaConfigException("must specify at least an emitterName"));
         }
 
-        if (StringUtils.isBlank(fetcherName) && ! StringUtils.isBlank(fetchKeyColumn)) {
+        if (StringUtils.isBlank(fetcherName) && !StringUtils.isBlank(fetchKeyColumn)) {
             throw new IOException(new TikaConfigException("If specifying a 'fetchKeyColumn', " +
                     "you must also specify a 'fetcherName'"));
         }
@@ -145,17 +146,17 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
         }
 
         //if a fetchkeycolumn is specified, make sure that it was found
-        if (! StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
-            throw new IOException(new TikaConfigException("Couldn't find fetchKeyColumn ("+
-                    fetchKeyColumn+" in header.\n" +
-                    "These are the headers I see: " + headers));
+        if (!StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
+            throw new IOException(new TikaConfigException(
+                    "Couldn't find fetchKeyColumn (" + fetchKeyColumn + " in header.\n" +
+                            "These are the headers I see: " + headers));
         }
 
         //if an emitkeycolumn is specified, make sure that it was found
-        if (! StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
-            throw new IOException(new TikaConfigException("Couldn't find emitKeyColumn ("+
-                    emitKeyColumn+" in header.\n" +
-                    "These are the headers I see: " + headers));
+        if (!StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
+            throw new IOException(new TikaConfigException(
+                    "Couldn't find emitKeyColumn (" + emitKeyColumn + " in header.\n" +
+                            "These are the headers I see: " + headers));
         }
 
         if (StringUtils.isBlank(emitKeyColumn)) {
@@ -179,7 +180,8 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
         return getFetchKey(fetchEmitKeyIndices, record);
     }
 
-    private Metadata loadMetadata(FetchEmitKeyIndices fetchEmitKeyIndices, List<String> headers, CSVRecord record) {
+    private Metadata loadMetadata(FetchEmitKeyIndices fetchEmitKeyIndices, List<String> headers,
+                                  CSVRecord record) {
         Metadata metadata = new Metadata();
         for (int i = 0; i < record.size(); i++) {
             if (fetchEmitKeyIndices.shouldSkip(i)) {
@@ -192,7 +194,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
 
 
     private FetchEmitKeyIndices loadHeaders(CSVRecord record, List<String> headers)
-        throws IOException {
+            throws IOException {
         int fetchKeyColumnIndex = -1;
         int emitKeyColumnIndex = -1;
 
@@ -200,7 +202,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
             String header = record.get(col);
             if (StringUtils.isBlank(header)) {
                 throw new IOException(
-                        new TikaException("Header in column (" +col +") must not be empty"));
+                        new TikaException("Header in column (" + col + ") must not be empty"));
             }
             headers.add(header);
             if (header.equals(fetchKeyColumn)) {
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
index 74f3ebe..fd86a05 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
@@ -14,10 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.apache.tika.pipes.fetchiterator.csv.CSVFetchIterator;
-import org.junit.Test;
+
+import static org.apache.tika.pipes.fetchiterator.FetchIterator.COMPLETED_SEMAPHORE;
+import static org.junit.Assert.assertEquals;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -25,14 +24,16 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.csv.CSVFetchIterator;
 
 public class TestCSVFetchIterator {
 
@@ -46,69 +47,62 @@ public class TestCSVFetchIterator {
         it.setCsvPath(p);
         it.setFetchKeyColumn("fetchKey");
         int numConsumers = 2;
-        ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
-        ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
+        ExecutorService es = Executors.newFixedThreadPool(numConsumers);
         ExecutorCompletionService c = new ExecutorCompletionService(es);
-        c.submit(it);
+        ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(100);
         List<MockFetcher> fetchers = new ArrayList<>();
         for (int i = 0; i < numConsumers; i++) {
             MockFetcher f = new MockFetcher(queue);
             fetchers.add(f);
             c.submit(f);
         }
+        for (FetchEmitTuple t : it) {
+            queue.offer(t);
+        }
+        for (int i = 0; i < numConsumers; i++) {
+            queue.offer(COMPLETED_SEMAPHORE);
+        }
         int finished = 0;
         int completed = 0;
         try {
-            while (finished++ < numConsumers+1) {
+            while (finished++ < numConsumers) {
                 Future<Integer> f = c.take();
                 completed += f.get();
             }
         } finally {
             es.shutdownNow();
         }
-        assertEquals(10, completed);
+        assertEquals(5, completed);
         for (MockFetcher f : fetchers) {
             for (FetchEmitTuple t : f.pairs) {
                 String id = t.getMetadata().get("id");
-                assertEquals("path/to/my/file"+id,
-                        t.getFetchKey().getKey());
-                assertEquals("project"+
-                                (Integer.parseInt(id) % 2 == 1 ? "a" : "b"),
+                assertEquals("path/to/my/file" + id, t.getFetchKey().getFetchKey());
+                assertEquals("project" + (Integer.parseInt(id) % 2 == 1 ? "a" : "b"),
                         t.getMetadata().get("project"));
             }
         }
     }
 
-    @Test(expected = ExecutionException.class)
+    @Test(expected = RuntimeException.class)
     public void testBadFetchKeyCol() throws Exception {
         Path p = get("test-simple.csv");
         CSVFetchIterator it = new CSVFetchIterator();
         it.setFetcherName("fs");
         it.setCsvPath(p);
         it.setFetchKeyColumn("fetchKeyDoesntExist");
-        ExecutorService es = Executors.newFixedThreadPool(2);
-        ExecutorCompletionService c = new ExecutorCompletionService(es);
-        c.submit(it);
-        c.submit(new MockFetcher(it.init(1)));
-        int finished = 0;
-        try {
-            while (finished++ < 2) {
-                Future f = c.take();
-                f.get();
-            }
-        } finally {
-            es.shutdownNow();
-        }
+        for (FetchEmitTuple t : it) {
 
+        }
     }
 
     private Path get(String testFileName) throws Exception {
-        return Paths.get(TestCSVFetchIterator.class.getResource("/"+testFileName).toURI());
+        return Paths.get(TestCSVFetchIterator.class.getResource("/" + testFileName).toURI());
     }
 
     private static class MockFetcher implements Callable<Integer> {
         private final ArrayBlockingQueue<FetchEmitTuple> queue;
         private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
         private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
             this.queue = queue;
         }
@@ -117,7 +111,7 @@ public class TestCSVFetchIterator {
         public Integer call() throws Exception {
             while (true) {
                 FetchEmitTuple t = queue.poll(1, TimeUnit.HOURS);
-                if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+                if (t == COMPLETED_SEMAPHORE) {
                     return pairs.size();
                 }
                 pairs.add(t);
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
index 2813ed4..702554a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.tika</groupId>
@@ -29,10 +29,10 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>tika-fetch-iterator-jdbc</artifactId>
-    
+
     <name>Apache Tika Fetch Iterator - jdbc</name>
     <url>http://tika.apache.org/</url>
-    
+
     <dependencies>
         <dependency>
             <groupId>${project.groupId}</groupId>
@@ -97,15 +97,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
index 19da544..f1ca52c 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
@@ -16,19 +16,7 @@
  */
 package org.apache.tika.pipes.fetchiterator.jdbc;
 
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.emitter.EmitKey;
-import org.apache.tika.pipes.fetcher.FetchKey;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
 
 import java.io.IOException;
 import java.sql.Connection;
@@ -42,7 +30,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeoutException;
 
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.utils.StringUtils;
 
 /**
  * Iterates through a the results from a sql call via jdbc. This adds all columns
@@ -50,8 +51,10 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
  * to the metadata object.
  * <p>
  *  <ul>
- *      <li>If a 'fetchKeyColumn' is specified, this will use that column's value as the fetchKey.</li>
- *      <li>If no 'fetchKeyColumn' is specified, this will send the metadata from the other columns.</li>
+ *      <li>If a 'fetchKeyColumn' is specified, this will use that
+ *      column's value as the fetchKey.</li>
+ *      <li>If no 'fetchKeyColumn' is specified, this will send the
+ *      metadata from the other columns.</li>
  *      <li>The 'fetchKeyColumn' value is not added to the metadata.</li>
  *  </ul>
  * <p>
@@ -59,7 +62,6 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
  *      <li>An 'emitKeyColumn' must be specified</li>
  *      <li>The 'emitKeyColumn' value is not added to the metadata.</li>
  *  </ul>
- *
  */
 public class JDBCFetchIterator extends FetchIterator implements Initializable {
 
@@ -88,6 +90,10 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
         this.connection = connection;
     }
 
+    public String getSelect() {
+        return select;
+    }
+
     @Field
     public void setSelect(String select) {
         this.select = select;
@@ -106,16 +112,17 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
                 while (rs.next()) {
                     if (headers.size() == 0) {
                         fetchEmitKeyIndices = loadHeaders(rs.getMetaData(), headers);
-                        checkFetchEmitValidity(fetcherName, emitterName, fetchEmitKeyIndices, headers);
+                        checkFetchEmitValidity(fetcherName, emitterName, fetchEmitKeyIndices,
+                                headers);
                     }
                     try {
                         processRow(fetcherName, emitterName, headers, fetchEmitKeyIndices, rs);
                     } catch (SQLException e) {
-                        LOGGER.warn("Failed to insert: "+rs, e);
+                        LOGGER.warn("Failed to insert: " + rs, e);
                     }
                     rowCount++;
                     if (rowCount % 1000 == 0) {
-                        LOGGER.info("added "+rowCount + " rows to the queue");
+                        LOGGER.info("added " + rowCount + " rows to the queue");
                     }
                 }
             }
@@ -125,21 +132,23 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
         } finally {
             try {
                 db.close();
-            } catch (SQLException e){
+            } catch (SQLException e) {
                 LOGGER.warn("failed to close connection", e);
             }
         }
     }
-    private void checkFetchEmitValidity(String fetcherName,
-                                        String emitterName,
+
+    private void checkFetchEmitValidity(String fetcherName, String emitterName,
                                         FetchEmitKeyIndices fetchEmitKeyIndices,
                                         List<String> headers) throws IOException {
 
-        if (! StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
-            throw new IOException(new TikaConfigException("Couldn't find column: "+fetchKeyColumn));
+        if (!StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
+            throw new IOException(
+                    new TikaConfigException("Couldn't find column: " + fetchKeyColumn));
         }
-        if (! StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
-            throw new IOException(new TikaConfigException("Couldn't find column: "+emitKeyColumn));
+        if (!StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
+            throw new IOException(
+                    new TikaConfigException("Couldn't find column: " + emitKeyColumn));
         }
     }
 
@@ -152,14 +161,18 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
         for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
             if (i == fetchEmitKeyIndices.fetchKeyIndex) {
                 fetchKey = getString(i, rs);
+                if (fetchKey == null) {
+                    LOGGER.debug("fetchKey is empty for record " + toString(rs));
+                }
                 fetchKey = (fetchKey == null) ? "" : fetchKey;
-                LOGGER.debug("fetchKey is empty for record "+toString(rs));
                 continue;
             }
             if (i == fetchEmitKeyIndices.emitKeyIndex) {
                 emitKey = getString(i, rs);
+                if (emitKey == null) {
+                    LOGGER.debug("emitKey is empty for record " + toString(rs));
+                }
                 emitKey = (emitKey == null) ? "" : emitKey;
-                LOGGER.warn("emitKey is empty for record "+toString(rs));
                 continue;
             }
             String val = getString(i, rs);
@@ -168,10 +181,8 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
             }
         }
 
-        tryToAdd(new FetchEmitTuple(
-                new FetchKey(fetcherName, fetchKey),
-                new EmitKey(emitterName, emitKey),
-                metadata, getOnParseException()));
+        tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, fetchKey),
+                new EmitKey(emitterName, emitKey), metadata, getOnParseException()));
     }
 
     private String toString(ResultSet rs) throws SQLException {
@@ -180,7 +191,7 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
             String val = rs.getString(i);
             val = (val == null) ? "" : val;
             val = (val.length() > 100) ? val.substring(0, 100) : val;
-            sb.append(rs.getMetaData().getColumnLabel(i)+":"+val+"\n");
+            sb.append(rs.getMetaData().getColumnLabel(i) + ":" + val + "\n");
         }
         return sb.toString();
     }
@@ -195,7 +206,8 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
     }
 
 
-    private FetchEmitKeyIndices loadHeaders(ResultSetMetaData metaData, List<String> headers) throws SQLException {
+    private FetchEmitKeyIndices loadHeaders(ResultSetMetaData metaData, List<String> headers)
+            throws SQLException {
         int fetchKeyIndex = -1;
         int emitKeyIndex = -1;
         for (int i = 1; i <= metaData.getColumnCount(); i++) {
@@ -228,7 +240,7 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
         mustNotBeEmpty("emitterName", this.getEmitterName());
         mustNotBeEmpty("emitKeyColumn", this.emitKeyColumn);
 
-        if (StringUtils.isBlank(getFetcherName()) && ! StringUtils.isBlank(fetchKeyColumn)) {
+        if (StringUtils.isBlank(getFetcherName()) && !StringUtils.isBlank(fetchKeyColumn)) {
             throw new TikaConfigException(
                     "If you specify a 'fetchKeyColumn', you must specify a 'fetcherName'");
         }
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
index ee3fcee..7eebe31 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
@@ -16,13 +16,9 @@
  */
 package org.apache.tika.pipes.fetchiterator.jdbc;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -44,35 +40,41 @@ import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
 
 public class TestJDBCFetchIterator {
 
     static final String TABLE = "fetchkeys";
-    static Connection CONNECTION;
-    static Path DB_DIR;
     static final String db = "mydb";
     private static final int NUM_ROWS = 1000;
+    static Connection CONNECTION;
+    static Path DB_DIR;
 
     @BeforeClass
     public static void setUp() throws Exception {
         DB_DIR = Files.createTempDirectory("tika-jdbc-fetchiterator-test-");
 
-        CONNECTION = DriverManager.getConnection("jdbc:h2:file:"+DB_DIR.toAbsolutePath()+"/"+db);
-        String sql = "create table "+TABLE +
-                " (id varchar(128), " +
+        CONNECTION =
+                DriverManager.getConnection("jdbc:h2:file:" +
+                        DB_DIR.toAbsolutePath() + "/" + db);
+        String sql = "create table " + TABLE + " (id varchar(128), " +
                 "project varchar(128), " +
                 "fetchKey varchar(128))";
         CONNECTION.createStatement().execute(sql);
 
         for (int i = 0; i < NUM_ROWS; i++) {
-            sql = "insert into "+TABLE + " (id, project, fetchKey) values ('id"+i+"','project"+
-                    (i%2 == 0 ? "a" : "b") +"','fk"+i+"')";
+            sql = "insert into " + TABLE + " (id, project, fetchKey) values ('id" + i +
+                    "','project" + (i % 2 == 0 ? "a" : "b") + "','fk" + i + "')";
             CONNECTION.createStatement().execute(sql);
         }
-        sql = "select count(1) from "+ TABLE;
+        sql = "select count(1) from " + TABLE;
         ResultSet rs = CONNECTION.createStatement().executeQuery(sql);
         while (rs.next()) {
             int cnt = rs.getInt(1);
@@ -90,39 +92,44 @@ public class TestJDBCFetchIterator {
         TikaConfig tk = getConfig();
         int numConsumers = 5;
         FetchIterator fetchIterator = tk.getFetchIterator();
-        ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
-        ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
-        ArrayBlockingQueue<FetchEmitTuple> queue = fetchIterator.init(numConsumers);
-        completionService.submit(fetchIterator);
+        ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
+        ExecutorCompletionService<Integer> completionService =
+                new ExecutorCompletionService<>(es);
+        ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(100);
         List<MockFetcher> fetchers = new ArrayList<>();
         for (int i = 0; i < numConsumers; i++) {
             MockFetcher mockFetcher = new MockFetcher(queue);
             fetchers.add(mockFetcher);
             completionService.submit(mockFetcher);
         }
+        for (FetchEmitTuple t : fetchIterator) {
+            queue.offer(t);
+        }
+        for (int i = 0; i < numConsumers; i++) {
+            queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
+        }
         int processed = 0;
         int completed = 0;
-        while (completed < numConsumers+1) {
+        while (completed < numConsumers) {
             Future<Integer> f = completionService.take();
             processed += f.get();
             completed++;
         }
-        //fetchiterator added + MockFetcher yields 2x
-        assertEquals(NUM_ROWS * 2, processed);
+        assertEquals(NUM_ROWS, processed);
         int cnt = 0;
         Matcher m = Pattern.compile("fk(\\d+)").matcher("");
         for (MockFetcher f : fetchers) {
             for (FetchEmitTuple p : f.pairs) {
-                String k = p.getFetchKey().getKey();
+                String k = p.getFetchKey().getFetchKey();
                 String num = "";
                 if (m.reset(k).find()) {
                     num = m.group(1);
                 } else {
-                    fail("failed to find key pattern: "+k);
+                    fail("failed to find key pattern: " + k);
                 }
                 String aOrB = Integer.parseInt(num) % 2 == 0 ? "a" : "b";
-                assertEquals("id"+num, p.getMetadata().get("MY_ID"));
-                assertEquals("project"+aOrB, p.getMetadata().get("MY_PROJECT"));
+                assertEquals("id" + num, p.getMetadata().get("MY_ID"));
+                assertEquals("project" + aOrB, p.getMetadata().get("MY_PROJECT"));
                 assertNull(p.getMetadata().get("fetchKey"));
                 assertNull(p.getMetadata().get("MY_FETCHKEY"));
                 cnt++;
@@ -134,17 +141,22 @@ public class TestJDBCFetchIterator {
     private TikaConfig getConfig() throws Exception {
         String config = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?><properties>\n" +
                 "    <fetchIterators>\n" +
-                "        <fetchIterator class=\"org.apache.tika.pipes.fetchiterator.jdbc.JDBCFetchIterator\">\n" +
+                "        <fetchIterator " +
+                "       class=\"org.apache.tika.pipes.fetchiterator.jdbc.JDBCFetchIterator\">\n" +
                 "            <params>\n" +
                 "                <param name=\"fetcherName\" type=\"string\">s3f</param>\n" +
                 "                <param name=\"emitterName\" type=\"string\">s3e</param>\n" +
                 "                <param name=\"queueSize\" type=\"int\">57</param>\n" +
-                "                <param name=\"fetchKeyColumn\" type=\"string\">my_fetchkey</param>\n" +
-                "                <param name=\"emitKeyColumn\" type=\"string\">my_fetchkey</param>\n" +
+                "                <param name=\"fetchKeyColumn\" " +
+                "                     type=\"string\">my_fetchkey</param>\n" +
+                "                <param name=\"emitKeyColumn\" " +
+                "                    type=\"string\">my_fetchkey</param>\n" +
                 "                <param name=\"select\" type=\"string\">" +
-                "select id as my_id, project as my_project, fetchKey as my_fetchKey from fetchkeys</param>\n" +
-                "                <param name=\"connection\" type=\"string\">jdbc:h2:file:"+
-                DB_DIR.toAbsolutePath()+"/"+db +"</param>\n" +
+                "select id as my_id, project as my_project, fetchKey as my_fetchKey " +
+                "from fetchkeys</param>\n" +
+                "                <param name=\"connection\" " +
+                "                type=\"string\">jdbc:h2:file:" + DB_DIR.toAbsolutePath() + "/" +
+                    db + "</param>\n" +
                 "            </params>\n" +
                 "        </fetchIterator>\n" +
                 "    </fetchIterators>\n" +
@@ -155,6 +167,7 @@ public class TestJDBCFetchIterator {
     private static class MockFetcher implements Callable<Integer> {
         private final ArrayBlockingQueue<FetchEmitTuple> queue;
         private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
         private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
             this.queue = queue;
         }
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #info,debug, error,fatal ...
 log4j.rootLogger=info,stderr
-
 #console
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
 log4j.appender.stderr.Target=System.err
-
 log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
index f8b2424..24be277 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.tika</groupId>
@@ -57,9 +57,18 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${commons.codec.version}</version>
+        </dependency>
+        <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <version>${jackson.version}</version>
@@ -124,15 +133,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
index dd14463..988f149 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
@@ -16,11 +16,23 @@
  */
 package org.apache.tika.pipes.fetchiterator.s3;
 
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.iterable.S3Objects;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
@@ -31,21 +43,13 @@ import org.apache.tika.pipes.emitter.EmitKey;
 import org.apache.tika.pipes.fetcher.FetchKey;
 import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
 import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
 
 public class S3FetchIterator extends FetchIterator implements Initializable {
 
-
     private static final Logger LOGGER = LoggerFactory.getLogger(S3FetchIterator.class);
-    private String s3PathPrefix = "";
+    private String prefix = "";
     private String region;
+    private String credentialsProvider;
     private String profile;
     private String bucket;
 
@@ -67,19 +71,45 @@ public class S3FetchIterator extends FetchIterator implements Initializable {
     }
 
     @Field
-    public void setS3PathPrefix(String s3PathPrefix) {
-        this.s3PathPrefix = s3PathPrefix;
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
     }
 
+    @Field
+    public void setCredentialsProvider(String credentialsProvider) {
+        if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+            throw new IllegalArgumentException(
+                    "credentialsProvider must be either 'profile' or instance'");
+        }
+        this.credentialsProvider = credentialsProvider;
+    }
+
+    /**
+     * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+     * e.g. AmazonClientException in a TikaConfigException.
+     *
+     * @param params params to use for initialization
+     * @throws TikaConfigException
+     */
     @Override
     public void initialize(Map<String, Param> params) throws TikaConfigException {
-        //params have already been set
-        //ignore them
+        //params have already been set...ignore them
+        AWSCredentialsProvider provider = null;
+        if ("instance".equals(credentialsProvider)) {
+            provider = InstanceProfileCredentialsProvider.getInstance();
+        } else if ("profile".equals(credentialsProvider)) {
+            provider = new ProfileCredentialsProvider(profile);
+        } else {
+            throw new TikaConfigException("credentialsProvider must be set and " +
+                    "must be either 'instance' or 'profile'");
+        }
 
-        s3Client = AmazonS3ClientBuilder.standard()
-                .withRegion(region)
-                .withCredentials(new ProfileCredentialsProvider(profile))
-                .build();
+        try {
+            s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+                    .build();
+        } catch (AmazonClientException e) {
+            throw new TikaConfigException("can't initialize s3 fetchiterator");
+        }
     }
 
     @Override
@@ -87,24 +117,22 @@ public class S3FetchIterator extends FetchIterator implements Initializable {
             throws TikaConfigException {
         super.checkInitialization(problemHandler);
         mustNotBeEmpty("bucket", this.bucket);
-        mustNotBeEmpty("profile", this.profile);
         mustNotBeEmpty("region", this.region);
     }
 
     @Override
     protected void enqueue() throws InterruptedException, IOException, TimeoutException {
         String fetcherName = getFetcherName();
+        String emitterName = getEmitterName();
         long start = System.currentTimeMillis();
         int count = 0;
-        for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, s3PathPrefix)) {
+        for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, prefix)) {
 
             long elapsed = System.currentTimeMillis() - start;
-            LOGGER.debug("adding ({}) {} in {} ms", count, summary.getKey(),
-                    elapsed);
-            tryToAdd(new FetchEmitTuple(
-                    new FetchKey(fetcherName, summary.getKey()),
-                    new EmitKey(fetcherName, summary.getKey()),
-                    new Metadata(), getOnParseException()));
+            LOGGER.debug("adding ({}) {} in {} ms", count, summary.getKey(), elapsed);
+            tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, summary.getKey()),
+                    new EmitKey(emitterName, summary.getKey()), new Metadata(),
+                    getOnParseException()));
             count++;
         }
         long elapsed = System.currentTimeMillis() - start;
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
index 51d20ac..50cb819 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
@@ -15,10 +15,8 @@
  * limitations under the License.
  */
 package org.apache.tika.pipes.fetchiterator.s3;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.Ignore;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -31,7 +29,11 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
 
 @Ignore("turn into an actual unit test")
 public class TestS3FetchIterator {
@@ -46,21 +48,26 @@ public class TestS3FetchIterator {
         it.setRegion("");//select one
         it.initialize(Collections.EMPTY_MAP);
         int numConsumers = 6;
-        ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
+        ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(10);
 
-        ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
+        ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
         ExecutorCompletionService c = new ExecutorCompletionService(es);
-        c.submit(it);
         List<MockFetcher> fetchers = new ArrayList<>();
         for (int i = 0; i < numConsumers; i++) {
             MockFetcher fetcher = new MockFetcher(queue);
             fetchers.add(fetcher);
             c.submit(fetcher);
         }
+        for (FetchEmitTuple t : it) {
+            queue.offer(t);
+        }
+        for (int i = 0; i < numConsumers; i++) {
+            queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
+        }
         int finished = 0;
         int completed = 0;
         try {
-            while (finished++ < numConsumers+1) {
+            while (finished++ < numConsumers + 1) {
                 Future<Integer> f = c.take();
                 completed += f.get();
             }
@@ -74,6 +81,7 @@ public class TestS3FetchIterator {
     private static class MockFetcher implements Callable<Integer> {
         private final ArrayBlockingQueue<FetchEmitTuple> queue;
         private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
         private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
             this.queue = queue;
         }
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #info,debug, error,fatal ...
 log4j.rootLogger=info,stderr
-
 #console
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
 log4j.appender.stderr.Target=System.err
-
 log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
index 09dcb97..ae76a52 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
@@ -17,8 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>tika-fetchers</artifactId>
@@ -97,15 +97,18 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/LICENSE</resource>
                                     <file>target/classes/META-INF/LICENSE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/NOTICE</resource>
                                     <file>target/classes/META-INF/NOTICE</file>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
                                     <resource>META-INF/DEPENDENCIES</resource>
                                     <file>target/classes/META-INF/DEPENDENCIES</file>
                                 </transformer>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index a6d2325..be39765 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -17,11 +17,19 @@
 package org.apache.tika.pipes.fetcher.http;
 
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.client.HttpClientFactory;
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
@@ -32,13 +40,6 @@ import org.apache.tika.exception.TikaException;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.pipes.fetcher.AbstractFetcher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
 
 /**
  * Based on Apache httpclient
@@ -52,9 +53,9 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
     public HttpFetcher() throws TikaConfigException {
         httpClientFactory = new HttpClientFactory();
     }
+
     @Override
-    public InputStream fetch(String fetchKey, Metadata metadata)
-            throws IOException, TikaException {
+    public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
         HttpGet get = new HttpGet(fetchKey);
         return get(get);
     }
@@ -62,7 +63,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
     public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
             throws IOException, TikaException {
         HttpGet get = new HttpGet(fetchKey);
-        get.setHeader("Range", "bytes="+startRange+"-"+endRange);
+        get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
         return get(get);
     }
 
@@ -70,21 +71,18 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
         HttpResponse response = httpClient.execute(get);
         int code = response.getStatusLine().getStatusCode();
         if (code < 200 || code > 299) {
-            throw new IOException("bad status code: "+
-                    code
-                    + " :: " +
+            throw new IOException("bad status code: " + code + " :: " +
                     responseToString(response.getEntity().getContent()));
         }
 
         //spool to local
         long start = System.currentTimeMillis();
-        TikaInputStream tis = TikaInputStream.get(
-                response.getEntity().getContent());
+        TikaInputStream tis = TikaInputStream.get(response.getEntity().getContent());
         tis.getPath();
         if (response instanceof CloseableHttpResponse) {
             ((CloseableHttpResponse) response).close();
         }
-        long elapsed = System.currentTimeMillis()-start;
+        long elapsed = System.currentTimeMillis() - start;
         LOG.debug("took {} ms to copy to local tmp file", elapsed);
         return tis;
     }
@@ -149,7 +147,8 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
     }
 
     @Override
-    public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+    public void checkInitialization(InitializableProblemHandler problemHandler)
+            throws TikaConfigException {
 
     }
 }
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
index 8a2d8a1..3a2e127 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
@@ -16,13 +16,7 @@
  */
 package org.apache.tika.pipes.fetcher.http;
 
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.metadata.Metadata;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.xml.sax.SAXException;
+import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -31,32 +25,40 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.zip.GZIPInputStream;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.metadata.Metadata;
 
 @Ignore("requires network connectivity")
 public class HttpFetcherTest {
 
-        @Test
-        public void testRange() throws Exception {
-            String url =
-                    "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
-            long start = 969596307;
-            long end = start + 1408 - 1;
-            Metadata metadata = new Metadata();
-            HttpFetcher httpFetcher = (HttpFetcher) getConfig("tika-config-http.xml")
-                    .getFetcherManager().getFetcher("http");
-            try (TemporaryResources tmp = new TemporaryResources()) {
-                Path tmpPath = tmp.createTempFile();
-                try (InputStream is = httpFetcher.fetch(url, start, end, metadata)) {
-                    Files.copy(new GZIPInputStream(is), tmpPath, StandardCopyOption.REPLACE_EXISTING);
-                }
-                assertEquals(2461, Files.size(tmpPath));
+    @Test
+    public void testRange() throws Exception {
+        String url =
+                "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
+        long start = 969596307;
+        long end = start + 1408 - 1;
+        Metadata metadata = new Metadata();
+        HttpFetcher httpFetcher =
+                (HttpFetcher) getConfig("tika-config-http.xml").getFetcherManager()
+                        .getFetcher("http");
+        try (TemporaryResources tmp = new TemporaryResources()) {
+            Path tmpPath = tmp.createTempFile();
+            try (InputStream is = httpFetcher.fetch(url, start, end, metadata)) {
+                Files.copy(new GZIPInputStream(is), tmpPath, StandardCopyOption.REPLACE_EXISTING);
             }
+            assertEquals(2461, Files.size(tmpPath));
         }
+    }
 
 
     TikaConfig getConfig(String path) throws TikaException, IOException, SAXException {
-            return new TikaConfig(HttpFetcherTest.class.getResourceAsStream("/"+path));
+        return new TikaConfig(HttpFetcherTest.class.getResourceAsStream("/" + path));
     }
 
 
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
index 158e701..7d8288c 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
@@ -47,8 +47,17 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${commons.codec.version}</version>
+        </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index 217881e..a6b34dd 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -16,29 +16,33 @@
  */
 package org.apache.tika.pipes.fetcher.s3;
 
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.config.Field;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.InitializableProblemHandler;
 import org.apache.tika.config.Param;
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.fetcher.AbstractFetcher;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+import org.apache.tika.pipes.fetcher.AbstractFetcher;
 
 /**
  * Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
@@ -52,64 +56,67 @@ public class S3Fetcher extends AbstractFetcher implements Initializable {
     private String region;
     private String bucket;
     private String profile;
+    private String credentialsProvider;
     private boolean extractUserMetadata = true;
     private AmazonS3 s3Client;
     private boolean spoolToTemp = true;
 
     @Override
-    public InputStream fetch(String fetchKey, Metadata metadata)
-            throws TikaException, IOException {
+    public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
 
-        LOGGER.debug("about to fetch fetchkey={} from bucket ({})",
-                fetchKey, bucket);
+        LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
 
-        S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
-        if (extractUserMetadata) {
-            for (Map.Entry<String, String> e :
-                    s3Object.getObjectMetadata().getUserMetadata().entrySet()) {
-                metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+        try {
+            S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
+            if (extractUserMetadata) {
+                for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
+                        .entrySet()) {
+                    metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+                }
             }
-        }
-        if (! spoolToTemp) {
-            return TikaInputStream.get(
-                    s3Object.getObjectContent());
-        } else {
-            long start = System.currentTimeMillis();
-            TikaInputStream tis = TikaInputStream.get(
-                    s3Object.getObjectContent());
-            tis.getPath();
-            long elapsed = System.currentTimeMillis()-start;
-            LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
-            return tis;
+            if (!spoolToTemp) {
+                return TikaInputStream.get(s3Object.getObjectContent());
+            } else {
+                long start = System.currentTimeMillis();
+                TikaInputStream tis = TikaInputStream.get(s3Object.getObjectContent());
+                tis.getPath();
+                long elapsed = System.currentTimeMillis() - start;
+                LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
+                return tis;
+            }
+        } catch (AmazonClientException e) {
+            throw new IOException("s3 client exception", e);
         }
     }
 
     public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
             throws TikaException, IOException {
         //TODO -- figure out how to integrate this
-        LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})",
-                fetchKey, startRange, endRange, bucket);
-
-        S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey)
-                .withRange(startRange, endRange));
-
-        if (extractUserMetadata) {
-            for (Map.Entry<String, String> e :
-                    s3Object.getObjectMetadata().getUserMetadata().entrySet()) {
-                metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+        LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})", fetchKey,
+                startRange, endRange, bucket);
+
+        try {
+            S3Object s3Object = s3Client.getObject(
+                    new GetObjectRequest(bucket, fetchKey).withRange(startRange, endRange));
+
+            if (extractUserMetadata) {
+                for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
+                        .entrySet()) {
+                    metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+                }
             }
-        }
-        if (! spoolToTemp) {
-            return TikaInputStream.get(
-                    s3Object.getObjectContent());
-        } else {
-            long start = System.currentTimeMillis();
-            TikaInputStream tis = TikaInputStream.get(
-                    s3Object.getObjectContent());
-            tis.getPath();
-            long elapsed = System.currentTimeMillis()-start;
-            LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
-            return tis;
+            if (!spoolToTemp) {
+                return TikaInputStream.get(s3Object.getObjectContent());
+            } else {
+                long start = System.currentTimeMillis();
+                TikaInputStream tis = TikaInputStream.get(s3Object.getObjectContent());
+                tis.getPath();
+                long elapsed = System.currentTimeMillis() - start;
+                LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
+                return tis;
+            }
+        } catch (AmazonClientException e) {
+            throw new IOException(e);
         }
     }
 
@@ -143,21 +150,47 @@ public class S3Fetcher extends AbstractFetcher implements Initializable {
         this.extractUserMetadata = extractUserMetadata;
     }
 
+    @Field
+    public void setCredentialsProvider(String credentialsProvider) {
+        if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+            throw new IllegalArgumentException(
+                    "credentialsProvider must be either 'profile' or instance'");
+        }
+        this.credentialsProvider = credentialsProvider;
+    }
+
+    /**
+     * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+     * e.g. AmazonClientException in a TikaConfigException.
+     *
+     * @param params params to use for initialization
+     * @throws TikaConfigException
+     */
     @Override
     public void initialize(Map<String, Param> params) throws TikaConfigException {
-        //params have already been set
-        //ignore them
-        s3Client = AmazonS3ClientBuilder.standard()
-                .withRegion(region)
-                .withCredentials(new ProfileCredentialsProvider(profile))
-                .build();
+        //params have already been set...ignore them
+        AWSCredentialsProvider provider = null;
+        if ("instance".equals(credentialsProvider)) {
+            provider = InstanceProfileCredentialsProvider.getInstance();
+        } else if ("profile".equals(credentialsProvider)) {
+            provider = new ProfileCredentialsProvider(profile);
+        } else {
+            throw new TikaConfigException("credentialsProvider must be set and " +
+                    "must be either 'instance' or 'profile'");
+        }
+
+        try {
+            s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+                    .build();
+        } catch (AmazonClientException e) {
+            throw new TikaConfigException("can't initialize s3 fetcher", e);
+        }
     }
 
     @Override
-    public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+    public void checkInitialization(InitializableProblemHandler problemHandler)
+            throws TikaConfigException {
         mustNotBeEmpty("bucket", this.bucket);
-        mustNotBeEmpty("profile", this.profile);
         mustNotBeEmpty("region", this.region);
-
     }
 }
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
index 450d933..9be1ab4 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
@@ -15,11 +15,6 @@
  * limitations under the License.
  */
 package org.apache.tika.pipes.fetcher.s3;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.pipes.fetcher.Fetcher;
-import org.apache.tika.metadata.Metadata;
-import org.junit.Ignore;
-import org.junit.Test;
 
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -28,12 +23,19 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.Collections;
 
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.Fetcher;
+
 @Ignore("write actual unit tests")
 public class TestS3Fetcher {
     private static final String FETCH_STRING = "";
-    private Path outputFile = Paths.get("");
-    private String region = "us-east-1";
-    private String profile = "";
+    private final Path outputFile = Paths.get("");
+    private final String region = "us-east-1";
+    private final String profile = "";
 
     @Test
     public void testBasic() throws Exception {
@@ -50,9 +52,8 @@ public class TestS3Fetcher {
 
     @Test
     public void testConfig() throws Exception {
-        TikaConfig config = new TikaConfig(
-                this.getClass().getResourceAsStream("/tika-config-s3.xml")
-        );
+        TikaConfig config =
+                new TikaConfig(this.getClass().getResourceAsStream("/tika-config-s3.xml"));
         Fetcher fetcher = config.getFetcherManager().getFetcher("s3");
         Metadata metadata = new Metadata();
         try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
index 740058a..176513b 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
@@ -22,6 +22,7 @@
                 <param name="name" type="string">s3</param>
                 <param name="region" type="string">us-east-1</param>
                 <param name="profile" type="string">my_profile</param>
+                <param name="credentialsProvider" type="string">profile</param>
             </params>
         </fetcher>
     </fetchers>
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
index 8cbc26b..77619f9 100644
--- a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
@@ -16,6 +16,27 @@
  */
 package org.apache.tika.client;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.Set;
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+import javax.net.ssl.SSLContext;
+
 import org.apache.http.Header;
 import org.apache.http.HeaderElement;
 import org.apache.http.HeaderElementIterator;
@@ -52,35 +73,15 @@ import org.apache.http.message.BasicHeaderElementIterator;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.ssl.SSLContexts;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.SecretKeySpec;
-import javax.net.ssl.SSLContext;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.HashSet;
-import java.util.Set;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.utils.StringUtils;
 
 /**
  * This holds quite a bit of state and is not thread safe.  Beware!
- *
+ * <p>
  * Also, we're currently ignoring the SSL checks.  Please open a ticket/PR
  * if you need robust SSL.
  */
@@ -104,7 +105,7 @@ public class HttpClientFactory {
     private String password;
     private String ntDomain;//if using nt credentials
     private String authScheme = "basic"; //ntlm or basic
-    private boolean credentialsAESEncrypted = false;
+    private final boolean credentialsAESEncrypted = false;
 
 
     public HttpClientFactory() throws TikaConfigException {
@@ -116,6 +117,7 @@ public class HttpClientFactory {
             aes = new AES();
         }
     }
+
     public String getProxyHost() {
         return proxyHost;
     }
@@ -218,6 +220,7 @@ public class HttpClientFactory {
 
     /**
      * only basic and ntlm are supported
+     *
      * @param authScheme
      */
     public void setAuthScheme(String authScheme) {
@@ -230,19 +233,18 @@ public class HttpClientFactory {
         TrustStrategy acceptingTrustStrategy = (cert, authType) -> true;
         SSLContext sslContext = null;
         try {
-            sslContext = SSLContexts.custom().loadTrustMaterial(null,
-                    acceptingTrustStrategy).build();
+            sslContext =
+                    SSLContexts.custom().loadTrustMaterial(
+                            null, acceptingTrustStrategy).build();
         } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
             throw new TikaConfigException("", e);
         }
-        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
-                NoopHostnameVerifier.INSTANCE);
+        SSLConnectionSocketFactory sslsf =
+                new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
 
         Registry<ConnectionSocketFactory> socketFactoryRegistry =
-                RegistryBuilder.<ConnectionSocketFactory>create()
-                        .register("https", sslsf)
-                        .register("http", new PlainConnectionSocketFactory())
-                        .build();
+                RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf)
+                        .register("http", new PlainConnectionSocketFactory()).build();
 
         PoolingHttpClientConnectionManager manager =
                 new PoolingHttpClientConnectionManager(socketFactoryRegistry);
@@ -253,19 +255,13 @@ public class HttpClientFactory {
         addCredentialsProvider(builder);
         addProxy(builder);
         return builder.setConnectionManager(manager)
-                .setRedirectStrategy(
-                        new CustomRedirectStrategy(allowedHostsForRedirect))
-                .setDefaultRequestConfig(RequestConfig.custom()
-                        .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC,
-                                AuthSchemes.NTLM))
-                        .setConnectionRequestTimeout((int) requestTimeout)
-                        .setConnectionRequestTimeout(connectTimeout)
-                        .setSocketTimeout(socketTimeout)
-                        .build()
-                )
-                .setKeepAliveStrategy(getKeepAliveStrategy())
-                .setSSLSocketFactory(sslsf)
-                .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                .setRedirectStrategy(new CustomRedirectStrategy(allowedHostsForRedirect))
+                .setDefaultRequestConfig(RequestConfig.custom().setTargetPreferredAuthSchemes(
+                        Arrays.asList(AuthSchemes.BASIC, AuthSchemes.NTLM))
+                        .setConnectionRequestTimeout(requestTimeout)
+                        .setConnectionRequestTimeout(connectTimeout).setSocketTimeout(socketTimeout)
+                        .build()).setKeepAliveStrategy(getKeepAliveStrategy())
+                .setSSLSocketFactory(sslsf).setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                 .build();
     }
 
@@ -285,33 +281,32 @@ public class HttpClientFactory {
 
         if ((StringUtils.isBlank(userName) && StringUtils.isBlank(password)) ||
                 (StringUtils.isBlank(password) && StringUtils.isBlank(userName))) {
-            throw new IllegalArgumentException("can't have one of 'username', " +
-                    "'password' null and the other not");
+            throw new IllegalArgumentException(
+                    "can't have one of 'username', " + "'password' null and the other not");
         }
 
         String finalUserName = decrypt(userName);
         String finalPassword = decrypt(password);
         String finalDomain = decrypt(ntDomain);
-            CredentialsProvider provider = new BasicCredentialsProvider();
-            Credentials credentials = null;
-            Registry<AuthSchemeProvider> authSchemeRegistry = null;
-            if (authScheme.equals("basic")) {
-                credentials = new UsernamePasswordCredentials(finalUserName, finalPassword);
-                authSchemeRegistry = RegistryBuilder
-                        .<AuthSchemeProvider>create()
-                        .register("basic", new BasicSchemeFactory())
-                        .build();
-            } else if (authScheme.equals("ntlm")) {
-                if (StringUtils.isBlank(ntDomain)) {
-                    throw new IllegalArgumentException("must specify 'ntDomain'");
-                }
-                credentials = new NTCredentials(finalUserName, finalPassword, null, finalDomain);
-                authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
-                        .register("ntlm", new NTLMSchemeFactory()).build();
+        CredentialsProvider provider = new BasicCredentialsProvider();
+        Credentials credentials = null;
+        Registry<AuthSchemeProvider> authSchemeRegistry = null;
+        if (authScheme.equals("basic")) {
+            credentials = new UsernamePasswordCredentials(finalUserName, finalPassword);
+            authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+                    .register("basic", new BasicSchemeFactory()).build();
+        } else if (authScheme.equals("ntlm")) {
+            if (StringUtils.isBlank(ntDomain)) {
+                throw new IllegalArgumentException("must specify 'ntDomain'");
             }
-            provider.setCredentials(AuthScope.ANY, credentials);
-            builder.setDefaultCredentialsProvider(provider);
-            builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+            credentials = new NTCredentials(finalUserName, finalPassword,
+                    null, finalDomain);
+            authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+                    .register("ntlm", new NTLMSchemeFactory()).build();
+        }
+        provider.setCredentials(AuthScope.ANY, credentials);
+        builder.setDefaultCredentialsProvider(provider);
+        builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
 
     }
 
@@ -350,7 +345,7 @@ public class HttpClientFactory {
     private static class CustomRedirectStrategy extends LaxRedirectStrategy {
 
         private static final Logger LOG = LoggerFactory.getLogger(CustomRedirectStrategy.class);
-        private Set<String> allowedHosts;
+        private final Set<String> allowedHosts;
 
         public CustomRedirectStrategy(Set<String> allowedHosts) {
             this.allowedHosts = allowedHosts;
@@ -373,7 +368,9 @@ public class HttpClientFactory {
         }
 
         @Override
-        public boolean isRedirected(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
+        public boolean isRedirected(HttpRequest request, HttpResponse response,
+                                    HttpContext context)
+                throws ProtocolException {
             boolean isRedirectedSuper = super.isRedirected(request, response, context);
             if (isRedirectedSuper) {
                 Header locationHeader = response.getFirstHeader("Location");
@@ -422,9 +419,10 @@ public class HttpClientFactory {
             try {
                 Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
                 cipher.init(Cipher.ENCRYPT_MODE, secretKey);
-                return Base64.getEncoder()
-                        .encodeToString(cipher.doFinal(strToEncrypt.getBytes(StandardCharsets.UTF_8)));
-            } catch (NoSuchAlgorithmException|InvalidKeyException|NoSuchPaddingException|BadPaddingException|IllegalBlockSizeException e) {
+                return Base64.getEncoder().encodeToString(
+                        cipher.doFinal(strToEncrypt.getBytes(StandardCharsets.UTF_8)));
+            } catch (NoSuchAlgorithmException | InvalidKeyException |
+                    NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException e) {
                 throw new TikaConfigException("bad encryption info", e);
             }
         }
@@ -435,11 +433,8 @@ public class HttpClientFactory {
                 cipher.init(Cipher.DECRYPT_MODE, secretKey);
                 return new String(cipher.doFinal(Base64.getDecoder().decode(strToDecrypt)),
                         StandardCharsets.UTF_8);
-            } catch (NoSuchAlgorithmException|
-                    InvalidKeyException|
-                    NoSuchPaddingException|
-                    BadPaddingException|
-                    IllegalBlockSizeException e) {
+            } catch (NoSuchAlgorithmException | InvalidKeyException |
+                    NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException e) {
                 throw new TikaConfigException("bad encryption info", e);
             }
         }
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
index 11ebb3c..3de32da 100644
--- a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
@@ -16,20 +16,19 @@
  */
 package org.apache.tika.client;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.util.EntityUtils;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
 public class HttpClientUtil {
 
-    public static boolean postJson(HttpClient client, String url, String json) throws IOException,
-            TikaClientException {
+    public static boolean postJson(HttpClient client, String url, String json)
+            throws IOException, TikaClientException {
         HttpPost post = new HttpPost(url);
         post.setHeader("Content-Encoding", "gzip");
         ByteArrayEntity entity = new ByteArrayEntity(json.getBytes(StandardCharsets.UTF_8));
@@ -41,9 +40,8 @@ public class HttpClientUtil {
 
         if (response.getStatusLine().getStatusCode() != 200) {
             String msg = EntityUtils.toString(response.getEntity());
-            throw new TikaClientException("Bad status: " +
-                    response.getStatusLine().getStatusCode() + " : "+
-                    msg);
+            throw new TikaClientException(
+                    "Bad status: " + response.getStatusLine().getStatusCode() + " : " + msg);
         } else {
             String msg = EntityUtils.toString(response.getEntity());
             System.out.println("httputil: " + msg);
@@ -51,9 +49,8 @@ public class HttpClientUtil {
         return true;
     }
 
-    public static boolean postJson(HttpClient client, String url,
-                                   byte[] bytes, boolean gzipped) throws IOException,
-            TikaClientException {
+    public static boolean postJson(HttpClient client, String url, byte[] bytes, boolean gzipped)
+            throws IOException, TikaClientException {
         HttpPost post = new HttpPost(url);
         if (gzipped) {
             post.setHeader("Content-Encoding", "gzip");
@@ -67,9 +64,8 @@ public class HttpClientUtil {
 
         if (response.getStatusLine().getStatusCode() != 200) {
             String msg = EntityUtils.toString(response.getEntity());
-            throw new TikaClientException("Bad status: " +
-                    response.getStatusLine().getStatusCode() + " : "+
-                    msg);
+            throw new TikaClientException(
+                    "Bad status: " + response.getStatusLine().getStatusCode() + " : " + msg);
         } else {
             String msg = EntityUtils.toString(response.getEntity());
             System.out.println("httputil: " + msg);
diff --git a/tika-pipes/tika-pipes-async/pom.xml b/tika-pipes/tika-pipes-async/pom.xml
new file mode 100644
index 0000000..5f4e418
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/pom.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>org.apache.tika</groupId>
+        <artifactId>tika-pipes</artifactId>
+        <version>2.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>tika-pipes-async</artifactId>
+    <packaging>pom</packaging>
+    <name>Apache Tika emitters</name>
+    <url>http://tika.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>tika-serialization</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>${commons.io.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>${h2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>net.jpountz.lz4</groupId>
+            <artifactId>lz4</artifactId>
+            <version>1.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>tika-emitter-fs</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <!--
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>tika-parsers-classic-package</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>-->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java
new file mode 100644
index 0000000..cb8347f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.EmptyFetchIterator;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
+public class AsyncCli {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncCli.class);
+
+    public static void main(String[] args) throws Exception {
+        Path configPath = Paths.get(args[0]);
+        int maxConsumers = 20;
+        AsyncCli asyncCli = new AsyncCli();
+        Path dbDir = Files.createTempDirectory("tika-async-db-");
+        try {
+            asyncCli.execute(dbDir, configPath, maxConsumers);
+        } finally {
+            FileUtils.deleteDirectory(dbDir.toFile());
+        }
+
+    }
+
+    private static List<Integer> getActiveWorkers(Connection connection) throws SQLException {
+        PreparedStatement findActiveWorkers =
+                connection.prepareStatement("select worker_id from workers");
+        List<Integer> workers = new ArrayList<>();
+        try (ResultSet rs = findActiveWorkers.executeQuery()) {
+            while (rs.next()) {
+                workers.add(rs.getInt(1));
+            }
+        }
+        return workers;
+    }
+
+    private void execute(Path dbDir, Path configPath, int maxConsumers) throws Exception {
+        TikaConfig tikaConfig = new TikaConfig(configPath);
+
+        String connectionString = setupTables(dbDir);
+
+        ExecutorService executorService = Executors.newFixedThreadPool(maxConsumers + 3);
+        ExecutorCompletionService<Integer> executorCompletionService =
+                new ExecutorCompletionService<>(executorService);
+
+        try (Connection connection = DriverManager.getConnection(connectionString)) {
+            FetchIterator fetchIterator = tikaConfig.getFetchIterator();
+            if (fetchIterator instanceof EmptyFetchIterator) {
+                throw new IllegalArgumentException("can't have empty fetch iterator");
+            }
+            ArrayBlockingQueue<FetchEmitTuple> q =
+                    new ArrayBlockingQueue<>(10000);//fetchIterator.init(maxConsumers);
+            AsyncTaskEnqueuer enqueuer = new AsyncTaskEnqueuer(q, connection);
+            executorCompletionService.submit(fetchIterator);
+            executorCompletionService.submit(enqueuer);
+            executorCompletionService.submit(new AssignmentManager(connection, enqueuer));
+
+            for (int i = 0; i < maxConsumers; i++) {
+                executorCompletionService
+                        .submit(new AsyncWorker(connection, connectionString, i, configPath));
+            }
+            int completed = 0;
+            while (completed < maxConsumers + 3) {
+                Future<Integer> future = executorCompletionService.take();
+                if (future != null) {
+                    int val = future.get();
+                    completed++;
+                    LOG.debug("finished " + val);
+                }
+            }
+        } finally {
+            executorService.shutdownNow();
+        }
+    }
+
+    private String setupTables(Path dbDir) throws SQLException {
+        Path dbFile = dbDir.resolve("tika-async");
+        String url = "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE";
+        Connection connection = DriverManager.getConnection(url);
+
+        String sql = "create table task_queue " + "(id bigint auto_increment primary key," +
+                "status tinyint," + //byte
+                "worker_id integer," + "retry smallint," + //short
+                "time_stamp timestamp," + "json varchar(64000))";
+        connection.createStatement().execute(sql);
+        //no clear benefit to creating an index on timestamp
+//        sql = "CREATE INDEX IF NOT EXISTS status_timestamp on status (time_stamp)";
+        sql = "create table workers (worker_id int primary key)";
+        connection.createStatement().execute(sql);
+
+        sql = "create table workers_shutdown (worker_id int primary key)";
+        connection.createStatement().execute(sql);
+
+        sql = "create table error_log (task_id bigint, " + "fetch_key varchar(10000)," +
+                "time_stamp timestamp," + "retry integer," + "error_code tinyint)";
+        connection.createStatement().execute(sql);
+
+        return url;
+    }
+
+    //this reads fetchemittuples from the queue and inserts them in the db
+    //for the workers to read
+    private static class AsyncTaskEnqueuer implements Callable<Integer> {
+        private final PreparedStatement insert;
+
+        private final ArrayBlockingQueue<FetchEmitTuple> queue;
+        private final Connection connection;
+        private final Random random = new Random();
+
+        private volatile boolean isComplete = false;
+
+        AsyncTaskEnqueuer(ArrayBlockingQueue<FetchEmitTuple> queue, Connection connection)
+                throws SQLException {
+            this.queue = queue;
+            this.connection = connection;
+            String sql = "insert into task_queue (status, time_stamp, worker_id, retry, json) " +
+                    "values (?,CURRENT_TIMESTAMP(),?,?,?)";
+            insert = connection.prepareStatement(sql);
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            List<Integer> workers = new ArrayList<>();
+            while (true) {
+                FetchEmitTuple t = queue.poll(1, TimeUnit.SECONDS);
+                LOG.debug("enqueing to db " + t);
+                if (t == null) {
+                    //log.trace?
+                } else if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+                    isComplete = true;
+                    return 1;
+                } else {
+                    long start = System.currentTimeMillis();
+                    long elapsed = System.currentTimeMillis() - start;
+                    //TODO -- fix this
+                    while (workers.size() == 0 && elapsed < 600000) {
+                        workers = getActiveWorkers(connection);
+                        Thread.sleep(100);
+                        elapsed = System.currentTimeMillis() - start;
+                    }
+                    insert(t, workers);
+                }
+            }
+        }
+
+        boolean isComplete() {
+            return isComplete;
+        }
+
+        private void insert(FetchEmitTuple t, List<Integer> workers)
+                throws IOException, SQLException {
+            int workerId = workers.size() == 1 ? workers.get(0) :
+                    workers.get(random.nextInt(workers.size()));
+            insert.clearParameters();
+            insert.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+            insert.setInt(2, workerId);
+            insert.setShort(3, (short) 0);
+            insert.setString(4, JsonFetchEmitTuple.toJson(t));
+            insert.execute();
+        }
+    }
+
+    private static class AssignmentManager implements Callable {
+
+        private final Connection connection;
+        private final AsyncTaskEnqueuer enqueuer;
+        private final PreparedStatement getQueueDistribution;
+        private final PreparedStatement findMissingWorkers;
+        private final PreparedStatement allocateNonworkersToWorkers;
+        private final PreparedStatement reallocate;
+        private final PreparedStatement countAvailableTasks;
+        private final PreparedStatement insertWorkersShutdown;
+        private final Random random = new Random();
+
+
+        public AssignmentManager(Connection connection, AsyncTaskEnqueuer enqueuer)
+                throws SQLException {
+            this.connection = connection;
+            this.enqueuer = enqueuer;
+            //this gets workers and # of tasks in desc order of number of tasks
+            String sql = "select w.worker_id, p.cnt " + "from workers w " +
+                    "left join (select worker_id, count(1) as cnt from task_queue " +
+                    "where status=0 group by worker_id)" +
+                    " p on p.worker_id=w.worker_id order by p.cnt desc";
+            getQueueDistribution = connection.prepareStatement(sql);
+            //find workers that have assigned tasks but are not in the
+            //workers table
+            sql = "select p.worker_id, count(1) as cnt from task_queue p " +
+                    "left join workers w on p.worker_id=w.worker_id " +
+                    "where w.worker_id is null group by p.worker_id";
+            findMissingWorkers = connection.prepareStatement(sql);
+
+            sql = "update task_queue set worker_id=? where worker_id=?";
+            allocateNonworkersToWorkers = connection.prepareStatement(sql);
+
+            //current strategy reallocate tasks from longest queue to shortest
+            //TODO: might consider randomly shuffling or other algorithms
+            sql = "update task_queue set worker_id= ? where id in " +
+                    "(select id from task_queue where " + "worker_id = ? and " + "rand() < 0.8 " +
+                    "and status=0 for update)";
+            reallocate = connection.prepareStatement(sql);
+
+            sql = "select count(1) from task_queue where status=" +
+                    AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal();
+            countAvailableTasks = connection.prepareStatement(sql);
+
+            sql = "insert into workers_shutdown (worker_id) values (?)";
+            insertWorkersShutdown = connection.prepareStatement(sql);
+        }
+
+        @Override
+        public Integer call() throws Exception {
+
+            while (true) {
+                List<Integer> missingWorkers = getMissingWorkers();
+                reallocateFromMissingWorkers(missingWorkers);
+                redistribute();
+                if (isComplete()) {
+                    notifyWorkers();
+                    return 1;
+                }
+                Thread.sleep(100);
+            }
+        }
+
+        private void notifyWorkers() throws SQLException {
+            for (int workerId : getActiveWorkers(connection)) {
+                insertWorkersShutdown.clearParameters();
+                insertWorkersShutdown.setInt(1, workerId);
+                insertWorkersShutdown.execute();
+            }
+        }
+
+        private boolean isComplete() throws SQLException {
+            if (!enqueuer.isComplete) {
+                return false;
+            }
+            try (ResultSet rs = countAvailableTasks.executeQuery()) {
+                while (rs.next()) {
+                    return rs.getInt(1) == 0;
+                }
+            }
+            return false;
+        }
+
+        private void redistribute() throws SQLException {
+            //parallel lists of workerid = task queue size
+            List<Integer> workerIds = new ArrayList<>();
+            List<Integer> queueSize = new ArrayList<>();
+            int totalTasks = 0;
+
+            try (ResultSet rs = getQueueDistribution.executeQuery()) {
+                while (rs.next()) {
+                    int workerId = rs.getInt(1);
+                    int numTasks = rs.getInt(2);
+                    workerIds.add(workerId);
+                    queueSize.add(numTasks);
+                    LOG.debug("workerId: ({}) numTasks: ({})", workerId, numTasks);
+                    totalTasks += numTasks;
+                }
+            }
+            if (workerIds.size() == 0) {
+                return;
+            }
+            int averagePerWorker = Math.round((float) totalTasks / (float) workerIds.size());
+            int midPoint = Math.round((float) queueSize.size() / 2) + 1;
+            for (int i = queueSize.size() - 1, j = 0; i > midPoint && j < midPoint; i--, j++) {
+                int shortestQueue = queueSize.get(i);
+                int longestQueue = queueSize.get(j);
+                if ((shortestQueue < 5 && longestQueue > 5) ||
+                        longestQueue > 5 && longestQueue > (int) (1.5 * averagePerWorker)) {
+                    int shortestQueueWorker = workerIds.get(i);
+                    int longestQueueWorker = workerIds.get(j);
+                    reallocate.clearParameters();
+                    reallocate.setLong(1, shortestQueueWorker);
+                    reallocate.setLong(2, longestQueueWorker);
+                    reallocate.execute();
+                }
+            }
+
+        }
+
+        private void reallocateFromMissingWorkers(List<Integer> missingWorkers)
+                throws SQLException {
+
+            if (missingWorkers.size() == 0) {
+                return;
+            }
+
+            List<Integer> activeWorkers = getActiveWorkers(connection);
+            if (activeWorkers.size() == 0) {
+                return;
+            }
+
+            for (int missing : missingWorkers) {
+                int active = activeWorkers.get(random.nextInt(activeWorkers.size()));
+                allocateNonworkersToWorkers.clearParameters();
+                allocateNonworkersToWorkers.setInt(1, active);
+                allocateNonworkersToWorkers.setInt(2, missing);
+                allocateNonworkersToWorkers.execute();
+                LOG.debug("allocating missing working ({}) to ({})", missing, active);
+            }
+        }
+
+        private List<Integer> getMissingWorkers() throws SQLException {
+            List<Integer> missingWorkers = new ArrayList<>();
+            try (ResultSet rs = findMissingWorkers.executeQuery()) {
+                while (rs.next()) {
+                    int workerId = rs.getInt(1);
+                    missingWorkers.add(workerId);
+                    LOG.debug("Worker ({}) no longer active", workerId);
+                }
+            }
+            return missingWorkers;
+        }
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java
new file mode 100644
index 0000000..33964b3
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.tika.utils.StringUtils;
+
+public class AsyncConfig {
+
+    private final int queueSize = 1000;
+    private final int numWorkers = 10;
+    private final int numEmitters = 1;
+    private String jdbcString;
+    private Path dbDir;
+
+    public static AsyncConfig load(Path p) throws IOException {
+        AsyncConfig asyncConfig = new AsyncConfig();
+
+        if (StringUtils.isBlank(asyncConfig.getJdbcString())) {
+            asyncConfig.dbDir = Files.createTempDirectory("tika-async-");
+            Path dbFile = asyncConfig.dbDir.resolve("tika-async");
+            asyncConfig.setJdbcString(
+                    "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE");
+        } else {
+            asyncConfig.dbDir = null;
+        }
+        return asyncConfig;
+    }
+
+    public int getQueueSize() {
+        return queueSize;
+    }
+
+    public int getNumWorkers() {
+        return numWorkers;
+    }
+
+    public int getNumEmitters() {
+        return numEmitters;
+    }
+
+    public String getJdbcString() {
+        return jdbcString;
+    }
+
+    public void setJdbcString(String jdbcString) {
+        this.jdbcString = jdbcString;
+    }
+
+    /**
+     * If no jdbc connection was specified, this
+     * dir contains the h2 database.  Otherwise, null.
+     *
+     * @return
+     */
+    public Path getTempDBDir() {
+        return dbDir;
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java
new file mode 100644
index 0000000..df80929
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncData extends EmitData {
+
+    private final long taskId;
+    private final FetchKey fetchKey;
+    private final FetchEmitTuple.ON_PARSE_EXCEPTION onParseException;
+
+    public AsyncData(@JsonProperty("taskId") long taskId,
+                     @JsonProperty("fetchKey") FetchKey fetchKey,
+                     @JsonProperty("emitKey") EmitKey emitKey, @JsonProperty("onParseException")
+                             FetchEmitTuple.ON_PARSE_EXCEPTION onParseException,
+                     @JsonProperty("metadataList") List<Metadata> metadataList) {
+        super(emitKey, metadataList);
+        this.taskId = taskId;
+        this.fetchKey = fetchKey;
+        this.onParseException = onParseException;
+    }
+
+    public FetchKey getFetchKey() {
+        return fetchKey;
+    }
+
+    public long getTaskId() {
+        return taskId;
+    }
+
+    public FetchEmitTuple.ON_PARSE_EXCEPTION getOnParseException() {
+        return onParseException;
+    }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
index bc9fa3b..502c8b4 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
@@ -14,21 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
 
-import java.io.IOException;
-import java.util.List;
+public interface AsyncEmitHook {
 
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
-    String getName();
-
-    void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
-    void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
-    //TODO -- add this later for xhtml?
-    //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+    void onSuccess(AsyncTask task);
 
+    void onFail(AsyncTask task);
 }
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java
new file mode 100644
index 0000000..4c44f9c
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY;
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_JDBC_KEY;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncEmitter implements Callable<Integer> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncEmitter.class);
+
+
+    private final String connectionString;
+    private final int emitterId;
+    private final Path tikaConfigPath;
+    private final Connection connection;
+    private final PreparedStatement finished;
+    private final PreparedStatement restarting;
+
+    public AsyncEmitter(Connection connection, String connectionString, int emitterId,
+                        Path tikaConfigPath) throws SQLException {
+        this.connectionString = connectionString;
+        this.emitterId = emitterId;
+        this.tikaConfigPath = tikaConfigPath;
+        this.connection = connection;
+        String sql = "update emitters set status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal() +
+                " where emitter_id = (" + emitterId + ")";
+        finished = connection.prepareStatement(sql);
+
+        sql = "update emitters set status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.RESTARTING.ordinal() +
+                " where emitter_id = (" + emitterId + ")";
+        restarting = connection.prepareStatement(sql);
+    }
+
+    @Override
+    public Integer call() throws Exception {
+        Process p = null;
+        try {
+            p = start();
+            int restarts = 0;
+            while (true) {
+                boolean finished = p.waitFor(60, TimeUnit.SECONDS);
+                if (finished) {
+                    int exitValue = p.exitValue();
+                    if (exitValue == 0) {
+                        LOG.debug("forked emitter process finished with exitValue=0");
+                        return 1;
+                    }
+                    reportCrash(++restarts, exitValue);
+                    p = start();
+                }
+            }
+        } finally {
+            if (p != null) {
+                p.destroyForcibly();
+            }
+            finished.execute();
+        }
+    }
+
+    private Process start() throws IOException {
+        String[] args = new String[]{"java", "-Djava.awt.headless=true", "-cp",
+                System.getProperty("java.class.path"),
+                "org.apache.tika.pipes.async.AsyncEmitterProcess", Integer.toString(emitterId)};
+        ProcessBuilder pb = new ProcessBuilder(args);
+        pb.environment().put(TIKA_ASYNC_JDBC_KEY, connectionString);
+        pb.environment()
+                .put(TIKA_ASYNC_CONFIG_FILE_KEY, tikaConfigPath.toAbsolutePath().toString());
+        pb.inheritIO();
+        return pb.start();
+    }
+
+    private void reportCrash(int numRestarts, int exitValue) throws SQLException, IOException {
+        LOG.warn("emitter id={} terminated, exitValue={}", emitterId, exitValue);
+        restarting.execute();
+        //should we unassign emit tasks here?
+    }
+
+ /*
+    static PreparedStatement prepareInsertErrorLog(Connection connection) throws SQLException {
+        //if not, this is called to insert into the error log
+        return connection.prepareStatement(
+                "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code) " +
+                        " values (?,?,CURRENT_TIMESTAMP(),?,?)"
+        );
+    }
+
+    static PreparedStatement prepareReset(Connection connection) throws SQLException {
+        //and this is called to reset the status on error
+        return connection.prepareStatement(
+                "update task_queue set " +
+                        "status=?, " +
+                        "time_stamp=CURRENT_TIMESTAMP(), " +
+                        "retry=? " +
+                        "where id=?");
+    }*/
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java
new file mode 100644
index 0000000..03f5b93
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.sql.Blob;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataDeserializer;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.EmitterManager;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.utils.ExceptionUtils;
+
+public class AsyncEmitterProcess {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncEmitterProcess.class);
+    private final LZ4FastDecompressor decompressor =
+            LZ4Factory.fastestInstance().fastDecompressor();
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final FutureTask<Integer> stdinWatcher;
+    int recordsPerPulse = 10;
+    //TODO -- parameterize these
+    private final long emitWithinMs = 10000;
+    private final long emitMaxBytes = 10_000_000;
+    private PreparedStatement markForSelecting;
+    private PreparedStatement selectForProcessing;
+    private PreparedStatement emitStatusUpdate;
+    private PreparedStatement checkForCanShutdown;
+    private PreparedStatement checkForShouldShutdown;
+    private PreparedStatement updateEmitterStatus;
+
+    private AsyncEmitterProcess(InputStream stdin) {
+        SimpleModule module = new SimpleModule();
+        module.addDeserializer(Metadata.class, new JsonMetadataDeserializer());
+        objectMapper.registerModule(module);
+        stdinWatcher = new FutureTask<>(new ForkWatcher(stdin));
+        new Thread(stdinWatcher).start();
+    }
+
+    public static void main(String[] args) throws Exception {
+        String db = System.getenv(AsyncProcessor.TIKA_ASYNC_JDBC_KEY);
+        TikaConfig tikaConfig =
+                new TikaConfig(Paths.get(System.getenv(AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY)));
+        int workerId = Integer.parseInt(args[0]);
+        LOG.debug("trying to get connection {} >{}<", workerId, db);
+        try (Connection connection = DriverManager.getConnection(db)) {
+            AsyncEmitterProcess asyncEmitter = new AsyncEmitterProcess(System.in);
+            asyncEmitter.execute(connection, workerId, tikaConfig);
+        }
+        System.exit(0);
+    }
+
+    private static void reportEmitStatus(List<Long> ids,
+                                         AsyncWorkerProcess.TASK_STATUS_CODES emitted,
+                                         PreparedStatement emitStatusUpdate) throws SQLException {
+        for (long id : ids) {
+            emitStatusUpdate.clearParameters();
+            emitStatusUpdate.setByte(1, (byte) emitted.ordinal());
+            emitStatusUpdate.setLong(2, id);
+            emitStatusUpdate.addBatch();
+        }
+        emitStatusUpdate.executeBatch();
+    }
+
+    private void execute(Connection connection, int workerId, TikaConfig tikaConfig)
+            throws SQLException, InterruptedException {
+        prepareStatements(connection, workerId);
+        updateStatus((byte) AsyncWorkerProcess.WORKER_STATUS_CODES.ACTIVE.ordinal());
+        EmitterManager emitterManager = tikaConfig.getEmitterManager();
+        EmitDataCache emitDataCache =
+                new EmitDataCache(emitterManager, emitMaxBytes, emitStatusUpdate);
+        try {
+            mainLoop(emitDataCache);
+        } finally {
+            emitDataCache.emitAll();
+            updateStatus((byte) AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal());
+        }
+    }
+
+    private void mainLoop(EmitDataCache emitDataCache) throws InterruptedException, SQLException {
+
+        while (true) {
+            if (shouldShutdown()) {
+                LOG.debug("received should shutdown signal");
+                return;
+            }
+            int toEmit = markForSelecting.executeUpdate();
+            if (toEmit == 0 && canShutdown()) {
+                //avoid race condition; double check there's nothing
+                //left to emit
+                toEmit = markForSelecting.executeUpdate();
+                if (toEmit == 0) {
+                    LOG.debug("received can shutdown and didn't update any for selecting");
+                    return;
+                }
+            }
+            if (toEmit > 0) {
+                try {
+                    tryToEmitNextBatch(emitDataCache);
+                } catch (IOException e) {
+                    LOG.warn("IOException trying to emit", e);
+                }
+            }
+            if (emitDataCache.exceedsEmitWithin(emitWithinMs)) {
+                emitDataCache.emitAll();
+            }
+            Thread.sleep(500);
+        }
+    }
+
+    private void tryToEmitNextBatch(EmitDataCache emitDataCache) throws IOException, SQLException {
+        List<AsyncEmitTuple> toEmitList = new ArrayList<>();
+        try (ResultSet rs = selectForProcessing.executeQuery()) {
+            while (rs.next()) {
+                long id = rs.getLong(1);
+                Timestamp ts = rs.getTimestamp(2);
+                int uncompressedSize = rs.getInt(3);
+                Blob blob = rs.getBlob(4);
+                ByteArrayOutputStream bos = new ByteArrayOutputStream();
+                IOUtils.copyLarge(blob.getBinaryStream(), bos);
+                byte[] bytes = bos.toByteArray();
+                toEmitList.add(new AsyncEmitTuple(id, ts, uncompressedSize, bytes));
+            }
+        }
+        List<Long> successes = new ArrayList<>();
+        List<Long> exceptions = new ArrayList<>();
+        for (AsyncEmitTuple tuple : toEmitList) {
+            try {
+                tryToEmit(tuple, emitDataCache);
+                successes.add(tuple.id);
+            } catch (IOException | SQLException e) {
+                exceptions.add(tuple.id);
+            }
+        }
+        reportEmitStatus(successes, AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS_EMIT,
+                emitStatusUpdate);
+        reportEmitStatus(exceptions, AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT,
+                emitStatusUpdate);
+
+    }
+
+    private void updateStatus(byte status) throws SQLException {
+        updateEmitterStatus.clearParameters();
+        updateEmitterStatus.setByte(1, status);
+        updateEmitterStatus.executeUpdate();
+    }
+
+    private void tryToEmit(AsyncEmitTuple asyncEmitTuple, EmitDataCache emitDataCache)
+            throws SQLException, IOException {
+        AsyncData asyncData = deserialize(asyncEmitTuple.bytes, asyncEmitTuple.uncompressedSize);
+        emitDataCache.add(asyncData);
+    }
+
+    boolean shouldShutdown() throws SQLException {
+        if (stdinWatcher.isDone()) {
+            LOG.info("parent inputstream closed; shutting down now");
+        }
+        try (ResultSet rs = checkForShouldShutdown.executeQuery()) {
+            if (rs.next()) {
+                int val = rs.getInt(1);
+                return val > 0;
+            }
+        }
+        return false;
+    }
+
+    boolean canShutdown() throws SQLException {
+        try (ResultSet rs = checkForCanShutdown.executeQuery()) {
+            if (rs.next()) {
+                int val = rs.getInt(1);
+                return val > 0;
+            }
+        }
+        return false;
+    }
+
+    private void prepareStatements(Connection connection, int workerId) throws SQLException {
+        String sql = "update task_queue set status=" +
+                AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED_EMIT.ordinal() + ", worker_id=" +
+                workerId + ", time_stamp=CURRENT_TIMESTAMP()" + " where id in " +
+                " (select id from task_queue " + //where worker_id = " + workerId +
+                " where status=" + AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE_EMIT.ordinal() +
+                " order by time_stamp asc limit " + recordsPerPulse + " for update)";
+        markForSelecting = connection.prepareStatement(sql);
+
+        sql = "select q.id, q.time_stamp, uncompressed_size, bytes " + "from emits e " +
+                "join task_queue q on e.id=q.id " + "where q.status=" +
+                AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED_EMIT.ordinal() + " and worker_id=" +
+                workerId + " order by time_stamp asc";
+        selectForProcessing = connection.prepareStatement(sql);
+
+        //only update the status if it is not already emitted or failed emit
+        sql = "update task_queue set status=?" + ", time_stamp=CURRENT_TIMESTAMP()" +
+                " where id=? and status not in (" +
+                AsyncWorkerProcess.TASK_STATUS_CODES.EMITTED.ordinal() + ", " +
+                AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT.ordinal() + ")";
+
+        emitStatusUpdate = connection.prepareStatement(sql);
+
+        sql = "select count(1) from emitters where emitter_id=" + workerId + " and status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.CAN_SHUTDOWN.ordinal();
+        checkForCanShutdown = connection.prepareStatement(sql);
+
+        sql = "select count(1) from emitters where emitter_id=" + workerId + " and status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal();
+        checkForShouldShutdown = connection.prepareStatement(sql);
+
+        sql = "merge into emitters key (emitter_id) " + "values (" + workerId + ", ? )";
+        updateEmitterStatus = connection.prepareStatement(sql);
+    }
+
+    private AsyncData deserialize(byte[] compressed, int decompressedLength) throws IOException {
+        byte[] restored = new byte[decompressedLength];
+        int compressedLength2 =
+                decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
+        return objectMapper.readerFor(AsyncData.class).readValue(restored);
+    }
+
+    private static class EmitDataCache {
+        private final EmitterManager emitterManager;
+        private final long maxBytes;
+        private final PreparedStatement emitStatusUpdate;
+        long estimatedSize = 0;
+        int size = 0;
+        Map<String, List<AsyncData>> map = new HashMap<>();
+        private Instant lastAdded = Instant.now();
+
+        public EmitDataCache(EmitterManager emitterManager, long maxBytes,
+                             PreparedStatement emitStatusUpdate) {
+            this.emitterManager = emitterManager;
+            this.maxBytes = maxBytes;
+            this.emitStatusUpdate = emitStatusUpdate;
+        }
+
+        void updateEstimatedSize(long newBytes) {
+            estimatedSize += newBytes;
+        }
+
+        void add(AsyncData data) {
+
+            size++;
+            long sz = AbstractEmitter
+                    .estimateSizeInBytes(data.getEmitKey().getEmitKey(), data.getMetadataList());
+            if (estimatedSize + sz > maxBytes) {
+                LOG.debug("estimated size ({}) > maxBytes({}), going to emitAll",
+                        (estimatedSize + sz), maxBytes);
+                emitAll();
+            }
+            List<AsyncData> cached = map.get(data.getEmitKey().getEmitterName());
+            if (cached == null) {
+                cached = new ArrayList<>();
+                map.put(data.getEmitKey().getEmitterName(), cached);
+            }
+            updateEstimatedSize(sz);
+            cached.add(data);
+            lastAdded = Instant.now();
+        }
+
+        private void emitAll() {
+            int emitted = 0;
+            LOG.debug("about to emit {}", size);
+            for (Map.Entry<String, List<AsyncData>> e : map.entrySet()) {
+                Emitter emitter = emitterManager.getEmitter(e.getKey());
+
+                try {
+                    tryToEmit(emitter, e.getValue());
+                } catch (SQLException ex) {
+                    ex.printStackTrace();
+                }
+                emitted += e.getValue().size();
+            }
+            LOG.debug("emitted: {}", emitted);
+            estimatedSize = 0;
+            size = 0;
+            map.clear();
+        }
+
+        private long tryToEmit(Emitter emitter, List<AsyncData> cachedEmitData)
+                throws SQLException {
+            List<Long> ids = new ArrayList<>();
+            for (AsyncData d : cachedEmitData) {
+                ids.add(d.getTaskId());
+            }
+            try {
+                emitter.emit(cachedEmitData);
+            } catch (IOException | TikaEmitterException e) {
+                LOG.warn("emitter class ({}): {}", emitter.getClass(),
+                        ExceptionUtils.getStackTrace(e));
+                reportEmitStatus(ids, AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT,
+                        emitStatusUpdate);
+            }
+            reportEmitStatus(ids, AsyncWorkerProcess.TASK_STATUS_CODES.EMITTED, emitStatusUpdate);
+            return 1;
+        }
+
+
+        public boolean exceedsEmitWithin(long emitWithinMs) {
+            return ChronoUnit.MILLIS.between(lastAdded, Instant.now()) > emitWithinMs;
+        }
+    }
+
+    private static class ForkWatcher implements Callable<Integer> {
+        private final InputStream in;
+
+        public ForkWatcher(InputStream in) {
+            this.in = in;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            //this should block forever
+            //if the forking process dies,
+            // this will either throw an IOException or read -1.
+            try {
+                int i = in.read();
+            } finally {
+                LOG.warn("forking process shutdown; exiting now");
+                System.exit(0);
+            }
+            return 1;
+        }
+    }
+
+    private static class AsyncEmitTuple {
+        final long id;
+        final Timestamp timestamp;
+        final int uncompressedSize;
+        final byte[] bytes;
+
+        public AsyncEmitTuple(long id, Timestamp timestamp, int uncompressedSize, byte[] bytes) {
+            this.id = id;
+            this.timestamp = timestamp;
+            this.uncompressedSize = uncompressedSize;
+            this.bytes = bytes;
+        }
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java
new file mode 100644
index 0000000..79ed80d
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncPipesEmitHook implements AsyncEmitHook {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncPipesEmitHook.class);
+
+    private final PreparedStatement markSuccess;
+    private final PreparedStatement markFailure;
+
+    public AsyncPipesEmitHook(Connection connection) throws SQLException {
+        String sql = "delete from task_queue where id=?";
+        markSuccess = connection.prepareStatement(sql);
+        //TODO --fix this
+        markFailure = connection.prepareStatement(sql);
+    }
+
+    @Override
+    public void onSuccess(AsyncTask task) {
+        try {
+            markSuccess.clearParameters();
+            markSuccess.setLong(1, task.getTaskId());
+            markSuccess.execute();
+        } catch (SQLException e) {
+            LOG.warn("problem with on success: " + task.getTaskId(), e);
+        }
+    }
+
+    @Override
+    public void onFail(AsyncTask task) {
+        try {
+            markFailure.clearParameters();
+            markFailure.setLong(1, task.getTaskId());
+            markFailure.execute();
+        } catch (SQLException e) {
+            LOG.warn("problem with on fail: " + task.getTaskId(), e);
+        }
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
new file mode 100644
index 0000000..efd6fec
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
+public class AsyncProcessor implements Closeable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncProcessor.class);
+    protected static String TIKA_ASYNC_JDBC_KEY = "TIKA_ASYC_JDBC_KEY";
+    protected static String TIKA_ASYNC_CONFIG_FILE_KEY = "TIKA_ASYNC_CONFIG_FILE_KEY";
+    private final Path tikaConfigPath;
+    private final ArrayBlockingQueue<FetchEmitTuple> queue;
+    private final Connection connection;
+    private final int totalThreads;
+    private final AsyncConfig asyncConfig;
+    private PreparedStatement emittersCanShutdown;
+    private volatile boolean isShuttingDown = false;
+    private AssignmentManager assignmentManager;
+    private int finishedThreads = 0;
+    private ExecutorService executorService;
+    private ExecutorCompletionService<Integer> executorCompletionService;
+
+    private AsyncProcessor(Path tikaConfigPath) throws SQLException, IOException {
+        this.tikaConfigPath = tikaConfigPath;
+        this.asyncConfig = AsyncConfig.load(tikaConfigPath);
+        this.queue = new ArrayBlockingQueue<>(asyncConfig.getQueueSize());
+        this.connection = DriverManager.getConnection(asyncConfig.getJdbcString());
+        this.totalThreads = asyncConfig.getNumWorkers() + asyncConfig.getNumEmitters() +
+                2;//assignment manager and enqueuer threads
+    }
+
+    public static AsyncProcessor build(Path tikaConfigPath) throws AsyncRuntimeException {
+        try {
+            AsyncProcessor processor = new AsyncProcessor(tikaConfigPath);
+            processor.init();
+            return processor;
+        } catch (SQLException | IOException e) {
+            throw new AsyncRuntimeException(e);
+        }
+    }
+
+    private static List<Integer> getActiveWorkers(Connection connection) throws SQLException {
+        PreparedStatement findActiveWorkers =
+                connection.prepareStatement("select worker_id from workers");
+        List<Integer> workers = new ArrayList<>();
+        try (ResultSet rs = findActiveWorkers.executeQuery()) {
+            while (rs.next()) {
+                workers.add(rs.getInt(1));
+            }
+        }
+        return workers;
+    }
+
+    public synchronized boolean offer(List<FetchEmitTuple> fetchEmitTuples, long offerMs)
+            throws AsyncRuntimeException, InterruptedException {
+        if (queue == null) {
+            throw new IllegalStateException("queue hasn't been initialized yet.");
+        } else if (isShuttingDown) {
+            throw new IllegalStateException(
+                    "Can't call offer after calling close() or " + "shutdownNow()");
+        }
+        long start = System.currentTimeMillis();
+        long elapsed = System.currentTimeMillis() - start;
+        while (elapsed < offerMs) {
+            checkActive();
+            if (queue.remainingCapacity() > fetchEmitTuples.size()) {
+                try {
+                    queue.addAll(fetchEmitTuples);
+                    return true;
+                } catch (IllegalStateException e) {
+                    //swallow
+                }
+            }
+            Thread.sleep(100);
+            elapsed = System.currentTimeMillis() - start;
+        }
+        return false;
+    }
+
+    public synchronized boolean offer(FetchEmitTuple t, long offerMs)
+            throws AsyncRuntimeException, InterruptedException {
+        if (queue == null) {
+            throw new IllegalStateException("queue hasn't been initialized yet.");
+        } else if (isShuttingDown) {
+            throw new IllegalStateException(
+                    "Can't call offer after calling close() or " + "shutdownNow()");
+        }
+        checkActive();
+        return queue.offer(t, offerMs, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * This polls the executorcompletionservice to check for execution exceptions
+     * and to make sure that some threads are still active.
+     *
+     * @return
+     * @throws AsyncRuntimeException
+     * @throws InterruptedException
+     */
+    public synchronized boolean checkActive() throws AsyncRuntimeException, InterruptedException {
+        Future<Integer> future = executorCompletionService.poll();
+        if (future != null) {
+            try {
+                future.get();
+            } catch (ExecutionException e) {
+                throw new AsyncRuntimeException(e);
+            }
+            finishedThreads++;
+        }
+        return finishedThreads != totalThreads;
+    }
+
+    private void init() throws SQLException {
+
+        setupTables();
+        String sql = "update emitters set status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.CAN_SHUTDOWN.ordinal();
+        this.emittersCanShutdown = connection.prepareStatement(sql);
+        executorService = Executors.newFixedThreadPool(totalThreads);
+        executorCompletionService = new ExecutorCompletionService<>(executorService);
+
+        AsyncTaskEnqueuer taskEnqueuer = new AsyncTaskEnqueuer(queue, connection);
+
+        executorCompletionService.submit(taskEnqueuer);
+
+        List<AsyncWorker> workers = buildWorkers(connection, asyncConfig, tikaConfigPath);
+        int maxRetries = 0;
+        for (AsyncWorker worker : workers) {
+            if (worker.getMaxRetries() > maxRetries) {
+                maxRetries = worker.getMaxRetries();
+            }
+            executorCompletionService.submit(worker);
+        }
+        assignmentManager = new AssignmentManager(connection, taskEnqueuer, maxRetries);
+        executorCompletionService.submit(assignmentManager);
+        for (int i = 0; i < asyncConfig.getNumEmitters(); i++) {
+            executorCompletionService
+                    .submit(new AsyncEmitter(connection, asyncConfig.getJdbcString(),
+                            asyncConfig.getNumWorkers() + i, tikaConfigPath));
+        }
+    }
+
+    private List<AsyncWorker> buildWorkers(Connection connection, AsyncConfig asyncConfig,
+                                           Path tikaConfigPath) throws SQLException {
+        //TODO -- make these workers configurable via the tika config, e.g. max retries
+        //and jvm args, etc.
+        List<AsyncWorker> workers = new ArrayList<>();
+        for (int i = 0; i < asyncConfig.getNumWorkers(); i++) {
+            workers.add(
+                    new AsyncWorker(connection, asyncConfig.getJdbcString(), i, tikaConfigPath));
+        }
+        return workers;
+    }
+
+    private void setupTables() throws SQLException {
+
+        String sql = "create table task_queue " + "(id bigint auto_increment primary key," +
+                "status tinyint," + //byte
+                "worker_id integer," + "retry smallint," + //short
+                "time_stamp timestamp," +
+                "json varchar(64000))";//this is the AsyncTask ... not the emit data!
+        try (Statement st = connection.createStatement()) {
+            st.execute(sql);
+        }
+        sql = "create table workers (worker_id int primary key, status tinyint)";
+        try (Statement st = connection.createStatement()) {
+            st.execute(sql);
+        }
+
+        sql = "create table emitters (emitter_id int primary key, status tinyint)";
+        try (Statement st = connection.createStatement()) {
+            st.execute(sql);
+        }
+
+        sql = "create table error_log (task_id bigint, " + "fetch_key varchar(10000)," +
+                "time_stamp timestamp," + "retry integer," + "error_code tinyint)";
+
+        try (Statement st = connection.createStatement()) {
+            st.execute(sql);
+        }
+
+        sql = "create table emits (" + "id bigint primary key, " + "time_stamp timestamp, " +
+                "uncompressed_size bigint, " + "bytes blob)";
+        try (Statement st = connection.createStatement()) {
+            st.execute(sql);
+        }
+    }
+
+    public void shutdownNow() throws IOException, AsyncRuntimeException {
+        isShuttingDown = true;
+        try {
+            executorService.shutdownNow();
+        } finally {
+            //close down processes and db
+            if (asyncConfig.getTempDBDir() != null) {
+                FileUtils.deleteDirectory(asyncConfig.getTempDBDir().toFile());
+            }
+        }
+    }
+
+    /**
+     * This is a blocking close.  It will wait for all tasks successfully submitted before this
+     * call to close() to complete before closing.  If you need to shutdown immediately, try
+     * {@link #shutdownNow()}.
+     *
+     * @throws IOException
+     */
+    @Override
+    public void close() throws IOException {
+        isShuttingDown = true;
+        try {
+            completeAndShutdown();
+        } catch (SQLException | InterruptedException e) {
+            throw new IOException(e);
+        } finally {
+            executorService.shutdownNow();
+            SQLException ex = null;
+            try {
+                connection.close();
+            } catch (SQLException e) {
+                ex = e;
+            }
+            //close down processes and db
+            if (asyncConfig.getTempDBDir() != null) {
+                FileUtils.deleteDirectory(asyncConfig.getTempDBDir().toFile());
+            }
+            if (ex != null) {
+                throw new IOException(ex);
+            }
+        }
+    }
+
+    //this will block until everything finishes
+    private void completeAndShutdown() throws SQLException, InterruptedException {
+
+        //blocking...notify taskEnqueuer
+        queue.put(FetchIterator.COMPLETED_SEMAPHORE);
+
+        //wait for assignmentManager to finish
+        //it will only complete after the task enqueuer has completed
+        //and there are no more parse tasks available, selected or in process
+        while (!assignmentManager.hasCompletedTasks()) {
+            Thread.sleep(100);
+        }
+
+        emittersCanShutdown.executeUpdate();
+
+        //wait for emitters to finish
+        long start = System.currentTimeMillis();
+        long elapsed = System.currentTimeMillis() - start;
+        try {
+            boolean isActive = checkActive();
+            while (isActive) {
+                isActive = checkActive();
+                elapsed = System.currentTimeMillis();
+            }
+        } catch (InterruptedException e) {
+            return;
+        }
+    }
+
+    //this reads fetchemittuples from the queue and inserts them in the db
+    //for the workers to read
+    private static class AsyncTaskEnqueuer implements Callable<Integer> {
+        private final PreparedStatement insert;
+
+        private final ArrayBlockingQueue<FetchEmitTuple> queue;
+        private final Connection connection;
+        private final Random random = new Random();
+
+        private volatile boolean isComplete = false;
+
+        AsyncTaskEnqueuer(ArrayBlockingQueue<FetchEmitTuple> queue, Connection connection)
+                throws SQLException {
+            this.queue = queue;
+            this.connection = connection;
+            String sql = "insert into task_queue (status, time_stamp, worker_id, retry, json) " +
+                    "values (?,CURRENT_TIMESTAMP(),?,?,?)";
+            insert = connection.prepareStatement(sql);
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            List<Integer> workers = new ArrayList<>();
+            while (true) {
+                FetchEmitTuple t = queue.poll(1, TimeUnit.SECONDS);
+                LOG.debug("enqueing to db " + t);
+                if (t == null) {
+                    //log.trace?
+                } else if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+                    isComplete = true;
+                    return 1;
+                } else {
+                    long start = System.currentTimeMillis();
+                    long elapsed = System.currentTimeMillis() - start;
+                    //TODO -- fix this -- this loop waits for workers to register
+                    while (workers.size() == 0 && elapsed < 600000) {
+                        workers = getActiveWorkers(connection);
+                        Thread.sleep(100);
+                        elapsed = System.currentTimeMillis() - start;
+                    }
+                    insert(t, workers);
+                }
+            }
+        }
+
+        boolean isComplete() {
+            return isComplete;
+        }
+
+        private void insert(FetchEmitTuple t, List<Integer> workers)
+                throws IOException, SQLException {
+            int workerId = workers.size() == 1 ? workers.get(0) :
+                    workers.get(random.nextInt(workers.size()));
+            insert.clearParameters();
+            insert.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+            insert.setInt(2, workerId);
+            insert.setShort(3, (short) 0);
+            insert.setString(4, JsonFetchEmitTuple.toJson(t));
+            insert.execute();
+        }
+    }
+
+    private static class AssignmentManager implements Callable {
+
+        private final Connection connection;
+        private final AsyncTaskEnqueuer enqueuer;
+        private final PreparedStatement getQueueDistribution;
+        private final PreparedStatement findMissingWorkers;
+        private final PreparedStatement allocateNonworkersToWorkers;
+        private final PreparedStatement reallocate;
+        private final PreparedStatement countAvailableTasks;
+        private final PreparedStatement shutdownWorker;
+        private final PreparedStatement findMaxRetrieds;
+        private final PreparedStatement logMaxRetrieds;
+        private final PreparedStatement removeMaxRetrieds;
+        private final Random random = new Random();
+        private final int maxRetries;
+        private volatile boolean hasCompleted = false;
+
+
+        public AssignmentManager(Connection connection, AsyncTaskEnqueuer enqueuer, int maxRetries)
+                throws SQLException {
+            this.connection = connection;
+            this.enqueuer = enqueuer;
+            this.maxRetries = maxRetries;
+            //this gets workers and # of tasks in desc order of number of tasks
+            String sql = "select w.worker_id, p.cnt " + "from workers w " +
+                    "left join (select worker_id, count(1) as cnt from task_queue " +
+                    "where status=" + AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal() +
+                    " group by worker_id)" + " p on p.worker_id=w.worker_id order by p.cnt desc";
+            getQueueDistribution = connection.prepareStatement(sql);
+            //find workers that have assigned tasks but are not in the
+            //workers table
+            sql = "select p.worker_id, count(1) as cnt from task_queue p " +
+                    "left join workers w on p.worker_id=w.worker_id " +
+                    "where w.worker_id is null group by p.worker_id";
+            findMissingWorkers = connection.prepareStatement(sql);
+
+            sql = "update task_queue set worker_id=? where worker_id=?";
+            allocateNonworkersToWorkers = connection.prepareStatement(sql);
+
+            //current strategy reallocate tasks from longest queue to shortest
+            //TODO: might consider randomly shuffling or other algorithms
+            sql = "update task_queue set worker_id= ? where id in " +
+                    "(select id from task_queue where " + "worker_id = ? and " + "rand() < 0.8 " +
+                    "and status=0 for update)";
+            reallocate = connection.prepareStatement(sql);
+
+            //get those tasks that are in the parse phase
+            //if they are selected or in process, it is possible that
+            //they'll need to be retried.  So, include all statuses
+            //meaning that the parse has not completed.
+            sql = "select count(1) from task_queue where status in (" +
+                    AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal() + ", " +
+                    AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED.ordinal() + ", " +
+                    AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS.ordinal() + ")";
+            countAvailableTasks = connection.prepareStatement(sql);
+
+            sql = "update workers set status=" +
+                    AsyncWorkerProcess.WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal() +
+                    " where worker_id = ?";
+            shutdownWorker = connection.prepareStatement(sql);
+
+            sql = "select id, retry, json from task_queue where retry >=" + maxRetries;
+            findMaxRetrieds = connection.prepareStatement(sql);
+
+            sql = "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code)" +
+                    "values (?,?,CURRENT_TIMESTAMP(), ?," +
+                    AsyncWorkerProcess.ERROR_CODES.MAX_RETRIES.ordinal() + ")";
+            logMaxRetrieds = connection.prepareStatement(sql);
+
+            sql = "delete from task_queue where id=?";
+            removeMaxRetrieds = connection.prepareStatement(sql);
+        }
+
+        protected boolean hasCompletedTasks() {
+            return hasCompleted;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+
+            while (true) {
+                removeMaxRetrieds();
+                List<Integer> missingWorkers = getMissingWorkers();
+                reallocateFromMissingWorkers(missingWorkers);
+                redistribute();
+                if (isComplete()) {
+                    notifyWorkers();
+                    return 1;
+                }
+                Thread.sleep(200);
+            }
+        }
+
+        private void removeMaxRetrieds() throws SQLException {
+            Set<Long> toRemove = new HashSet<>();
+            try (ResultSet rs = findMaxRetrieds.executeQuery()) {
+                while (rs.next()) {
+                    long id = rs.getLong(1);
+                    String json = rs.getString(2);
+                    int retries = rs.getInt(3);
+                    toRemove.add(id);
+                    FetchEmitTuple t;
+                    try (Reader reader = new StringReader(json)) {
+                        t = JsonFetchEmitTuple.fromJson(reader);
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                        //need to log this in the error_logs table
+                        continue;
+                    }
+                    logMaxRetrieds.clearParameters();
+                    logMaxRetrieds.setLong(1, id);
+                    logMaxRetrieds.setString(2, t.getFetchKey().getFetchKey());
+                    logMaxRetrieds.setInt(3, retries);
+                    logMaxRetrieds.addBatch();
+                }
+            }
+            logMaxRetrieds.executeBatch();
+
+            for (Long id : toRemove) {
+                removeMaxRetrieds.clearParameters();
+                removeMaxRetrieds.setLong(1, id);
+                removeMaxRetrieds.addBatch();
+            }
+            removeMaxRetrieds.executeBatch();
+        }
+
+        private void notifyWorkers() throws SQLException {
+            for (int workerId : getActiveWorkers(connection)) {
+                shutdownWorker.clearParameters();
+                shutdownWorker.setInt(1, workerId);
+                shutdownWorker.execute();
+            }
+        }
+
+        private boolean isComplete() throws SQLException {
+            if (hasCompleted) {
+                return hasCompleted;
+            }
+            if (!enqueuer.isComplete) {
+                return false;
+            }
+            try (ResultSet rs = countAvailableTasks.executeQuery()) {
+                while (rs.next()) {
+                    int availTasks = rs.getInt(1);
+                    if (availTasks == 0) {
+                        hasCompleted = true;
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
+        private void redistribute() throws SQLException {
+            //parallel lists of workerid = task queue size
+            List<Integer> workerIds = new ArrayList<>();
+            List<Integer> queueSize = new ArrayList<>();
+            int totalTasks = 0;
+
+            try (ResultSet rs = getQueueDistribution.executeQuery()) {
+                while (rs.next()) {
+                    int workerId = rs.getInt(1);
+                    int numTasks = rs.getInt(2);
+                    workerIds.add(workerId);
+                    queueSize.add(numTasks);
+                    LOG.debug("workerId: ({}) numTasks: ({})", workerId, numTasks);
+                    totalTasks += numTasks;
+                }
+            }
+            if (workerIds.size() == 0) {
+                return;
+            }
+            int averagePerWorker = Math.round((float) totalTasks / (float) workerIds.size());
+            int midPoint = Math.round((float) queueSize.size() / 2) + 1;
+            for (int i = queueSize.size() - 1, j = 0; i > midPoint && j < midPoint; i--, j++) {
+                int shortestQueue = queueSize.get(i);
+                int longestQueue = queueSize.get(j);
+                if ((shortestQueue < 5 && longestQueue > 5) ||
+                        longestQueue > 5 && longestQueue > (int) (1.5 * averagePerWorker)) {
+                    int shortestQueueWorker = workerIds.get(i);
+                    int longestQueueWorker = workerIds.get(j);
+                    reallocate.clearParameters();
+                    reallocate.setLong(1, shortestQueueWorker);
+                    reallocate.setLong(2, longestQueueWorker);
+                    reallocate.execute();
+                }
+            }
+
+        }
+
+        private void reallocateFromMissingWorkers(List<Integer> missingWorkers)
+                throws SQLException {
+
+            if (missingWorkers.size() == 0) {
+                return;
+            }
+
+            List<Integer> activeWorkers = getActiveWorkers(connection);
+            if (activeWorkers.size() == 0) {
+                return;
+            }
+
+            for (int missing : missingWorkers) {
+                int active = activeWorkers.get(random.nextInt(activeWorkers.size()));
+                allocateNonworkersToWorkers.clearParameters();
+                allocateNonworkersToWorkers.setInt(1, active);
+                allocateNonworkersToWorkers.setInt(2, missing);
+                allocateNonworkersToWorkers.execute();
+                LOG.debug("allocating missing working ({}) to ({})", missing, active);
+            }
+        }
+
+        private List<Integer> getMissingWorkers() throws SQLException {
+            List<Integer> missingWorkers = new ArrayList<>();
+            try (ResultSet rs = findMissingWorkers.executeQuery()) {
+                while (rs.next()) {
+                    int workerId = rs.getInt(1);
+                    missingWorkers.add(workerId);
+                    LOG.debug("Worker ({}) no longer active", workerId);
+                }
+            }
+            return missingWorkers;
+        }
+    }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
index bc9fa3b..875cc90 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
@@ -14,21 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
 
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
-    String getName();
-
-    void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
-    void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
-    //TODO -- add this later for xhtml?
-    //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+/**
+ * Fatal exception that means that something went seriously wrong.
+ */
+public class AsyncRuntimeException extends RuntimeException {
 
+    public AsyncRuntimeException(Throwable t) {
+        super(t);
+    }
 }
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java
new file mode 100644
index 0000000..e0c214a
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncTask extends FetchEmitTuple {
+
+    public static final AsyncTask SHUTDOWN_SEMAPHORE =
+            new AsyncTask(-1, (short) -1, new FetchEmitTuple(null, null, null));
+    private final short retry;
+    private long taskId;
+
+    public AsyncTask(@JsonProperty("taskId") long taskId, @JsonProperty("retry") short retry,
+                     @JsonProperty("fetchEmitTuple") FetchEmitTuple fetchEmitTuple) {
+        super(fetchEmitTuple.getFetchKey(), fetchEmitTuple.getEmitKey(),
+                fetchEmitTuple.getMetadata());
+        this.taskId = taskId;
+        this.retry = retry;
+    }
+
+    public long getTaskId() {
+        return taskId;
+    }
+
+    public void setTaskId(long taskId) {
+        this.taskId = taskId;
+    }
+
+    public short getRetry() {
+        return retry;
+    }
+
+    @Override
+    public String toString() {
+        return "AsyncTask{" + "taskId=" + taskId + ", retry=" + retry + '}';
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java
new file mode 100644
index 0000000..c26ee0f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY;
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_JDBC_KEY;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+/**
+ * This controls monitoring of the AsyncWorkerProcess
+ * and updates to the db on crashes etc.
+ */
+public class AsyncWorker implements Callable<Integer> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncWorker.class);
+
+
+    private final String connectionString;
+    private final int workerId;
+    private final Path tikaConfigPath;
+    private final Connection connection;
+    private final PreparedStatement finished;
+    private final PreparedStatement restarting;
+    private final PreparedStatement selectActiveTasks;
+    private final PreparedStatement insertErrorLog;
+    private final PreparedStatement resetStatus;
+    //TODO: make this configurable
+    private final int maxRetries = 2;
+
+    public AsyncWorker(Connection connection, String connectionString, int workerId,
+                       Path tikaConfigPath) throws SQLException {
+        this.connectionString = connectionString;
+        this.workerId = workerId;
+        this.tikaConfigPath = tikaConfigPath;
+        this.connection = connection;
+        String sql = "update workers set status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal() +
+                " where worker_id = (" + workerId + ")";
+        finished = connection.prepareStatement(sql);
+
+        sql = "update workers set status=" +
+                AsyncWorkerProcess.WORKER_STATUS_CODES.RESTARTING.ordinal() +
+                " where worker_id = (" + workerId + ")";
+        restarting = connection.prepareStatement(sql);
+        //this checks if the process was able to reset the status
+        sql = "select id, retry, json from task_queue where worker_id=" + workerId +
+                " and status=" + AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS.ordinal();
+        selectActiveTasks = connection.prepareStatement(sql);
+
+        //if not, this is called to insert into the error log
+        insertErrorLog = prepareInsertErrorLog(connection);
+
+        //and this is called to reset the status on error
+        resetStatus = prepareReset(connection);
+    }
+
+    static void reportAndReset(AsyncTask task, AsyncWorkerProcess.ERROR_CODES errorCode,
+                               PreparedStatement insertErrorLog, PreparedStatement resetStatus,
+                               Logger logger) {
+        try {
+            insertErrorLog.clearParameters();
+            insertErrorLog.setLong(1, task.getTaskId());
+            insertErrorLog.setString(2, task.getFetchKey().getFetchKey());
+            insertErrorLog.setInt(3, task.getRetry());
+            insertErrorLog.setByte(4, (byte) errorCode.ordinal());
+            insertErrorLog.execute();
+        } catch (SQLException e) {
+            logger.error("Can't update error log", e);
+        }
+
+        try {
+            resetStatus.clearParameters();
+            resetStatus.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+            resetStatus.setShort(2, (short) (task.getRetry() + 1));
+            resetStatus.setLong(3, task.getTaskId());
+            resetStatus.execute();
+        } catch (SQLException e) {
+            logger.error("Can't reset try status", e);
+        }
+    }
+
+    static PreparedStatement prepareInsertErrorLog(Connection connection) throws SQLException {
+        //if not, this is called to insert into the error log
+        return connection.prepareStatement(
+                "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code) " +
+                        " values (?,?,CURRENT_TIMESTAMP(),?,?)");
+    }
+
+    static PreparedStatement prepareReset(Connection connection) throws SQLException {
+        //and this is called to reset the status on error
+        return connection.prepareStatement(
+                "update task_queue set " + "status=?, " + "time_stamp=CURRENT_TIMESTAMP(), " +
+                        "retry=? " + "where id=?");
+    }
+
+    @Override
+    public Integer call() throws Exception {
+        Process p = null;
+        try {
+            p = start();
+            int restarts = 0;
+            while (true) {
+                boolean finished = p.waitFor(60, TimeUnit.SECONDS);
+                if (finished) {
+                    int exitValue = p.exitValue();
+                    if (exitValue == 0) {
+                        LOG.debug("forked worker process finished with exitValue=0");
+                        return 1;
+                    }
+                    reportCrash(++restarts, exitValue);
+                    p = start();
+                }
+            }
+        } finally {
+            if (p != null) {
+                p.destroyForcibly();
+            }
+            finished.execute();
+        }
+    }
+
+    public int getMaxRetries() {
+        return maxRetries;
+    }
+
+    private Process start() throws IOException {
+        String[] args = new String[]{"java", "-Djava.awt.headless=true", "-cp",
+                System.getProperty("java.class.path"),
+                "org.apache.tika.pipes.async.AsyncWorkerProcess", Integer.toString(workerId)};
+        ProcessBuilder pb = new ProcessBuilder(args);
+        pb.environment().put(TIKA_ASYNC_JDBC_KEY, connectionString);
+        pb.environment()
+                .put(TIKA_ASYNC_CONFIG_FILE_KEY, tikaConfigPath.toAbsolutePath().toString());
+        pb.inheritIO();
+        return pb.start();
+    }
+
+    private void reportCrash(int numRestarts, int exitValue) throws SQLException, IOException {
+        LOG.warn("worker id={} terminated, exitValue={}", workerId, exitValue);
+        restarting.execute();
+        List<AsyncTask> activeTasks = new ArrayList<>();
+        try (ResultSet rs = selectActiveTasks.executeQuery()) {
+            long taskId = rs.getLong(1);
+            short retry = rs.getShort(2);
+            String json = rs.getString(3);
+            FetchEmitTuple tuple = JsonFetchEmitTuple.fromJson(new StringReader(json));
+            activeTasks.add(new AsyncTask(taskId, retry, tuple));
+        }
+        if (activeTasks.size() == 0) {
+            LOG.debug("worker reset active tasks, nothing extra to report");
+            return;
+        }
+
+        if (activeTasks.size() > 1) {
+            LOG.warn("more than one active task? this should never happen!");
+        }
+
+        for (AsyncTask t : activeTasks) {
+            reportAndReset(t, AsyncWorkerProcess.ERROR_CODES.UNKNOWN_PARSE, insertErrorLog,
+                    resetStatus, LOG);
+        }
+
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java
new file mode 100644
index 0000000..988748f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.apache.tika.pipes.async.AsyncTask.SHUTDOWN_SEMAPHORE;
+import static org.apache.tika.pipes.async.AsyncWorker.prepareInsertErrorLog;
+import static org.apache.tika.pipes.async.AsyncWorker.prepareReset;
+import static org.apache.tika.pipes.async.AsyncWorker.reportAndReset;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import net.jpountz.lz4.LZ4Factory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.EncryptedDocumentException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.metadata.serialization.JsonMetadataSerializer;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.sax.BasicContentHandlerFactory;
+import org.apache.tika.sax.RecursiveParserWrapperHandler;
+import org.apache.tika.utils.StringUtils;
+
+public class AsyncWorkerProcess {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncWorkerProcess.class);
+    //make these all configurable
+    private static final long SHUTDOWN_AFTER_MS = 120000;
+    private static final long PULSE_MS = 1000;
+    private final long parseTimeoutMs = 60000;
+
+    public static void main(String[] args) throws Exception {
+        Path tikaConfigPath = Paths.get(System.getenv(AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY));
+        String db = System.getenv(AsyncProcessor.TIKA_ASYNC_JDBC_KEY);
+        TikaConfig tikaConfig = new TikaConfig(tikaConfigPath);
+        int workerId = Integer.parseInt(args[0]);
+        LOG.debug("trying to get connection {} >{}<", workerId, db);
+        try (Connection connection = DriverManager.getConnection(db)) {
+            AsyncWorkerProcess asyncWorker = new AsyncWorkerProcess();
+            asyncWorker.execute(connection, workerId, tikaConfig);
+        }
+        System.exit(0);
+    }
+
+    private void execute(Connection connection, int workerId, TikaConfig tikaConfig)
+            throws SQLException {
+        //3 = worker + forkwatcher + active task
+        ExecutorService service = Executors.newFixedThreadPool(3);
+        ExecutorCompletionService<Integer> executorCompletionService =
+                new ExecutorCompletionService<>(service);
+
+        executorCompletionService
+                .submit(new Worker(connection, workerId, tikaConfig, parseTimeoutMs));
+        executorCompletionService.submit(new ForkWatcher(System.in));
+
+        int completed = 0;
+
+        //if either one stops, we need to stop
+        try {
+            while (completed < 1) {
+                Future<Integer> future = executorCompletionService.poll(60, TimeUnit.SECONDS);
+                if (future != null) {
+                    completed++;
+                    future.get();
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            LOG.error("worker " + workerId + " had a mainloop exception", e);
+        } finally {
+            service.shutdownNow();
+        }
+        return;
+    }
+
+    enum TASK_STATUS_CODES {
+        AVAILABLE, SELECTED, IN_PROCESS, AVAILABLE_EMIT, SELECTED_EMIT, IN_PROCESS_EMIT,
+        FAILED_EMIT, EMITTED
+    }
+
+    public enum WORKER_STATUS_CODES {
+        ACTIVE, RESTARTING, HIBERNATING, CAN_SHUTDOWN,//if there's nothing else to process, shutdown
+        SHOULD_SHUTDOWN, //shutdown now whether or not there's anything else to process
+        HAS_SHUTDOWN
+    }
+
+    enum ERROR_CODES {
+        TIMEOUT, SECURITY_EXCEPTION, OTHER_EXCEPTION, OOM, OTHER_ERROR, UNKNOWN_PARSE, MAX_RETRIES,
+        EMIT_SERIALIZATION, EMIT_SQL_INSERT_EXCEPTION, EMIT_SQL_SELECT_EXCEPTION,
+        EMIT_DESERIALIZATION, EMIT_EXCEPTION
+    }
+
+    private static class TaskQueue {
+        private final Connection connection;
+        private final int workerId;
+
+        private final PreparedStatement markForSelecting;
+        private final PreparedStatement selectForProcessing;
+        private final PreparedStatement markForProcessing;
+        private final PreparedStatement checkForShutdown;
+
+
+        TaskQueue(Connection connection, int workerId) throws SQLException {
+            this.connection = connection;
+            this.workerId = workerId;
+            //TODO -- need to update timestamp
+            String sql = "update task_queue set status=" + TASK_STATUS_CODES.SELECTED.ordinal() +
+                    ", time_stamp=CURRENT_TIMESTAMP()" + " where id = " +
+                    " (select id from task_queue where worker_id = " + workerId + " and status=" +
+                    TASK_STATUS_CODES.AVAILABLE.ordinal() +
+                    " order by time_stamp asc limit 1 for update)";
+            markForSelecting = connection.prepareStatement(sql);
+            sql = "select id, retry, json from task_queue where status=" +
+                    TASK_STATUS_CODES.SELECTED.ordinal() + " and " + " worker_id=" + workerId +
+                    " order by time_stamp asc limit 1";
+            selectForProcessing = connection.prepareStatement(sql);
+            sql = "update task_queue set status=" + TASK_STATUS_CODES.IN_PROCESS.ordinal() +
+                    ", time_stamp=CURRENT_TIMESTAMP()" + " where id=?";
+            markForProcessing = connection.prepareStatement(sql);
+
+            sql = "select count(1) from workers where worker_id=" + workerId + " and status=" +
+                    WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal();
+            checkForShutdown = connection.prepareStatement(sql);
+        }
+
+        AsyncTask poll(long pollMs) throws InterruptedException, IOException, SQLException {
+            long start = System.currentTimeMillis();
+            long elapsed = System.currentTimeMillis() - start;
+            while (elapsed < pollMs) {
+                if (shouldShutdown()) {
+                    return SHUTDOWN_SEMAPHORE;
+                }
+                int i = markForSelecting.executeUpdate();
+                if (i == 0) {
+//                   debugQueue();
+                    Thread.sleep(PULSE_MS);
+                } else {
+                    long taskId = -1;
+                    short retry = -1;
+                    String json = "";
+                    try (ResultSet rs = selectForProcessing.executeQuery()) {
+                        while (rs.next()) {
+                            taskId = rs.getLong(1);
+                            retry = rs.getShort(2);
+                            json = rs.getString(3);
+                        }
+                    }
+                    markForProcessing.clearParameters();
+                    markForProcessing.setLong(1, taskId);
+                    markForProcessing.execute();
+
+                    FetchEmitTuple t = null;
+                    try (Reader reader = new StringReader(json)) {
+                        t = JsonFetchEmitTuple.fromJson(reader);
+                    }
+                    AsyncTask task = new AsyncTask(taskId, retry, t);
+                    return task;
+                }
+                elapsed = System.currentTimeMillis() - start;
+            }
+            return null;
+        }
+
+        private void debugQueue() throws SQLException {
+            try (ResultSet rs = connection.createStatement()
+                    .executeQuery("select id, status, worker_id from task_queue limit 10")) {
+                while (rs.next()) {
+                    System.out.println(
+                            "id: " + rs.getInt(1) + " status: " + rs.getInt(2) + " worker_id: " +
+                                    rs.getInt(3));
+                }
+            }
+        }
+
+        boolean shouldShutdown() throws SQLException {
+            try (ResultSet rs = checkForShutdown.executeQuery()) {
+                if (rs.next()) {
+                    int val = rs.getInt(1);
+                    return val > 0;
+                }
+            }
+            return false;
+        }
+    }
+
+
+    private static class Worker implements Callable<Integer> {
+
+        private final Connection connection;
+        private final int workerId;
+        private final RecursiveParserWrapper parser;
+        private final TikaConfig tikaConfig;
+        private final long parseTimeoutMs;
+        private final PreparedStatement insertErrorLog;
+        private final PreparedStatement resetStatus;
+        private final PreparedStatement insertEmitData;
+        private final PreparedStatement updateStatusForEmit;
+        private final ObjectMapper objectMapper = new ObjectMapper();
+        LZ4Factory factory = LZ4Factory.fastestInstance();
+        private final ExecutorService executorService;
+        private final ExecutorCompletionService<AsyncData> executorCompletionService;
+
+        public Worker(Connection connection, int workerId, TikaConfig tikaConfig,
+                      long parseTimeoutMs) throws SQLException {
+            this.connection = connection;
+            this.workerId = workerId;
+            this.parser = new RecursiveParserWrapper(tikaConfig.getParser());
+            this.tikaConfig = tikaConfig;
+            this.executorService = Executors.newFixedThreadPool(1);
+            this.executorCompletionService = new ExecutorCompletionService<>(executorService);
+            this.parseTimeoutMs = parseTimeoutMs;
+
+            SimpleModule module = new SimpleModule();
+            module.addSerializer(Metadata.class, new JsonMetadataSerializer());
+            objectMapper.registerModule(module);
+            String sql = "merge into workers key (worker_id) " + "values (" + workerId + ", " +
+                    WORKER_STATUS_CODES.ACTIVE.ordinal() + ")";
+            connection.createStatement().execute(sql);
+            insertErrorLog = prepareInsertErrorLog(connection);
+            resetStatus = prepareReset(connection);
+            insertEmitData = prepareInsertEmitData(connection);
+            sql = "update task_queue set status=" + TASK_STATUS_CODES.AVAILABLE_EMIT.ordinal() +
+                    ", time_stamp=CURRENT_TIMESTAMP()" + " where id=?";
+            updateStatusForEmit = connection.prepareStatement(sql);
+        }
+
+        static PreparedStatement prepareInsertEmitData(Connection connection) throws SQLException {
+            return connection.prepareStatement(
+                    "insert into emits (id, time_stamp, uncompressed_size, bytes) " +
+                            " values (?,CURRENT_TIMESTAMP(),?,?)");
+        }
+
+        public Integer call() throws Exception {
+            AsyncTask task = null;
+            try {
+
+                TaskQueue queue = new TaskQueue(connection, workerId);
+
+                long lastProcessed = System.currentTimeMillis();
+
+                while (true) {
+
+                    task = queue.poll(1000);
+                    if (task == null) {
+                        long elapsed = System.currentTimeMillis() - lastProcessed;
+                        if (elapsed > SHUTDOWN_AFTER_MS) {
+                            LOG.debug("shutting down after no assignments in {}ms", elapsed);
+                            return 1;
+                        }
+                    } else if (task == SHUTDOWN_SEMAPHORE) {
+                        break;
+                    } else {
+                        processTask(task);
+                        lastProcessed = System.currentTimeMillis();
+                    }
+                }
+            } catch (TimeoutException e) {
+                LOG.warn(task.getFetchKey().getFetchKey(), e);
+                reportAndReset(task, ERROR_CODES.TIMEOUT, insertErrorLog, resetStatus, LOG);
+            } catch (SecurityException e) {
+                LOG.warn(task.getFetchKey().getFetchKey(), e);
+                reportAndReset(task, ERROR_CODES.SECURITY_EXCEPTION, insertErrorLog, resetStatus,
+                        LOG);
+            } catch (Exception e) {
+                e.printStackTrace();
+                LOG.warn(task.getFetchKey().getFetchKey(), e);
+                reportAndReset(task, ERROR_CODES.OTHER_EXCEPTION, insertErrorLog, resetStatus, LOG);
+            } catch (OutOfMemoryError e) {
+                LOG.warn(task.getFetchKey().getFetchKey(), e);
+                reportAndReset(task, ERROR_CODES.OOM, insertErrorLog, resetStatus, LOG);
+            } catch (Error e) {
+                LOG.warn(task.getFetchKey().getFetchKey(), e);
+                reportAndReset(task, ERROR_CODES.OTHER_ERROR, insertErrorLog, resetStatus, LOG);
+            } finally {
+                executorService.shutdownNow();
+                return 1;
+            }
+        }
+
+        private void processTask(AsyncTask task) throws Exception {
+
+            if (task == SHUTDOWN_SEMAPHORE) {
+                LOG.debug("received shutdown notification");
+                return;
+            } else {
+                executorCompletionService
+                        .submit(new TaskProcessor(task, tikaConfig, parser, workerId));
+                Future<AsyncData> future =
+                        executorCompletionService.poll(parseTimeoutMs, TimeUnit.MILLISECONDS);
+                if (future == null) {
+                    handleTimeout(task.getTaskId(), task.getFetchKey().getFetchKey());
+                } else {
+                    AsyncData asyncData = future.get(1000, TimeUnit.MILLISECONDS);
+                    if (asyncData == null) {
+                        handleTimeout(task.getTaskId(), task.getFetchKey().getFetchKey());
+                    }
+                    boolean shouldEmit = checkForParseException(asyncData);
+                    if (shouldEmit) {
+                        try {
+                            emit(asyncData);
+                        } catch (JsonProcessingException e) {
+                            e.printStackTrace();
+                            recordBadEmit(task.getTaskId(), task.getFetchKey().getFetchKey(),
+                                    ERROR_CODES.EMIT_SERIALIZATION.ordinal());
+                        } catch (SQLException e) {
+                            e.printStackTrace();
+                            recordBadEmit(task.getTaskId(), task.getFetchKey().getFetchKey(),
+                                    ERROR_CODES.EMIT_SQL_INSERT_EXCEPTION.ordinal());
+                        }
+                    }
+                }
+            }
+        }
+
+        private void recordBadEmit(long taskId, String key, int ordinal) {
+            //stub
+        }
+
+        private void emit(AsyncData asyncData) throws SQLException, JsonProcessingException {
+            insertEmitData.clearParameters();
+            insertEmitData.setLong(1, asyncData.getTaskId());
+            byte[] bytes = objectMapper.writeValueAsBytes(asyncData);
+            byte[] compressed = factory.fastCompressor().compress(bytes);
+            insertEmitData.setLong(2, bytes.length);
+            insertEmitData.setBlob(3, new ByteArrayInputStream(compressed));
+            insertEmitData.execute();
+            updateStatusForEmit.clearParameters();
+            updateStatusForEmit.setLong(1, asyncData.getTaskId());
+            updateStatusForEmit.execute();
+        }
+
+        private void handleTimeout(long taskId, String key) throws TimeoutException {
+            LOG.warn("timeout taskid:{} fetchKey:{}", taskId, key);
+            throw new TimeoutException(key);
+        }
+
+        private boolean checkForParseException(AsyncData asyncData) {
+            if (asyncData == null || asyncData.getMetadataList() == null ||
+                    asyncData.getMetadataList().size() == 0) {
+                LOG.warn("empty or null emit data ({})", asyncData.getFetchKey().getFetchKey());
+                return false;
+            }
+            boolean shouldEmit = true;
+            Metadata container = asyncData.getMetadataList().get(0);
+            String stack = container.get(TikaCoreProperties.CONTAINER_EXCEPTION);
+            if (stack != null) {
+                LOG.warn("fetchKey ({}) container parse exception ({})",
+                        asyncData.getFetchKey().getFetchKey(), stack);
+                if (asyncData.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
+                    shouldEmit = false;
+                }
+            }
+
+            for (int i = 1; i < asyncData.getMetadataList().size(); i++) {
+                Metadata m = asyncData.getMetadataList().get(i);
+                String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
+                if (embeddedStack != null) {
+                    LOG.warn("fetchKey ({}) embedded parse exception ({})",
+                            asyncData.getFetchKey().getFetchKey(), embeddedStack);
+                }
+            }
+            return shouldEmit;
+        }
+    }
+
+    private static class TaskProcessor implements Callable<AsyncData> {
+
+        private final AsyncTask task;
+        private final Parser parser;
+        private final TikaConfig tikaConfig;
+        private final int workerId;
+
+        public TaskProcessor(AsyncTask task, TikaConfig tikaConfig, Parser parser, int workerId) {
+            this.task = task;
+            this.parser = parser;
+            this.tikaConfig = tikaConfig;
+            this.workerId = workerId;
+        }
+
+        public AsyncData call() throws Exception {
+            Metadata userMetadata = task.getMetadata();
+            Metadata metadata = new Metadata();
+            String fetcherName = task.getFetchKey().getFetcherName();
+            String fetchKey = task.getFetchKey().getFetchKey();
+            List<Metadata> metadataList = null;
+            try (InputStream stream = tikaConfig.getFetcherManager().getFetcher(fetcherName)
+                    .fetch(fetchKey, metadata)) {
+                metadataList = parseMetadata(task.getFetchKey(), stream, metadata);
+            } catch (SecurityException e) {
+                throw e;
+            }
+            injectUserMetadata(userMetadata, metadataList);
+            EmitKey emitKey = task.getEmitKey();
+            if (StringUtils.isBlank(emitKey.getEmitKey())) {
+                emitKey = new EmitKey(emitKey.getEmitterName(), fetchKey);
+                task.setEmitKey(emitKey);
+            }
+            return new AsyncData(task.getTaskId(), task.getFetchKey(), task.getEmitKey(),
+                    task.getOnParseException(), metadataList);
+        }
+
+        private List<Metadata> parseMetadata(FetchKey fetchKey, InputStream stream,
+                                             Metadata metadata) {
+            //make these configurable
+            BasicContentHandlerFactory.HANDLER_TYPE type =
+                    BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+            int writeLimit = -1;
+            int maxEmbeddedResources = 1000;
+
+            RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
+                    new BasicContentHandlerFactory(type, writeLimit), maxEmbeddedResources,
+                    tikaConfig.getMetadataFilter());
+            ParseContext parseContext = new ParseContext();
+            try {
+                parser.parse(stream, handler, metadata, parseContext);
+            } catch (SAXException e) {
+                LOG.warn("problem:" + fetchKey.getFetchKey(), e);
+            } catch (EncryptedDocumentException e) {
+                LOG.warn("encrypted:" + fetchKey.getFetchKey(), e);
+            } catch (SecurityException e) {
+                LOG.warn("security exception: " + fetchKey.getFetchKey());
+                throw e;
+            } catch (Exception e) {
+                LOG.warn("exception: " + fetchKey.getFetchKey());
+            } catch (OutOfMemoryError e) {
+                LOG.error("oom: " + fetchKey.getFetchKey());
+                throw e;
+            }
+            return handler.getMetadataList();
+        }
+
+        private void injectUserMetadata(Metadata userMetadata, List<Metadata> metadataList) {
+            for (String n : userMetadata.names()) {
+                //overwrite whatever was there
+                metadataList.get(0).set(n, null);
+                for (String val : userMetadata.getValues(n)) {
+                    metadataList.get(0).add(n, val);
+                }
+            }
+        }
+    }
+
+    private static class ForkWatcher implements Callable<Integer> {
+        private final InputStream in;
+
+        public ForkWatcher(InputStream in) {
+            this.in = in;
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            //this should block forever
+            //if the forking process dies,
+            // this will either throw an IOException or read -1.
+            int i = in.read();
+            LOG.info("forking process notified forked to shutdown");
+            return 1;
+        }
+    }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties b/tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
similarity index 93%
copy from tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
copy to tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
index 92b6d56..43553a4 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #info,debug, error,fatal ...
 log4j.rootLogger=debug,stderr
-
 #console
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
 log4j.appender.stderr.Target=System.err
-
-log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java
new file mode 100644
index 0000000..04890b3
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncProcessorTest {
+
+    private Path dbDir;
+    private Path dbFile;
+    private Connection connection;
+    private Path tikaConfigPath;
+
+    @Before
+    public void setUp() throws SQLException, IOException {
+        dbDir = Files.createTempDirectory("async-db");
+        dbFile = dbDir.resolve("emitted-db");
+        String jdbc = "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE";
+        String sql = "create table emitted (id int auto_increment primary key, " +
+                "emitkey varchar(2000), json varchar(20000))";
+
+        connection = DriverManager.getConnection(jdbc);
+        connection.createStatement().execute(sql);
+        tikaConfigPath = dbDir.resolve("tika-config.xml");
+        String xml = "" + "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + "<properties>" +
+                "  <emitters>" + "  <emitter class=\"org.apache.tika.pipes.async.MockEmitter\">\n" +
+                "    <params>\n" + "      <param name=\"name\" type=\"string\">mock</param>\n" +
+                "      <param name=\"jdbc\" type=\"string\">" + jdbc + "</param>\n" +
+                "    </params>" + "  </emitter>" + "  </emitters>" + "  <fetchers>" +
+                "    <fetcher class=\"org.apache.tika.pipes.async.MockFetcher\">" +
+                "      <param name=\"name\" type=\"string\">mock</param>\n" + "    </fetcher>" +
+                "  </fetchers>" + "</properties>";
+        Files.write(tikaConfigPath, xml.getBytes(StandardCharsets.UTF_8));
+    }
+
+    @After
+    public void tearDown() throws SQLException, IOException {
+        connection.createStatement().execute("drop table emitted");
+        connection.close();
+        FileUtils.deleteDirectory(dbDir.toFile());
+    }
+
+    @Test
+    public void testBasic() throws Exception {
+
+
+        AsyncProcessor processor = AsyncProcessor.build(tikaConfigPath);
+        int max = 100;
+        for (int i = 0; i < max; i++) {
+            FetchEmitTuple t = new FetchEmitTuple(new FetchKey("mock", "key-" + i),
+                    new EmitKey("mock", "emit-" + i), new Metadata());
+            processor.offer(t, 1000);
+        }
+        processor.close();
+        String sql = "select emitkey from emitted";
+        Set<String> emitKeys = new HashSet<>();
+        try (Statement st = connection.createStatement(); ResultSet rs = st.executeQuery(sql)) {
+            while (rs.next()) {
+                String emitKey = rs.getString(1);
+                emitKeys.add(emitKey);
+            }
+        }
+        assertEquals(max, emitKeys.size());
+        for (int i = 0; i < max; i++) {
+            assertTrue(emitKeys.contains("emit-" + i));
+        }
+    }
+}
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java
new file mode 100644
index 0000000..5576f73
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataSerializer;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+
+public class MockEmitter implements Initializable, Emitter {
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private Connection connection;
+    private String jdbc;
+    private PreparedStatement insert;
+
+    public MockEmitter() {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(Metadata.class, new JsonMetadataSerializer());
+        objectMapper.registerModule(module);
+    }
+
+    @Field
+    public void setJdbc(String jdbc) {
+        this.jdbc = jdbc;
+    }
+
+    @Override
+    public String getName() {
+        return "mock";
+    }
+
+    @Override
+    public void emit(String emitKey, List<Metadata> metadataList)
+            throws IOException, TikaEmitterException {
+        emit(Collections
+                .singletonList(new EmitData(new EmitKey(getName(), emitKey), metadataList)));
+    }
+
+    @Override
+    public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
+        for (EmitData d : emitData) {
+            String json = objectMapper.writeValueAsString(d);
+            try {
+                insert.clearParameters();
+                insert.setString(1, d.getEmitKey().getEmitKey());
+                insert.setString(2, json);
+                insert.execute();
+            } catch (SQLException e) {
+                throw new TikaEmitterException("problem inserting", e);
+            }
+        }
+    }
+
+    @Override
+    public void initialize(Map<String, Param> params) throws TikaConfigException {
+        try {
+            connection = DriverManager.getConnection(jdbc);
+            String sql = "insert into emitted (emitkey, json) values (?, ?)";
+            insert = connection.prepareStatement(sql);
+        } catch (SQLException e) {
+            throw new TikaConfigException("problem w connection", e);
+        }
+    }
+
+    @Override
+    public void checkInitialization(InitializableProblemHandler problemHandler)
+            throws TikaConfigException {
+
+    }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
similarity index 55%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
index 95bb462..7a74f2a 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
@@ -14,28 +14,31 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.util.List;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 
+import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.Fetcher;
 
-public class EmptyEmitter implements Emitter {
+public class MockFetcher implements Fetcher {
 
-    @Override
-    public String getName() {
-        return "empty";
-    }
+    private static final byte[] BYTES = ("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + "<mock>" +
+            "<metadata action=\"add\" name=\"dc:creator\">Nikolai Lobachevsky</metadata>" +
+            "<write element=\"p\">main_content</write>" + "</mock>")
+            .getBytes(StandardCharsets.UTF_8);
 
     @Override
-    public void emit(String emitKey, List<Metadata> metadataList)
-            throws IOException, TikaEmitterException {
-
+    public String getName() {
+        return "mock";
     }
 
     @Override
-    public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+    public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
+        return new ByteArrayInputStream(BYTES);
     }
 }
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java
new file mode 100644
index 0000000..ebbd8f0
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.junit.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataDeserializer;
+
+public class SerializationTest {
+
+    @Test
+    public void testBasic() throws Exception {
+        String json = "{\"taskId\":49,\"fetchKey\":{\"fetcherName\":\"mock\"," +
+                "\"fetchKey\":\"key-48\"},\"emitKey\":{\"emitterName\":\"mock\"," +
+                "\"emitKey\":\"emit-48\"},\"onParseException\":\"EMIT\",\"metadataList\":" +
+                "[{\"X-TIKA:Parsed-By\":" +
+                "\"org.apache.tika.parser.EmptyParser\",\"X-TIKA:parse_time_millis\":" +
+                "\"0\",\"X-TIKA:embedded_depth\":\"0\"}]}";
+
+        ObjectMapper mapper = new ObjectMapper();
+        SimpleModule module = new SimpleModule();
+        module.addDeserializer(Metadata.class, new JsonMetadataDeserializer());
+        mapper.registerModule(module);
+        AsyncData asyncData = mapper.readValue(json, AsyncData.class);
+        assertEquals(49, asyncData.getTaskId());
+        assertEquals("mock", asyncData.getFetchKey().getFetcherName());
+        assertEquals(1, asyncData.getMetadataList().size());
+    }
+
+}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
index 3831f07..7fbba83 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
@@ -17,23 +17,6 @@
 
 package org.apache.tika.pipes;
 
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.iterable.S3Objects;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.emitter.Emitter;
-import org.apache.tika.pipes.emitter.s3.S3Emitter;
-import org.apache.tika.pipes.fetcher.Fetcher;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.Ignore;
-import org.junit.Test;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -49,6 +32,24 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.iterable.S3Objects;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.s3.S3Emitter;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
 @Ignore("turn these into actual tests")
 public class PipeIntegrationTests {
 
@@ -59,10 +60,8 @@ public class PipeIntegrationTests {
         String region = "";
         String profile = "";
         String bucket = "";
-        AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
-                .withRegion(region)
-                .withCredentials(new ProfileCredentialsProvider(profile))
-                .build();
+        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(region)
+                .withCredentials(new ProfileCredentialsProvider(profile)).build();
         s3Client.listObjects(bucket);
         int cnt = 0;
         long sz = 0;
@@ -72,17 +71,18 @@ public class PipeIntegrationTests {
             if (Files.isRegularFile(targ)) {
                 continue;
             }
-            if (! Files.isDirectory(targ.getParent())) {
+            if (!Files.isDirectory(targ.getParent())) {
                 Files.createDirectories(targ.getParent());
             }
-            System.out.println("id: " + cnt + " :: " + summary.getKey() + " : " + summary.getSize());
+            System.out
+                    .println("id: " + cnt + " :: " + summary.getKey() + " : " + summary.getSize());
             S3Object s3Object = s3Client.getObject(bucket, summary.getKey());
             Files.copy(s3Object.getObjectContent(), targ);
             summary.getSize();
             cnt++;
             sz += summary.getSize();
         }
-        System.out.println("iterated: "+cnt + " sz: "+sz);
+        System.out.println("iterated: " + cnt + " sz: " + sz);
     }
 
     @Test
@@ -92,15 +92,20 @@ public class PipeIntegrationTests {
         int numConsumers = 1;
         ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
         ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
-        ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
-        completionService.submit(it);
+        ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(1000);
+        for (int i = 0; i < numConsumers; i++) {
+            completionService.submit(new FSFetcherEmitter(queue,
+                    tikaConfig.getFetcherManager().getFetcher("s3"), null));
+        }
+        for (FetchEmitTuple t : it) {
+            queue.offer(t);
+        }
         for (int i = 0; i < numConsumers; i++) {
-            completionService.submit(new FSFetcherEmitter(
-                    queue, tikaConfig.getFetcherManager().getFetcher("s3"), null));
+            queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
         }
         int finished = 0;
         try {
-            while (finished++ < numConsumers+1) {
+            while (finished++ < numConsumers + 1) {
                 Future<Integer> future = completionService.take();
                 future.get();
             }
@@ -112,20 +117,25 @@ public class PipeIntegrationTests {
     @Test
     public void testS3ToS3() throws Exception {
         TikaConfig tikaConfig = getConfig("tika-config-s3Tos3.xml");
-        FetchIterator it = tikaConfig.getFetchIterator();
         int numConsumers = 20;
         ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
         ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
-        ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
-        completionService.submit(it);
+        ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(1000);
+        for (int i = 0; i < numConsumers; i++) {
+            completionService.submit(new S3FetcherEmitter(queue,
+                    tikaConfig.getFetcherManager().getFetcher("s3f"),
+                    (S3Emitter) tikaConfig.getEmitterManager().getEmitter("s3e")));
+        }
+        FetchIterator it = tikaConfig.getFetchIterator();
+        for (FetchEmitTuple t : it) {
+            queue.offer(t);
+        }
         for (int i = 0; i < numConsumers; i++) {
-            completionService.submit(new S3FetcherEmitter(
-                    queue, tikaConfig.getFetcherManager().getFetcher("s3f"),
-                    (S3Emitter)tikaConfig.getEmitterManager().getEmitter("s3e")));
+            queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
         }
         int finished = 0;
         try {
-            while (finished++ < numConsumers+1) {
+            while (finished++ < numConsumers + 1) {
                 Future<Integer> future = completionService.take();
                 future.get();
             }
@@ -135,8 +145,7 @@ public class PipeIntegrationTests {
     }
 
     private TikaConfig getConfig(String fileName) throws Exception {
-        try (InputStream is =
-                     PipeIntegrationTests.class.getResourceAsStream("/"+fileName)) {
+        try (InputStream is = PipeIntegrationTests.class.getResourceAsStream("/" + fileName)) {
             return new TikaConfig(is);
         }
     }
@@ -149,8 +158,8 @@ public class PipeIntegrationTests {
         private final Emitter emitter;
         private final ArrayBlockingQueue<FetchEmitTuple> queue;
 
-        FSFetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher
-                fetcher, Emitter emitter) {
+        FSFetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher fetcher,
+                         Emitter emitter) {
             this.queue = queue;
             this.fetcher = fetcher;
             this.emitter = emitter;
@@ -172,12 +181,12 @@ public class PipeIntegrationTests {
         }
 
         private void process(FetchEmitTuple t) throws IOException, TikaException {
-            Path targ = OUTDIR.resolve(t.getFetchKey().getKey());
+            Path targ = OUTDIR.resolve(t.getFetchKey().getFetchKey());
             if (Files.isRegularFile(targ)) {
                 return;
             }
-            try (InputStream is = fetcher.fetch(t.getFetchKey().getKey(), t.getMetadata())) {
-                System.out.println(counter.getAndIncrement() + " : "+t );
+            try (InputStream is = fetcher.fetch(t.getFetchKey().getFetchKey(), t.getMetadata())) {
+                System.out.println(counter.getAndIncrement() + " : " + t);
                 Files.createDirectories(targ.getParent());
                 Files.copy(is, targ);
             }
@@ -191,8 +200,8 @@ public class PipeIntegrationTests {
         private final S3Emitter emitter;
         private final ArrayBlockingQueue<FetchEmitTuple> queue;
 
-        S3FetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher
-                fetcher, S3Emitter emitter) {
+        S3FetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher fetcher,
+                         S3Emitter emitter) {
             this.queue = queue;
             this.fetcher = fetcher;
             this.emitter = emitter;
@@ -217,8 +226,8 @@ public class PipeIntegrationTests {
             Metadata userMetadata = new Metadata();
             userMetadata.set("project", "my-project");
 
-            try (InputStream is = fetcher.fetch(t.getFetchKey().getKey(), t.getMetadata())) {
-                emitter.emit(t.getEmitKey().getKey(), is, userMetadata);
+            try (InputStream is = fetcher.fetch(t.getFetchKey().getFetchKey(), t.getMetadata())) {
+                emitter.emit(t.getEmitKey().getEmitKey(), is, userMetadata);
             }
         }
     }
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #info,debug, error,fatal ...
 log4j.rootLogger=info,stderr
-
 #console
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
 log4j.appender.stderr.Target=System.err
-
 log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-serialization/pom.xml b/tika-serialization/pom.xml
index 64e3aa2..2ad8035 100644
--- a/tika-serialization/pom.xml
+++ b/tika-serialization/pom.xml
@@ -56,6 +56,11 @@
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
 
     <!-- Test dependencies -->
     <dependency>
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
index 21b6377..e256d7d 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
@@ -32,7 +32,7 @@ public class JsonEmitData {
             jsonGenerator.writeStartObject();
             EmitKey key = emitData.getEmitKey();
             jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITTER, key.getEmitterName());
-            jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITKEY, key.getKey());
+            jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITKEY, key.getEmitKey());
             jsonGenerator.writeFieldName("data");
             jsonGenerator.writeStartArray();
             for (Metadata m : emitData.getMetadataList()) {
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
index 55d10ef..78ba9d2 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
@@ -133,10 +133,10 @@ public class JsonFetchEmitTuple {
     static void writeTuple(FetchEmitTuple t, JsonGenerator jsonGenerator) throws IOException {
         jsonGenerator.writeStartObject();
         jsonGenerator.writeStringField(FETCHER, t.getFetchKey().getFetcherName());
-        jsonGenerator.writeStringField(FETCHKEY, t.getFetchKey().getKey());
+        jsonGenerator.writeStringField(FETCHKEY, t.getFetchKey().getFetchKey());
         jsonGenerator.writeStringField(EMITTER, t.getEmitKey().getEmitterName());
-        if (!StringUtils.isBlank(t.getEmitKey().getKey())) {
-            jsonGenerator.writeStringField(EMITKEY, t.getEmitKey().getKey());
+        if (!StringUtils.isBlank(t.getEmitKey().getEmitKey())) {
+            jsonGenerator.writeStringField(EMITKEY, t.getEmitKey().getEmitKey());
         }
         if (t.getMetadata().size() > 0) {
             jsonGenerator.writeFieldName(METADATAKEY);
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
similarity index 54%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
index 95bb462..5a9b7d0 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
@@ -14,28 +14,30 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.metadata.serialization;
+
+import static org.apache.tika.metadata.serialization.JsonMetadata.readMetadataObject;
 
 import java.io.IOException;
-import java.util.List;
 
-import org.apache.tika.metadata.Metadata;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
 
-public class EmptyEmitter implements Emitter {
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 
-    @Override
-    public String getName() {
-        return "empty";
-    }
+import org.apache.tika.metadata.Metadata;
 
-    @Override
-    public void emit(String emitKey, List<Metadata> metadataList)
-            throws IOException, TikaEmitterException {
+public class JsonMetadataDeserializer extends StdDeserializer<Metadata> {
 
+    public JsonMetadataDeserializer() {
+        super(Metadata.class);
     }
 
     @Override
-    public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+    public Metadata deserialize(JsonParser jsonParser,
+                                DeserializationContext deserializationContext)
+            throws IOException, JsonProcessingException {
+        return readMetadataObject(jsonParser);
     }
 }
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
similarity index 56%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
index 95bb462..afcf012 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
@@ -14,28 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.metadata.serialization;
 
-import java.io.IOException;
-import java.util.List;
+import static org.apache.tika.metadata.serialization.JsonMetadata.writeMetadataObject;
 
-import org.apache.tika.metadata.Metadata;
+import java.io.IOException;
 
-public class EmptyEmitter implements Emitter {
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
 
-    @Override
-    public String getName() {
-        return "empty";
-    }
+import org.apache.tika.metadata.Metadata;
 
-    @Override
-    public void emit(String emitKey, List<Metadata> metadataList)
-            throws IOException, TikaEmitterException {
+public class JsonMetadataSerializer extends StdSerializer<Metadata> {
 
+    public JsonMetadataSerializer() {
+        super(Metadata.class);
     }
 
     @Override
-    public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+    public void serialize(Metadata metadata,
+                          JsonGenerator jsonGenerator,
+                          SerializerProvider serializerProvider) throws IOException {
+        writeMetadataObject(metadata, jsonGenerator, false);
     }
 }
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
index 67bbbde..2c19afb 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
@@ -40,12 +40,12 @@ public class JsonFetchEmitTupleTest {
         m.add("m3", "v4");
 
         FetchEmitTuple t = new FetchEmitTuple(
-                new FetchKey("fetcher1", "fetchkey1"),
-                        new EmitKey("emitter1", "emitKey1"),
+                new FetchKey("my_fetcher", "fetchKey1"),
+                        new EmitKey("my_emitter", "emitKey1"),
                         m);
         StringWriter writer = new StringWriter();
         JsonFetchEmitTuple.toJson(t, writer);
-
+        System.out.println(writer.toString());
         Reader reader = new StringReader(writer.toString());
         FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader);
         assertEquals(t, deserialized);
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
index 3b1fb25..a9f7319 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
@@ -31,7 +31,7 @@ import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
 public class TikaClient {
 
     private final Random random = new Random();
-    private List<TikaHttpClient> clients;
+    private final List<TikaHttpClient> clients;
 
 
     private TikaClient(List<TikaHttpClient> clients) {
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
index b959839..5e583db 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
@@ -45,8 +45,9 @@ import org.apache.tika.pipes.fetchiterator.FetchIterator;
 public class TikaClientCLI {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(TikaClientCLI.class);
+    private static final int QUEUE_SIZE = 10000;
 
-    private long maxWaitMs = 300000;
+    private final long maxWaitMs = 300000;
 
     public static void main(String[] args) throws Exception {
         //TODO -- add an actual commandline,
@@ -65,9 +66,10 @@ public class TikaClientCLI {
         ExecutorCompletionService<Integer> completionService =
                 new ExecutorCompletionService<>(executorService);
         final FetchIterator fetchIterator = config.getFetchIterator();
-        final ArrayBlockingQueue<FetchEmitTuple> queue = fetchIterator.init(numThreads);
+        final ArrayBlockingQueue<FetchEmitTuple> queue =
+                new ArrayBlockingQueue<>(QUEUE_SIZE);
 
-        completionService.submit(fetchIterator);
+        completionService.submit(new FetchIteratorWrapper(fetchIterator, queue, numThreads));
         if (tikaServerUrls.size() == numThreads) {
             logDiffSizes(tikaServerUrls.size(), numThreads);
             for (int i = 0; i < numThreads; i++) {
@@ -179,4 +181,31 @@ public class TikaClientCLI {
             }
         }
     }
+
+    private class FetchIteratorWrapper implements Callable<Integer> {
+        private final FetchIterator fetchIterator;
+        private final ArrayBlockingQueue<FetchEmitTuple> queue;
+        private final int numThreads;
+
+        public FetchIteratorWrapper(FetchIterator fetchIterator,
+                                    ArrayBlockingQueue<FetchEmitTuple> queue,
+                                    int numThreads) {
+            this.fetchIterator = fetchIterator;
+            this.queue = queue;
+            this.numThreads = numThreads;
+
+        }
+
+        @Override
+        public Integer call() throws Exception {
+            for (FetchEmitTuple t : fetchIterator) {
+                //potentially blocks forever
+                queue.put(t);
+            }
+            for (int i = 0; i < numThreads; i ++) {
+                queue.put(FetchIterator.COMPLETED_SEMAPHORE);
+            }
+            return 1;
+        }
+    }
 }
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
index 1438885..4cb134c 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
@@ -19,9 +19,10 @@ package org.apache.tika.server.client;
 public class TikaEmitterResult {
 
 
-    private STATUS status;
-    private String msg;//used for exceptions. will be null for status ok
-    private long timeElapsed;
+    private final STATUS status;
+    private final String msg;//used for exceptions. will be null for status ok
+    private final long timeElapsed;
+
     public TikaEmitterResult(STATUS status, long timeElapsed, String msg) {
         this.status = status;
         this.timeElapsed = timeElapsed;
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
index 6d6517f..9a5ff7f 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
@@ -48,11 +48,11 @@ class TikaHttpClient {
     private final String emitEndPointUrl;
     private final String asyncEndPointUrl;
     private final String tikaUrl;
-    private int maxRetries = 3;
+    private final int maxRetries = 3;
     //if can't make contact with Tika server, max wait time in ms
-    private long maxWaitForTikaMs = 120000;
+    private final long maxWaitForTikaMs = 120000;
     //how often to ping /tika (in ms) to see if the server is up and running
-    private long pulseWaitForTikaMs = 1000;
+    private final long pulseWaitForTikaMs = 1000;
 
     /**
      * @param baseUrl    url to base endpoint
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
index 81f9a7e..054a1d2 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
@@ -46,9 +46,12 @@ public class AsyncEmitter implements Callable<Integer> {
 
     //TODO -- need to configure these
     private final long emitWithinMs = 1000;
+
+    private long maxEstimatedBytes = 10_000_000;
+
     private final ArrayBlockingQueue<EmitData> emitDataQueue;
+
     Instant lastEmitted = Instant.now();
-    private long maxEstimatedBytes = 10_000_000;
 
     public AsyncEmitter(ArrayBlockingQueue<EmitData> emitData) {
         this.emitDataQueue = emitData;
@@ -94,7 +97,7 @@ public class AsyncEmitter implements Callable<Integer> {
         void add(EmitData data) {
             size++;
             long sz = AbstractEmitter
-                    .estimateSizeInBytes(data.getEmitKey().getKey(), data.getMetadataList());
+                    .estimateSizeInBytes(data.getEmitKey().getEmitKey(), data.getMetadataList());
             if (estimatedSize + sz > maxBytes) {
                 LOG.debug("estimated size ({}) > maxBytes({}), going to emitAll",
                         (estimatedSize + sz), maxBytes);
@@ -113,8 +116,8 @@ public class AsyncEmitter implements Callable<Integer> {
             int emitted = 0;
             LOG.debug("about to emit {}", size);
             for (Map.Entry<String, List<EmitData>> e : map.entrySet()) {
-                Emitter emitter =
-                        TikaResource.getConfig().getEmitterManager().getEmitter(e.getKey());
+                Emitter emitter = TikaResource.getConfig()
+                        .getEmitterManager().getEmitter(e.getKey());
                 tryToEmit(emitter, e.getValue());
                 emitted += e.getValue().size();
             }
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
index 7a9f4ad..bf1019e 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
@@ -64,7 +64,7 @@ public class AsyncParser implements Callable<Integer> {
                     if (!offered) {
                         //TODO: deal with this
                         LOG.warn("Failed to add ({}) " + "to emit queue after 10 minutes.",
-                                request.getFetchKey().getKey());
+                                request.getFetchKey().getFetchKey());
                     }
                 }
             } else {
@@ -76,14 +76,15 @@ public class AsyncParser implements Callable<Integer> {
     private boolean checkForParseException(FetchEmitTuple request, EmitData emitData) {
         if (emitData == null || emitData.getMetadataList() == null ||
                 emitData.getMetadataList().size() == 0) {
-            LOG.warn("empty or null emit data ({})", request.getFetchKey().getKey());
+            LOG.warn("empty or null emit data ({})", request.getFetchKey().getFetchKey());
             return false;
         }
         boolean shouldEmit = true;
         Metadata container = emitData.getMetadataList().get(0);
         String stack = container.get(TikaCoreProperties.CONTAINER_EXCEPTION);
         if (stack != null) {
-            LOG.warn("fetchKey ({}) container parse exception ({})", request.getFetchKey().getKey(),
+            LOG.warn("fetchKey ({}) container parse exception ({})",
+                    request.getFetchKey().getFetchKey(),
                     stack);
             if (request.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
                 shouldEmit = false;
@@ -95,7 +96,7 @@ public class AsyncParser implements Callable<Integer> {
             String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
             if (embeddedStack != null) {
                 LOG.warn("fetchKey ({}) embedded parse exception ({})",
-                        request.getFetchKey().getKey(), embeddedStack);
+                        request.getFetchKey().getFetchKey(), embeddedStack);
             }
         }
         return shouldEmit;
@@ -105,7 +106,7 @@ public class AsyncParser implements Callable<Integer> {
         Metadata userMetadata = t.getMetadata();
         Metadata metadata = new Metadata();
         String fetcherName = t.getFetchKey().getFetcherName();
-        String fetchKey = t.getFetchKey().getKey();
+        String fetchKey = t.getFetchKey().getFetchKey();
         List<Metadata> metadataList = null;
         try (InputStream stream = TikaResource.getConfig().getFetcherManager()
                 .getFetcher(fetcherName).fetch(fetchKey, metadata)) {
@@ -120,7 +121,7 @@ public class AsyncParser implements Callable<Integer> {
 
         injectUserMetadata(userMetadata, metadataList);
         EmitKey emitKey = t.getEmitKey();
-        if (StringUtils.isBlank(emitKey.getKey())) {
+        if (StringUtils.isBlank(emitKey.getEmitKey())) {
             emitKey = new EmitKey(emitKey.getEmitterName(), fetchKey);
         }
         return new EmitData(emitKey, metadataList);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
index 47dbc50..001926b 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
@@ -138,7 +138,7 @@ public class AsyncResource {
     private Map<String, Object> throttle(List<FetchEmitTuple> notAdded, int added) {
         List<String> fetchKeys = new ArrayList<>();
         for (FetchEmitTuple t : notAdded) {
-            fetchKeys.add(t.getFetchKey().getKey());
+            fetchKeys.add(t.getFetchKey().getFetchKey());
         }
         Map<String, Object> map = new HashMap<>();
         map.put("status", "throttled");
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
index 96c2e30..16aa76e 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
@@ -63,14 +63,15 @@ public class EmitterResource {
     private static final String FETCHER_NAME_ABBREV = "fn";
     private static final String FETCH_KEY_ABBREV = "fk";
     private static final String EMIT_KEY_ABBREV = "ek";
+
     private static final Logger LOG = LoggerFactory.getLogger(EmitterResource.class);
 
     static EmitKey calcEmitKey(FetchEmitTuple t) {
         //use fetch key if emitter key is not specified
         //TODO: clean this up?
         EmitKey emitKey = t.getEmitKey();
-        if (StringUtils.isBlank(emitKey.getKey())) {
-            emitKey = new EmitKey(emitKey.getEmitterName(), t.getFetchKey().getKey());
+        if (StringUtils.isBlank(emitKey.getEmitKey())) {
+            emitKey = new EmitKey(emitKey.getEmitterName(), t.getFetchKey().getFetchKey());
         }
         return emitKey;
     }
@@ -126,15 +127,18 @@ public class EmitterResource {
     @PUT
     @Produces("application/json")
     @Path("{" + EMITTER_PARAM + " : (\\w+)?}")
-    public Map<String, String> putRmeta(InputStream is, @Context HttpHeaders httpHeaders,
+    public Map<String, String> putRmeta(InputStream is,
+                                        @Context HttpHeaders httpHeaders,
                                         @Context UriInfo info,
-                                        @PathParam(EMITTER_PARAM) String emitterName)
-            throws Exception {
+                                        @PathParam(EMITTER_PARAM) String emitterName
+    ) throws Exception {
 
         Metadata metadata = new Metadata();
         String emitKey = httpHeaders.getHeaderString(EMIT_KEY_FOR_HTTP_HEADER);
-        List<Metadata> metadataList = RecursiveMetadataResource
-                .parseMetadata(is, metadata, httpHeaders.getRequestHeaders(), info, "text");
+        List<Metadata> metadataList =
+                RecursiveMetadataResource.parseMetadata(is,
+                        metadata,
+                        httpHeaders.getRequestHeaders(), info, "text");
         return emit(new EmitKey(emitterName, emitKey), metadataList);
     }
 
@@ -167,7 +171,7 @@ public class EmitterResource {
         List<Metadata> metadataList = null;
         try (InputStream stream = TikaResource.getConfig().getFetcherManager()
                 .getFetcher(t.getFetchKey().getFetcherName())
-                .fetch(t.getFetchKey().getKey(), metadata)) {
+                .fetch(t.getFetchKey().getFetchKey(), metadata)) {
 
             metadataList = RecursiveMetadataResource
                     .parseMetadata(stream, metadata, httpHeaders.getRequestHeaders(), info, "text");
@@ -192,7 +196,7 @@ public class EmitterResource {
         Map<String, String> statusMap = new HashMap<>();
         statusMap.put("status", "ok");
         statusMap.put("emitter", t.getEmitKey().getEmitterName());
-        statusMap.put("emitKey", t.getEmitKey().getKey());
+        statusMap.put("emitKey", t.getEmitKey().getEmitKey());
         String msg = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
         statusMap.put("parse_exception", msg);
         return statusMap;
@@ -209,7 +213,7 @@ public class EmitterResource {
                 shouldEmit = false;
             }
             LOG.warn("fetchKey ({}) caught container parse exception ({})",
-                    t.getFetchKey().getKey(), stack);
+                    t.getFetchKey().getFetchKey(), stack);
         }
 
         for (int i = 1; i < metadataList.size(); i++) {
@@ -217,7 +221,7 @@ public class EmitterResource {
             String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
             if (embeddedStack != null) {
                 LOG.warn("fetchKey ({}) caught embedded parse exception ({})",
-                        t.getFetchKey().getKey(), embeddedStack);
+                        t.getFetchKey().getFetchKey(), embeddedStack);
             }
         }
 
@@ -250,9 +254,9 @@ public class EmitterResource {
         String status = "ok";
         String exceptionMsg = "";
         try {
-            emitter.emit(emitKey.getKey(), metadataList);
+            emitter.emit(emitKey.getEmitKey(), metadataList);
         } catch (IOException | TikaEmitterException e) {
-            LOG.warn("problem emitting (" + emitKey.getKey() + ")", e);
+            LOG.warn("problem emitting (" + emitKey.getEmitterName() + ")", e);
             status = "emitter_exception";
             exceptionMsg = ExceptionUtils.getStackTrace(e);
         }
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
index fda802f..a99cea1 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
@@ -242,7 +242,10 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
                              FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) throws Exception {
 
         awaitServerStartup();
-        Response response = WebClient.create(endPoint + "/emit").accept("application/json")
+        System.out.println(getJsonString(fileName, onParseException));
+        Response response = WebClient
+                .create(endPoint + "/emit")
+                .accept("application/json")
                 .post(getJsonString(fileName, onParseException));
         if (response.getStatus() == 200) {
             Path targFile = TMP_OUTPUT_DIR.resolve(fileName + ".json");