You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/03/26 21:51:57 UTC
[tika] branch main updated: TIKA-3304 (#420)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new e47c6cd TIKA-3304 (#420)
e47c6cd is described below
commit e47c6cd62e587fdaae7e2e999f37122d09449754
Author: Tim Allison <ta...@apache.org>
AuthorDate: Fri Mar 26 17:51:49 2021 -0400
TIKA-3304 (#420)
* fix npe in when OCR is available and fix incorrect mark offset exception
* fix indentation
* Avoid reporting of temporary ocr-based mime type in xhtml output
* TIKA-3304 -- experimental... WIP -- not to be merged
* allow instance credentials in s3 components
* WIP -- do not merge...allow instance credentials in s3 components
* WIP -- do not merge...still a bunch to do
* WIP -- do not merge...still a bunch to do
* merge from main and required updates/conflict resolution
* TIKA-3304 -- basically works...lots more remains
* checkstyle
---
.gitignore | 1 +
pom.xml | 2 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 188 +++----
.../java/org/apache/tika/config/TikaConfig.java | 2 +-
.../TikaTimeoutException.java} | 24 +-
.../apache/tika/pipes/emitter/AbstractEmitter.java | 4 +-
.../org/apache/tika/pipes/emitter/EmitKey.java | 10 +-
.../org/apache/tika/pipes/emitter/Emitter.java | 2 +-
.../apache/tika/pipes/emitter/EmptyEmitter.java | 2 +-
.../org/apache/tika/pipes/fetcher/FetchKey.java | 11 +-
.../tika/pipes/fetchiterator/FetchEmitTuple.java | 5 +-
.../tika/pipes/fetchiterator/FetchIterator.java | 129 +++--
.../fetchiterator/FileSystemFetchIterator.java | 4 +-
.../fetchiterator/FileSystemFetchIteratorTest.java | 26 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 3 +-
tika-pipes/pom.xml | 38 +-
tika-pipes/tika-emitters/tika-emitter-fs/pom.xml | 13 +-
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 44 +-
tika-pipes/tika-emitters/tika-emitter-s3/pom.xml | 22 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 162 +++---
tika-pipes/tika-emitters/tika-emitter-solr/pom.xml | 13 +-
.../tika/pipes/emitter/solr/SolrEmitter.java | 133 +++--
.../apache/tika/pipes/emitter/solr/TestBasic.java | 34 +-
.../src/test/resources/log4j.properties | 6 +-
.../pipes/fetchiterator/csv/CSVFetchIterator.java | 82 +--
.../src/test/java/TestCSVFetchIterator.java | 56 +-
.../tika-fetch-iterator-jdbc/pom.xml | 17 +-
.../fetchiterator/jdbc/JDBCFetchIterator.java | 84 +--
.../fetchiterator/jdbc/TestJDBCFetchIterator.java | 83 +--
.../src/test/resources/log4j.properties | 4 +-
.../tika-fetch-iterator-s3/pom.xml | 22 +-
.../pipes/fetchiterator/s3/S3FetchIterator.java | 80 ++-
.../fetchiterator/s3/TestS3FetchIterator.java | 26 +-
.../src/test/resources/log4j.properties | 4 +-
tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml | 13 +-
.../tika/pipes/fetcher/http/HttpFetcher.java | 33 +-
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 50 +-
tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml | 9 +
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 157 +++---
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 23 +-
.../src/test/resources/tika-config-s3.xml | 1 +
.../org/apache/tika/client/HttpClientFactory.java | 147 +++--
.../org/apache/tika/client/HttpClientUtil.java | 26 +-
tika-pipes/tika-pipes-async/pom.xml | 101 ++++
.../java/org/apache/tika/pipes/async/AsyncCli.java | 358 +++++++++++++
.../org/apache/tika/pipes/async/AsyncConfig.java | 76 +++
.../org/apache/tika/pipes/async/AsyncData.java | 57 ++
.../org/apache/tika/pipes/async/AsyncEmitHook.java | 18 +-
.../org/apache/tika/pipes/async/AsyncEmitter.java | 125 +++++
.../tika/pipes/async/AsyncEmitterProcess.java | 379 +++++++++++++
.../tika/pipes/async/AsyncPipesEmitHook.java | 61 +++
.../apache/tika/pipes/async/AsyncProcessor.java | 594 +++++++++++++++++++++
.../tika/pipes/async/AsyncRuntimeException.java | 23 +-
.../org/apache/tika/pipes/async/AsyncTask.java | 54 ++
.../org/apache/tika/pipes/async/AsyncWorker.java | 195 +++++++
.../tika/pipes/async/AsyncWorkerProcess.java | 505 ++++++++++++++++++
.../src/main}/resources/log4j.properties | 6 +-
.../tika/pipes/async/AsyncProcessorTest.java | 105 ++++
.../org/apache/tika/pipes/async/MockEmitter.java | 103 ++++
.../org/apache/tika/pipes/async/MockFetcher.java | 27 +-
.../apache/tika/pipes/async/SerializationTest.java | 49 ++
.../apache/tika/pipes/PipeIntegrationTests.java | 103 ++--
.../src/test/resources/log4j.properties | 4 +-
tika-serialization/pom.xml | 5 +
.../tika/metadata/serialization/JsonEmitData.java | 2 +-
.../metadata/serialization/JsonFetchEmitTuple.java | 6 +-
.../serialization/JsonMetadataDeserializer.java | 28 +-
.../serialization/JsonMetadataSerializer.java | 28 +-
.../serialization/JsonFetchEmitTupleTest.java | 6 +-
.../org/apache/tika/server/client/TikaClient.java | 2 +-
.../apache/tika/server/client/TikaClientCLI.java | 35 +-
.../tika/server/client/TikaEmitterResult.java | 7 +-
.../apache/tika/server/client/TikaHttpClient.java | 6 +-
.../tika/server/core/resource/AsyncEmitter.java | 11 +-
.../tika/server/core/resource/AsyncParser.java | 13 +-
.../tika/server/core/resource/AsyncResource.java | 2 +-
.../tika/server/core/resource/EmitterResource.java | 30 +-
.../core/TikaServerEmitterIntegrationTest.java | 5 +-
78 files changed, 3978 insertions(+), 946 deletions(-)
diff --git a/.gitignore b/.gitignore
index dda6180..c608dc2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
.svn
target
dependency-reduced-pom.xml
+.editorconfig
.idea
.classpath
.project
diff --git a/pom.xml b/pom.xml
index 17bc12d..1ad3c85 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,11 +37,11 @@
<modules>
<module>tika-parent</module>
<module>tika-core</module>
+ <module>tika-serialization</module>
<module>tika-pipes</module>
<module>tika-parsers</module>
<module>tika-bundles</module>
<module>tika-xmp</module>
- <module>tika-serialization</module>
<module>tika-batch</module>
<module>tika-langdetect</module>
<module>tika-app</module>
diff --git a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
index 50b3c22..3f9587b 100644
--- a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
+++ b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
@@ -72,6 +72,7 @@ import org.apache.tika.metadata.serialization.JsonMetadataList;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.DelegatingParser;
import org.apache.tika.parser.DigestingParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -99,7 +100,7 @@ public class TikaGUI extends JFrame
implements ActionListener, HyperlinkListener {
//maximum length to allow for mark for reparse to get JSON
- private static final int MAX_MARK = 20*1024*1024;//20MB
+ private static final int MAX_MARK = 20 * 1024 * 1024;//20MB
/**
* Serial version UID.
@@ -129,7 +130,7 @@ public class TikaGUI extends JFrame
new CommonsDigester(MAX_MARK,
CommonsDigester.DigestAlgorithm.MD5,
CommonsDigester.DigestAlgorithm.SHA256)
- )).setVisible(true);
+ )).setVisible(true);
}
});
}
@@ -143,7 +144,7 @@ public class TikaGUI extends JFrame
* Configured parser instance.
*/
private final Parser parser;
-
+
/**
* Captures requested embedded images
*/
@@ -173,7 +174,7 @@ public class TikaGUI extends JFrame
* Main content output.
*/
private final JEditorPane textMain;
-
+
/**
* Raw XHTML source.
*/
@@ -341,14 +342,13 @@ public class TikaGUI extends JFrame
context.set(DocumentSelector.class, new ImageDocumentSelector());
- input = TikaInputStream.get(new ProgressMonitorInputStream(
- this, "Parsing stream", input));
+ input = TikaInputStream.get(input);
if (input.markSupported()) {
int mark = -1;
if (input instanceof TikaInputStream) {
- if (((TikaInputStream)input).hasFile()) {
- mark = (int)((TikaInputStream)input).getLength();
+ if (((TikaInputStream) input).hasFile()) {
+ mark = (int) ((TikaInputStream) input).getLength();
}
}
if (mark == -1) {
@@ -391,8 +391,8 @@ public class TikaGUI extends JFrame
input.reset();
isReset = true;
} catch (IOException e) {
- setText(json, "Error during stream reset.\n"+
- "There's a limit of "+MAX_MARK + " bytes for this type of processing in the GUI.\n"+
+ setText(json, "Error during stream reset.\n" +
+ "There's a limit of " + MAX_MARK + " bytes for this type of processing in the GUI.\n" +
"Try the app with command line argument of -J."
);
}
@@ -420,7 +420,7 @@ public class TikaGUI extends JFrame
t.printStackTrace(new PrintWriter(writer));
JEditorPane editor =
- new JEditorPane("text/plain", writer.toString());
+ new JEditorPane("text/plain", writer.toString());
editor.setEditable(false);
editor.setBackground(Color.WHITE);
editor.setCaretPosition(0);
@@ -435,7 +435,7 @@ public class TikaGUI extends JFrame
private void addWelcomeCard(JPanel panel, String name) {
try {
JEditorPane editor =
- new JEditorPane(TikaGUI.class.getResource("welcome.html"));
+ new JEditorPane(TikaGUI.class.getResource("welcome.html"));
editor.setContentType("text/html");
editor.setEditable(false);
editor.setBackground(Color.WHITE);
@@ -480,7 +480,7 @@ public class TikaGUI extends JFrame
URL url = e.getURL();
try (InputStream stream = url.openStream()) {
JEditorPane editor =
- new JEditorPane("text/plain", IOUtils.toString(stream, UTF_8));
+ new JEditorPane("text/plain", IOUtils.toString(stream, UTF_8));
editor.setEditable(false);
editor.setBackground(Color.WHITE);
editor.setCaretPosition(0);
@@ -519,7 +519,7 @@ public class TikaGUI extends JFrame
* {@link JEditorPane} fail thinking that the document character set
* is inconsistent.
* <p>
- * Additionally, it will use ImageSavingParser to re-write embedded:(image)
+ * Additionally, it will use ImageSavingParser to re-write embedded:(image)
* image links to be file:///(temporary file) so that they can be loaded.
*
* @param writer output writer
@@ -529,7 +529,7 @@ public class TikaGUI extends JFrame
private ContentHandler getHtmlHandler(Writer writer)
throws TransformerConfigurationException {
SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
handler.setResult(new StreamResult(writer));
@@ -542,36 +542,37 @@ public class TikaGUI extends JFrame
uri = null;
}
if (!"head".equals(localName)) {
- if("img".equals(localName)) {
- AttributesImpl newAttrs;
- if(atts instanceof AttributesImpl) {
- newAttrs = (AttributesImpl)atts;
- } else {
- newAttrs = new AttributesImpl(atts);
- }
-
- for(int i=0; i<newAttrs.getLength(); i++) {
- if("src".equals(newAttrs.getLocalName(i))) {
- String src = newAttrs.getValue(i);
- if(src.startsWith("embedded:")) {
- String filename = src.substring(src.indexOf(':')+1);
- try {
- File img = imageParser.requestSave(filename);
- String newSrc = img.toURI().toString();
- newAttrs.setValue(i, newSrc);
- } catch(IOException e) {
- System.err.println("Error creating temp image file " + filename);
- // The html viewer will show a broken image too to alert them
+ if ("img".equals(localName)) {
+ AttributesImpl newAttrs;
+ if (atts instanceof AttributesImpl) {
+ newAttrs = (AttributesImpl) atts;
+ } else {
+ newAttrs = new AttributesImpl(atts);
+ }
+
+ for (int i = 0; i < newAttrs.getLength(); i++) {
+ if ("src".equals(newAttrs.getLocalName(i))) {
+ String src = newAttrs.getValue(i);
+ if (src.startsWith("embedded:")) {
+ String filename = src.substring(src.indexOf(':') + 1);
+ try {
+ File img = imageParser.requestSave(filename);
+ String newSrc = img.toURI().toString();
+ newAttrs.setValue(i, newSrc);
+ } catch (IOException e) {
+ System.err.println("Error creating temp image file " + filename);
+ // The html viewer will show a broken image too to alert them
+ }
}
- }
- }
- }
- super.startElement(uri, localName, name, newAttrs);
+ }
+ }
+ super.startElement(uri, localName, name, newAttrs);
} else {
- super.startElement(uri, localName, name, atts);
+ super.startElement(uri, localName, name, atts);
}
}
}
+
@Override
public void endElement(String uri, String localName, String name)
throws SAXException {
@@ -582,9 +583,11 @@ public class TikaGUI extends JFrame
super.endElement(uri, localName, name);
}
}
+
@Override
public void startPrefixMapping(String prefix, String uri) {
}
+
@Override
public void endPrefixMapping(String prefix) {
}
@@ -594,6 +597,7 @@ public class TikaGUI extends JFrame
private ContentHandler getTextContentHandler(Writer writer) {
return new BodyContentHandler(writer);
}
+
private ContentHandler getTextMainContentHandler(Writer writer) {
return new BoilerpipeContentHandler(writer);
}
@@ -601,7 +605,7 @@ public class TikaGUI extends JFrame
private ContentHandler getXmlContentHandler(Writer writer)
throws TransformerConfigurationException {
SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
handler.setResult(new StreamResult(writer));
@@ -612,63 +616,63 @@ public class TikaGUI extends JFrame
* A {@link DocumentSelector} that accepts only images.
*/
private static class ImageDocumentSelector implements DocumentSelector {
- public boolean select(Metadata metadata) {
- String type = metadata.get(Metadata.CONTENT_TYPE);
- return type != null && type.startsWith("image/");
- }
+ public boolean select(Metadata metadata) {
+ String type = metadata.get(Metadata.CONTENT_TYPE);
+ return type != null && type.startsWith("image/");
+ }
}
-
+
/**
* A recursive parser that saves certain images into the temporary
- * directory, and delegates everything else to another downstream
- * parser.
+ * directory, and delegates everything else to another downstream
+ * parser.
*/
private static class ImageSavingParser extends AbstractParser {
- private Map<String,File> wanted = new HashMap<String,File>();
- private Parser downstreamParser;
- private File tmpDir;
-
- private ImageSavingParser(Parser downstreamParser) {
- this.downstreamParser = downstreamParser;
-
- try {
- File t = File.createTempFile("tika", ".test");
- tmpDir = t.getParentFile();
- } catch(IOException e) {}
- }
-
- public File requestSave(String embeddedName) throws IOException {
- String suffix = ".tika";
-
- int splitAt = embeddedName.lastIndexOf('.');
- if (splitAt > 0) {
- embeddedName.substring(splitAt);
- }
-
- File tmp = File.createTempFile("tika-embedded-", suffix);
- wanted.put(embeddedName, tmp);
- return tmp;
- }
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- // Never used in an auto setup
- return null;
- }
-
- public void parse(InputStream stream, ContentHandler handler,
- Metadata metadata, ParseContext context) throws IOException,
- SAXException, TikaException {
- String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- if(name != null && wanted.containsKey(name)) {
- FileOutputStream out = new FileOutputStream(wanted.get(name));
- IOUtils.copy(stream, out);
- out.close();
- } else {
- if(downstreamParser != null) {
- downstreamParser.parse(stream, handler, metadata, context);
+ private Map<String, File> wanted = new HashMap<String, File>();
+ private Parser downstreamParser;
+ private File tmpDir;
+
+ private ImageSavingParser(Parser downstreamParser) {
+ this.downstreamParser = downstreamParser;
+
+ try {
+ File t = File.createTempFile("tika", ".test");
+ tmpDir = t.getParentFile();
+ } catch (IOException e) {
}
- }
- }
+ }
+
+ public File requestSave(String embeddedName) throws IOException {
+ String suffix = ".tika";
+
+ int splitAt = embeddedName.lastIndexOf('.');
+ if (splitAt > 0) {
+ embeddedName.substring(splitAt);
+ }
+
+ File tmp = File.createTempFile("tika-embedded-", suffix);
+ wanted.put(embeddedName, tmp);
+ return tmp;
+ }
+
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return downstreamParser.getSupportedTypes(context);
+ }
+
+ public void parse(InputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext context) throws IOException,
+ SAXException, TikaException {
+ String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+ if (name != null && wanted.containsKey(name)) {
+ FileOutputStream out = new FileOutputStream(wanted.get(name));
+ IOUtils.copy(stream, out);
+ out.close();
+ } else {
+ if (downstreamParser != null) {
+ downstreamParser.parse(stream, handler, metadata, context);
+ }
+ }
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index b987b4e..8c3ad86 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -467,7 +467,7 @@ public class TikaConfig {
if (serviceLoaderElement != null) {
boolean dynamic = Boolean.parseBoolean(serviceLoaderElement.getAttribute("dynamic"));
- LoadErrorHandler loadErrorHandler = LoadErrorHandler.IGNORE;
+ LoadErrorHandler loadErrorHandler = LoadErrorHandler.THROW;
String loadErrorHandleConfig = serviceLoaderElement.getAttribute("loadErrorHandler");
if (LoadErrorHandler.WARN.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
loadErrorHandler = LoadErrorHandler.WARN;
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
index bc9fa3b..a53dbd6 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-core/src/main/java/org/apache/tika/exception/TikaTimeoutException.java
@@ -14,21 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
- String getName();
-
- void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
- void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
- //TODO -- add this later for xhtml?
- //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+package org.apache.tika.exception;
+/**
+ * Runtime/unchecked version of {@link java.util.concurrent.TimeoutException}
+ */
+public class TikaTimeoutException extends RuntimeException {
+ public TikaTimeoutException(String message) {
+ super(message);
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
index db6dd4c..dda0054 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
@@ -59,9 +59,9 @@ public abstract class AbstractEmitter implements Emitter {
* @throws TikaEmitterException
*/
@Override
- public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+ public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
for (EmitData d : emitData) {
- emit(d.getEmitKey().getKey(), d.getMetadataList());
+ emit(d.getEmitKey().getEmitKey(), d.getMetadataList());
}
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
index 65271e3..b565a43 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
@@ -18,9 +18,13 @@ package org.apache.tika.pipes.emitter;
public class EmitKey {
- private final String emitterName;
- private final String emitKey;
+ private String emitterName;
+ private String emitKey;
+ //for serialization only...yuck.
+ public EmitKey() {
+
+ }
public EmitKey(String emitterName, String emitKey) {
this.emitterName = emitterName;
this.emitKey = emitKey;
@@ -30,7 +34,7 @@ public class EmitKey {
return emitterName;
}
- public String getKey() {
+ public String getEmitKey() {
return emitKey;
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
index bc9fa3b..f60ef3b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
@@ -27,7 +27,7 @@ public interface Emitter {
void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
- void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
+ void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException;
//TODO -- add this later for xhtml?
//void emit(String txt, Metadata metadata) throws IOException, TikaException;
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
index 95bb462..b77107b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
@@ -35,7 +35,7 @@ public class EmptyEmitter implements Emitter {
}
@Override
- public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+ public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
index 2992bb1..2c0ea64 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
@@ -21,8 +21,13 @@ package org.apache.tika.pipes.fetcher;
* to send to that fetcher to retrieve a specific file.
*/
public class FetchKey {
- private final String fetcherName;
- private final String fetchKey;
+ private String fetcherName;
+ private String fetchKey;
+
+ //this is for serialization...yuck
+ public FetchKey(){
+
+ }
public FetchKey(String fetcherName, String fetchKey) {
this.fetcherName = fetcherName;
@@ -33,7 +38,7 @@ public class FetchKey {
return fetcherName;
}
- public String getKey() {
+ public String getFetchKey() {
return fetchKey;
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
index 1f7d5b9..974aa3b 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
@@ -24,7 +24,7 @@ public class FetchEmitTuple {
public static final ON_PARSE_EXCEPTION DEFAULT_ON_PARSE_EXCEPTION = ON_PARSE_EXCEPTION.EMIT;
private final FetchKey fetchKey;
- private final EmitKey emitKey;
+ private EmitKey emitKey;
private final Metadata metadata;
private final ON_PARSE_EXCEPTION onParseException;
public FetchEmitTuple(FetchKey fetchKey, EmitKey emitKey, Metadata metadata) {
@@ -55,6 +55,9 @@ public class FetchEmitTuple {
return onParseException;
}
+ public void setEmitKey(EmitKey emitKey) {
+ this.emitKey = emitKey;
+ }
@Override
public String toString() {
return "FetchEmitTuple{" + "fetchKey=" + fetchKey + ", emitKey=" + emitKey + ", metadata=" +
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
index efc1723..2b9273d 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
@@ -17,32 +17,42 @@
package org.apache.tika.pipes.fetchiterator;
import java.io.IOException;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.config.Param;
import org.apache.tika.exception.TikaConfigException;
-
+import org.apache.tika.exception.TikaTimeoutException;
/**
* Abstract class that handles the testing for timeouts/thread safety
* issues. Concrete classes implement the blocking {@link #enqueue()}.
- * <p>
- * This must be "called", obviously...
+ * If there's an exception in the enqueuing thread, this will throw
+ * a RuntimeException. It will throw an IllegalStateException if
+ * next() is called after hasNext() has returned false.
*/
-public abstract class FetchIterator implements Callable<Integer>, Initializable {
+public abstract class FetchIterator
+ implements Callable<Integer>, Iterable<FetchEmitTuple>, Initializable {
public static final long DEFAULT_MAX_WAIT_MS = 300_000;
public static final int DEFAULT_QUEUE_SIZE = 1000;
+
public static final FetchEmitTuple COMPLETED_SEMAPHORE = new FetchEmitTuple(null, null, null);
+ private static final Logger LOGGER = LoggerFactory.getLogger(FetchIterator.class);
+
private long maxWaitMs = DEFAULT_MAX_WAIT_MS;
- private int numConsumers = -1;
private ArrayBlockingQueue<FetchEmitTuple> queue = null;
private int queueSize = DEFAULT_QUEUE_SIZE;
private String fetcherName;
@@ -50,25 +60,7 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
private int added = 0;
private FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
-
- public FetchIterator() {
-
- }
-
- public FetchIterator(String fetcherName) {
- this.fetcherName = fetcherName;
- }
-
- /**
- * This must be called before 'calling' this object.
- *
- * @param numConsumers
- */
- public ArrayBlockingQueue<FetchEmitTuple> init(int numConsumers) {
- this.queue = new ArrayBlockingQueue<>(queueSize);
- this.numConsumers = numConsumers;
- return queue;
- }
+ private FutureTask<Integer> futureTask;
public String getFetcherName() {
return fetcherName;
@@ -117,29 +109,16 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
this.onParseException = onParseException;
}
- @Override
public Integer call() throws Exception {
- if (queue == null || numConsumers < 0) {
- throw new IllegalStateException("Must call 'init' before calling this object");
- }
-
enqueue();
- for (int i = 0; i < numConsumers; i++) {
- try {
- tryToAdd(COMPLETED_SEMAPHORE);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
+ tryToAdd(COMPLETED_SEMAPHORE);
return added;
}
protected abstract void enqueue() throws IOException, TimeoutException, InterruptedException;
protected void tryToAdd(FetchEmitTuple p) throws InterruptedException, TimeoutException {
- if (p != COMPLETED_SEMAPHORE) {
- added++;
- }
+ added++;
boolean offered = queue.offer(p, maxWaitMs, TimeUnit.MILLISECONDS);
if (!offered) {
throw new TimeoutException("timed out while offering");
@@ -157,4 +136,76 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
//no-op
}
+ @Override
+ public Iterator<FetchEmitTuple> iterator() {
+ if (futureTask != null) {
+ throw new IllegalStateException("Can't call iterator more than once!");
+ }
+ futureTask = new FutureTask<>(this);
+ queue = new ArrayBlockingQueue<>(queueSize);
+ new Thread(futureTask).start();
+ return new TupleIterator();
+ }
+
+ private class TupleIterator implements Iterator<FetchEmitTuple> {
+ FetchEmitTuple next = null;
+
+ @Override
+ public boolean hasNext() {
+ if (next == null) {
+ next = pollNext();
+ }
+ return next != COMPLETED_SEMAPHORE;
+ }
+
+ @Override
+ public FetchEmitTuple next() {
+ if (next == COMPLETED_SEMAPHORE) {
+ throw new IllegalStateException(
+ "don't call next() after hasNext() has returned false!");
+ }
+ FetchEmitTuple ret = next;
+ next = pollNext();
+ return ret;
+ }
+
+ private FetchEmitTuple pollNext() throws TikaTimeoutException {
+
+ FetchEmitTuple t = null;
+ long start = System.currentTimeMillis();
+ try {
+ long elapsed = System.currentTimeMillis() - start;
+ while (t == null && elapsed < maxWaitMs) {
+ checkThreadOk();
+ t = queue.poll(100, TimeUnit.MILLISECONDS);
+ elapsed = System.currentTimeMillis() - start;
+ }
+ } catch (InterruptedException e) {
+ LOGGER.warn("interrupted");
+ return COMPLETED_SEMAPHORE;
+ }
+ if (t == null) {
+ throw new TikaTimeoutException(
+ "waited longer than " + maxWaitMs + "ms for the next tuple");
+ }
+ return t;
+ }
+
+ /**
+ * this checks to make sure that the thread hasn't terminated early.
+ * Will return true if the thread has successfully completed or if
+ * it has not completed. Will return false if there has been a thread
+ * interrupt. Will throw a RuntimeException if there's been
+ * an execution exception in the thread.
+ */
+ private void checkThreadOk() throws InterruptedException {
+ if (futureTask.isDone()) {
+ try {
+ futureTask.get();
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
+ }
+ }
+ }
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
index 2d08090..3487b63 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
@@ -39,11 +39,9 @@ public class FileSystemFetchIterator extends FetchIterator implements Initializa
private Path basePath;
public FileSystemFetchIterator() {
-
}
- public FileSystemFetchIterator(String fetcherName, Path basePath) {
- super(fetcherName);
+ public FileSystemFetchIterator(Path basePath) {
this.basePath = basePath;
}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
index 8bfb432..4e314bc 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
@@ -25,11 +25,6 @@ import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -60,24 +55,13 @@ public class FileSystemFetchIteratorTest {
}
String fetcherName = "fs";
- ExecutorService es = Executors.newFixedThreadPool(1);
- ExecutorCompletionService<Integer> cs = new ExecutorCompletionService<>(es);
- FetchIterator it = new FileSystemFetchIterator(fetcherName, root);
- it.setQueueSize(20000);
- ArrayBlockingQueue<FetchEmitTuple> q = it.init(1);
-
- cs.submit(it);
-
-
- Future<Integer> f = cs.take();
- f.get();
+ FetchIterator it = new FileSystemFetchIterator(root);
+ it.setFetcherName(fetcherName);
+ it.setQueueSize(2);
Set<String> iteratorSet = new HashSet<>();
- for (FetchEmitTuple p : q) {
- if (p == FetchIterator.COMPLETED_SEMAPHORE) {
- break;
- }
- iteratorSet.add(p.getFetchKey().getKey());
+ for (FetchEmitTuple p : it) {
+ iteratorSet.add(p.getFetchKey().getFetchKey());
}
assertEquals(truthSet, iteratorSet);
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index c75b430..187cc3e 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -169,7 +169,6 @@ public class TesseractOCRParserTest extends TikaTest {
@Test
public void testSingleImage() throws Exception {
Assume.assumeTrue("can run OCR", canRun());
-
String xml = getXML("testOCR.jpg").xml;
assertContains("OCR Testing", xml);
//test metadata extraction
@@ -182,6 +181,8 @@ public class TesseractOCRParserTest extends TikaTest {
assertContainsCount("<body", xml, 1);
assertContainsCount("</body", xml, 1);
assertContainsCount("</html", xml, 1);
+
+ assertNotContained("<meta name=\"Content-Type\" content=\"image/ocr-jpeg\" />", xml);
}
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index b90d3c3..21a5a49 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -14,7 +14,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
@@ -35,7 +36,42 @@
<module>tika-fetchers</module>
<module>tika-emitters</module>
<module>tika-fetch-iterators</module>
+ <module>tika-pipes-async</module>
<module>tika-pipes-integration-tests</module>
</modules>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${checkstyle.plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>8.41</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>validate</id>
+ <phase>validate</phase>
+ <configuration>
+ <configLocation>checkstyle.xml</configLocation>
+ <encoding>UTF-8</encoding>
+ <consoleOutput>false</consoleOutput>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
+ <violationSeverity>error</violationSeverity>
+ <failOnViolation>true</failOnViolation>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
index cf214da..df61b96 100644
--- a/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tika-emitters</artifactId>
@@ -90,15 +90,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
index d4a0448..285142b 100644
--- a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
@@ -16,16 +16,6 @@
*/
package org.apache.tika.pipes.emitter.fs;
-import org.apache.tika.config.Field;
-import org.apache.tika.pipes.emitter.AbstractEmitter;
-import org.apache.tika.pipes.emitter.Emitter;
-import org.apache.tika.pipes.emitter.StreamEmitter;
-import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.metadata.serialization.JsonMetadataList;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
@@ -35,9 +25,15 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
-import java.util.Collections;
import java.util.List;
-import java.util.Set;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.StreamEmitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
/**
* Emitter to write to a file system.
@@ -56,7 +52,8 @@ import java.util.Set;
* <param name="basePath" type="string">/path/to/output</param>
* <!-- optional; default is 'json' -->
* <param name="fileExtension" type="string">json</param>
- * <!-- optional; if the file already exists, options ('skip', 'replace', 'exception')
+ * <!-- optional; if the file already exists,
+ * options ('skip', 'replace', 'exception')
* default is 'exception' -->
* <param name="onExists" type="string">skip</param>
* </params>
@@ -66,17 +63,13 @@ import java.util.Set;
*/
public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter {
- enum ON_EXISTS {
- SKIP, EXCEPTION, REPLACE
- }
-
private Path basePath = null;
private String fileExtension = "json";
private ON_EXISTS onExists = ON_EXISTS.EXCEPTION;
-
@Override
- public void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException {
+ public void emit(String emitKey, List<Metadata> metadataList)
+ throws IOException, TikaEmitterException {
Path output;
if (metadataList == null || metadataList.size() == 0) {
throw new TikaEmitterException("metadata list must not be null or of size 0");
@@ -124,15 +117,14 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
} else if (onExists.equals("exception")) {
this.onExists = ON_EXISTS.EXCEPTION;
} else {
- throw new IllegalArgumentException(
- "Don't understand '" + onExists +
- "'; must be one of: 'skip', 'replace', 'exception'");
+ throw new IllegalArgumentException("Don't understand '" + onExists +
+ "'; must be one of: 'skip', 'replace', 'exception'");
}
}
@Override
- public void emit(String path, InputStream inputStream, Metadata userMetadata) throws IOException,
- TikaEmitterException {
+ public void emit(String path, InputStream inputStream, Metadata userMetadata)
+ throws IOException, TikaEmitterException {
Path target = basePath.resolve(path);
if (!Files.isDirectory(target.getParent())) {
@@ -152,4 +144,8 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
}
}
}
+
+ enum ON_EXISTS {
+ SKIP, EXCEPTION, REPLACE
+ }
}
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
index 62a79f0..773865c 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tika-emitters</artifactId>
@@ -60,9 +60,18 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons.codec.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
@@ -128,15 +137,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
index cb97d20..6f9c745 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
@@ -16,11 +16,31 @@
*/
package org.apache.tika.pipes.emitter.s3;
-import com.amazonaws.SdkClientException;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Map;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
@@ -29,31 +49,13 @@ import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.serialization.JsonMetadataList;
import org.apache.tika.pipes.emitter.AbstractEmitter;
import org.apache.tika.pipes.emitter.StreamEmitter;
import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.apache.tika.metadata.Metadata;
import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedWriter;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
/**
* Emits to existing s3 bucket
@@ -67,15 +69,21 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
* <!-- required -->
* <param name="region" type="string">us-east-1</param>
* <!-- required -->
+ * <param name="credentialsProvider"
+ * type="string">(profile|instance)</param>
+ * <!-- required if credentialsProvider=profile-->
* <param name="profile" type="string">my-profile</param>
* <!-- required -->
* <param name="bucket" type="string">my-bucket</param>
- * <!-- optional; prefix to add to the path before emitting; default is no prefix -->
+ * <!-- optional; prefix to add to the path before emitting;
+ * default is no prefix -->
* <param name="prefix" type="string">my-prefix</param>
* <!-- optional; default is 'json' this will be added to the SOURCE_PATH
- * if no emitter key is specified -->
+ * if no emitter key is specified. Do not add a "."
+ * before the extension -->
* <param name="fileExtension" type="string">json</param>
- * <!-- optional; default is 'true'-- whether to copy the json to a local file before putting to s3 -->
+ * <!-- optional; default is 'true'-- whether to copy the
+ * json to a local file before putting to s3 -->
* <param name="spoolToTemp" type="bool">true</param>
* </params>
* </emitter>
@@ -88,6 +96,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
private String region;
private String profile;
private String bucket;
+ private String credentialsProvider;
private String fileExtension = "json";
private boolean spoolToTemp = true;
private String prefix = null;
@@ -101,15 +110,16 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
* @throws TikaException
*/
@Override
- public void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException {
+ public void emit(String emitKey, List<Metadata> metadataList)
+ throws IOException, TikaEmitterException {
if (metadataList == null || metadataList.size() == 0) {
throw new TikaEmitterException("metadata list must not be null or of size 0");
}
- if (! spoolToTemp) {
+ if (!spoolToTemp) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
- try (Writer writer =
- new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
+ try (Writer writer = new BufferedWriter(
+ new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
JsonMetadataList.toJson(metadataList, writer);
} catch (IOException e) {
throw new TikaEmitterException("can't jsonify", e);
@@ -122,8 +132,8 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
TemporaryResources tmp = new TemporaryResources();
try {
Path tmpPath = tmp.createTempFile();
- try (Writer writer = Files.newBufferedWriter(tmpPath,
- StandardCharsets.UTF_8, StandardOpenOption.CREATE)) {
+ try (Writer writer = Files.newBufferedWriter(tmpPath, StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE)) {
JsonMetadataList.toJson(metadataList, writer);
} catch (IOException e) {
throw new TikaEmitterException("can't jsonify", e);
@@ -138,28 +148,27 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
}
/**
- *
- * @param path -- object path, not including the bucket
- * @param is inputStream to copy
+ * @param path -- object path, not including the bucket
+ * @param is inputStream to copy
* @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
- * @throws TikaEmitterException
+ * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
*/
@Override
- public void emit(String path, InputStream is, Metadata userMetadata) throws IOException, TikaEmitterException {
+ public void emit(String path, InputStream is, Metadata userMetadata)
+ throws IOException, TikaEmitterException {
if (!StringUtils.isBlank(prefix)) {
path = prefix + "/" + path;
}
- if (! StringUtils.isBlank(fileExtension)) {
+ if (!StringUtils.isBlank(fileExtension)) {
path += "." + fileExtension;
}
- LOGGER.debug("about to emit to target bucket: ({}) path:({})",
- bucket, path);
+ LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
long length = -1;
if (is instanceof TikaInputStream) {
- if (((TikaInputStream)is).hasFile()) {
+ if (((TikaInputStream) is).hasFile()) {
try {
length = ((TikaInputStream) is).getLength();
} catch (IOException e) {
@@ -167,22 +176,23 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
}
}
}
- ObjectMetadata objectMetadata = new ObjectMetadata();
- if (length > 0) {
- objectMetadata.setContentLength(length);
- }
- for (String n : userMetadata.names()) {
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.",
- n, vals.length);
- }
- objectMetadata.addUserMetadata(n, vals[0]);
- }
try {
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ if (length > 0) {
+ objectMetadata.setContentLength(length);
+ }
+ for (String n : userMetadata.names()) {
+ String[] vals = userMetadata.getValues(n);
+ if (vals.length > 1) {
+ LOGGER.warn("Can only write the first value for key {}. I see {} values.",
+ n,
+ vals.length);
+ }
+ objectMetadata.addUserMetadata(n, vals[0]);
+ }
s3Client.putObject(bucket, path, is, objectMetadata);
- } catch (SdkClientException e) {
- throw new TikaEmitterException("problem writing s3object", e);
+ } catch (AmazonClientException e) {
+ throw new IOException("problem writing s3object", e);
}
}
@@ -190,6 +200,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
* Whether or not to spool the metadatalist to a tmp file before putting object.
* Default: <code>true</code>. If this is set to <code>false</code>,
* this emitter writes the json object to memory and then puts that into s3.
+ *
* @param spoolToTemp
*/
@Field
@@ -216,15 +227,25 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
public void setPrefix(String prefix) {
//strip final "/" if it exists
if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length()-1);
+ this.prefix = prefix.substring(0, prefix.length() - 1);
} else {
this.prefix = prefix;
}
}
+ @Field
+ public void setCredentialsProvider(String credentialsProvider) {
+ if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+ throw new IllegalArgumentException(
+ "credentialsProvider must be either 'profile' or instance'");
+ }
+ this.credentialsProvider = credentialsProvider;
+ }
+
/**
* If you want to customize the output file's file extension.
* Do not include the "."
+ *
* @param fileExtension
*/
@Field
@@ -232,20 +253,39 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
this.fileExtension = fileExtension;
}
+
+ /**
+ * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+ * e.g. AmazonClientException in a TikaConfigException.
+ *
+ * @param params params to use for initialization
+ * @throws TikaConfigException
+ */
@Override
public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set
- //ignore them
- s3Client = AmazonS3ClientBuilder.standard()
- .withRegion(region)
- .withCredentials(new ProfileCredentialsProvider(profile))
- .build();
+ //params have already been set...ignore them
+ AWSCredentialsProvider provider = null;
+ if ("instance".equals(credentialsProvider)) {
+ provider = InstanceProfileCredentialsProvider.getInstance();
+ } else if ("profile".equals(credentialsProvider)) {
+ provider = new ProfileCredentialsProvider(profile);
+ } else {
+ throw new TikaConfigException("credentialsProvider must be set and " +
+ "must be either 'instance' or 'profile'");
+ }
+
+ try {
+ s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+ .build();
+ } catch (AmazonClientException e) {
+ throw new TikaConfigException("can't initialize s3 emitter", e);
+ }
}
@Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("profile", this.profile);
mustNotBeEmpty("region", this.region);
}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
index e36d1c8..a831fa9 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tika-emitters</artifactId>
@@ -102,15 +102,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
index 0e09e9c..f43963e 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
@@ -16,9 +16,23 @@
*/
package org.apache.tika.pipes.emitter.solr;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.zip.GZIPOutputStream;
+
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.http.client.HttpClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.client.HttpClientFactory;
import org.apache.tika.client.HttpClientUtil;
import org.apache.tika.client.TikaClientException;
@@ -26,41 +40,19 @@ import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.config.Param;
-import org.apache.tika.pipes.emitter.AbstractEmitter;
-import org.apache.tika.pipes.emitter.EmitData;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.EmitData;
import org.apache.tika.pipes.emitter.TikaEmitterException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.zip.GZIPOutputStream;
public class SolrEmitter extends AbstractEmitter implements Initializable {
- enum AttachmentStrategy {
- SKIP,
- CONCATENATE_CONTENT,
- PARENT_CHILD,
- //anything else?
- }
private static final String ATTACHMENTS = "attachments";
private static final String UPDATE_PATH = "/update";
private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
//one day this will be allowed or can be configured?
private final boolean gzipJson = false;
-
private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
private String url;
private String contentField = "content";
@@ -68,77 +60,71 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
private int commitWithin = 100;
private HttpClientFactory httpClientFactory;
private HttpClient httpClient;
-
public SolrEmitter() throws TikaConfigException {
httpClientFactory = new HttpClientFactory();
}
+
@Override
- public void emit(String emitKey, List<Metadata> metadataList) throws IOException,
- TikaEmitterException {
+ public void emit(String emitKey, List<Metadata> metadataList)
+ throws IOException, TikaEmitterException {
if (metadataList == null || metadataList.size() == 0) {
LOG.warn("metadataList is null or empty");
return;
}
ByteArrayOutputStream bos = new ByteArrayOutputStream();
- Writer writer = gzipJson ?
- new BufferedWriter(
- new OutputStreamWriter(
- new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
- new BufferedWriter(
- new OutputStreamWriter(bos, StandardCharsets.UTF_8));
- try (
- JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ Writer writer = gzipJson ? new BufferedWriter(
+ new OutputStreamWriter(new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+ new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
jsonGenerator.writeStartArray();
jsonify(jsonGenerator, emitKey, metadataList);
jsonGenerator.writeEndArray();
}
- LOG.debug("emitting json ({})",
- new String(bos.toByteArray(), StandardCharsets.UTF_8));
+ LOG.debug("emitting json ({})", new String(bos.toByteArray(), StandardCharsets.UTF_8));
try {
- HttpClientUtil.postJson(httpClient,
- url+UPDATE_PATH+"?commitWithin="+getCommitWithin(), bos.toByteArray(), gzipJson);
+ HttpClientUtil
+ .postJson(httpClient, url + UPDATE_PATH +
+ "?commitWithin=" + getCommitWithin(),
+ bos.toByteArray(), gzipJson);
} catch (TikaClientException e) {
throw new TikaEmitterException("can't post", e);
}
}
@Override
- public void emit(List<EmitData> batch) throws IOException,
- TikaEmitterException {
+ public void emit(List<? extends EmitData> batch) throws IOException, TikaEmitterException {
if (batch == null || batch.size() == 0) {
LOG.warn("batch is null or empty");
return;
}
ByteArrayOutputStream bos = new ByteArrayOutputStream();
- Writer writer = gzipJson ?
- new BufferedWriter(
- new OutputStreamWriter(
- new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
- new BufferedWriter(
- new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+ Writer writer = gzipJson ? new BufferedWriter(
+ new OutputStreamWriter(new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+ new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8));
try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
jsonGenerator.writeStartArray();
for (EmitData d : batch) {
- jsonify(jsonGenerator, d.getEmitKey().getKey(), d.getMetadataList());
+ jsonify(jsonGenerator, d.getEmitKey().getEmitKey(), d.getMetadataList());
}
jsonGenerator.writeEndArray();
}
- LOG.debug("emitting json ({})",
- new String(bos.toByteArray(), StandardCharsets.UTF_8));
+ LOG.debug("emitting json ({})", new String(bos.toByteArray(), StandardCharsets.UTF_8));
try {
- HttpClientUtil.postJson(httpClient,
- url+UPDATE_PATH+"?commitWithin="+getCommitWithin(),
- bos.toByteArray(), gzipJson);
+ HttpClientUtil
+ .postJson(httpClient, url + UPDATE_PATH +
+ "?commitWithin=" + getCommitWithin(),
+ bos.toByteArray(), gzipJson);
} catch (TikaClientException e) {
throw new TikaEmitterException("can't post", e);
}
}
- private void jsonify(JsonGenerator jsonGenerator, String emitKey, List<Metadata> metadataList) throws IOException {
+ private void jsonify(JsonGenerator jsonGenerator, String emitKey,
+ List<Metadata> metadataList)
+ throws IOException {
metadataList.get(0).set(idField, emitKey);
- if (attachmentStrategy == AttachmentStrategy.SKIP ||
- metadataList.size() == 1) {
+ if (attachmentStrategy == AttachmentStrategy.SKIP || metadataList.size() == 1) {
jsonify(metadataList.get(0), jsonGenerator);
} else if (attachmentStrategy == AttachmentStrategy.CONCATENATE_CONTENT) {
//this only handles text for now, not xhtml
@@ -164,12 +150,13 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
} else {
- throw new IllegalArgumentException("I don't yet support this attachment strategy: "
- + attachmentStrategy);
+ throw new IllegalArgumentException(
+ "I don't yet support this attachment strategy: " + attachmentStrategy);
}
}
- private void jsonify(Metadata metadata, JsonGenerator jsonGenerator, boolean writeEndObject) throws IOException {
+ private void jsonify(Metadata metadata, JsonGenerator jsonGenerator, boolean writeEndObject)
+ throws IOException {
jsonGenerator.writeStartObject();
for (String n : metadata.names()) {
String[] vals = metadata.getValues(n);
@@ -215,29 +202,35 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
} else if (attachmentStrategy.equals("parent-child")) {
this.attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
} else {
- throw new IllegalArgumentException("Expected 'skip', 'concatenate-content' or "+
+ throw new IllegalArgumentException("Expected 'skip', 'concatenate-content' or " +
"'parent-child'. I regret I do not recognize: " + attachmentStrategy);
}
}
/**
* Specify the url for Solr
+ *
* @param url
*/
@Field
public void setUrl(String url) {
if (url.endsWith("/")) {
- url = url.substring(0, url.length()-1);
+ url = url.substring(0, url.length() - 1);
}
this.url = url;
}
+ public String getContentField() {
+ return contentField;
+ }
+
/**
* This is the field _after_ metadata mappings have been applied
* that contains the "content" for each metadata object.
- *
+ * <p>
* This is the field that is used if {@link #attachmentStrategy}
* is {@link AttachmentStrategy#CONCATENATE_CONTENT}.
+ *
* @param contentField
*/
@Field
@@ -245,8 +238,8 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
this.contentField = contentField;
}
- public String getContentField() {
- return contentField;
+ public int getCommitWithin() {
+ return commitWithin;
}
@Field
@@ -254,10 +247,6 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
this.commitWithin = commitWithin;
}
- public int getCommitWithin() {
- return commitWithin;
- }
-
/**
* Specify the field in the first Metadata that should be
* used as the id field for the document.
@@ -302,8 +291,14 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
}
@Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
}
+ enum AttachmentStrategy {
+ SKIP, CONCATENATE_CONTENT, PARENT_CHILD,
+ //anything else?
+ }
+
}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
index 069e7a2..049fe96 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
@@ -17,19 +17,20 @@
package org.apache.tika.pipes.emitter.solr;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.emitter.EmitData;
-import org.apache.tika.pipes.emitter.EmitKey;
-import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.filter.MetadataFilter;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.emitter.Emitter;
@Ignore("requires solr to be up and running")
public class TestBasic {
@@ -39,8 +40,7 @@ public class TestBasic {
TikaConfig tikaConfig = new TikaConfig(
TestBasic.class.getResourceAsStream("/tika-config-simple-emitter.xml"));
Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr1");
- List<Metadata> metadataList = getParentChild(tikaConfig,
- "id1", 2);
+ List<Metadata> metadataList = getParentChild(tikaConfig, "id1", 2);
emitter.emit("1", metadataList);
}
@@ -52,17 +52,15 @@ public class TestBasic {
Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr2");
List<EmitData> emitData = new ArrayList<>();
for (int i = 0; i < 100; i++) {
- List<Metadata> metadataList = getParentChild(tikaConfig,
- "batch_"+i, 4);
- emitData.add(new EmitData(
- new EmitKey(emitter.getName(), "batch_"+i),
- metadataList));
+ List<Metadata> metadataList = getParentChild(tikaConfig, "batch_" + i, 4);
+ emitData.add(new EmitData(new EmitKey(emitter.getName(),
+ "batch_" + i), metadataList));
}
emitter.emit(emitData);
}
- private List<Metadata> getParentChild(TikaConfig tikaConfig,
- String id, int numChildren) throws TikaException {
+ private List<Metadata> getParentChild(TikaConfig tikaConfig, String id, int numChildren)
+ throws TikaException {
List<Metadata> metadataList = new ArrayList<>();
MetadataFilter filter = tikaConfig.getMetadataFilter();
@@ -75,7 +73,7 @@ public class TestBasic {
m1.add(TikaCoreProperties.CREATOR, "secondAuthor");
filter.filter(m1);
metadataList.add(m1);
- for (int i = 1; i < numChildren; i++ ) {
+ for (int i = 1; i < numChildren; i++) {
Metadata m2 = new Metadata();
m2.set(TikaCoreProperties.EMBEDDED_RESOURCE_PATH, "/path_to_this.txt");
m2.set(TikaCoreProperties.TIKA_CONTENT, "fox jumped over the lazy " + i);
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
index 92b6d56..11e5887 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#info,debug, error,fatal ...
log4j.rootLogger=debug,stderr
-
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
-log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
+log4j.appender.stderr.layout.ConversionPattern=%-5p %m%n
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
index 7cb0c02..3063124 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
@@ -16,8 +16,24 @@
*/
package org.apache.tika.pipes.fetchiterator.csv;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
@@ -29,21 +45,6 @@ import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.apache.tika.pipes.fetchiterator.FetchIterator;
import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
/**
* Iterates through a UTF-8 CSV file. This adds all columns
@@ -51,17 +52,20 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
* to the metadata object.
* <p>
* <ul>
- * <li>If a 'fetchKeyColumn' is specified, this will use that column's value as the fetchKey.</li>
- * <li>If no 'fetchKeyColumn' is specified, this will send the metadata from the other columns.</li>
+ * <li>If a 'fetchKeyColumn' is specified, this will use that
+ * column's value as the fetchKey.</li>
+ * <li>If no 'fetchKeyColumn' is specified, this will send the
+ * metadata from the other columns.</li>
* <li>The 'fetchKeyColumn' value is not added to the metadata.</li>
* </ul>
* <p>
* <ul>
- * <li>If an 'emitKeyColumn' is specified, this will use that column's value as the emit key.</li>
- * <li>If an 'emitKeyColumn' is not specified, this will use the value from the 'fetchKeyColumn'.</li>
+ * <li>If an 'emitKeyColumn' is specified, this will use that
+ * column's value as the emit key.</li>
+ * <li>If an 'emitKeyColumn' is not specified, this will use
+ * the value from the 'fetchKeyColumn'.</li>
* <li>The 'emitKeyColumn' value is not added to the metadata.</li>
* </ul>
- *
*/
public class CSVFetchIterator extends FetchIterator implements Initializable {
@@ -110,24 +114,21 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
for (CSVRecord record : records) {
String fetchKey = getFetchKey(fetchEmitKeyIndices, record);
String emitKey = getEmitKey(fetchEmitKeyIndices, record);
- if (StringUtils.isBlank(fetchKey) && ! StringUtils.isBlank(fetcherName)) {
+ if (StringUtils.isBlank(fetchKey) && !StringUtils.isBlank(fetcherName)) {
LOGGER.debug("Fetcher specified ({}), but no fetchkey was found in ({})",
fetcherName, record);
}
if (StringUtils.isBlank(emitKey)) {
- throw new IOException("emitKey must not be blank in :"+record);
+ throw new IOException("emitKey must not be blank in :" + record);
}
Metadata metadata = loadMetadata(fetchEmitKeyIndices, headers, record);
- tryToAdd(new FetchEmitTuple(
- new FetchKey(fetcherName, fetchKey),
- new EmitKey(emitterName, emitKey), metadata,
- getOnParseException()));
+ tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, fetchKey),
+ new EmitKey(emitterName, emitKey), metadata, getOnParseException()));
}
}
}
- private void checkFetchEmitValidity(String fetcherName,
- String emitterName,
+ private void checkFetchEmitValidity(String fetcherName, String emitterName,
FetchEmitKeyIndices fetchEmitKeyIndices,
List<String> headers) throws IOException {
@@ -135,7 +136,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
throw new IOException(new TikaConfigException("must specify at least an emitterName"));
}
- if (StringUtils.isBlank(fetcherName) && ! StringUtils.isBlank(fetchKeyColumn)) {
+ if (StringUtils.isBlank(fetcherName) && !StringUtils.isBlank(fetchKeyColumn)) {
throw new IOException(new TikaConfigException("If specifying a 'fetchKeyColumn', " +
"you must also specify a 'fetcherName'"));
}
@@ -145,17 +146,17 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
}
//if a fetchkeycolumn is specified, make sure that it was found
- if (! StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
- throw new IOException(new TikaConfigException("Couldn't find fetchKeyColumn ("+
- fetchKeyColumn+" in header.\n" +
- "These are the headers I see: " + headers));
+ if (!StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
+ throw new IOException(new TikaConfigException(
+ "Couldn't find fetchKeyColumn (" + fetchKeyColumn + " in header.\n" +
+ "These are the headers I see: " + headers));
}
//if an emitkeycolumn is specified, make sure that it was found
- if (! StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
- throw new IOException(new TikaConfigException("Couldn't find emitKeyColumn ("+
- emitKeyColumn+" in header.\n" +
- "These are the headers I see: " + headers));
+ if (!StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
+ throw new IOException(new TikaConfigException(
+ "Couldn't find emitKeyColumn (" + emitKeyColumn + " in header.\n" +
+ "These are the headers I see: " + headers));
}
if (StringUtils.isBlank(emitKeyColumn)) {
@@ -179,7 +180,8 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
return getFetchKey(fetchEmitKeyIndices, record);
}
- private Metadata loadMetadata(FetchEmitKeyIndices fetchEmitKeyIndices, List<String> headers, CSVRecord record) {
+ private Metadata loadMetadata(FetchEmitKeyIndices fetchEmitKeyIndices, List<String> headers,
+ CSVRecord record) {
Metadata metadata = new Metadata();
for (int i = 0; i < record.size(); i++) {
if (fetchEmitKeyIndices.shouldSkip(i)) {
@@ -192,7 +194,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
private FetchEmitKeyIndices loadHeaders(CSVRecord record, List<String> headers)
- throws IOException {
+ throws IOException {
int fetchKeyColumnIndex = -1;
int emitKeyColumnIndex = -1;
@@ -200,7 +202,7 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
String header = record.get(col);
if (StringUtils.isBlank(header)) {
throw new IOException(
- new TikaException("Header in column (" +col +") must not be empty"));
+ new TikaException("Header in column (" + col + ") must not be empty"));
}
headers.add(header);
if (header.equals(fetchKeyColumn)) {
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
index 74f3ebe..fd86a05 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
@@ -14,10 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.apache.tika.pipes.fetchiterator.csv.CSVFetchIterator;
-import org.junit.Test;
+
+import static org.apache.tika.pipes.fetchiterator.FetchIterator.COMPLETED_SEMAPHORE;
+import static org.junit.Assert.assertEquals;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -25,14 +24,16 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.csv.CSVFetchIterator;
public class TestCSVFetchIterator {
@@ -46,69 +47,62 @@ public class TestCSVFetchIterator {
it.setCsvPath(p);
it.setFetchKeyColumn("fetchKey");
int numConsumers = 2;
- ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
- ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers);
ExecutorCompletionService c = new ExecutorCompletionService(es);
- c.submit(it);
+ ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(100);
List<MockFetcher> fetchers = new ArrayList<>();
for (int i = 0; i < numConsumers; i++) {
MockFetcher f = new MockFetcher(queue);
fetchers.add(f);
c.submit(f);
}
+ for (FetchEmitTuple t : it) {
+ queue.offer(t);
+ }
+ for (int i = 0; i < numConsumers; i++) {
+ queue.offer(COMPLETED_SEMAPHORE);
+ }
int finished = 0;
int completed = 0;
try {
- while (finished++ < numConsumers+1) {
+ while (finished++ < numConsumers) {
Future<Integer> f = c.take();
completed += f.get();
}
} finally {
es.shutdownNow();
}
- assertEquals(10, completed);
+ assertEquals(5, completed);
for (MockFetcher f : fetchers) {
for (FetchEmitTuple t : f.pairs) {
String id = t.getMetadata().get("id");
- assertEquals("path/to/my/file"+id,
- t.getFetchKey().getKey());
- assertEquals("project"+
- (Integer.parseInt(id) % 2 == 1 ? "a" : "b"),
+ assertEquals("path/to/my/file" + id, t.getFetchKey().getFetchKey());
+ assertEquals("project" + (Integer.parseInt(id) % 2 == 1 ? "a" : "b"),
t.getMetadata().get("project"));
}
}
}
- @Test(expected = ExecutionException.class)
+ @Test(expected = RuntimeException.class)
public void testBadFetchKeyCol() throws Exception {
Path p = get("test-simple.csv");
CSVFetchIterator it = new CSVFetchIterator();
it.setFetcherName("fs");
it.setCsvPath(p);
it.setFetchKeyColumn("fetchKeyDoesntExist");
- ExecutorService es = Executors.newFixedThreadPool(2);
- ExecutorCompletionService c = new ExecutorCompletionService(es);
- c.submit(it);
- c.submit(new MockFetcher(it.init(1)));
- int finished = 0;
- try {
- while (finished++ < 2) {
- Future f = c.take();
- f.get();
- }
- } finally {
- es.shutdownNow();
- }
+ for (FetchEmitTuple t : it) {
+ }
}
private Path get(String testFileName) throws Exception {
- return Paths.get(TestCSVFetchIterator.class.getResource("/"+testFileName).toURI());
+ return Paths.get(TestCSVFetchIterator.class.getResource("/" + testFileName).toURI());
}
private static class MockFetcher implements Callable<Integer> {
private final ArrayBlockingQueue<FetchEmitTuple> queue;
private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
this.queue = queue;
}
@@ -117,7 +111,7 @@ public class TestCSVFetchIterator {
public Integer call() throws Exception {
while (true) {
FetchEmitTuple t = queue.poll(1, TimeUnit.HOURS);
- if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+ if (t == COMPLETED_SEMAPHORE) {
return pairs.size();
}
pairs.add(t);
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
index 2813ed4..702554a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.tika</groupId>
@@ -29,10 +29,10 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>tika-fetch-iterator-jdbc</artifactId>
-
+
<name>Apache Tika Fetch Iterator - jdbc</name>
<url>http://tika.apache.org/</url>
-
+
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -97,15 +97,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
index 19da544..f1ca52c 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
@@ -16,19 +16,7 @@
*/
package org.apache.tika.pipes.fetchiterator.jdbc;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.emitter.EmitKey;
-import org.apache.tika.pipes.fetcher.FetchKey;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.apache.tika.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
import java.io.IOException;
import java.sql.Connection;
@@ -42,7 +30,20 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.utils.StringUtils;
/**
* Iterates through a the results from a sql call via jdbc. This adds all columns
@@ -50,8 +51,10 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
* to the metadata object.
* <p>
* <ul>
- * <li>If a 'fetchKeyColumn' is specified, this will use that column's value as the fetchKey.</li>
- * <li>If no 'fetchKeyColumn' is specified, this will send the metadata from the other columns.</li>
+ * <li>If a 'fetchKeyColumn' is specified, this will use that
+ * column's value as the fetchKey.</li>
+ * <li>If no 'fetchKeyColumn' is specified, this will send the
+ * metadata from the other columns.</li>
* <li>The 'fetchKeyColumn' value is not added to the metadata.</li>
* </ul>
* <p>
@@ -59,7 +62,6 @@ import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
* <li>An 'emitKeyColumn' must be specified</li>
* <li>The 'emitKeyColumn' value is not added to the metadata.</li>
* </ul>
- *
*/
public class JDBCFetchIterator extends FetchIterator implements Initializable {
@@ -88,6 +90,10 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
this.connection = connection;
}
+ public String getSelect() {
+ return select;
+ }
+
@Field
public void setSelect(String select) {
this.select = select;
@@ -106,16 +112,17 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
while (rs.next()) {
if (headers.size() == 0) {
fetchEmitKeyIndices = loadHeaders(rs.getMetaData(), headers);
- checkFetchEmitValidity(fetcherName, emitterName, fetchEmitKeyIndices, headers);
+ checkFetchEmitValidity(fetcherName, emitterName, fetchEmitKeyIndices,
+ headers);
}
try {
processRow(fetcherName, emitterName, headers, fetchEmitKeyIndices, rs);
} catch (SQLException e) {
- LOGGER.warn("Failed to insert: "+rs, e);
+ LOGGER.warn("Failed to insert: " + rs, e);
}
rowCount++;
if (rowCount % 1000 == 0) {
- LOGGER.info("added "+rowCount + " rows to the queue");
+ LOGGER.info("added " + rowCount + " rows to the queue");
}
}
}
@@ -125,21 +132,23 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
} finally {
try {
db.close();
- } catch (SQLException e){
+ } catch (SQLException e) {
LOGGER.warn("failed to close connection", e);
}
}
}
- private void checkFetchEmitValidity(String fetcherName,
- String emitterName,
+
+ private void checkFetchEmitValidity(String fetcherName, String emitterName,
FetchEmitKeyIndices fetchEmitKeyIndices,
List<String> headers) throws IOException {
- if (! StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
- throw new IOException(new TikaConfigException("Couldn't find column: "+fetchKeyColumn));
+ if (!StringUtils.isBlank(fetchKeyColumn) && fetchEmitKeyIndices.fetchKeyIndex < 0) {
+ throw new IOException(
+ new TikaConfigException("Couldn't find column: " + fetchKeyColumn));
}
- if (! StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
- throw new IOException(new TikaConfigException("Couldn't find column: "+emitKeyColumn));
+ if (!StringUtils.isBlank(emitKeyColumn) && fetchEmitKeyIndices.emitKeyIndex < 0) {
+ throw new IOException(
+ new TikaConfigException("Couldn't find column: " + emitKeyColumn));
}
}
@@ -152,14 +161,18 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
if (i == fetchEmitKeyIndices.fetchKeyIndex) {
fetchKey = getString(i, rs);
+ if (fetchKey == null) {
+ LOGGER.debug("fetchKey is empty for record " + toString(rs));
+ }
fetchKey = (fetchKey == null) ? "" : fetchKey;
- LOGGER.debug("fetchKey is empty for record "+toString(rs));
continue;
}
if (i == fetchEmitKeyIndices.emitKeyIndex) {
emitKey = getString(i, rs);
+ if (emitKey == null) {
+ LOGGER.debug("emitKey is empty for record " + toString(rs));
+ }
emitKey = (emitKey == null) ? "" : emitKey;
- LOGGER.warn("emitKey is empty for record "+toString(rs));
continue;
}
String val = getString(i, rs);
@@ -168,10 +181,8 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
}
}
- tryToAdd(new FetchEmitTuple(
- new FetchKey(fetcherName, fetchKey),
- new EmitKey(emitterName, emitKey),
- metadata, getOnParseException()));
+ tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, fetchKey),
+ new EmitKey(emitterName, emitKey), metadata, getOnParseException()));
}
private String toString(ResultSet rs) throws SQLException {
@@ -180,7 +191,7 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
String val = rs.getString(i);
val = (val == null) ? "" : val;
val = (val.length() > 100) ? val.substring(0, 100) : val;
- sb.append(rs.getMetaData().getColumnLabel(i)+":"+val+"\n");
+ sb.append(rs.getMetaData().getColumnLabel(i) + ":" + val + "\n");
}
return sb.toString();
}
@@ -195,7 +206,8 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
}
- private FetchEmitKeyIndices loadHeaders(ResultSetMetaData metaData, List<String> headers) throws SQLException {
+ private FetchEmitKeyIndices loadHeaders(ResultSetMetaData metaData, List<String> headers)
+ throws SQLException {
int fetchKeyIndex = -1;
int emitKeyIndex = -1;
for (int i = 1; i <= metaData.getColumnCount(); i++) {
@@ -228,7 +240,7 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
mustNotBeEmpty("emitterName", this.getEmitterName());
mustNotBeEmpty("emitKeyColumn", this.emitKeyColumn);
- if (StringUtils.isBlank(getFetcherName()) && ! StringUtils.isBlank(fetchKeyColumn)) {
+ if (StringUtils.isBlank(getFetcherName()) && !StringUtils.isBlank(fetchKeyColumn)) {
throw new TikaConfigException(
"If you specify a 'fetchKeyColumn', you must specify a 'fetcherName'");
}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
index ee3fcee..7eebe31 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
@@ -16,13 +16,9 @@
*/
package org.apache.tika.pipes.fetchiterator.jdbc;
-import org.apache.commons.io.FileUtils;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -44,35 +40,41 @@ import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
public class TestJDBCFetchIterator {
static final String TABLE = "fetchkeys";
- static Connection CONNECTION;
- static Path DB_DIR;
static final String db = "mydb";
private static final int NUM_ROWS = 1000;
+ static Connection CONNECTION;
+ static Path DB_DIR;
@BeforeClass
public static void setUp() throws Exception {
DB_DIR = Files.createTempDirectory("tika-jdbc-fetchiterator-test-");
- CONNECTION = DriverManager.getConnection("jdbc:h2:file:"+DB_DIR.toAbsolutePath()+"/"+db);
- String sql = "create table "+TABLE +
- " (id varchar(128), " +
+ CONNECTION =
+ DriverManager.getConnection("jdbc:h2:file:" +
+ DB_DIR.toAbsolutePath() + "/" + db);
+ String sql = "create table " + TABLE + " (id varchar(128), " +
"project varchar(128), " +
"fetchKey varchar(128))";
CONNECTION.createStatement().execute(sql);
for (int i = 0; i < NUM_ROWS; i++) {
- sql = "insert into "+TABLE + " (id, project, fetchKey) values ('id"+i+"','project"+
- (i%2 == 0 ? "a" : "b") +"','fk"+i+"')";
+ sql = "insert into " + TABLE + " (id, project, fetchKey) values ('id" + i +
+ "','project" + (i % 2 == 0 ? "a" : "b") + "','fk" + i + "')";
CONNECTION.createStatement().execute(sql);
}
- sql = "select count(1) from "+ TABLE;
+ sql = "select count(1) from " + TABLE;
ResultSet rs = CONNECTION.createStatement().executeQuery(sql);
while (rs.next()) {
int cnt = rs.getInt(1);
@@ -90,39 +92,44 @@ public class TestJDBCFetchIterator {
TikaConfig tk = getConfig();
int numConsumers = 5;
FetchIterator fetchIterator = tk.getFetchIterator();
- ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
- ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
- ArrayBlockingQueue<FetchEmitTuple> queue = fetchIterator.init(numConsumers);
- completionService.submit(fetchIterator);
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
+ ExecutorCompletionService<Integer> completionService =
+ new ExecutorCompletionService<>(es);
+ ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(100);
List<MockFetcher> fetchers = new ArrayList<>();
for (int i = 0; i < numConsumers; i++) {
MockFetcher mockFetcher = new MockFetcher(queue);
fetchers.add(mockFetcher);
completionService.submit(mockFetcher);
}
+ for (FetchEmitTuple t : fetchIterator) {
+ queue.offer(t);
+ }
+ for (int i = 0; i < numConsumers; i++) {
+ queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
+ }
int processed = 0;
int completed = 0;
- while (completed < numConsumers+1) {
+ while (completed < numConsumers) {
Future<Integer> f = completionService.take();
processed += f.get();
completed++;
}
- //fetchiterator added + MockFetcher yields 2x
- assertEquals(NUM_ROWS * 2, processed);
+ assertEquals(NUM_ROWS, processed);
int cnt = 0;
Matcher m = Pattern.compile("fk(\\d+)").matcher("");
for (MockFetcher f : fetchers) {
for (FetchEmitTuple p : f.pairs) {
- String k = p.getFetchKey().getKey();
+ String k = p.getFetchKey().getFetchKey();
String num = "";
if (m.reset(k).find()) {
num = m.group(1);
} else {
- fail("failed to find key pattern: "+k);
+ fail("failed to find key pattern: " + k);
}
String aOrB = Integer.parseInt(num) % 2 == 0 ? "a" : "b";
- assertEquals("id"+num, p.getMetadata().get("MY_ID"));
- assertEquals("project"+aOrB, p.getMetadata().get("MY_PROJECT"));
+ assertEquals("id" + num, p.getMetadata().get("MY_ID"));
+ assertEquals("project" + aOrB, p.getMetadata().get("MY_PROJECT"));
assertNull(p.getMetadata().get("fetchKey"));
assertNull(p.getMetadata().get("MY_FETCHKEY"));
cnt++;
@@ -134,17 +141,22 @@ public class TestJDBCFetchIterator {
private TikaConfig getConfig() throws Exception {
String config = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?><properties>\n" +
" <fetchIterators>\n" +
- " <fetchIterator class=\"org.apache.tika.pipes.fetchiterator.jdbc.JDBCFetchIterator\">\n" +
+ " <fetchIterator " +
+ " class=\"org.apache.tika.pipes.fetchiterator.jdbc.JDBCFetchIterator\">\n" +
" <params>\n" +
" <param name=\"fetcherName\" type=\"string\">s3f</param>\n" +
" <param name=\"emitterName\" type=\"string\">s3e</param>\n" +
" <param name=\"queueSize\" type=\"int\">57</param>\n" +
- " <param name=\"fetchKeyColumn\" type=\"string\">my_fetchkey</param>\n" +
- " <param name=\"emitKeyColumn\" type=\"string\">my_fetchkey</param>\n" +
+ " <param name=\"fetchKeyColumn\" " +
+ " type=\"string\">my_fetchkey</param>\n" +
+ " <param name=\"emitKeyColumn\" " +
+ " type=\"string\">my_fetchkey</param>\n" +
" <param name=\"select\" type=\"string\">" +
- "select id as my_id, project as my_project, fetchKey as my_fetchKey from fetchkeys</param>\n" +
- " <param name=\"connection\" type=\"string\">jdbc:h2:file:"+
- DB_DIR.toAbsolutePath()+"/"+db +"</param>\n" +
+ "select id as my_id, project as my_project, fetchKey as my_fetchKey " +
+ "from fetchkeys</param>\n" +
+ " <param name=\"connection\" " +
+ " type=\"string\">jdbc:h2:file:" + DB_DIR.toAbsolutePath() + "/" +
+ db + "</param>\n" +
" </params>\n" +
" </fetchIterator>\n" +
" </fetchIterators>\n" +
@@ -155,6 +167,7 @@ public class TestJDBCFetchIterator {
private static class MockFetcher implements Callable<Integer> {
private final ArrayBlockingQueue<FetchEmitTuple> queue;
private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
this.queue = queue;
}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#info,debug, error,fatal ...
log4j.rootLogger=info,stderr
-
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
index f8b2424..24be277 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.tika</groupId>
@@ -57,9 +57,18 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons.codec.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
@@ -124,15 +133,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
index dd14463..988f149 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
@@ -16,11 +16,23 @@
*/
package org.apache.tika.pipes.fetchiterator.s3;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.iterable.S3Objects;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
@@ -31,21 +43,13 @@ import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
public class S3FetchIterator extends FetchIterator implements Initializable {
-
private static final Logger LOGGER = LoggerFactory.getLogger(S3FetchIterator.class);
- private String s3PathPrefix = "";
+ private String prefix = "";
private String region;
+ private String credentialsProvider;
private String profile;
private String bucket;
@@ -67,19 +71,45 @@ public class S3FetchIterator extends FetchIterator implements Initializable {
}
@Field
- public void setS3PathPrefix(String s3PathPrefix) {
- this.s3PathPrefix = s3PathPrefix;
+ public void setPrefix(String prefix) {
+ this.prefix = prefix;
}
+ @Field
+ public void setCredentialsProvider(String credentialsProvider) {
+ if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+ throw new IllegalArgumentException(
+ "credentialsProvider must be either 'profile' or instance'");
+ }
+ this.credentialsProvider = credentialsProvider;
+ }
+
+ /**
+ * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+ * e.g. AmazonClientException in a TikaConfigException.
+ *
+ * @param params params to use for initialization
+ * @throws TikaConfigException
+ */
@Override
public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set
- //ignore them
+ //params have already been set...ignore them
+ AWSCredentialsProvider provider = null;
+ if ("instance".equals(credentialsProvider)) {
+ provider = InstanceProfileCredentialsProvider.getInstance();
+ } else if ("profile".equals(credentialsProvider)) {
+ provider = new ProfileCredentialsProvider(profile);
+ } else {
+ throw new TikaConfigException("credentialsProvider must be set and " +
+ "must be either 'instance' or 'profile'");
+ }
- s3Client = AmazonS3ClientBuilder.standard()
- .withRegion(region)
- .withCredentials(new ProfileCredentialsProvider(profile))
- .build();
+ try {
+ s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+ .build();
+ } catch (AmazonClientException e) {
+ throw new TikaConfigException("can't initialize s3 fetchiterator");
+ }
}
@Override
@@ -87,24 +117,22 @@ public class S3FetchIterator extends FetchIterator implements Initializable {
throws TikaConfigException {
super.checkInitialization(problemHandler);
mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("profile", this.profile);
mustNotBeEmpty("region", this.region);
}
@Override
protected void enqueue() throws InterruptedException, IOException, TimeoutException {
String fetcherName = getFetcherName();
+ String emitterName = getEmitterName();
long start = System.currentTimeMillis();
int count = 0;
- for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, s3PathPrefix)) {
+ for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, prefix)) {
long elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("adding ({}) {} in {} ms", count, summary.getKey(),
- elapsed);
- tryToAdd(new FetchEmitTuple(
- new FetchKey(fetcherName, summary.getKey()),
- new EmitKey(fetcherName, summary.getKey()),
- new Metadata(), getOnParseException()));
+ LOGGER.debug("adding ({}) {} in {} ms", count, summary.getKey(), elapsed);
+ tryToAdd(new FetchEmitTuple(new FetchKey(fetcherName, summary.getKey()),
+ new EmitKey(emitterName, summary.getKey()), new Metadata(),
+ getOnParseException()));
count++;
}
long elapsed = System.currentTimeMillis() - start;
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
index 51d20ac..50cb819 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
@@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.tika.pipes.fetchiterator.s3;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.Ignore;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Collections;
@@ -31,7 +29,11 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import static org.junit.Assert.assertEquals;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
@Ignore("turn into an actual unit test")
public class TestS3FetchIterator {
@@ -46,21 +48,26 @@ public class TestS3FetchIterator {
it.setRegion("");//select one
it.initialize(Collections.EMPTY_MAP);
int numConsumers = 6;
- ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
+ ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(10);
- ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
ExecutorCompletionService c = new ExecutorCompletionService(es);
- c.submit(it);
List<MockFetcher> fetchers = new ArrayList<>();
for (int i = 0; i < numConsumers; i++) {
MockFetcher fetcher = new MockFetcher(queue);
fetchers.add(fetcher);
c.submit(fetcher);
}
+ for (FetchEmitTuple t : it) {
+ queue.offer(t);
+ }
+ for (int i = 0; i < numConsumers; i++) {
+ queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
+ }
int finished = 0;
int completed = 0;
try {
- while (finished++ < numConsumers+1) {
+ while (finished++ < numConsumers + 1) {
Future<Integer> f = c.take();
completed += f.get();
}
@@ -74,6 +81,7 @@ public class TestS3FetchIterator {
private static class MockFetcher implements Callable<Integer> {
private final ArrayBlockingQueue<FetchEmitTuple> queue;
private final List<FetchEmitTuple> pairs = new ArrayList<>();
+
private MockFetcher(ArrayBlockingQueue<FetchEmitTuple> queue) {
this.queue = queue;
}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#info,debug, error,fatal ...
log4j.rootLogger=info,stderr
-
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
index 09dcb97..ae76a52 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
@@ -17,8 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tika-fetchers</artifactId>
@@ -97,15 +97,18 @@
</filter>
</filters>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index a6d2325..be39765 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -17,11 +17,19 @@
package org.apache.tika.pipes.fetcher.http;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.client.HttpClientFactory;
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
@@ -32,13 +40,6 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
/**
* Based on Apache httpclient
@@ -52,9 +53,9 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
public HttpFetcher() throws TikaConfigException {
httpClientFactory = new HttpClientFactory();
}
+
@Override
- public InputStream fetch(String fetchKey, Metadata metadata)
- throws IOException, TikaException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
return get(get);
}
@@ -62,7 +63,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
- get.setHeader("Range", "bytes="+startRange+"-"+endRange);
+ get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
return get(get);
}
@@ -70,21 +71,18 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
HttpResponse response = httpClient.execute(get);
int code = response.getStatusLine().getStatusCode();
if (code < 200 || code > 299) {
- throw new IOException("bad status code: "+
- code
- + " :: " +
+ throw new IOException("bad status code: " + code + " :: " +
responseToString(response.getEntity().getContent()));
}
//spool to local
long start = System.currentTimeMillis();
- TikaInputStream tis = TikaInputStream.get(
- response.getEntity().getContent());
+ TikaInputStream tis = TikaInputStream.get(response.getEntity().getContent());
tis.getPath();
if (response instanceof CloseableHttpResponse) {
((CloseableHttpResponse) response).close();
}
- long elapsed = System.currentTimeMillis()-start;
+ long elapsed = System.currentTimeMillis() - start;
LOG.debug("took {} ms to copy to local tmp file", elapsed);
return tis;
}
@@ -149,7 +147,8 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
}
@Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
index 8a2d8a1..3a2e127 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
@@ -16,13 +16,7 @@
*/
package org.apache.tika.pipes.fetcher.http;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.metadata.Metadata;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.xml.sax.SAXException;
+import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.InputStream;
@@ -31,32 +25,40 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.zip.GZIPInputStream;
-import static org.junit.Assert.assertEquals;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.metadata.Metadata;
@Ignore("requires network connectivity")
public class HttpFetcherTest {
- @Test
- public void testRange() throws Exception {
- String url =
- "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
- long start = 969596307;
- long end = start + 1408 - 1;
- Metadata metadata = new Metadata();
- HttpFetcher httpFetcher = (HttpFetcher) getConfig("tika-config-http.xml")
- .getFetcherManager().getFetcher("http");
- try (TemporaryResources tmp = new TemporaryResources()) {
- Path tmpPath = tmp.createTempFile();
- try (InputStream is = httpFetcher.fetch(url, start, end, metadata)) {
- Files.copy(new GZIPInputStream(is), tmpPath, StandardCopyOption.REPLACE_EXISTING);
- }
- assertEquals(2461, Files.size(tmpPath));
+ @Test
+ public void testRange() throws Exception {
+ String url =
+ "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
+ long start = 969596307;
+ long end = start + 1408 - 1;
+ Metadata metadata = new Metadata();
+ HttpFetcher httpFetcher =
+ (HttpFetcher) getConfig("tika-config-http.xml").getFetcherManager()
+ .getFetcher("http");
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ Path tmpPath = tmp.createTempFile();
+ try (InputStream is = httpFetcher.fetch(url, start, end, metadata)) {
+ Files.copy(new GZIPInputStream(is), tmpPath, StandardCopyOption.REPLACE_EXISTING);
}
+ assertEquals(2461, Files.size(tmpPath));
}
+ }
TikaConfig getConfig(String path) throws TikaException, IOException, SAXException {
- return new TikaConfig(HttpFetcherTest.class.getResourceAsStream("/"+path));
+ return new TikaConfig(HttpFetcherTest.class.getResourceAsStream("/" + path));
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
index 158e701..7d8288c 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
@@ -47,8 +47,17 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons.codec.version}</version>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index 217881e..a6b34dd 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -16,29 +16,33 @@
*/
package org.apache.tika.pipes.fetcher.s3;
+import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.tika.config.Field;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.config.Param;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
+import org.apache.tika.pipes.fetcher.AbstractFetcher;
/**
* Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
@@ -52,64 +56,67 @@ public class S3Fetcher extends AbstractFetcher implements Initializable {
private String region;
private String bucket;
private String profile;
+ private String credentialsProvider;
private boolean extractUserMetadata = true;
private AmazonS3 s3Client;
private boolean spoolToTemp = true;
@Override
- public InputStream fetch(String fetchKey, Metadata metadata)
- throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
- LOGGER.debug("about to fetch fetchkey={} from bucket ({})",
- fetchKey, bucket);
+ LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
- S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
- if (extractUserMetadata) {
- for (Map.Entry<String, String> e :
- s3Object.getObjectMetadata().getUserMetadata().entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ try {
+ S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
+ if (extractUserMetadata) {
+ for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
+ .entrySet()) {
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ }
}
- }
- if (! spoolToTemp) {
- return TikaInputStream.get(
- s3Object.getObjectContent());
- } else {
- long start = System.currentTimeMillis();
- TikaInputStream tis = TikaInputStream.get(
- s3Object.getObjectContent());
- tis.getPath();
- long elapsed = System.currentTimeMillis()-start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
+ if (!spoolToTemp) {
+ return TikaInputStream.get(s3Object.getObjectContent());
+ } else {
+ long start = System.currentTimeMillis();
+ TikaInputStream tis = TikaInputStream.get(s3Object.getObjectContent());
+ tis.getPath();
+ long elapsed = System.currentTimeMillis() - start;
+ LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
+ return tis;
+ }
+ } catch (AmazonClientException e) {
+ throw new IOException("s3 client exception", e);
}
}
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws TikaException, IOException {
//TODO -- figure out how to integrate this
- LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})",
- fetchKey, startRange, endRange, bucket);
-
- S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey)
- .withRange(startRange, endRange));
-
- if (extractUserMetadata) {
- for (Map.Entry<String, String> e :
- s3Object.getObjectMetadata().getUserMetadata().entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})", fetchKey,
+ startRange, endRange, bucket);
+
+ try {
+ S3Object s3Object = s3Client.getObject(
+ new GetObjectRequest(bucket, fetchKey).withRange(startRange, endRange));
+
+ if (extractUserMetadata) {
+ for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
+ .entrySet()) {
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ }
}
- }
- if (! spoolToTemp) {
- return TikaInputStream.get(
- s3Object.getObjectContent());
- } else {
- long start = System.currentTimeMillis();
- TikaInputStream tis = TikaInputStream.get(
- s3Object.getObjectContent());
- tis.getPath();
- long elapsed = System.currentTimeMillis()-start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
+ if (!spoolToTemp) {
+ return TikaInputStream.get(s3Object.getObjectContent());
+ } else {
+ long start = System.currentTimeMillis();
+ TikaInputStream tis = TikaInputStream.get(s3Object.getObjectContent());
+ tis.getPath();
+ long elapsed = System.currentTimeMillis() - start;
+ LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
+ return tis;
+ }
+ } catch (AmazonClientException e) {
+ throw new IOException(e);
}
}
@@ -143,21 +150,47 @@ public class S3Fetcher extends AbstractFetcher implements Initializable {
this.extractUserMetadata = extractUserMetadata;
}
+ @Field
+ public void setCredentialsProvider(String credentialsProvider) {
+ if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance")) {
+ throw new IllegalArgumentException(
+ "credentialsProvider must be either 'profile' or instance'");
+ }
+ this.credentialsProvider = credentialsProvider;
+ }
+
+ /**
+ * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
+ * e.g. AmazonClientException in a TikaConfigException.
+ *
+ * @param params params to use for initialization
+ * @throws TikaConfigException
+ */
@Override
public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set
- //ignore them
- s3Client = AmazonS3ClientBuilder.standard()
- .withRegion(region)
- .withCredentials(new ProfileCredentialsProvider(profile))
- .build();
+ //params have already been set...ignore them
+ AWSCredentialsProvider provider = null;
+ if ("instance".equals(credentialsProvider)) {
+ provider = InstanceProfileCredentialsProvider.getInstance();
+ } else if ("profile".equals(credentialsProvider)) {
+ provider = new ProfileCredentialsProvider(profile);
+ } else {
+ throw new TikaConfigException("credentialsProvider must be set and " +
+ "must be either 'instance' or 'profile'");
+ }
+
+ try {
+ s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
+ .build();
+ } catch (AmazonClientException e) {
+ throw new TikaConfigException("can't initialize s3 fetcher", e);
+ }
}
@Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("profile", this.profile);
mustNotBeEmpty("region", this.region);
-
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
index 450d933..9be1ab4 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
@@ -15,11 +15,6 @@
* limitations under the License.
*/
package org.apache.tika.pipes.fetcher.s3;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.pipes.fetcher.Fetcher;
-import org.apache.tika.metadata.Metadata;
-import org.junit.Ignore;
-import org.junit.Test;
import java.io.InputStream;
import java.nio.file.Files;
@@ -28,12 +23,19 @@ import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Collections;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.Fetcher;
+
@Ignore("write actual unit tests")
public class TestS3Fetcher {
private static final String FETCH_STRING = "";
- private Path outputFile = Paths.get("");
- private String region = "us-east-1";
- private String profile = "";
+ private final Path outputFile = Paths.get("");
+ private final String region = "us-east-1";
+ private final String profile = "";
@Test
public void testBasic() throws Exception {
@@ -50,9 +52,8 @@ public class TestS3Fetcher {
@Test
public void testConfig() throws Exception {
- TikaConfig config = new TikaConfig(
- this.getClass().getResourceAsStream("/tika-config-s3.xml")
- );
+ TikaConfig config =
+ new TikaConfig(this.getClass().getResourceAsStream("/tika-config-s3.xml"));
Fetcher fetcher = config.getFetcherManager().getFetcher("s3");
Metadata metadata = new Metadata();
try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
index 740058a..176513b 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
@@ -22,6 +22,7 @@
<param name="name" type="string">s3</param>
<param name="region" type="string">us-east-1</param>
<param name="profile" type="string">my_profile</param>
+ <param name="credentialsProvider" type="string">profile</param>
</params>
</fetcher>
</fetchers>
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
index 8cbc26b..77619f9 100644
--- a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientFactory.java
@@ -16,6 +16,27 @@
*/
package org.apache.tika.client;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.Set;
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+import javax.net.ssl.SSLContext;
+
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
@@ -52,35 +73,15 @@ import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.SecretKeySpec;
-import javax.net.ssl.SSLContext;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.HashSet;
-import java.util.Set;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.utils.StringUtils;
/**
* This holds quite a bit of state and is not thread safe. Beware!
- *
+ * <p>
* Also, we're currently ignoring the SSL checks. Please open a ticket/PR
* if you need robust SSL.
*/
@@ -104,7 +105,7 @@ public class HttpClientFactory {
private String password;
private String ntDomain;//if using nt credentials
private String authScheme = "basic"; //ntlm or basic
- private boolean credentialsAESEncrypted = false;
+ private final boolean credentialsAESEncrypted = false;
public HttpClientFactory() throws TikaConfigException {
@@ -116,6 +117,7 @@ public class HttpClientFactory {
aes = new AES();
}
}
+
public String getProxyHost() {
return proxyHost;
}
@@ -218,6 +220,7 @@ public class HttpClientFactory {
/**
* only basic and ntlm are supported
+ *
* @param authScheme
*/
public void setAuthScheme(String authScheme) {
@@ -230,19 +233,18 @@ public class HttpClientFactory {
TrustStrategy acceptingTrustStrategy = (cert, authType) -> true;
SSLContext sslContext = null;
try {
- sslContext = SSLContexts.custom().loadTrustMaterial(null,
- acceptingTrustStrategy).build();
+ sslContext =
+ SSLContexts.custom().loadTrustMaterial(
+ null, acceptingTrustStrategy).build();
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
throw new TikaConfigException("", e);
}
- SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
- NoopHostnameVerifier.INSTANCE);
+ SSLConnectionSocketFactory sslsf =
+ new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
Registry<ConnectionSocketFactory> socketFactoryRegistry =
- RegistryBuilder.<ConnectionSocketFactory>create()
- .register("https", sslsf)
- .register("http", new PlainConnectionSocketFactory())
- .build();
+ RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf)
+ .register("http", new PlainConnectionSocketFactory()).build();
PoolingHttpClientConnectionManager manager =
new PoolingHttpClientConnectionManager(socketFactoryRegistry);
@@ -253,19 +255,13 @@ public class HttpClientFactory {
addCredentialsProvider(builder);
addProxy(builder);
return builder.setConnectionManager(manager)
- .setRedirectStrategy(
- new CustomRedirectStrategy(allowedHostsForRedirect))
- .setDefaultRequestConfig(RequestConfig.custom()
- .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC,
- AuthSchemes.NTLM))
- .setConnectionRequestTimeout((int) requestTimeout)
- .setConnectionRequestTimeout(connectTimeout)
- .setSocketTimeout(socketTimeout)
- .build()
- )
- .setKeepAliveStrategy(getKeepAliveStrategy())
- .setSSLSocketFactory(sslsf)
- .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+ .setRedirectStrategy(new CustomRedirectStrategy(allowedHostsForRedirect))
+ .setDefaultRequestConfig(RequestConfig.custom().setTargetPreferredAuthSchemes(
+ Arrays.asList(AuthSchemes.BASIC, AuthSchemes.NTLM))
+ .setConnectionRequestTimeout(requestTimeout)
+ .setConnectionRequestTimeout(connectTimeout).setSocketTimeout(socketTimeout)
+ .build()).setKeepAliveStrategy(getKeepAliveStrategy())
+ .setSSLSocketFactory(sslsf).setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
.build();
}
@@ -285,33 +281,32 @@ public class HttpClientFactory {
if ((StringUtils.isBlank(userName) && StringUtils.isBlank(password)) ||
(StringUtils.isBlank(password) && StringUtils.isBlank(userName))) {
- throw new IllegalArgumentException("can't have one of 'username', " +
- "'password' null and the other not");
+ throw new IllegalArgumentException(
+ "can't have one of 'username', " + "'password' null and the other not");
}
String finalUserName = decrypt(userName);
String finalPassword = decrypt(password);
String finalDomain = decrypt(ntDomain);
- CredentialsProvider provider = new BasicCredentialsProvider();
- Credentials credentials = null;
- Registry<AuthSchemeProvider> authSchemeRegistry = null;
- if (authScheme.equals("basic")) {
- credentials = new UsernamePasswordCredentials(finalUserName, finalPassword);
- authSchemeRegistry = RegistryBuilder
- .<AuthSchemeProvider>create()
- .register("basic", new BasicSchemeFactory())
- .build();
- } else if (authScheme.equals("ntlm")) {
- if (StringUtils.isBlank(ntDomain)) {
- throw new IllegalArgumentException("must specify 'ntDomain'");
- }
- credentials = new NTCredentials(finalUserName, finalPassword, null, finalDomain);
- authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
- .register("ntlm", new NTLMSchemeFactory()).build();
+ CredentialsProvider provider = new BasicCredentialsProvider();
+ Credentials credentials = null;
+ Registry<AuthSchemeProvider> authSchemeRegistry = null;
+ if (authScheme.equals("basic")) {
+ credentials = new UsernamePasswordCredentials(finalUserName, finalPassword);
+ authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+ .register("basic", new BasicSchemeFactory()).build();
+ } else if (authScheme.equals("ntlm")) {
+ if (StringUtils.isBlank(ntDomain)) {
+ throw new IllegalArgumentException("must specify 'ntDomain'");
}
- provider.setCredentials(AuthScope.ANY, credentials);
- builder.setDefaultCredentialsProvider(provider);
- builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+ credentials = new NTCredentials(finalUserName, finalPassword,
+ null, finalDomain);
+ authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+ .register("ntlm", new NTLMSchemeFactory()).build();
+ }
+ provider.setCredentials(AuthScope.ANY, credentials);
+ builder.setDefaultCredentialsProvider(provider);
+ builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
}
@@ -350,7 +345,7 @@ public class HttpClientFactory {
private static class CustomRedirectStrategy extends LaxRedirectStrategy {
private static final Logger LOG = LoggerFactory.getLogger(CustomRedirectStrategy.class);
- private Set<String> allowedHosts;
+ private final Set<String> allowedHosts;
public CustomRedirectStrategy(Set<String> allowedHosts) {
this.allowedHosts = allowedHosts;
@@ -373,7 +368,9 @@ public class HttpClientFactory {
}
@Override
- public boolean isRedirected(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
+ public boolean isRedirected(HttpRequest request, HttpResponse response,
+ HttpContext context)
+ throws ProtocolException {
boolean isRedirectedSuper = super.isRedirected(request, response, context);
if (isRedirectedSuper) {
Header locationHeader = response.getFirstHeader("Location");
@@ -422,9 +419,10 @@ public class HttpClientFactory {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
- return Base64.getEncoder()
- .encodeToString(cipher.doFinal(strToEncrypt.getBytes(StandardCharsets.UTF_8)));
- } catch (NoSuchAlgorithmException|InvalidKeyException|NoSuchPaddingException|BadPaddingException|IllegalBlockSizeException e) {
+ return Base64.getEncoder().encodeToString(
+ cipher.doFinal(strToEncrypt.getBytes(StandardCharsets.UTF_8)));
+ } catch (NoSuchAlgorithmException | InvalidKeyException |
+ NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException e) {
throw new TikaConfigException("bad encryption info", e);
}
}
@@ -435,11 +433,8 @@ public class HttpClientFactory {
cipher.init(Cipher.DECRYPT_MODE, secretKey);
return new String(cipher.doFinal(Base64.getDecoder().decode(strToDecrypt)),
StandardCharsets.UTF_8);
- } catch (NoSuchAlgorithmException|
- InvalidKeyException|
- NoSuchPaddingException|
- BadPaddingException|
- IllegalBlockSizeException e) {
+ } catch (NoSuchAlgorithmException | InvalidKeyException |
+ NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException e) {
throw new TikaConfigException("bad encryption info", e);
}
}
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
index 11ebb3c..3de32da 100644
--- a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
@@ -16,20 +16,19 @@
*/
package org.apache.tika.client;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.InputStreamEntity;
import org.apache.http.util.EntityUtils;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
public class HttpClientUtil {
- public static boolean postJson(HttpClient client, String url, String json) throws IOException,
- TikaClientException {
+ public static boolean postJson(HttpClient client, String url, String json)
+ throws IOException, TikaClientException {
HttpPost post = new HttpPost(url);
post.setHeader("Content-Encoding", "gzip");
ByteArrayEntity entity = new ByteArrayEntity(json.getBytes(StandardCharsets.UTF_8));
@@ -41,9 +40,8 @@ public class HttpClientUtil {
if (response.getStatusLine().getStatusCode() != 200) {
String msg = EntityUtils.toString(response.getEntity());
- throw new TikaClientException("Bad status: " +
- response.getStatusLine().getStatusCode() + " : "+
- msg);
+ throw new TikaClientException(
+ "Bad status: " + response.getStatusLine().getStatusCode() + " : " + msg);
} else {
String msg = EntityUtils.toString(response.getEntity());
System.out.println("httputil: " + msg);
@@ -51,9 +49,8 @@ public class HttpClientUtil {
return true;
}
- public static boolean postJson(HttpClient client, String url,
- byte[] bytes, boolean gzipped) throws IOException,
- TikaClientException {
+ public static boolean postJson(HttpClient client, String url, byte[] bytes, boolean gzipped)
+ throws IOException, TikaClientException {
HttpPost post = new HttpPost(url);
if (gzipped) {
post.setHeader("Content-Encoding", "gzip");
@@ -67,9 +64,8 @@ public class HttpClientUtil {
if (response.getStatusLine().getStatusCode() != 200) {
String msg = EntityUtils.toString(response.getEntity());
- throw new TikaClientException("Bad status: " +
- response.getStatusLine().getStatusCode() + " : "+
- msg);
+ throw new TikaClientException(
+ "Bad status: " + response.getStatusLine().getStatusCode() + " : " + msg);
} else {
String msg = EntityUtils.toString(response.getEntity());
System.out.println("httputil: " + msg);
diff --git a/tika-pipes/tika-pipes-async/pom.xml b/tika-pipes/tika-pipes-async/pom.xml
new file mode 100644
index 0000000..5f4e418
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/pom.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-pipes</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-pipes-async</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Tika emitters</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-serialization</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>${h2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-emitter-fs</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <!--
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-parsers-classic-package</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>-->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java
new file mode 100644
index 0000000..cb8347f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncCli.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.EmptyFetchIterator;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
+public class AsyncCli {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncCli.class);
+
+ public static void main(String[] args) throws Exception {
+ Path configPath = Paths.get(args[0]);
+ int maxConsumers = 20;
+ AsyncCli asyncCli = new AsyncCli();
+ Path dbDir = Files.createTempDirectory("tika-async-db-");
+ try {
+ asyncCli.execute(dbDir, configPath, maxConsumers);
+ } finally {
+ FileUtils.deleteDirectory(dbDir.toFile());
+ }
+
+ }
+
+ private static List<Integer> getActiveWorkers(Connection connection) throws SQLException {
+ PreparedStatement findActiveWorkers =
+ connection.prepareStatement("select worker_id from workers");
+ List<Integer> workers = new ArrayList<>();
+ try (ResultSet rs = findActiveWorkers.executeQuery()) {
+ while (rs.next()) {
+ workers.add(rs.getInt(1));
+ }
+ }
+ return workers;
+ }
+
+ private void execute(Path dbDir, Path configPath, int maxConsumers) throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(configPath);
+
+ String connectionString = setupTables(dbDir);
+
+ ExecutorService executorService = Executors.newFixedThreadPool(maxConsumers + 3);
+ ExecutorCompletionService<Integer> executorCompletionService =
+ new ExecutorCompletionService<>(executorService);
+
+ try (Connection connection = DriverManager.getConnection(connectionString)) {
+ FetchIterator fetchIterator = tikaConfig.getFetchIterator();
+ if (fetchIterator instanceof EmptyFetchIterator) {
+ throw new IllegalArgumentException("can't have empty fetch iterator");
+ }
+ ArrayBlockingQueue<FetchEmitTuple> q =
+ new ArrayBlockingQueue<>(10000);//fetchIterator.init(maxConsumers);
+ AsyncTaskEnqueuer enqueuer = new AsyncTaskEnqueuer(q, connection);
+ executorCompletionService.submit(fetchIterator);
+ executorCompletionService.submit(enqueuer);
+ executorCompletionService.submit(new AssignmentManager(connection, enqueuer));
+
+ for (int i = 0; i < maxConsumers; i++) {
+ executorCompletionService
+ .submit(new AsyncWorker(connection, connectionString, i, configPath));
+ }
+ int completed = 0;
+ while (completed < maxConsumers + 3) {
+ Future<Integer> future = executorCompletionService.take();
+ if (future != null) {
+ int val = future.get();
+ completed++;
+ LOG.debug("finished " + val);
+ }
+ }
+ } finally {
+ executorService.shutdownNow();
+ }
+ }
+
+ private String setupTables(Path dbDir) throws SQLException {
+ Path dbFile = dbDir.resolve("tika-async");
+ String url = "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE";
+ Connection connection = DriverManager.getConnection(url);
+
+ String sql = "create table task_queue " + "(id bigint auto_increment primary key," +
+ "status tinyint," + //byte
+ "worker_id integer," + "retry smallint," + //short
+ "time_stamp timestamp," + "json varchar(64000))";
+ connection.createStatement().execute(sql);
+ //no clear benefit to creating an index on timestamp
+// sql = "CREATE INDEX IF NOT EXISTS status_timestamp on status (time_stamp)";
+ sql = "create table workers (worker_id int primary key)";
+ connection.createStatement().execute(sql);
+
+ sql = "create table workers_shutdown (worker_id int primary key)";
+ connection.createStatement().execute(sql);
+
+ sql = "create table error_log (task_id bigint, " + "fetch_key varchar(10000)," +
+ "time_stamp timestamp," + "retry integer," + "error_code tinyint)";
+ connection.createStatement().execute(sql);
+
+ return url;
+ }
+
+ //this reads fetchemittuples from the queue and inserts them in the db
+ //for the workers to read
+ private static class AsyncTaskEnqueuer implements Callable<Integer> {
+ private final PreparedStatement insert;
+
+ private final ArrayBlockingQueue<FetchEmitTuple> queue;
+ private final Connection connection;
+ private final Random random = new Random();
+
+ private volatile boolean isComplete = false;
+
+ AsyncTaskEnqueuer(ArrayBlockingQueue<FetchEmitTuple> queue, Connection connection)
+ throws SQLException {
+ this.queue = queue;
+ this.connection = connection;
+ String sql = "insert into task_queue (status, time_stamp, worker_id, retry, json) " +
+ "values (?,CURRENT_TIMESTAMP(),?,?,?)";
+ insert = connection.prepareStatement(sql);
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ List<Integer> workers = new ArrayList<>();
+ while (true) {
+ FetchEmitTuple t = queue.poll(1, TimeUnit.SECONDS);
+ LOG.debug("enqueing to db " + t);
+ if (t == null) {
+ //log.trace?
+ } else if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+ isComplete = true;
+ return 1;
+ } else {
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ //TODO -- fix this
+ while (workers.size() == 0 && elapsed < 600000) {
+ workers = getActiveWorkers(connection);
+ Thread.sleep(100);
+ elapsed = System.currentTimeMillis() - start;
+ }
+ insert(t, workers);
+ }
+ }
+ }
+
+ boolean isComplete() {
+ return isComplete;
+ }
+
+ private void insert(FetchEmitTuple t, List<Integer> workers)
+ throws IOException, SQLException {
+ int workerId = workers.size() == 1 ? workers.get(0) :
+ workers.get(random.nextInt(workers.size()));
+ insert.clearParameters();
+ insert.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+ insert.setInt(2, workerId);
+ insert.setShort(3, (short) 0);
+ insert.setString(4, JsonFetchEmitTuple.toJson(t));
+ insert.execute();
+ }
+ }
+
+ private static class AssignmentManager implements Callable {
+
+ private final Connection connection;
+ private final AsyncTaskEnqueuer enqueuer;
+ private final PreparedStatement getQueueDistribution;
+ private final PreparedStatement findMissingWorkers;
+ private final PreparedStatement allocateNonworkersToWorkers;
+ private final PreparedStatement reallocate;
+ private final PreparedStatement countAvailableTasks;
+ private final PreparedStatement insertWorkersShutdown;
+ private final Random random = new Random();
+
+
+ public AssignmentManager(Connection connection, AsyncTaskEnqueuer enqueuer)
+ throws SQLException {
+ this.connection = connection;
+ this.enqueuer = enqueuer;
+ //this gets workers and # of tasks in desc order of number of tasks
+ String sql = "select w.worker_id, p.cnt " + "from workers w " +
+ "left join (select worker_id, count(1) as cnt from task_queue " +
+ "where status=0 group by worker_id)" +
+ " p on p.worker_id=w.worker_id order by p.cnt desc";
+ getQueueDistribution = connection.prepareStatement(sql);
+ //find workers that have assigned tasks but are not in the
+ //workers table
+ sql = "select p.worker_id, count(1) as cnt from task_queue p " +
+ "left join workers w on p.worker_id=w.worker_id " +
+ "where w.worker_id is null group by p.worker_id";
+ findMissingWorkers = connection.prepareStatement(sql);
+
+ sql = "update task_queue set worker_id=? where worker_id=?";
+ allocateNonworkersToWorkers = connection.prepareStatement(sql);
+
+ //current strategy reallocate tasks from longest queue to shortest
+ //TODO: might consider randomly shuffling or other algorithms
+ sql = "update task_queue set worker_id= ? where id in " +
+ "(select id from task_queue where " + "worker_id = ? and " + "rand() < 0.8 " +
+ "and status=0 for update)";
+ reallocate = connection.prepareStatement(sql);
+
+ sql = "select count(1) from task_queue where status=" +
+ AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal();
+ countAvailableTasks = connection.prepareStatement(sql);
+
+ sql = "insert into workers_shutdown (worker_id) values (?)";
+ insertWorkersShutdown = connection.prepareStatement(sql);
+ }
+
+ @Override
+ public Integer call() throws Exception {
+
+ while (true) {
+ List<Integer> missingWorkers = getMissingWorkers();
+ reallocateFromMissingWorkers(missingWorkers);
+ redistribute();
+ if (isComplete()) {
+ notifyWorkers();
+ return 1;
+ }
+ Thread.sleep(100);
+ }
+ }
+
+ private void notifyWorkers() throws SQLException {
+ for (int workerId : getActiveWorkers(connection)) {
+ insertWorkersShutdown.clearParameters();
+ insertWorkersShutdown.setInt(1, workerId);
+ insertWorkersShutdown.execute();
+ }
+ }
+
+ private boolean isComplete() throws SQLException {
+ if (!enqueuer.isComplete) {
+ return false;
+ }
+ try (ResultSet rs = countAvailableTasks.executeQuery()) {
+ while (rs.next()) {
+ return rs.getInt(1) == 0;
+ }
+ }
+ return false;
+ }
+
+ private void redistribute() throws SQLException {
+ //parallel lists of workerid = task queue size
+ List<Integer> workerIds = new ArrayList<>();
+ List<Integer> queueSize = new ArrayList<>();
+ int totalTasks = 0;
+
+ try (ResultSet rs = getQueueDistribution.executeQuery()) {
+ while (rs.next()) {
+ int workerId = rs.getInt(1);
+ int numTasks = rs.getInt(2);
+ workerIds.add(workerId);
+ queueSize.add(numTasks);
+ LOG.debug("workerId: ({}) numTasks: ({})", workerId, numTasks);
+ totalTasks += numTasks;
+ }
+ }
+ if (workerIds.size() == 0) {
+ return;
+ }
+ int averagePerWorker = Math.round((float) totalTasks / (float) workerIds.size());
+ int midPoint = Math.round((float) queueSize.size() / 2) + 1;
+ for (int i = queueSize.size() - 1, j = 0; i > midPoint && j < midPoint; i--, j++) {
+ int shortestQueue = queueSize.get(i);
+ int longestQueue = queueSize.get(j);
+ if ((shortestQueue < 5 && longestQueue > 5) ||
+ longestQueue > 5 && longestQueue > (int) (1.5 * averagePerWorker)) {
+ int shortestQueueWorker = workerIds.get(i);
+ int longestQueueWorker = workerIds.get(j);
+ reallocate.clearParameters();
+ reallocate.setLong(1, shortestQueueWorker);
+ reallocate.setLong(2, longestQueueWorker);
+ reallocate.execute();
+ }
+ }
+
+ }
+
+ private void reallocateFromMissingWorkers(List<Integer> missingWorkers)
+ throws SQLException {
+
+ if (missingWorkers.size() == 0) {
+ return;
+ }
+
+ List<Integer> activeWorkers = getActiveWorkers(connection);
+ if (activeWorkers.size() == 0) {
+ return;
+ }
+
+ for (int missing : missingWorkers) {
+ int active = activeWorkers.get(random.nextInt(activeWorkers.size()));
+ allocateNonworkersToWorkers.clearParameters();
+ allocateNonworkersToWorkers.setInt(1, active);
+ allocateNonworkersToWorkers.setInt(2, missing);
+ allocateNonworkersToWorkers.execute();
+ LOG.debug("allocating missing working ({}) to ({})", missing, active);
+ }
+ }
+
+ private List<Integer> getMissingWorkers() throws SQLException {
+ List<Integer> missingWorkers = new ArrayList<>();
+ try (ResultSet rs = findMissingWorkers.executeQuery()) {
+ while (rs.next()) {
+ int workerId = rs.getInt(1);
+ missingWorkers.add(workerId);
+ LOG.debug("Worker ({}) no longer active", workerId);
+ }
+ }
+ return missingWorkers;
+ }
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java
new file mode 100644
index 0000000..33964b3
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.tika.utils.StringUtils;
+
+public class AsyncConfig {
+
+ private final int queueSize = 1000;
+ private final int numWorkers = 10;
+ private final int numEmitters = 1;
+ private String jdbcString;
+ private Path dbDir;
+
+ public static AsyncConfig load(Path p) throws IOException {
+ AsyncConfig asyncConfig = new AsyncConfig();
+
+ if (StringUtils.isBlank(asyncConfig.getJdbcString())) {
+ asyncConfig.dbDir = Files.createTempDirectory("tika-async-");
+ Path dbFile = asyncConfig.dbDir.resolve("tika-async");
+ asyncConfig.setJdbcString(
+ "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE");
+ } else {
+ asyncConfig.dbDir = null;
+ }
+ return asyncConfig;
+ }
+
+ public int getQueueSize() {
+ return queueSize;
+ }
+
+ public int getNumWorkers() {
+ return numWorkers;
+ }
+
+ public int getNumEmitters() {
+ return numEmitters;
+ }
+
+ public String getJdbcString() {
+ return jdbcString;
+ }
+
+ public void setJdbcString(String jdbcString) {
+ this.jdbcString = jdbcString;
+ }
+
+ /**
+ * If no jdbc connection was specified, this
+ * dir contains the h2 database. Otherwise, null.
+ *
+ * @return
+ */
+ public Path getTempDBDir() {
+ return dbDir;
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java
new file mode 100644
index 0000000..df80929
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncData.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncData extends EmitData {
+
+ private final long taskId;
+ private final FetchKey fetchKey;
+ private final FetchEmitTuple.ON_PARSE_EXCEPTION onParseException;
+
+ public AsyncData(@JsonProperty("taskId") long taskId,
+ @JsonProperty("fetchKey") FetchKey fetchKey,
+ @JsonProperty("emitKey") EmitKey emitKey, @JsonProperty("onParseException")
+ FetchEmitTuple.ON_PARSE_EXCEPTION onParseException,
+ @JsonProperty("metadataList") List<Metadata> metadataList) {
+ super(emitKey, metadataList);
+ this.taskId = taskId;
+ this.fetchKey = fetchKey;
+ this.onParseException = onParseException;
+ }
+
+ public FetchKey getFetchKey() {
+ return fetchKey;
+ }
+
+ public long getTaskId() {
+ return taskId;
+ }
+
+ public FetchEmitTuple.ON_PARSE_EXCEPTION getOnParseException() {
+ return onParseException;
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
index bc9fa3b..502c8b4 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitHook.java
@@ -14,21 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
-import java.io.IOException;
-import java.util.List;
+public interface AsyncEmitHook {
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
- String getName();
-
- void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
- void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
- //TODO -- add this later for xhtml?
- //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+ void onSuccess(AsyncTask task);
+ void onFail(AsyncTask task);
}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java
new file mode 100644
index 0000000..4c44f9c
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY;
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_JDBC_KEY;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncEmitter implements Callable<Integer> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncEmitter.class);
+
+
+ private final String connectionString;
+ private final int emitterId;
+ private final Path tikaConfigPath;
+ private final Connection connection;
+ private final PreparedStatement finished;
+ private final PreparedStatement restarting;
+
+ public AsyncEmitter(Connection connection, String connectionString, int emitterId,
+ Path tikaConfigPath) throws SQLException {
+ this.connectionString = connectionString;
+ this.emitterId = emitterId;
+ this.tikaConfigPath = tikaConfigPath;
+ this.connection = connection;
+ String sql = "update emitters set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal() +
+ " where emitter_id = (" + emitterId + ")";
+ finished = connection.prepareStatement(sql);
+
+ sql = "update emitters set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.RESTARTING.ordinal() +
+ " where emitter_id = (" + emitterId + ")";
+ restarting = connection.prepareStatement(sql);
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ Process p = null;
+ try {
+ p = start();
+ int restarts = 0;
+ while (true) {
+ boolean finished = p.waitFor(60, TimeUnit.SECONDS);
+ if (finished) {
+ int exitValue = p.exitValue();
+ if (exitValue == 0) {
+ LOG.debug("forked emitter process finished with exitValue=0");
+ return 1;
+ }
+ reportCrash(++restarts, exitValue);
+ p = start();
+ }
+ }
+ } finally {
+ if (p != null) {
+ p.destroyForcibly();
+ }
+ finished.execute();
+ }
+ }
+
+ private Process start() throws IOException {
+ String[] args = new String[]{"java", "-Djava.awt.headless=true", "-cp",
+ System.getProperty("java.class.path"),
+ "org.apache.tika.pipes.async.AsyncEmitterProcess", Integer.toString(emitterId)};
+ ProcessBuilder pb = new ProcessBuilder(args);
+ pb.environment().put(TIKA_ASYNC_JDBC_KEY, connectionString);
+ pb.environment()
+ .put(TIKA_ASYNC_CONFIG_FILE_KEY, tikaConfigPath.toAbsolutePath().toString());
+ pb.inheritIO();
+ return pb.start();
+ }
+
+ private void reportCrash(int numRestarts, int exitValue) throws SQLException, IOException {
+ LOG.warn("emitter id={} terminated, exitValue={}", emitterId, exitValue);
+ restarting.execute();
+ //should we unassign emit tasks here?
+ }
+
+ /*
+ static PreparedStatement prepareInsertErrorLog(Connection connection) throws SQLException {
+ //if not, this is called to insert into the error log
+ return connection.prepareStatement(
+ "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code) " +
+ " values (?,?,CURRENT_TIMESTAMP(),?,?)"
+ );
+ }
+
+ static PreparedStatement prepareReset(Connection connection) throws SQLException {
+ //and this is called to reset the status on error
+ return connection.prepareStatement(
+ "update task_queue set " +
+ "status=?, " +
+ "time_stamp=CURRENT_TIMESTAMP(), " +
+ "retry=? " +
+ "where id=?");
+ }*/
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java
new file mode 100644
index 0000000..03f5b93
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncEmitterProcess.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.sql.Blob;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataDeserializer;
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.EmitterManager;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.utils.ExceptionUtils;
+
+public class AsyncEmitterProcess {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncEmitterProcess.class);
+ private final LZ4FastDecompressor decompressor =
+ LZ4Factory.fastestInstance().fastDecompressor();
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final FutureTask<Integer> stdinWatcher;
+ int recordsPerPulse = 10;
+ //TODO -- parameterize these
+ private final long emitWithinMs = 10000;
+ private final long emitMaxBytes = 10_000_000;
+ private PreparedStatement markForSelecting;
+ private PreparedStatement selectForProcessing;
+ private PreparedStatement emitStatusUpdate;
+ private PreparedStatement checkForCanShutdown;
+ private PreparedStatement checkForShouldShutdown;
+ private PreparedStatement updateEmitterStatus;
+
+ private AsyncEmitterProcess(InputStream stdin) {
+ SimpleModule module = new SimpleModule();
+ module.addDeserializer(Metadata.class, new JsonMetadataDeserializer());
+ objectMapper.registerModule(module);
+ stdinWatcher = new FutureTask<>(new ForkWatcher(stdin));
+ new Thread(stdinWatcher).start();
+ }
+
+ public static void main(String[] args) throws Exception {
+ String db = System.getenv(AsyncProcessor.TIKA_ASYNC_JDBC_KEY);
+ TikaConfig tikaConfig =
+ new TikaConfig(Paths.get(System.getenv(AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY)));
+ int workerId = Integer.parseInt(args[0]);
+ LOG.debug("trying to get connection {} >{}<", workerId, db);
+ try (Connection connection = DriverManager.getConnection(db)) {
+ AsyncEmitterProcess asyncEmitter = new AsyncEmitterProcess(System.in);
+ asyncEmitter.execute(connection, workerId, tikaConfig);
+ }
+ System.exit(0);
+ }
+
+ private static void reportEmitStatus(List<Long> ids,
+ AsyncWorkerProcess.TASK_STATUS_CODES emitted,
+ PreparedStatement emitStatusUpdate) throws SQLException {
+ for (long id : ids) {
+ emitStatusUpdate.clearParameters();
+ emitStatusUpdate.setByte(1, (byte) emitted.ordinal());
+ emitStatusUpdate.setLong(2, id);
+ emitStatusUpdate.addBatch();
+ }
+ emitStatusUpdate.executeBatch();
+ }
+
+ private void execute(Connection connection, int workerId, TikaConfig tikaConfig)
+ throws SQLException, InterruptedException {
+ prepareStatements(connection, workerId);
+ updateStatus((byte) AsyncWorkerProcess.WORKER_STATUS_CODES.ACTIVE.ordinal());
+ EmitterManager emitterManager = tikaConfig.getEmitterManager();
+ EmitDataCache emitDataCache =
+ new EmitDataCache(emitterManager, emitMaxBytes, emitStatusUpdate);
+ try {
+ mainLoop(emitDataCache);
+ } finally {
+ emitDataCache.emitAll();
+ updateStatus((byte) AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal());
+ }
+ }
+
+ private void mainLoop(EmitDataCache emitDataCache) throws InterruptedException, SQLException {
+
+ while (true) {
+ if (shouldShutdown()) {
+ LOG.debug("received should shutdown signal");
+ return;
+ }
+ int toEmit = markForSelecting.executeUpdate();
+ if (toEmit == 0 && canShutdown()) {
+ //avoid race condition; double check there's nothing
+ //left to emit
+ toEmit = markForSelecting.executeUpdate();
+ if (toEmit == 0) {
+ LOG.debug("received can shutdown and didn't update any for selecting");
+ return;
+ }
+ }
+ if (toEmit > 0) {
+ try {
+ tryToEmitNextBatch(emitDataCache);
+ } catch (IOException e) {
+ LOG.warn("IOException trying to emit", e);
+ }
+ }
+ if (emitDataCache.exceedsEmitWithin(emitWithinMs)) {
+ emitDataCache.emitAll();
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void tryToEmitNextBatch(EmitDataCache emitDataCache) throws IOException, SQLException {
+ List<AsyncEmitTuple> toEmitList = new ArrayList<>();
+ try (ResultSet rs = selectForProcessing.executeQuery()) {
+ while (rs.next()) {
+ long id = rs.getLong(1);
+ Timestamp ts = rs.getTimestamp(2);
+ int uncompressedSize = rs.getInt(3);
+ Blob blob = rs.getBlob(4);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ IOUtils.copyLarge(blob.getBinaryStream(), bos);
+ byte[] bytes = bos.toByteArray();
+ toEmitList.add(new AsyncEmitTuple(id, ts, uncompressedSize, bytes));
+ }
+ }
+ List<Long> successes = new ArrayList<>();
+ List<Long> exceptions = new ArrayList<>();
+ for (AsyncEmitTuple tuple : toEmitList) {
+ try {
+ tryToEmit(tuple, emitDataCache);
+ successes.add(tuple.id);
+ } catch (IOException | SQLException e) {
+ exceptions.add(tuple.id);
+ }
+ }
+ reportEmitStatus(successes, AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS_EMIT,
+ emitStatusUpdate);
+ reportEmitStatus(exceptions, AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT,
+ emitStatusUpdate);
+
+ }
+
+ private void updateStatus(byte status) throws SQLException {
+ updateEmitterStatus.clearParameters();
+ updateEmitterStatus.setByte(1, status);
+ updateEmitterStatus.executeUpdate();
+ }
+
+ private void tryToEmit(AsyncEmitTuple asyncEmitTuple, EmitDataCache emitDataCache)
+ throws SQLException, IOException {
+ AsyncData asyncData = deserialize(asyncEmitTuple.bytes, asyncEmitTuple.uncompressedSize);
+ emitDataCache.add(asyncData);
+ }
+
+ boolean shouldShutdown() throws SQLException {
+ if (stdinWatcher.isDone()) {
+ LOG.info("parent inputstream closed; shutting down now");
+ }
+ try (ResultSet rs = checkForShouldShutdown.executeQuery()) {
+ if (rs.next()) {
+ int val = rs.getInt(1);
+ return val > 0;
+ }
+ }
+ return false;
+ }
+
+ boolean canShutdown() throws SQLException {
+ try (ResultSet rs = checkForCanShutdown.executeQuery()) {
+ if (rs.next()) {
+ int val = rs.getInt(1);
+ return val > 0;
+ }
+ }
+ return false;
+ }
+
+ private void prepareStatements(Connection connection, int workerId) throws SQLException {
+ String sql = "update task_queue set status=" +
+ AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED_EMIT.ordinal() + ", worker_id=" +
+ workerId + ", time_stamp=CURRENT_TIMESTAMP()" + " where id in " +
+ " (select id from task_queue " + //where worker_id = " + workerId +
+ " where status=" + AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE_EMIT.ordinal() +
+ " order by time_stamp asc limit " + recordsPerPulse + " for update)";
+ markForSelecting = connection.prepareStatement(sql);
+
+ sql = "select q.id, q.time_stamp, uncompressed_size, bytes " + "from emits e " +
+ "join task_queue q on e.id=q.id " + "where q.status=" +
+ AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED_EMIT.ordinal() + " and worker_id=" +
+ workerId + " order by time_stamp asc";
+ selectForProcessing = connection.prepareStatement(sql);
+
+ //only update the status if it is not already emitted or failed emit
+ sql = "update task_queue set status=?" + ", time_stamp=CURRENT_TIMESTAMP()" +
+ " where id=? and status not in (" +
+ AsyncWorkerProcess.TASK_STATUS_CODES.EMITTED.ordinal() + ", " +
+ AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT.ordinal() + ")";
+
+ emitStatusUpdate = connection.prepareStatement(sql);
+
+ sql = "select count(1) from emitters where emitter_id=" + workerId + " and status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.CAN_SHUTDOWN.ordinal();
+ checkForCanShutdown = connection.prepareStatement(sql);
+
+ sql = "select count(1) from emitters where emitter_id=" + workerId + " and status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal();
+ checkForShouldShutdown = connection.prepareStatement(sql);
+
+ sql = "merge into emitters key (emitter_id) " + "values (" + workerId + ", ? )";
+ updateEmitterStatus = connection.prepareStatement(sql);
+ }
+
+ private AsyncData deserialize(byte[] compressed, int decompressedLength) throws IOException {
+ byte[] restored = new byte[decompressedLength];
+ int compressedLength2 =
+ decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
+ return objectMapper.readerFor(AsyncData.class).readValue(restored);
+ }
+
+ private static class EmitDataCache {
+ private final EmitterManager emitterManager;
+ private final long maxBytes;
+ private final PreparedStatement emitStatusUpdate;
+ long estimatedSize = 0;
+ int size = 0;
+ Map<String, List<AsyncData>> map = new HashMap<>();
+ private Instant lastAdded = Instant.now();
+
+ public EmitDataCache(EmitterManager emitterManager, long maxBytes,
+ PreparedStatement emitStatusUpdate) {
+ this.emitterManager = emitterManager;
+ this.maxBytes = maxBytes;
+ this.emitStatusUpdate = emitStatusUpdate;
+ }
+
+ void updateEstimatedSize(long newBytes) {
+ estimatedSize += newBytes;
+ }
+
+ void add(AsyncData data) {
+
+ size++;
+ long sz = AbstractEmitter
+ .estimateSizeInBytes(data.getEmitKey().getEmitKey(), data.getMetadataList());
+ if (estimatedSize + sz > maxBytes) {
+ LOG.debug("estimated size ({}) > maxBytes({}), going to emitAll",
+ (estimatedSize + sz), maxBytes);
+ emitAll();
+ }
+ List<AsyncData> cached = map.get(data.getEmitKey().getEmitterName());
+ if (cached == null) {
+ cached = new ArrayList<>();
+ map.put(data.getEmitKey().getEmitterName(), cached);
+ }
+ updateEstimatedSize(sz);
+ cached.add(data);
+ lastAdded = Instant.now();
+ }
+
+ private void emitAll() {
+ int emitted = 0;
+ LOG.debug("about to emit {}", size);
+ for (Map.Entry<String, List<AsyncData>> e : map.entrySet()) {
+ Emitter emitter = emitterManager.getEmitter(e.getKey());
+
+ try {
+ tryToEmit(emitter, e.getValue());
+ } catch (SQLException ex) {
+ ex.printStackTrace();
+ }
+ emitted += e.getValue().size();
+ }
+ LOG.debug("emitted: {}", emitted);
+ estimatedSize = 0;
+ size = 0;
+ map.clear();
+ }
+
+ private long tryToEmit(Emitter emitter, List<AsyncData> cachedEmitData)
+ throws SQLException {
+ List<Long> ids = new ArrayList<>();
+ for (AsyncData d : cachedEmitData) {
+ ids.add(d.getTaskId());
+ }
+ try {
+ emitter.emit(cachedEmitData);
+ } catch (IOException | TikaEmitterException e) {
+ LOG.warn("emitter class ({}): {}", emitter.getClass(),
+ ExceptionUtils.getStackTrace(e));
+ reportEmitStatus(ids, AsyncWorkerProcess.TASK_STATUS_CODES.FAILED_EMIT,
+ emitStatusUpdate);
+ }
+ reportEmitStatus(ids, AsyncWorkerProcess.TASK_STATUS_CODES.EMITTED, emitStatusUpdate);
+ return 1;
+ }
+
+
+ public boolean exceedsEmitWithin(long emitWithinMs) {
+ return ChronoUnit.MILLIS.between(lastAdded, Instant.now()) > emitWithinMs;
+ }
+ }
+
+ private static class ForkWatcher implements Callable<Integer> {
+ private final InputStream in;
+
+ public ForkWatcher(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ //this should block forever
+ //if the forking process dies,
+ // this will either throw an IOException or read -1.
+ try {
+ int i = in.read();
+ } finally {
+ LOG.warn("forking process shutdown; exiting now");
+ System.exit(0);
+ }
+ return 1;
+ }
+ }
+
+ private static class AsyncEmitTuple {
+ final long id;
+ final Timestamp timestamp;
+ final int uncompressedSize;
+ final byte[] bytes;
+
+ public AsyncEmitTuple(long id, Timestamp timestamp, int uncompressedSize, byte[] bytes) {
+ this.id = id;
+ this.timestamp = timestamp;
+ this.uncompressedSize = uncompressedSize;
+ this.bytes = bytes;
+ }
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java
new file mode 100644
index 0000000..79ed80d
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncPipesEmitHook.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncPipesEmitHook implements AsyncEmitHook {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncPipesEmitHook.class);
+
+ private final PreparedStatement markSuccess;
+ private final PreparedStatement markFailure;
+
+ public AsyncPipesEmitHook(Connection connection) throws SQLException {
+ String sql = "delete from task_queue where id=?";
+ markSuccess = connection.prepareStatement(sql);
+ //TODO --fix this
+ markFailure = connection.prepareStatement(sql);
+ }
+
+ @Override
+ public void onSuccess(AsyncTask task) {
+ try {
+ markSuccess.clearParameters();
+ markSuccess.setLong(1, task.getTaskId());
+ markSuccess.execute();
+ } catch (SQLException e) {
+ LOG.warn("problem with on success: " + task.getTaskId(), e);
+ }
+ }
+
+ @Override
+ public void onFail(AsyncTask task) {
+ try {
+ markFailure.clearParameters();
+ markFailure.setLong(1, task.getTaskId());
+ markFailure.execute();
+ } catch (SQLException e) {
+ LOG.warn("problem with on fail: " + task.getTaskId(), e);
+ }
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
new file mode 100644
index 0000000..efd6fec
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
+public class AsyncProcessor implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncProcessor.class);
+ protected static String TIKA_ASYNC_JDBC_KEY = "TIKA_ASYC_JDBC_KEY";
+ protected static String TIKA_ASYNC_CONFIG_FILE_KEY = "TIKA_ASYNC_CONFIG_FILE_KEY";
+ private final Path tikaConfigPath;
+ private final ArrayBlockingQueue<FetchEmitTuple> queue;
+ private final Connection connection;
+ private final int totalThreads;
+ private final AsyncConfig asyncConfig;
+ private PreparedStatement emittersCanShutdown;
+ private volatile boolean isShuttingDown = false;
+ private AssignmentManager assignmentManager;
+ private int finishedThreads = 0;
+ private ExecutorService executorService;
+ private ExecutorCompletionService<Integer> executorCompletionService;
+
+ private AsyncProcessor(Path tikaConfigPath) throws SQLException, IOException {
+ this.tikaConfigPath = tikaConfigPath;
+ this.asyncConfig = AsyncConfig.load(tikaConfigPath);
+ this.queue = new ArrayBlockingQueue<>(asyncConfig.getQueueSize());
+ this.connection = DriverManager.getConnection(asyncConfig.getJdbcString());
+ this.totalThreads = asyncConfig.getNumWorkers() + asyncConfig.getNumEmitters() +
+ 2;//assignment manager and enqueuer threads
+ }
+
+ public static AsyncProcessor build(Path tikaConfigPath) throws AsyncRuntimeException {
+ try {
+ AsyncProcessor processor = new AsyncProcessor(tikaConfigPath);
+ processor.init();
+ return processor;
+ } catch (SQLException | IOException e) {
+ throw new AsyncRuntimeException(e);
+ }
+ }
+
+ private static List<Integer> getActiveWorkers(Connection connection) throws SQLException {
+ PreparedStatement findActiveWorkers =
+ connection.prepareStatement("select worker_id from workers");
+ List<Integer> workers = new ArrayList<>();
+ try (ResultSet rs = findActiveWorkers.executeQuery()) {
+ while (rs.next()) {
+ workers.add(rs.getInt(1));
+ }
+ }
+ return workers;
+ }
+
+ public synchronized boolean offer(List<FetchEmitTuple> fetchEmitTuples, long offerMs)
+ throws AsyncRuntimeException, InterruptedException {
+ if (queue == null) {
+ throw new IllegalStateException("queue hasn't been initialized yet.");
+ } else if (isShuttingDown) {
+ throw new IllegalStateException(
+ "Can't call offer after calling close() or " + "shutdownNow()");
+ }
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ while (elapsed < offerMs) {
+ checkActive();
+ if (queue.remainingCapacity() > fetchEmitTuples.size()) {
+ try {
+ queue.addAll(fetchEmitTuples);
+ return true;
+ } catch (IllegalStateException e) {
+ //swallow
+ }
+ }
+ Thread.sleep(100);
+ elapsed = System.currentTimeMillis() - start;
+ }
+ return false;
+ }
+
+ public synchronized boolean offer(FetchEmitTuple t, long offerMs)
+ throws AsyncRuntimeException, InterruptedException {
+ if (queue == null) {
+ throw new IllegalStateException("queue hasn't been initialized yet.");
+ } else if (isShuttingDown) {
+ throw new IllegalStateException(
+ "Can't call offer after calling close() or " + "shutdownNow()");
+ }
+ checkActive();
+ return queue.offer(t, offerMs, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * This polls the executorcompletionservice to check for execution exceptions
+ * and to make sure that some threads are still active.
+ *
+ * @return
+ * @throws AsyncRuntimeException
+ * @throws InterruptedException
+ */
+ public synchronized boolean checkActive() throws AsyncRuntimeException, InterruptedException {
+ Future<Integer> future = executorCompletionService.poll();
+ if (future != null) {
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ throw new AsyncRuntimeException(e);
+ }
+ finishedThreads++;
+ }
+ return finishedThreads != totalThreads;
+ }
+
+ private void init() throws SQLException {
+
+ setupTables();
+ String sql = "update emitters set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.CAN_SHUTDOWN.ordinal();
+ this.emittersCanShutdown = connection.prepareStatement(sql);
+ executorService = Executors.newFixedThreadPool(totalThreads);
+ executorCompletionService = new ExecutorCompletionService<>(executorService);
+
+ AsyncTaskEnqueuer taskEnqueuer = new AsyncTaskEnqueuer(queue, connection);
+
+ executorCompletionService.submit(taskEnqueuer);
+
+ List<AsyncWorker> workers = buildWorkers(connection, asyncConfig, tikaConfigPath);
+ int maxRetries = 0;
+ for (AsyncWorker worker : workers) {
+ if (worker.getMaxRetries() > maxRetries) {
+ maxRetries = worker.getMaxRetries();
+ }
+ executorCompletionService.submit(worker);
+ }
+ assignmentManager = new AssignmentManager(connection, taskEnqueuer, maxRetries);
+ executorCompletionService.submit(assignmentManager);
+ for (int i = 0; i < asyncConfig.getNumEmitters(); i++) {
+ executorCompletionService
+ .submit(new AsyncEmitter(connection, asyncConfig.getJdbcString(),
+ asyncConfig.getNumWorkers() + i, tikaConfigPath));
+ }
+ }
+
+ private List<AsyncWorker> buildWorkers(Connection connection, AsyncConfig asyncConfig,
+ Path tikaConfigPath) throws SQLException {
+ //TODO -- make these workers configurable via the tika config, e.g. max retries
+ //and jvm args, etc.
+ List<AsyncWorker> workers = new ArrayList<>();
+ for (int i = 0; i < asyncConfig.getNumWorkers(); i++) {
+ workers.add(
+ new AsyncWorker(connection, asyncConfig.getJdbcString(), i, tikaConfigPath));
+ }
+ return workers;
+ }
+
+ private void setupTables() throws SQLException {
+
+ String sql = "create table task_queue " + "(id bigint auto_increment primary key," +
+ "status tinyint," + //byte
+ "worker_id integer," + "retry smallint," + //short
+ "time_stamp timestamp," +
+ "json varchar(64000))";//this is the AsyncTask ... not the emit data!
+ try (Statement st = connection.createStatement()) {
+ st.execute(sql);
+ }
+ sql = "create table workers (worker_id int primary key, status tinyint)";
+ try (Statement st = connection.createStatement()) {
+ st.execute(sql);
+ }
+
+ sql = "create table emitters (emitter_id int primary key, status tinyint)";
+ try (Statement st = connection.createStatement()) {
+ st.execute(sql);
+ }
+
+ sql = "create table error_log (task_id bigint, " + "fetch_key varchar(10000)," +
+ "time_stamp timestamp," + "retry integer," + "error_code tinyint)";
+
+ try (Statement st = connection.createStatement()) {
+ st.execute(sql);
+ }
+
+ sql = "create table emits (" + "id bigint primary key, " + "time_stamp timestamp, " +
+ "uncompressed_size bigint, " + "bytes blob)";
+ try (Statement st = connection.createStatement()) {
+ st.execute(sql);
+ }
+ }
+
+ public void shutdownNow() throws IOException, AsyncRuntimeException {
+ isShuttingDown = true;
+ try {
+ executorService.shutdownNow();
+ } finally {
+ //close down processes and db
+ if (asyncConfig.getTempDBDir() != null) {
+ FileUtils.deleteDirectory(asyncConfig.getTempDBDir().toFile());
+ }
+ }
+ }
+
+ /**
+ * This is a blocking close. It will wait for all tasks successfully submitted before this
+ * call to close() to complete before closing. If you need to shutdown immediately, try
+ * {@link #shutdownNow()}.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ isShuttingDown = true;
+ try {
+ completeAndShutdown();
+ } catch (SQLException | InterruptedException e) {
+ throw new IOException(e);
+ } finally {
+ executorService.shutdownNow();
+ SQLException ex = null;
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ ex = e;
+ }
+ //close down processes and db
+ if (asyncConfig.getTempDBDir() != null) {
+ FileUtils.deleteDirectory(asyncConfig.getTempDBDir().toFile());
+ }
+ if (ex != null) {
+ throw new IOException(ex);
+ }
+ }
+ }
+
+ //this will block until everything finishes
+ private void completeAndShutdown() throws SQLException, InterruptedException {
+
+ //blocking...notify taskEnqueuer
+ queue.put(FetchIterator.COMPLETED_SEMAPHORE);
+
+ //wait for assignmentManager to finish
+ //it will only complete after the task enqueuer has completed
+ //and there are no more parse tasks available, selected or in process
+ while (!assignmentManager.hasCompletedTasks()) {
+ Thread.sleep(100);
+ }
+
+ emittersCanShutdown.executeUpdate();
+
+ //wait for emitters to finish
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ try {
+ boolean isActive = checkActive();
+ while (isActive) {
+ isActive = checkActive();
+ elapsed = System.currentTimeMillis();
+ }
+ } catch (InterruptedException e) {
+ return;
+ }
+ }
+
+ //this reads fetchemittuples from the queue and inserts them in the db
+ //for the workers to read
+ private static class AsyncTaskEnqueuer implements Callable<Integer> {
+ private final PreparedStatement insert;
+
+ private final ArrayBlockingQueue<FetchEmitTuple> queue;
+ private final Connection connection;
+ private final Random random = new Random();
+
+ private volatile boolean isComplete = false;
+
+ AsyncTaskEnqueuer(ArrayBlockingQueue<FetchEmitTuple> queue, Connection connection)
+ throws SQLException {
+ this.queue = queue;
+ this.connection = connection;
+ String sql = "insert into task_queue (status, time_stamp, worker_id, retry, json) " +
+ "values (?,CURRENT_TIMESTAMP(),?,?,?)";
+ insert = connection.prepareStatement(sql);
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ List<Integer> workers = new ArrayList<>();
+ while (true) {
+ FetchEmitTuple t = queue.poll(1, TimeUnit.SECONDS);
+ LOG.debug("enqueing to db " + t);
+ if (t == null) {
+ //log.trace?
+ } else if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+ isComplete = true;
+ return 1;
+ } else {
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ //TODO -- fix this -- this loop waits for workers to register
+ while (workers.size() == 0 && elapsed < 600000) {
+ workers = getActiveWorkers(connection);
+ Thread.sleep(100);
+ elapsed = System.currentTimeMillis() - start;
+ }
+ insert(t, workers);
+ }
+ }
+ }
+
+ boolean isComplete() {
+ return isComplete;
+ }
+
+ private void insert(FetchEmitTuple t, List<Integer> workers)
+ throws IOException, SQLException {
+ int workerId = workers.size() == 1 ? workers.get(0) :
+ workers.get(random.nextInt(workers.size()));
+ insert.clearParameters();
+ insert.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+ insert.setInt(2, workerId);
+ insert.setShort(3, (short) 0);
+ insert.setString(4, JsonFetchEmitTuple.toJson(t));
+ insert.execute();
+ }
+ }
+
+ private static class AssignmentManager implements Callable {
+
+ private final Connection connection;
+ private final AsyncTaskEnqueuer enqueuer;
+ private final PreparedStatement getQueueDistribution;
+ private final PreparedStatement findMissingWorkers;
+ private final PreparedStatement allocateNonworkersToWorkers;
+ private final PreparedStatement reallocate;
+ private final PreparedStatement countAvailableTasks;
+ private final PreparedStatement shutdownWorker;
+ private final PreparedStatement findMaxRetrieds;
+ private final PreparedStatement logMaxRetrieds;
+ private final PreparedStatement removeMaxRetrieds;
+ private final Random random = new Random();
+ private final int maxRetries;
+ private volatile boolean hasCompleted = false;
+
+
+ public AssignmentManager(Connection connection, AsyncTaskEnqueuer enqueuer, int maxRetries)
+ throws SQLException {
+ this.connection = connection;
+ this.enqueuer = enqueuer;
+ this.maxRetries = maxRetries;
+ //this gets workers and # of tasks in desc order of number of tasks
+ String sql = "select w.worker_id, p.cnt " + "from workers w " +
+ "left join (select worker_id, count(1) as cnt from task_queue " +
+ "where status=" + AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal() +
+ " group by worker_id)" + " p on p.worker_id=w.worker_id order by p.cnt desc";
+ getQueueDistribution = connection.prepareStatement(sql);
+ //find workers that have assigned tasks but are not in the
+ //workers table
+ sql = "select p.worker_id, count(1) as cnt from task_queue p " +
+ "left join workers w on p.worker_id=w.worker_id " +
+ "where w.worker_id is null group by p.worker_id";
+ findMissingWorkers = connection.prepareStatement(sql);
+
+ sql = "update task_queue set worker_id=? where worker_id=?";
+ allocateNonworkersToWorkers = connection.prepareStatement(sql);
+
+ //current strategy reallocate tasks from longest queue to shortest
+ //TODO: might consider randomly shuffling or other algorithms
+ sql = "update task_queue set worker_id= ? where id in " +
+ "(select id from task_queue where " + "worker_id = ? and " + "rand() < 0.8 " +
+ "and status=0 for update)";
+ reallocate = connection.prepareStatement(sql);
+
+ //get those tasks that are in the parse phase
+ //if they are selected or in process, it is possible that
+ //they'll need to be retried. So, include all statuses
+ //meaning that the parse has not completed.
+ sql = "select count(1) from task_queue where status in (" +
+ AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal() + ", " +
+ AsyncWorkerProcess.TASK_STATUS_CODES.SELECTED.ordinal() + ", " +
+ AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS.ordinal() + ")";
+ countAvailableTasks = connection.prepareStatement(sql);
+
+ sql = "update workers set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal() +
+ " where worker_id = ?";
+ shutdownWorker = connection.prepareStatement(sql);
+
+ sql = "select id, retry, json from task_queue where retry >=" + maxRetries;
+ findMaxRetrieds = connection.prepareStatement(sql);
+
+ sql = "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code)" +
+ "values (?,?,CURRENT_TIMESTAMP(), ?," +
+ AsyncWorkerProcess.ERROR_CODES.MAX_RETRIES.ordinal() + ")";
+ logMaxRetrieds = connection.prepareStatement(sql);
+
+ sql = "delete from task_queue where id=?";
+ removeMaxRetrieds = connection.prepareStatement(sql);
+ }
+
+ protected boolean hasCompletedTasks() {
+ return hasCompleted;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+
+ while (true) {
+ removeMaxRetrieds();
+ List<Integer> missingWorkers = getMissingWorkers();
+ reallocateFromMissingWorkers(missingWorkers);
+ redistribute();
+ if (isComplete()) {
+ notifyWorkers();
+ return 1;
+ }
+ Thread.sleep(200);
+ }
+ }
+
+ private void removeMaxRetrieds() throws SQLException {
+ Set<Long> toRemove = new HashSet<>();
+ try (ResultSet rs = findMaxRetrieds.executeQuery()) {
+ while (rs.next()) {
+ long id = rs.getLong(1);
+ String json = rs.getString(2);
+ int retries = rs.getInt(3);
+ toRemove.add(id);
+ FetchEmitTuple t;
+ try (Reader reader = new StringReader(json)) {
+ t = JsonFetchEmitTuple.fromJson(reader);
+ } catch (Exception e) {
+ e.printStackTrace();
+ //need to log this in the error_logs table
+ continue;
+ }
+ logMaxRetrieds.clearParameters();
+ logMaxRetrieds.setLong(1, id);
+ logMaxRetrieds.setString(2, t.getFetchKey().getFetchKey());
+ logMaxRetrieds.setInt(3, retries);
+ logMaxRetrieds.addBatch();
+ }
+ }
+ logMaxRetrieds.executeBatch();
+
+ for (Long id : toRemove) {
+ removeMaxRetrieds.clearParameters();
+ removeMaxRetrieds.setLong(1, id);
+ removeMaxRetrieds.addBatch();
+ }
+ removeMaxRetrieds.executeBatch();
+ }
+
+ private void notifyWorkers() throws SQLException {
+ for (int workerId : getActiveWorkers(connection)) {
+ shutdownWorker.clearParameters();
+ shutdownWorker.setInt(1, workerId);
+ shutdownWorker.execute();
+ }
+ }
+
+ private boolean isComplete() throws SQLException {
+ if (hasCompleted) {
+ return hasCompleted;
+ }
+ if (!enqueuer.isComplete) {
+ return false;
+ }
+ try (ResultSet rs = countAvailableTasks.executeQuery()) {
+ while (rs.next()) {
+ int availTasks = rs.getInt(1);
+ if (availTasks == 0) {
+ hasCompleted = true;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void redistribute() throws SQLException {
+ //parallel lists of workerid = task queue size
+ List<Integer> workerIds = new ArrayList<>();
+ List<Integer> queueSize = new ArrayList<>();
+ int totalTasks = 0;
+
+ try (ResultSet rs = getQueueDistribution.executeQuery()) {
+ while (rs.next()) {
+ int workerId = rs.getInt(1);
+ int numTasks = rs.getInt(2);
+ workerIds.add(workerId);
+ queueSize.add(numTasks);
+ LOG.debug("workerId: ({}) numTasks: ({})", workerId, numTasks);
+ totalTasks += numTasks;
+ }
+ }
+ if (workerIds.size() == 0) {
+ return;
+ }
+ int averagePerWorker = Math.round((float) totalTasks / (float) workerIds.size());
+ int midPoint = Math.round((float) queueSize.size() / 2) + 1;
+ for (int i = queueSize.size() - 1, j = 0; i > midPoint && j < midPoint; i--, j++) {
+ int shortestQueue = queueSize.get(i);
+ int longestQueue = queueSize.get(j);
+ if ((shortestQueue < 5 && longestQueue > 5) ||
+ longestQueue > 5 && longestQueue > (int) (1.5 * averagePerWorker)) {
+ int shortestQueueWorker = workerIds.get(i);
+ int longestQueueWorker = workerIds.get(j);
+ reallocate.clearParameters();
+ reallocate.setLong(1, shortestQueueWorker);
+ reallocate.setLong(2, longestQueueWorker);
+ reallocate.execute();
+ }
+ }
+
+ }
+
+ private void reallocateFromMissingWorkers(List<Integer> missingWorkers)
+ throws SQLException {
+
+ if (missingWorkers.size() == 0) {
+ return;
+ }
+
+ List<Integer> activeWorkers = getActiveWorkers(connection);
+ if (activeWorkers.size() == 0) {
+ return;
+ }
+
+ for (int missing : missingWorkers) {
+ int active = activeWorkers.get(random.nextInt(activeWorkers.size()));
+ allocateNonworkersToWorkers.clearParameters();
+ allocateNonworkersToWorkers.setInt(1, active);
+ allocateNonworkersToWorkers.setInt(2, missing);
+ allocateNonworkersToWorkers.execute();
+ LOG.debug("allocating missing working ({}) to ({})", missing, active);
+ }
+ }
+
+ private List<Integer> getMissingWorkers() throws SQLException {
+ List<Integer> missingWorkers = new ArrayList<>();
+ try (ResultSet rs = findMissingWorkers.executeQuery()) {
+ while (rs.next()) {
+ int workerId = rs.getInt(1);
+ missingWorkers.add(workerId);
+ LOG.debug("Worker ({}) no longer active", workerId);
+ }
+ }
+ return missingWorkers;
+ }
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
similarity index 62%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
copy to tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
index bc9fa3b..875cc90 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncRuntimeException.java
@@ -14,21 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.tika.metadata.Metadata;
-
-public interface Emitter {
-
- String getName();
-
- void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
- void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
- //TODO -- add this later for xhtml?
- //void emit(String txt, Metadata metadata) throws IOException, TikaException;
+/**
+ * Fatal exception that means that something went seriously wrong.
+ */
+public class AsyncRuntimeException extends RuntimeException {
+ public AsyncRuntimeException(Throwable t) {
+ super(t);
+ }
}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java
new file mode 100644
index 0000000..e0c214a
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncTask.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncTask extends FetchEmitTuple {
+
+ public static final AsyncTask SHUTDOWN_SEMAPHORE =
+ new AsyncTask(-1, (short) -1, new FetchEmitTuple(null, null, null));
+ private final short retry;
+ private long taskId;
+
+ public AsyncTask(@JsonProperty("taskId") long taskId, @JsonProperty("retry") short retry,
+ @JsonProperty("fetchEmitTuple") FetchEmitTuple fetchEmitTuple) {
+ super(fetchEmitTuple.getFetchKey(), fetchEmitTuple.getEmitKey(),
+ fetchEmitTuple.getMetadata());
+ this.taskId = taskId;
+ this.retry = retry;
+ }
+
+ public long getTaskId() {
+ return taskId;
+ }
+
+ public void setTaskId(long taskId) {
+ this.taskId = taskId;
+ }
+
+ public short getRetry() {
+ return retry;
+ }
+
+ @Override
+ public String toString() {
+ return "AsyncTask{" + "taskId=" + taskId + ", retry=" + retry + '}';
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java
new file mode 100644
index 0000000..c26ee0f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorker.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY;
+import static org.apache.tika.pipes.async.AsyncProcessor.TIKA_ASYNC_JDBC_KEY;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+/**
+ * This controls monitoring of the AsyncWorkerProcess
+ * and updates to the db on crashes etc.
+ */
+public class AsyncWorker implements Callable<Integer> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncWorker.class);
+
+
+ private final String connectionString;
+ private final int workerId;
+ private final Path tikaConfigPath;
+ private final Connection connection;
+ private final PreparedStatement finished;
+ private final PreparedStatement restarting;
+ private final PreparedStatement selectActiveTasks;
+ private final PreparedStatement insertErrorLog;
+ private final PreparedStatement resetStatus;
+ //TODO: make this configurable
+ private final int maxRetries = 2;
+
+ public AsyncWorker(Connection connection, String connectionString, int workerId,
+ Path tikaConfigPath) throws SQLException {
+ this.connectionString = connectionString;
+ this.workerId = workerId;
+ this.tikaConfigPath = tikaConfigPath;
+ this.connection = connection;
+ String sql = "update workers set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.HAS_SHUTDOWN.ordinal() +
+ " where worker_id = (" + workerId + ")";
+ finished = connection.prepareStatement(sql);
+
+ sql = "update workers set status=" +
+ AsyncWorkerProcess.WORKER_STATUS_CODES.RESTARTING.ordinal() +
+ " where worker_id = (" + workerId + ")";
+ restarting = connection.prepareStatement(sql);
+ //this checks if the process was able to reset the status
+ sql = "select id, retry, json from task_queue where worker_id=" + workerId +
+ " and status=" + AsyncWorkerProcess.TASK_STATUS_CODES.IN_PROCESS.ordinal();
+ selectActiveTasks = connection.prepareStatement(sql);
+
+ //if not, this is called to insert into the error log
+ insertErrorLog = prepareInsertErrorLog(connection);
+
+ //and this is called to reset the status on error
+ resetStatus = prepareReset(connection);
+ }
+
+ static void reportAndReset(AsyncTask task, AsyncWorkerProcess.ERROR_CODES errorCode,
+ PreparedStatement insertErrorLog, PreparedStatement resetStatus,
+ Logger logger) {
+ try {
+ insertErrorLog.clearParameters();
+ insertErrorLog.setLong(1, task.getTaskId());
+ insertErrorLog.setString(2, task.getFetchKey().getFetchKey());
+ insertErrorLog.setInt(3, task.getRetry());
+ insertErrorLog.setByte(4, (byte) errorCode.ordinal());
+ insertErrorLog.execute();
+ } catch (SQLException e) {
+ logger.error("Can't update error log", e);
+ }
+
+ try {
+ resetStatus.clearParameters();
+ resetStatus.setByte(1, (byte) AsyncWorkerProcess.TASK_STATUS_CODES.AVAILABLE.ordinal());
+ resetStatus.setShort(2, (short) (task.getRetry() + 1));
+ resetStatus.setLong(3, task.getTaskId());
+ resetStatus.execute();
+ } catch (SQLException e) {
+ logger.error("Can't reset try status", e);
+ }
+ }
+
+ static PreparedStatement prepareInsertErrorLog(Connection connection) throws SQLException {
+ //if not, this is called to insert into the error log
+ return connection.prepareStatement(
+ "insert into error_log (task_id, fetch_key, time_stamp, retry, error_code) " +
+ " values (?,?,CURRENT_TIMESTAMP(),?,?)");
+ }
+
+ static PreparedStatement prepareReset(Connection connection) throws SQLException {
+ //and this is called to reset the status on error
+ return connection.prepareStatement(
+ "update task_queue set " + "status=?, " + "time_stamp=CURRENT_TIMESTAMP(), " +
+ "retry=? " + "where id=?");
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ Process p = null;
+ try {
+ p = start();
+ int restarts = 0;
+ while (true) {
+ boolean finished = p.waitFor(60, TimeUnit.SECONDS);
+ if (finished) {
+ int exitValue = p.exitValue();
+ if (exitValue == 0) {
+ LOG.debug("forked worker process finished with exitValue=0");
+ return 1;
+ }
+ reportCrash(++restarts, exitValue);
+ p = start();
+ }
+ }
+ } finally {
+ if (p != null) {
+ p.destroyForcibly();
+ }
+ finished.execute();
+ }
+ }
+
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ private Process start() throws IOException {
+ String[] args = new String[]{"java", "-Djava.awt.headless=true", "-cp",
+ System.getProperty("java.class.path"),
+ "org.apache.tika.pipes.async.AsyncWorkerProcess", Integer.toString(workerId)};
+ ProcessBuilder pb = new ProcessBuilder(args);
+ pb.environment().put(TIKA_ASYNC_JDBC_KEY, connectionString);
+ pb.environment()
+ .put(TIKA_ASYNC_CONFIG_FILE_KEY, tikaConfigPath.toAbsolutePath().toString());
+ pb.inheritIO();
+ return pb.start();
+ }
+
+ private void reportCrash(int numRestarts, int exitValue) throws SQLException, IOException {
+ LOG.warn("worker id={} terminated, exitValue={}", workerId, exitValue);
+ restarting.execute();
+ List<AsyncTask> activeTasks = new ArrayList<>();
+ try (ResultSet rs = selectActiveTasks.executeQuery()) {
+ long taskId = rs.getLong(1);
+ short retry = rs.getShort(2);
+ String json = rs.getString(3);
+ FetchEmitTuple tuple = JsonFetchEmitTuple.fromJson(new StringReader(json));
+ activeTasks.add(new AsyncTask(taskId, retry, tuple));
+ }
+ if (activeTasks.size() == 0) {
+ LOG.debug("worker reset active tasks, nothing extra to report");
+ return;
+ }
+
+ if (activeTasks.size() > 1) {
+ LOG.warn("more than one active task? this should never happen!");
+ }
+
+ for (AsyncTask t : activeTasks) {
+ reportAndReset(t, AsyncWorkerProcess.ERROR_CODES.UNKNOWN_PARSE, insertErrorLog,
+ resetStatus, LOG);
+ }
+
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java
new file mode 100644
index 0000000..988748f
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/main/java/org/apache/tika/pipes/async/AsyncWorkerProcess.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.apache.tika.pipes.async.AsyncTask.SHUTDOWN_SEMAPHORE;
+import static org.apache.tika.pipes.async.AsyncWorker.prepareInsertErrorLog;
+import static org.apache.tika.pipes.async.AsyncWorker.prepareReset;
+import static org.apache.tika.pipes.async.AsyncWorker.reportAndReset;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import net.jpountz.lz4.LZ4Factory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.EncryptedDocumentException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.metadata.serialization.JsonMetadataSerializer;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.sax.BasicContentHandlerFactory;
+import org.apache.tika.sax.RecursiveParserWrapperHandler;
+import org.apache.tika.utils.StringUtils;
+
+public class AsyncWorkerProcess {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncWorkerProcess.class);
+ //make these all configurable
+ private static final long SHUTDOWN_AFTER_MS = 120000;
+ private static final long PULSE_MS = 1000;
+ private final long parseTimeoutMs = 60000;
+
+ public static void main(String[] args) throws Exception {
+ Path tikaConfigPath = Paths.get(System.getenv(AsyncProcessor.TIKA_ASYNC_CONFIG_FILE_KEY));
+ String db = System.getenv(AsyncProcessor.TIKA_ASYNC_JDBC_KEY);
+ TikaConfig tikaConfig = new TikaConfig(tikaConfigPath);
+ int workerId = Integer.parseInt(args[0]);
+ LOG.debug("trying to get connection {} >{}<", workerId, db);
+ try (Connection connection = DriverManager.getConnection(db)) {
+ AsyncWorkerProcess asyncWorker = new AsyncWorkerProcess();
+ asyncWorker.execute(connection, workerId, tikaConfig);
+ }
+ System.exit(0);
+ }
+
+ private void execute(Connection connection, int workerId, TikaConfig tikaConfig)
+ throws SQLException {
+ //3 = worker + forkwatcher + active task
+ ExecutorService service = Executors.newFixedThreadPool(3);
+ ExecutorCompletionService<Integer> executorCompletionService =
+ new ExecutorCompletionService<>(service);
+
+ executorCompletionService
+ .submit(new Worker(connection, workerId, tikaConfig, parseTimeoutMs));
+ executorCompletionService.submit(new ForkWatcher(System.in));
+
+ int completed = 0;
+
+ //if either one stops, we need to stop
+ try {
+ while (completed < 1) {
+ Future<Integer> future = executorCompletionService.poll(60, TimeUnit.SECONDS);
+ if (future != null) {
+ completed++;
+ future.get();
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.error("worker " + workerId + " had a mainloop exception", e);
+ } finally {
+ service.shutdownNow();
+ }
+ return;
+ }
+
+ enum TASK_STATUS_CODES {
+ AVAILABLE, SELECTED, IN_PROCESS, AVAILABLE_EMIT, SELECTED_EMIT, IN_PROCESS_EMIT,
+ FAILED_EMIT, EMITTED
+ }
+
+ public enum WORKER_STATUS_CODES {
+ ACTIVE, RESTARTING, HIBERNATING, CAN_SHUTDOWN,//if there's nothing else to process, shutdown
+ SHOULD_SHUTDOWN, //shutdown now whether or not there's anything else to process
+ HAS_SHUTDOWN
+ }
+
+ enum ERROR_CODES {
+ TIMEOUT, SECURITY_EXCEPTION, OTHER_EXCEPTION, OOM, OTHER_ERROR, UNKNOWN_PARSE, MAX_RETRIES,
+ EMIT_SERIALIZATION, EMIT_SQL_INSERT_EXCEPTION, EMIT_SQL_SELECT_EXCEPTION,
+ EMIT_DESERIALIZATION, EMIT_EXCEPTION
+ }
+
+ private static class TaskQueue {
+ private final Connection connection;
+ private final int workerId;
+
+ private final PreparedStatement markForSelecting;
+ private final PreparedStatement selectForProcessing;
+ private final PreparedStatement markForProcessing;
+ private final PreparedStatement checkForShutdown;
+
+
+ TaskQueue(Connection connection, int workerId) throws SQLException {
+ this.connection = connection;
+ this.workerId = workerId;
+ //TODO -- need to update timestamp
+ String sql = "update task_queue set status=" + TASK_STATUS_CODES.SELECTED.ordinal() +
+ ", time_stamp=CURRENT_TIMESTAMP()" + " where id = " +
+ " (select id from task_queue where worker_id = " + workerId + " and status=" +
+ TASK_STATUS_CODES.AVAILABLE.ordinal() +
+ " order by time_stamp asc limit 1 for update)";
+ markForSelecting = connection.prepareStatement(sql);
+ sql = "select id, retry, json from task_queue where status=" +
+ TASK_STATUS_CODES.SELECTED.ordinal() + " and " + " worker_id=" + workerId +
+ " order by time_stamp asc limit 1";
+ selectForProcessing = connection.prepareStatement(sql);
+ sql = "update task_queue set status=" + TASK_STATUS_CODES.IN_PROCESS.ordinal() +
+ ", time_stamp=CURRENT_TIMESTAMP()" + " where id=?";
+ markForProcessing = connection.prepareStatement(sql);
+
+ sql = "select count(1) from workers where worker_id=" + workerId + " and status=" +
+ WORKER_STATUS_CODES.SHOULD_SHUTDOWN.ordinal();
+ checkForShutdown = connection.prepareStatement(sql);
+ }
+
+ AsyncTask poll(long pollMs) throws InterruptedException, IOException, SQLException {
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ while (elapsed < pollMs) {
+ if (shouldShutdown()) {
+ return SHUTDOWN_SEMAPHORE;
+ }
+ int i = markForSelecting.executeUpdate();
+ if (i == 0) {
+// debugQueue();
+ Thread.sleep(PULSE_MS);
+ } else {
+ long taskId = -1;
+ short retry = -1;
+ String json = "";
+ try (ResultSet rs = selectForProcessing.executeQuery()) {
+ while (rs.next()) {
+ taskId = rs.getLong(1);
+ retry = rs.getShort(2);
+ json = rs.getString(3);
+ }
+ }
+ markForProcessing.clearParameters();
+ markForProcessing.setLong(1, taskId);
+ markForProcessing.execute();
+
+ FetchEmitTuple t = null;
+ try (Reader reader = new StringReader(json)) {
+ t = JsonFetchEmitTuple.fromJson(reader);
+ }
+ AsyncTask task = new AsyncTask(taskId, retry, t);
+ return task;
+ }
+ elapsed = System.currentTimeMillis() - start;
+ }
+ return null;
+ }
+
+ private void debugQueue() throws SQLException {
+ try (ResultSet rs = connection.createStatement()
+ .executeQuery("select id, status, worker_id from task_queue limit 10")) {
+ while (rs.next()) {
+ System.out.println(
+ "id: " + rs.getInt(1) + " status: " + rs.getInt(2) + " worker_id: " +
+ rs.getInt(3));
+ }
+ }
+ }
+
+ boolean shouldShutdown() throws SQLException {
+ try (ResultSet rs = checkForShutdown.executeQuery()) {
+ if (rs.next()) {
+ int val = rs.getInt(1);
+ return val > 0;
+ }
+ }
+ return false;
+ }
+ }
+
+
+ private static class Worker implements Callable<Integer> {
+
+ private final Connection connection;
+ private final int workerId;
+ private final RecursiveParserWrapper parser;
+ private final TikaConfig tikaConfig;
+ private final long parseTimeoutMs;
+ private final PreparedStatement insertErrorLog;
+ private final PreparedStatement resetStatus;
+ private final PreparedStatement insertEmitData;
+ private final PreparedStatement updateStatusForEmit;
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ LZ4Factory factory = LZ4Factory.fastestInstance();
+ private final ExecutorService executorService;
+ private final ExecutorCompletionService<AsyncData> executorCompletionService;
+
+ public Worker(Connection connection, int workerId, TikaConfig tikaConfig,
+ long parseTimeoutMs) throws SQLException {
+ this.connection = connection;
+ this.workerId = workerId;
+ this.parser = new RecursiveParserWrapper(tikaConfig.getParser());
+ this.tikaConfig = tikaConfig;
+ this.executorService = Executors.newFixedThreadPool(1);
+ this.executorCompletionService = new ExecutorCompletionService<>(executorService);
+ this.parseTimeoutMs = parseTimeoutMs;
+
+ SimpleModule module = new SimpleModule();
+ module.addSerializer(Metadata.class, new JsonMetadataSerializer());
+ objectMapper.registerModule(module);
+ String sql = "merge into workers key (worker_id) " + "values (" + workerId + ", " +
+ WORKER_STATUS_CODES.ACTIVE.ordinal() + ")";
+ connection.createStatement().execute(sql);
+ insertErrorLog = prepareInsertErrorLog(connection);
+ resetStatus = prepareReset(connection);
+ insertEmitData = prepareInsertEmitData(connection);
+ sql = "update task_queue set status=" + TASK_STATUS_CODES.AVAILABLE_EMIT.ordinal() +
+ ", time_stamp=CURRENT_TIMESTAMP()" + " where id=?";
+ updateStatusForEmit = connection.prepareStatement(sql);
+ }
+
+ static PreparedStatement prepareInsertEmitData(Connection connection) throws SQLException {
+ return connection.prepareStatement(
+ "insert into emits (id, time_stamp, uncompressed_size, bytes) " +
+ " values (?,CURRENT_TIMESTAMP(),?,?)");
+ }
+
+ public Integer call() throws Exception {
+ AsyncTask task = null;
+ try {
+
+ TaskQueue queue = new TaskQueue(connection, workerId);
+
+ long lastProcessed = System.currentTimeMillis();
+
+ while (true) {
+
+ task = queue.poll(1000);
+ if (task == null) {
+ long elapsed = System.currentTimeMillis() - lastProcessed;
+ if (elapsed > SHUTDOWN_AFTER_MS) {
+ LOG.debug("shutting down after no assignments in {}ms", elapsed);
+ return 1;
+ }
+ } else if (task == SHUTDOWN_SEMAPHORE) {
+ break;
+ } else {
+ processTask(task);
+ lastProcessed = System.currentTimeMillis();
+ }
+ }
+ } catch (TimeoutException e) {
+ LOG.warn(task.getFetchKey().getFetchKey(), e);
+ reportAndReset(task, ERROR_CODES.TIMEOUT, insertErrorLog, resetStatus, LOG);
+ } catch (SecurityException e) {
+ LOG.warn(task.getFetchKey().getFetchKey(), e);
+ reportAndReset(task, ERROR_CODES.SECURITY_EXCEPTION, insertErrorLog, resetStatus,
+ LOG);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.warn(task.getFetchKey().getFetchKey(), e);
+ reportAndReset(task, ERROR_CODES.OTHER_EXCEPTION, insertErrorLog, resetStatus, LOG);
+ } catch (OutOfMemoryError e) {
+ LOG.warn(task.getFetchKey().getFetchKey(), e);
+ reportAndReset(task, ERROR_CODES.OOM, insertErrorLog, resetStatus, LOG);
+ } catch (Error e) {
+ LOG.warn(task.getFetchKey().getFetchKey(), e);
+ reportAndReset(task, ERROR_CODES.OTHER_ERROR, insertErrorLog, resetStatus, LOG);
+ } finally {
+ executorService.shutdownNow();
+ return 1;
+ }
+ }
+
+ private void processTask(AsyncTask task) throws Exception {
+
+ if (task == SHUTDOWN_SEMAPHORE) {
+ LOG.debug("received shutdown notification");
+ return;
+ } else {
+ executorCompletionService
+ .submit(new TaskProcessor(task, tikaConfig, parser, workerId));
+ Future<AsyncData> future =
+ executorCompletionService.poll(parseTimeoutMs, TimeUnit.MILLISECONDS);
+ if (future == null) {
+ handleTimeout(task.getTaskId(), task.getFetchKey().getFetchKey());
+ } else {
+ AsyncData asyncData = future.get(1000, TimeUnit.MILLISECONDS);
+ if (asyncData == null) {
+ handleTimeout(task.getTaskId(), task.getFetchKey().getFetchKey());
+ }
+ boolean shouldEmit = checkForParseException(asyncData);
+ if (shouldEmit) {
+ try {
+ emit(asyncData);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ recordBadEmit(task.getTaskId(), task.getFetchKey().getFetchKey(),
+ ERROR_CODES.EMIT_SERIALIZATION.ordinal());
+ } catch (SQLException e) {
+ e.printStackTrace();
+ recordBadEmit(task.getTaskId(), task.getFetchKey().getFetchKey(),
+ ERROR_CODES.EMIT_SQL_INSERT_EXCEPTION.ordinal());
+ }
+ }
+ }
+ }
+ }
+
+ private void recordBadEmit(long taskId, String key, int ordinal) {
+ //stub
+ }
+
+ private void emit(AsyncData asyncData) throws SQLException, JsonProcessingException {
+ insertEmitData.clearParameters();
+ insertEmitData.setLong(1, asyncData.getTaskId());
+ byte[] bytes = objectMapper.writeValueAsBytes(asyncData);
+ byte[] compressed = factory.fastCompressor().compress(bytes);
+ insertEmitData.setLong(2, bytes.length);
+ insertEmitData.setBlob(3, new ByteArrayInputStream(compressed));
+ insertEmitData.execute();
+ updateStatusForEmit.clearParameters();
+ updateStatusForEmit.setLong(1, asyncData.getTaskId());
+ updateStatusForEmit.execute();
+ }
+
+ private void handleTimeout(long taskId, String key) throws TimeoutException {
+ LOG.warn("timeout taskid:{} fetchKey:{}", taskId, key);
+ throw new TimeoutException(key);
+ }
+
+ private boolean checkForParseException(AsyncData asyncData) {
+ if (asyncData == null || asyncData.getMetadataList() == null ||
+ asyncData.getMetadataList().size() == 0) {
+ LOG.warn("empty or null emit data ({})", asyncData.getFetchKey().getFetchKey());
+ return false;
+ }
+ boolean shouldEmit = true;
+ Metadata container = asyncData.getMetadataList().get(0);
+ String stack = container.get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ if (stack != null) {
+ LOG.warn("fetchKey ({}) container parse exception ({})",
+ asyncData.getFetchKey().getFetchKey(), stack);
+ if (asyncData.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
+ shouldEmit = false;
+ }
+ }
+
+ for (int i = 1; i < asyncData.getMetadataList().size(); i++) {
+ Metadata m = asyncData.getMetadataList().get(i);
+ String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
+ if (embeddedStack != null) {
+ LOG.warn("fetchKey ({}) embedded parse exception ({})",
+ asyncData.getFetchKey().getFetchKey(), embeddedStack);
+ }
+ }
+ return shouldEmit;
+ }
+ }
+
+ private static class TaskProcessor implements Callable<AsyncData> {
+
+ private final AsyncTask task;
+ private final Parser parser;
+ private final TikaConfig tikaConfig;
+ private final int workerId;
+
+ public TaskProcessor(AsyncTask task, TikaConfig tikaConfig, Parser parser, int workerId) {
+ this.task = task;
+ this.parser = parser;
+ this.tikaConfig = tikaConfig;
+ this.workerId = workerId;
+ }
+
+ public AsyncData call() throws Exception {
+ Metadata userMetadata = task.getMetadata();
+ Metadata metadata = new Metadata();
+ String fetcherName = task.getFetchKey().getFetcherName();
+ String fetchKey = task.getFetchKey().getFetchKey();
+ List<Metadata> metadataList = null;
+ try (InputStream stream = tikaConfig.getFetcherManager().getFetcher(fetcherName)
+ .fetch(fetchKey, metadata)) {
+ metadataList = parseMetadata(task.getFetchKey(), stream, metadata);
+ } catch (SecurityException e) {
+ throw e;
+ }
+ injectUserMetadata(userMetadata, metadataList);
+ EmitKey emitKey = task.getEmitKey();
+ if (StringUtils.isBlank(emitKey.getEmitKey())) {
+ emitKey = new EmitKey(emitKey.getEmitterName(), fetchKey);
+ task.setEmitKey(emitKey);
+ }
+ return new AsyncData(task.getTaskId(), task.getFetchKey(), task.getEmitKey(),
+ task.getOnParseException(), metadataList);
+ }
+
+ private List<Metadata> parseMetadata(FetchKey fetchKey, InputStream stream,
+ Metadata metadata) {
+ //make these configurable
+ BasicContentHandlerFactory.HANDLER_TYPE type =
+ BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+ int writeLimit = -1;
+ int maxEmbeddedResources = 1000;
+
+ RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
+ new BasicContentHandlerFactory(type, writeLimit), maxEmbeddedResources,
+ tikaConfig.getMetadataFilter());
+ ParseContext parseContext = new ParseContext();
+ try {
+ parser.parse(stream, handler, metadata, parseContext);
+ } catch (SAXException e) {
+ LOG.warn("problem:" + fetchKey.getFetchKey(), e);
+ } catch (EncryptedDocumentException e) {
+ LOG.warn("encrypted:" + fetchKey.getFetchKey(), e);
+ } catch (SecurityException e) {
+ LOG.warn("security exception: " + fetchKey.getFetchKey());
+ throw e;
+ } catch (Exception e) {
+ LOG.warn("exception: " + fetchKey.getFetchKey());
+ } catch (OutOfMemoryError e) {
+ LOG.error("oom: " + fetchKey.getFetchKey());
+ throw e;
+ }
+ return handler.getMetadataList();
+ }
+
+ private void injectUserMetadata(Metadata userMetadata, List<Metadata> metadataList) {
+ for (String n : userMetadata.names()) {
+ //overwrite whatever was there
+ metadataList.get(0).set(n, null);
+ for (String val : userMetadata.getValues(n)) {
+ metadataList.get(0).add(n, val);
+ }
+ }
+ }
+ }
+
+ private static class ForkWatcher implements Callable<Integer> {
+ private final InputStream in;
+
+ public ForkWatcher(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ //this should block forever
+ //if the forking process dies,
+ // this will either throw an IOException or read -1.
+ int i = in.read();
+ LOG.info("forking process notified forked to shutdown");
+ return 1;
+ }
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties b/tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
similarity index 93%
copy from tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
copy to tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
index 92b6d56..43553a4 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-pipes-async/src/main/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#info,debug, error,fatal ...
log4j.rootLogger=debug,stderr
-
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
-log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java
new file mode 100644
index 0000000..04890b3
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/AsyncProcessorTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+
+public class AsyncProcessorTest {
+
+ private Path dbDir;
+ private Path dbFile;
+ private Connection connection;
+ private Path tikaConfigPath;
+
+ @Before
+ public void setUp() throws SQLException, IOException {
+ dbDir = Files.createTempDirectory("async-db");
+ dbFile = dbDir.resolve("emitted-db");
+ String jdbc = "jdbc:h2:file:" + dbFile.toAbsolutePath().toString() + ";AUTO_SERVER=TRUE";
+ String sql = "create table emitted (id int auto_increment primary key, " +
+ "emitkey varchar(2000), json varchar(20000))";
+
+ connection = DriverManager.getConnection(jdbc);
+ connection.createStatement().execute(sql);
+ tikaConfigPath = dbDir.resolve("tika-config.xml");
+ String xml = "" + "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + "<properties>" +
+ " <emitters>" + " <emitter class=\"org.apache.tika.pipes.async.MockEmitter\">\n" +
+ " <params>\n" + " <param name=\"name\" type=\"string\">mock</param>\n" +
+ " <param name=\"jdbc\" type=\"string\">" + jdbc + "</param>\n" +
+ " </params>" + " </emitter>" + " </emitters>" + " <fetchers>" +
+ " <fetcher class=\"org.apache.tika.pipes.async.MockFetcher\">" +
+ " <param name=\"name\" type=\"string\">mock</param>\n" + " </fetcher>" +
+ " </fetchers>" + "</properties>";
+ Files.write(tikaConfigPath, xml.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @After
+ public void tearDown() throws SQLException, IOException {
+ connection.createStatement().execute("drop table emitted");
+ connection.close();
+ FileUtils.deleteDirectory(dbDir.toFile());
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+
+
+ AsyncProcessor processor = AsyncProcessor.build(tikaConfigPath);
+ int max = 100;
+ for (int i = 0; i < max; i++) {
+ FetchEmitTuple t = new FetchEmitTuple(new FetchKey("mock", "key-" + i),
+ new EmitKey("mock", "emit-" + i), new Metadata());
+ processor.offer(t, 1000);
+ }
+ processor.close();
+ String sql = "select emitkey from emitted";
+ Set<String> emitKeys = new HashSet<>();
+ try (Statement st = connection.createStatement(); ResultSet rs = st.executeQuery(sql)) {
+ while (rs.next()) {
+ String emitKey = rs.getString(1);
+ emitKeys.add(emitKey);
+ }
+ }
+ assertEquals(max, emitKeys.size());
+ for (int i = 0; i < max; i++) {
+ assertTrue(emitKeys.contains("emit-" + i));
+ }
+ }
+}
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java
new file mode 100644
index 0000000..5576f73
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockEmitter.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataSerializer;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+
+public class MockEmitter implements Initializable, Emitter {
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private Connection connection;
+ private String jdbc;
+ private PreparedStatement insert;
+
+ public MockEmitter() {
+ SimpleModule module = new SimpleModule();
+ module.addSerializer(Metadata.class, new JsonMetadataSerializer());
+ objectMapper.registerModule(module);
+ }
+
+ @Field
+ public void setJdbc(String jdbc) {
+ this.jdbc = jdbc;
+ }
+
+ @Override
+ public String getName() {
+ return "mock";
+ }
+
+ @Override
+ public void emit(String emitKey, List<Metadata> metadataList)
+ throws IOException, TikaEmitterException {
+ emit(Collections
+ .singletonList(new EmitData(new EmitKey(getName(), emitKey), metadataList)));
+ }
+
+ @Override
+ public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
+ for (EmitData d : emitData) {
+ String json = objectMapper.writeValueAsString(d);
+ try {
+ insert.clearParameters();
+ insert.setString(1, d.getEmitKey().getEmitKey());
+ insert.setString(2, json);
+ insert.execute();
+ } catch (SQLException e) {
+ throw new TikaEmitterException("problem inserting", e);
+ }
+ }
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ try {
+ connection = DriverManager.getConnection(jdbc);
+ String sql = "insert into emitted (emitkey, json) values (?, ?)";
+ insert = connection.prepareStatement(sql);
+ } catch (SQLException e) {
+ throw new TikaConfigException("problem w connection", e);
+ }
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
+
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
similarity index 55%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
index 95bb462..7a74f2a 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
@@ -14,28 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.pipes.async;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.util.List;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.Fetcher;
-public class EmptyEmitter implements Emitter {
+public class MockFetcher implements Fetcher {
- @Override
- public String getName() {
- return "empty";
- }
+ private static final byte[] BYTES = ("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + "<mock>" +
+ "<metadata action=\"add\" name=\"dc:creator\">Nikolai Lobachevsky</metadata>" +
+ "<write element=\"p\">main_content</write>" + "</mock>")
+ .getBytes(StandardCharsets.UTF_8);
@Override
- public void emit(String emitKey, List<Metadata> metadataList)
- throws IOException, TikaEmitterException {
-
+ public String getName() {
+ return "mock";
}
@Override
- public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
+ return new ByteArrayInputStream(BYTES);
}
}
diff --git a/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java
new file mode 100644
index 0000000..ebbd8f0
--- /dev/null
+++ b/tika-pipes/tika-pipes-async/src/test/java/org/apache/tika/pipes/async/SerializationTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.async;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.junit.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataDeserializer;
+
+public class SerializationTest {
+
+ @Test
+ public void testBasic() throws Exception {
+ String json = "{\"taskId\":49,\"fetchKey\":{\"fetcherName\":\"mock\"," +
+ "\"fetchKey\":\"key-48\"},\"emitKey\":{\"emitterName\":\"mock\"," +
+ "\"emitKey\":\"emit-48\"},\"onParseException\":\"EMIT\",\"metadataList\":" +
+ "[{\"X-TIKA:Parsed-By\":" +
+ "\"org.apache.tika.parser.EmptyParser\",\"X-TIKA:parse_time_millis\":" +
+ "\"0\",\"X-TIKA:embedded_depth\":\"0\"}]}";
+
+ ObjectMapper mapper = new ObjectMapper();
+ SimpleModule module = new SimpleModule();
+ module.addDeserializer(Metadata.class, new JsonMetadataDeserializer());
+ mapper.registerModule(module);
+ AsyncData asyncData = mapper.readValue(json, AsyncData.class);
+ assertEquals(49, asyncData.getTaskId());
+ assertEquals("mock", asyncData.getFetchKey().getFetcherName());
+ assertEquals(1, asyncData.getMetadataList().size());
+ }
+
+}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
index 3831f07..7fbba83 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
@@ -17,23 +17,6 @@
package org.apache.tika.pipes;
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.iterable.S3Objects;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.emitter.Emitter;
-import org.apache.tika.pipes.emitter.s3.S3Emitter;
-import org.apache.tika.pipes.fetcher.Fetcher;
-import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-import org.apache.tika.pipes.fetchiterator.FetchIterator;
-import org.junit.Ignore;
-import org.junit.Test;
-
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
@@ -49,6 +32,24 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.iterable.S3Objects;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.s3.S3Emitter;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
@Ignore("turn these into actual tests")
public class PipeIntegrationTests {
@@ -59,10 +60,8 @@ public class PipeIntegrationTests {
String region = "";
String profile = "";
String bucket = "";
- AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
- .withRegion(region)
- .withCredentials(new ProfileCredentialsProvider(profile))
- .build();
+ AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(region)
+ .withCredentials(new ProfileCredentialsProvider(profile)).build();
s3Client.listObjects(bucket);
int cnt = 0;
long sz = 0;
@@ -72,17 +71,18 @@ public class PipeIntegrationTests {
if (Files.isRegularFile(targ)) {
continue;
}
- if (! Files.isDirectory(targ.getParent())) {
+ if (!Files.isDirectory(targ.getParent())) {
Files.createDirectories(targ.getParent());
}
- System.out.println("id: " + cnt + " :: " + summary.getKey() + " : " + summary.getSize());
+ System.out
+ .println("id: " + cnt + " :: " + summary.getKey() + " : " + summary.getSize());
S3Object s3Object = s3Client.getObject(bucket, summary.getKey());
Files.copy(s3Object.getObjectContent(), targ);
summary.getSize();
cnt++;
sz += summary.getSize();
}
- System.out.println("iterated: "+cnt + " sz: "+sz);
+ System.out.println("iterated: " + cnt + " sz: " + sz);
}
@Test
@@ -92,15 +92,20 @@ public class PipeIntegrationTests {
int numConsumers = 1;
ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
- ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
- completionService.submit(it);
+ ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(1000);
+ for (int i = 0; i < numConsumers; i++) {
+ completionService.submit(new FSFetcherEmitter(queue,
+ tikaConfig.getFetcherManager().getFetcher("s3"), null));
+ }
+ for (FetchEmitTuple t : it) {
+ queue.offer(t);
+ }
for (int i = 0; i < numConsumers; i++) {
- completionService.submit(new FSFetcherEmitter(
- queue, tikaConfig.getFetcherManager().getFetcher("s3"), null));
+ queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
}
int finished = 0;
try {
- while (finished++ < numConsumers+1) {
+ while (finished++ < numConsumers + 1) {
Future<Integer> future = completionService.take();
future.get();
}
@@ -112,20 +117,25 @@ public class PipeIntegrationTests {
@Test
public void testS3ToS3() throws Exception {
TikaConfig tikaConfig = getConfig("tika-config-s3Tos3.xml");
- FetchIterator it = tikaConfig.getFetchIterator();
int numConsumers = 20;
ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
- ArrayBlockingQueue<FetchEmitTuple> queue = it.init(numConsumers);
- completionService.submit(it);
+ ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(1000);
+ for (int i = 0; i < numConsumers; i++) {
+ completionService.submit(new S3FetcherEmitter(queue,
+ tikaConfig.getFetcherManager().getFetcher("s3f"),
+ (S3Emitter) tikaConfig.getEmitterManager().getEmitter("s3e")));
+ }
+ FetchIterator it = tikaConfig.getFetchIterator();
+ for (FetchEmitTuple t : it) {
+ queue.offer(t);
+ }
for (int i = 0; i < numConsumers; i++) {
- completionService.submit(new S3FetcherEmitter(
- queue, tikaConfig.getFetcherManager().getFetcher("s3f"),
- (S3Emitter)tikaConfig.getEmitterManager().getEmitter("s3e")));
+ queue.offer(FetchIterator.COMPLETED_SEMAPHORE);
}
int finished = 0;
try {
- while (finished++ < numConsumers+1) {
+ while (finished++ < numConsumers + 1) {
Future<Integer> future = completionService.take();
future.get();
}
@@ -135,8 +145,7 @@ public class PipeIntegrationTests {
}
private TikaConfig getConfig(String fileName) throws Exception {
- try (InputStream is =
- PipeIntegrationTests.class.getResourceAsStream("/"+fileName)) {
+ try (InputStream is = PipeIntegrationTests.class.getResourceAsStream("/" + fileName)) {
return new TikaConfig(is);
}
}
@@ -149,8 +158,8 @@ public class PipeIntegrationTests {
private final Emitter emitter;
private final ArrayBlockingQueue<FetchEmitTuple> queue;
- FSFetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher
- fetcher, Emitter emitter) {
+ FSFetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher fetcher,
+ Emitter emitter) {
this.queue = queue;
this.fetcher = fetcher;
this.emitter = emitter;
@@ -172,12 +181,12 @@ public class PipeIntegrationTests {
}
private void process(FetchEmitTuple t) throws IOException, TikaException {
- Path targ = OUTDIR.resolve(t.getFetchKey().getKey());
+ Path targ = OUTDIR.resolve(t.getFetchKey().getFetchKey());
if (Files.isRegularFile(targ)) {
return;
}
- try (InputStream is = fetcher.fetch(t.getFetchKey().getKey(), t.getMetadata())) {
- System.out.println(counter.getAndIncrement() + " : "+t );
+ try (InputStream is = fetcher.fetch(t.getFetchKey().getFetchKey(), t.getMetadata())) {
+ System.out.println(counter.getAndIncrement() + " : " + t);
Files.createDirectories(targ.getParent());
Files.copy(is, targ);
}
@@ -191,8 +200,8 @@ public class PipeIntegrationTests {
private final S3Emitter emitter;
private final ArrayBlockingQueue<FetchEmitTuple> queue;
- S3FetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher
- fetcher, S3Emitter emitter) {
+ S3FetcherEmitter(ArrayBlockingQueue<FetchEmitTuple> queue, Fetcher fetcher,
+ S3Emitter emitter) {
this.queue = queue;
this.fetcher = fetcher;
this.emitter = emitter;
@@ -217,8 +226,8 @@ public class PipeIntegrationTests {
Metadata userMetadata = new Metadata();
userMetadata.set("project", "my-project");
- try (InputStream is = fetcher.fetch(t.getFetchKey().getKey(), t.getMetadata())) {
- emitter.emit(t.getEmitKey().getKey(), is, userMetadata);
+ try (InputStream is = fetcher.fetch(t.getFetchKey().getFetchKey(), t.getMetadata())) {
+ emitter.emit(t.getEmitKey().getEmitKey(), is, userMetadata);
}
}
}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
index 1bee240..2b2da1a 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,13 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#info,debug, error,fatal ...
log4j.rootLogger=info,stderr
-
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-serialization/pom.xml b/tika-serialization/pom.xml
index 64e3aa2..2ad8035 100644
--- a/tika-serialization/pom.xml
+++ b/tika-serialization/pom.xml
@@ -56,6 +56,11 @@
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<!-- Test dependencies -->
<dependency>
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
index 21b6377..e256d7d 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
@@ -32,7 +32,7 @@ public class JsonEmitData {
jsonGenerator.writeStartObject();
EmitKey key = emitData.getEmitKey();
jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITTER, key.getEmitterName());
- jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITKEY, key.getKey());
+ jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITKEY, key.getEmitKey());
jsonGenerator.writeFieldName("data");
jsonGenerator.writeStartArray();
for (Metadata m : emitData.getMetadataList()) {
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
index 55d10ef..78ba9d2 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
@@ -133,10 +133,10 @@ public class JsonFetchEmitTuple {
static void writeTuple(FetchEmitTuple t, JsonGenerator jsonGenerator) throws IOException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField(FETCHER, t.getFetchKey().getFetcherName());
- jsonGenerator.writeStringField(FETCHKEY, t.getFetchKey().getKey());
+ jsonGenerator.writeStringField(FETCHKEY, t.getFetchKey().getFetchKey());
jsonGenerator.writeStringField(EMITTER, t.getEmitKey().getEmitterName());
- if (!StringUtils.isBlank(t.getEmitKey().getKey())) {
- jsonGenerator.writeStringField(EMITKEY, t.getEmitKey().getKey());
+ if (!StringUtils.isBlank(t.getEmitKey().getEmitKey())) {
+ jsonGenerator.writeStringField(EMITKEY, t.getEmitKey().getEmitKey());
}
if (t.getMetadata().size() > 0) {
jsonGenerator.writeFieldName(METADATAKEY);
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
similarity index 54%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
index 95bb462..5a9b7d0 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
@@ -14,28 +14,30 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.metadata.serialization;
+
+import static org.apache.tika.metadata.serialization.JsonMetadata.readMetadataObject;
import java.io.IOException;
-import java.util.List;
-import org.apache.tika.metadata.Metadata;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
-public class EmptyEmitter implements Emitter {
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
- @Override
- public String getName() {
- return "empty";
- }
+import org.apache.tika.metadata.Metadata;
- @Override
- public void emit(String emitKey, List<Metadata> metadataList)
- throws IOException, TikaEmitterException {
+public class JsonMetadataDeserializer extends StdDeserializer<Metadata> {
+ public JsonMetadataDeserializer() {
+ super(Metadata.class);
}
@Override
- public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+ public Metadata deserialize(JsonParser jsonParser,
+ DeserializationContext deserializationContext)
+ throws IOException, JsonProcessingException {
+ return readMetadataObject(jsonParser);
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
similarity index 56%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
copy to tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
index 95bb462..afcf012 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
@@ -14,28 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.metadata.serialization;
-import java.io.IOException;
-import java.util.List;
+import static org.apache.tika.metadata.serialization.JsonMetadata.writeMetadataObject;
-import org.apache.tika.metadata.Metadata;
+import java.io.IOException;
-public class EmptyEmitter implements Emitter {
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
- @Override
- public String getName() {
- return "empty";
- }
+import org.apache.tika.metadata.Metadata;
- @Override
- public void emit(String emitKey, List<Metadata> metadataList)
- throws IOException, TikaEmitterException {
+public class JsonMetadataSerializer extends StdSerializer<Metadata> {
+ public JsonMetadataSerializer() {
+ super(Metadata.class);
}
@Override
- public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
-
+ public void serialize(Metadata metadata,
+ JsonGenerator jsonGenerator,
+ SerializerProvider serializerProvider) throws IOException {
+ writeMetadataObject(metadata, jsonGenerator, false);
}
}
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
index 67bbbde..2c19afb 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
@@ -40,12 +40,12 @@ public class JsonFetchEmitTupleTest {
m.add("m3", "v4");
FetchEmitTuple t = new FetchEmitTuple(
- new FetchKey("fetcher1", "fetchkey1"),
- new EmitKey("emitter1", "emitKey1"),
+ new FetchKey("my_fetcher", "fetchKey1"),
+ new EmitKey("my_emitter", "emitKey1"),
m);
StringWriter writer = new StringWriter();
JsonFetchEmitTuple.toJson(t, writer);
-
+ System.out.println(writer.toString());
Reader reader = new StringReader(writer.toString());
FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader);
assertEquals(t, deserialized);
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
index 3b1fb25..a9f7319 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
@@ -31,7 +31,7 @@ import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
public class TikaClient {
private final Random random = new Random();
- private List<TikaHttpClient> clients;
+ private final List<TikaHttpClient> clients;
private TikaClient(List<TikaHttpClient> clients) {
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
index b959839..5e583db 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
@@ -45,8 +45,9 @@ import org.apache.tika.pipes.fetchiterator.FetchIterator;
public class TikaClientCLI {
private static final Logger LOGGER = LoggerFactory.getLogger(TikaClientCLI.class);
+ private static final int QUEUE_SIZE = 10000;
- private long maxWaitMs = 300000;
+ private final long maxWaitMs = 300000;
public static void main(String[] args) throws Exception {
//TODO -- add an actual commandline,
@@ -65,9 +66,10 @@ public class TikaClientCLI {
ExecutorCompletionService<Integer> completionService =
new ExecutorCompletionService<>(executorService);
final FetchIterator fetchIterator = config.getFetchIterator();
- final ArrayBlockingQueue<FetchEmitTuple> queue = fetchIterator.init(numThreads);
+ final ArrayBlockingQueue<FetchEmitTuple> queue =
+ new ArrayBlockingQueue<>(QUEUE_SIZE);
- completionService.submit(fetchIterator);
+ completionService.submit(new FetchIteratorWrapper(fetchIterator, queue, numThreads));
if (tikaServerUrls.size() == numThreads) {
logDiffSizes(tikaServerUrls.size(), numThreads);
for (int i = 0; i < numThreads; i++) {
@@ -179,4 +181,31 @@ public class TikaClientCLI {
}
}
}
+
+ private class FetchIteratorWrapper implements Callable<Integer> {
+ private final FetchIterator fetchIterator;
+ private final ArrayBlockingQueue<FetchEmitTuple> queue;
+ private final int numThreads;
+
+ public FetchIteratorWrapper(FetchIterator fetchIterator,
+ ArrayBlockingQueue<FetchEmitTuple> queue,
+ int numThreads) {
+ this.fetchIterator = fetchIterator;
+ this.queue = queue;
+ this.numThreads = numThreads;
+
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ for (FetchEmitTuple t : fetchIterator) {
+ //potentially blocks forever
+ queue.put(t);
+ }
+ for (int i = 0; i < numThreads; i ++) {
+ queue.put(FetchIterator.COMPLETED_SEMAPHORE);
+ }
+ return 1;
+ }
+ }
}
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
index 1438885..4cb134c 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
@@ -19,9 +19,10 @@ package org.apache.tika.server.client;
public class TikaEmitterResult {
- private STATUS status;
- private String msg;//used for exceptions. will be null for status ok
- private long timeElapsed;
+ private final STATUS status;
+ private final String msg;//used for exceptions. will be null for status ok
+ private final long timeElapsed;
+
public TikaEmitterResult(STATUS status, long timeElapsed, String msg) {
this.status = status;
this.timeElapsed = timeElapsed;
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
index 6d6517f..9a5ff7f 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
@@ -48,11 +48,11 @@ class TikaHttpClient {
private final String emitEndPointUrl;
private final String asyncEndPointUrl;
private final String tikaUrl;
- private int maxRetries = 3;
+ private final int maxRetries = 3;
//if can't make contact with Tika server, max wait time in ms
- private long maxWaitForTikaMs = 120000;
+ private final long maxWaitForTikaMs = 120000;
//how often to ping /tika (in ms) to see if the server is up and running
- private long pulseWaitForTikaMs = 1000;
+ private final long pulseWaitForTikaMs = 1000;
/**
* @param baseUrl url to base endpoint
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
index 81f9a7e..054a1d2 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
@@ -46,9 +46,12 @@ public class AsyncEmitter implements Callable<Integer> {
//TODO -- need to configure these
private final long emitWithinMs = 1000;
+
+ private long maxEstimatedBytes = 10_000_000;
+
private final ArrayBlockingQueue<EmitData> emitDataQueue;
+
Instant lastEmitted = Instant.now();
- private long maxEstimatedBytes = 10_000_000;
public AsyncEmitter(ArrayBlockingQueue<EmitData> emitData) {
this.emitDataQueue = emitData;
@@ -94,7 +97,7 @@ public class AsyncEmitter implements Callable<Integer> {
void add(EmitData data) {
size++;
long sz = AbstractEmitter
- .estimateSizeInBytes(data.getEmitKey().getKey(), data.getMetadataList());
+ .estimateSizeInBytes(data.getEmitKey().getEmitKey(), data.getMetadataList());
if (estimatedSize + sz > maxBytes) {
LOG.debug("estimated size ({}) > maxBytes({}), going to emitAll",
(estimatedSize + sz), maxBytes);
@@ -113,8 +116,8 @@ public class AsyncEmitter implements Callable<Integer> {
int emitted = 0;
LOG.debug("about to emit {}", size);
for (Map.Entry<String, List<EmitData>> e : map.entrySet()) {
- Emitter emitter =
- TikaResource.getConfig().getEmitterManager().getEmitter(e.getKey());
+ Emitter emitter = TikaResource.getConfig()
+ .getEmitterManager().getEmitter(e.getKey());
tryToEmit(emitter, e.getValue());
emitted += e.getValue().size();
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
index 7a9f4ad..bf1019e 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
@@ -64,7 +64,7 @@ public class AsyncParser implements Callable<Integer> {
if (!offered) {
//TODO: deal with this
LOG.warn("Failed to add ({}) " + "to emit queue after 10 minutes.",
- request.getFetchKey().getKey());
+ request.getFetchKey().getFetchKey());
}
}
} else {
@@ -76,14 +76,15 @@ public class AsyncParser implements Callable<Integer> {
private boolean checkForParseException(FetchEmitTuple request, EmitData emitData) {
if (emitData == null || emitData.getMetadataList() == null ||
emitData.getMetadataList().size() == 0) {
- LOG.warn("empty or null emit data ({})", request.getFetchKey().getKey());
+ LOG.warn("empty or null emit data ({})", request.getFetchKey().getFetchKey());
return false;
}
boolean shouldEmit = true;
Metadata container = emitData.getMetadataList().get(0);
String stack = container.get(TikaCoreProperties.CONTAINER_EXCEPTION);
if (stack != null) {
- LOG.warn("fetchKey ({}) container parse exception ({})", request.getFetchKey().getKey(),
+ LOG.warn("fetchKey ({}) container parse exception ({})",
+ request.getFetchKey().getFetchKey(),
stack);
if (request.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
shouldEmit = false;
@@ -95,7 +96,7 @@ public class AsyncParser implements Callable<Integer> {
String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
if (embeddedStack != null) {
LOG.warn("fetchKey ({}) embedded parse exception ({})",
- request.getFetchKey().getKey(), embeddedStack);
+ request.getFetchKey().getFetchKey(), embeddedStack);
}
}
return shouldEmit;
@@ -105,7 +106,7 @@ public class AsyncParser implements Callable<Integer> {
Metadata userMetadata = t.getMetadata();
Metadata metadata = new Metadata();
String fetcherName = t.getFetchKey().getFetcherName();
- String fetchKey = t.getFetchKey().getKey();
+ String fetchKey = t.getFetchKey().getFetchKey();
List<Metadata> metadataList = null;
try (InputStream stream = TikaResource.getConfig().getFetcherManager()
.getFetcher(fetcherName).fetch(fetchKey, metadata)) {
@@ -120,7 +121,7 @@ public class AsyncParser implements Callable<Integer> {
injectUserMetadata(userMetadata, metadataList);
EmitKey emitKey = t.getEmitKey();
- if (StringUtils.isBlank(emitKey.getKey())) {
+ if (StringUtils.isBlank(emitKey.getEmitKey())) {
emitKey = new EmitKey(emitKey.getEmitterName(), fetchKey);
}
return new EmitData(emitKey, metadataList);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
index 47dbc50..001926b 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
@@ -138,7 +138,7 @@ public class AsyncResource {
private Map<String, Object> throttle(List<FetchEmitTuple> notAdded, int added) {
List<String> fetchKeys = new ArrayList<>();
for (FetchEmitTuple t : notAdded) {
- fetchKeys.add(t.getFetchKey().getKey());
+ fetchKeys.add(t.getFetchKey().getFetchKey());
}
Map<String, Object> map = new HashMap<>();
map.put("status", "throttled");
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
index 96c2e30..16aa76e 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
@@ -63,14 +63,15 @@ public class EmitterResource {
private static final String FETCHER_NAME_ABBREV = "fn";
private static final String FETCH_KEY_ABBREV = "fk";
private static final String EMIT_KEY_ABBREV = "ek";
+
private static final Logger LOG = LoggerFactory.getLogger(EmitterResource.class);
static EmitKey calcEmitKey(FetchEmitTuple t) {
//use fetch key if emitter key is not specified
//TODO: clean this up?
EmitKey emitKey = t.getEmitKey();
- if (StringUtils.isBlank(emitKey.getKey())) {
- emitKey = new EmitKey(emitKey.getEmitterName(), t.getFetchKey().getKey());
+ if (StringUtils.isBlank(emitKey.getEmitKey())) {
+ emitKey = new EmitKey(emitKey.getEmitterName(), t.getFetchKey().getFetchKey());
}
return emitKey;
}
@@ -126,15 +127,18 @@ public class EmitterResource {
@PUT
@Produces("application/json")
@Path("{" + EMITTER_PARAM + " : (\\w+)?}")
- public Map<String, String> putRmeta(InputStream is, @Context HttpHeaders httpHeaders,
+ public Map<String, String> putRmeta(InputStream is,
+ @Context HttpHeaders httpHeaders,
@Context UriInfo info,
- @PathParam(EMITTER_PARAM) String emitterName)
- throws Exception {
+ @PathParam(EMITTER_PARAM) String emitterName
+ ) throws Exception {
Metadata metadata = new Metadata();
String emitKey = httpHeaders.getHeaderString(EMIT_KEY_FOR_HTTP_HEADER);
- List<Metadata> metadataList = RecursiveMetadataResource
- .parseMetadata(is, metadata, httpHeaders.getRequestHeaders(), info, "text");
+ List<Metadata> metadataList =
+ RecursiveMetadataResource.parseMetadata(is,
+ metadata,
+ httpHeaders.getRequestHeaders(), info, "text");
return emit(new EmitKey(emitterName, emitKey), metadataList);
}
@@ -167,7 +171,7 @@ public class EmitterResource {
List<Metadata> metadataList = null;
try (InputStream stream = TikaResource.getConfig().getFetcherManager()
.getFetcher(t.getFetchKey().getFetcherName())
- .fetch(t.getFetchKey().getKey(), metadata)) {
+ .fetch(t.getFetchKey().getFetchKey(), metadata)) {
metadataList = RecursiveMetadataResource
.parseMetadata(stream, metadata, httpHeaders.getRequestHeaders(), info, "text");
@@ -192,7 +196,7 @@ public class EmitterResource {
Map<String, String> statusMap = new HashMap<>();
statusMap.put("status", "ok");
statusMap.put("emitter", t.getEmitKey().getEmitterName());
- statusMap.put("emitKey", t.getEmitKey().getKey());
+ statusMap.put("emitKey", t.getEmitKey().getEmitKey());
String msg = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
statusMap.put("parse_exception", msg);
return statusMap;
@@ -209,7 +213,7 @@ public class EmitterResource {
shouldEmit = false;
}
LOG.warn("fetchKey ({}) caught container parse exception ({})",
- t.getFetchKey().getKey(), stack);
+ t.getFetchKey().getFetchKey(), stack);
}
for (int i = 1; i < metadataList.size(); i++) {
@@ -217,7 +221,7 @@ public class EmitterResource {
String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
if (embeddedStack != null) {
LOG.warn("fetchKey ({}) caught embedded parse exception ({})",
- t.getFetchKey().getKey(), embeddedStack);
+ t.getFetchKey().getFetchKey(), embeddedStack);
}
}
@@ -250,9 +254,9 @@ public class EmitterResource {
String status = "ok";
String exceptionMsg = "";
try {
- emitter.emit(emitKey.getKey(), metadataList);
+ emitter.emit(emitKey.getEmitKey(), metadataList);
} catch (IOException | TikaEmitterException e) {
- LOG.warn("problem emitting (" + emitKey.getKey() + ")", e);
+ LOG.warn("problem emitting (" + emitKey.getEmitterName() + ")", e);
status = "emitter_exception";
exceptionMsg = ExceptionUtils.getStackTrace(e);
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
index fda802f..a99cea1 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
@@ -242,7 +242,10 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) throws Exception {
awaitServerStartup();
- Response response = WebClient.create(endPoint + "/emit").accept("application/json")
+ System.out.println(getJsonString(fileName, onParseException));
+ Response response = WebClient
+ .create(endPoint + "/emit")
+ .accept("application/json")
.post(getJsonString(fileName, onParseException));
if (response.getStatus() == 200) {
Path targFile = TMP_OUTPUT_DIR.resolve(fileName + ".json");