You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/01/26 20:31:56 UTC
[tika] branch main updated: TIKA-3226 (#399)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 4ec97b8 TIKA-3226 (#399)
4ec97b8 is described below
commit 4ec97b87e14593e7120f73dc1d614eb5b6b7ec78
Author: Tim Allison <ta...@apache.org>
AuthorDate: Tue Jan 26 15:31:47 2021 -0500
TIKA-3226 (#399)
TIKA-3226 -- initial pipes module contribution
---
pom.xml | 3 +-
.../main/java/org/apache/tika/config/Param.java | 10 +-
.../java/org/apache/tika/config/TikaConfig.java | 651 ++++++++++++++++-----
.../apache/tika/metadata/TikaCoreProperties.java | 14 +-
.../metadata/filter/FieldNameMappingFilter.java | 82 +++
.../apache/tika/pipes/emitter/AbstractEmitter.java | 25 +-
.../org/apache/tika/pipes/emitter/Emitter.java | 26 +-
.../apache/tika/pipes/emitter/EmitterManager.java | 60 ++
.../apache/tika/pipes/emitter/EmptyEmitter.java | 21 +-
.../tika/pipes/emitter/TikaEmitterException.java | 25 +-
.../apache/tika/pipes/fetcher/AbstractFetcher.java | 32 +-
.../apache/tika/pipes/fetcher/EmptyFetcher.java | 18 +-
.../org/apache/tika/pipes/fetcher/FetchId.java | 65 ++
.../tika/pipes/fetcher/FetchIdMetadataPair.java | 64 ++
.../org/apache/tika/pipes/fetcher/Fetcher.java | 25 +-
.../apache/tika/pipes/fetcher/FetcherManager.java | 67 +++
.../tika/pipes/fetcher/FetcherStringException.java | 23 +-
.../tika/pipes/fetcher/FileSystemFetcher.java | 110 ++++
.../tika/pipes/fetcher/SimpleUrlFetcher.java | 59 ++
.../pipes/fetchiterator/EmptyFetchIterator.java | 25 +-
.../tika/pipes/fetchiterator/FetchIterator.java | 135 +++++
.../fetchiterator/FileSystemFetchIterator.java | 127 ++++
.../org/apache/tika/config/TikaConfigTest.java | 67 +++
.../org/apache/tika/pipes/emitter/MockEmitter.java | 19 +-
.../tika/pipes/fetcher/FileSystemFetcherTest.java | 54 ++
.../fetchiterator/FileSystemFetchIteratorTest.java | 87 +++
.../org/apache/tika/config/emitters-config.xml | 28 +-
.../tika/config/emitters-duplicate-config.xml | 28 +-
.../apache/tika/config/fetch-iterator-config.xml | 24 +-
.../tika/config/fetch-iterator-multiple-config.xml | 30 +-
.../org/apache/tika/config/fetchers-config.xml | 30 +-
.../tika/config/fetchers-duplicate-config.xml | 30 +-
.../tika/config/fetchers-nobasepath-config.xml | 29 +-
.../apache/tika/config/fetchers-noname-config.xml | 29 +-
tika-parent/pom.xml | 1 +
.../tika/config/TikaTranslatorConfigTest.java | 5 +-
{tika-server => tika-pipes}/pom.xml | 30 +-
.../tika-emitters/pom.xml | 29 +-
tika-pipes/tika-emitters/tika-emitter-fs/pom.xml | 48 ++
.../apache/tika/emitter/fs/FileSystemEmitter.java | 102 ++++
tika-pipes/tika-emitters/tika-emitter-solr/pom.xml | 55 ++
.../org/apache/tika/emitter/solr/SolrEmitter.java | 232 ++++++++
.../org/apache/tika/emitter/solr/TestBasic.java | 60 ++
.../src/test/resources/log4j.properties | 24 +
.../test/resources/tika-config-simple-emitter.xml | 54 ++
tika-pipes/tika-fetch-iterators/pom.xml | 43 ++
.../tika-fetch-iterator-csv/pom.xml | 54 ++
.../pipes/fetchiterator/csv/CSVFetchIterator.java | 121 ++++
.../src/test/java/TestCSVFetchIterator.java | 129 ++++
.../src/test/resources/test-simple.csv | 6 +
.../tika-fetch-iterator-jdbc/pom.xml | 55 ++
.../fetchiterator/jdbc/JDBCFetchIterator.java | 166 ++++++
.../fetchiterator/jdbc/TestJDBCFetchIterator.java | 178 ++++++
.../src/test/resources/log4j.properties | 24 +
.../tika-fetch-iterator-s3/pom.xml | 94 +++
.../pipes/fetchiterator/s3/S3FetchIterator.java | 109 ++++
.../fetchiterator/s3/TestS3FetchIterator.java | 98 ++++
.../src/test/resources/log4j.properties | 24 +
.../tika-fetchers/pom.xml | 27 +-
tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml | 150 +++++
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 143 +++++
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 62 ++
.../src/test/resources/tika-config-s3.xml | 25 +-
tika-pipes/tika-httpclient-commons/pom.xml | 52 ++
.../org/apache/tika/client/HttpClientUtil.java | 38 ++
.../apache/tika/client/TikaClientException.java | 14 +
tika-pipes/tika-pipes-integration-tests/pom.xml | 64 ++
.../apache/tika/pipes/PipeIntegrationTests.java | 162 +++++
.../src/test/resources/log4j.properties | 24 +
.../src/test/resources/tika-config-s3ToFs.xml | 22 +
tika-server/pom.xml | 14 -
tika-server/tika-server-classic/pom.xml | 7 +-
.../apache/tika/server/classic/FetcherTest.java | 98 ++++
.../resources/config/tika-config-url-fetcher.xml} | 18 +-
tika-server/tika-server-client/pom.xml | 32 +
.../org/apache/tika/server/client/TikaClient.java | 99 ++++
.../apache/tika/server/client/TikaClientCLI.java | 150 +++++
.../server/client/TikaClientConfigException.java | 13 +
.../tika/server/client/TikaEmitterResult.java | 61 ++
.../apache/tika/server/client/TikaHttpClient.java | 170 ++++++
.../src/main/resources/log4j.properties | 24 +
.../org/apache/tika/server/client/TestBasic.java} | 36 +-
.../src/test/resources/log4j.properties | 24 +
.../resources/tika-config-simple-fs-emitter.xml | 78 +++
tika-server/tika-server-core/pom.xml | 6 +
.../server/core/DefaultInputStreamFactory.java | 5 -
...treamFactory.java => FetcherStreamFactory.java} | 46 +-
.../tika/server/core/InputStreamFactory.java | 1 -
.../org/apache/tika/server/core/TikaServerCli.java | 79 ++-
.../tika/server/core/resource/EmitterResource.java | 239 ++++++++
.../core/resource/RecursiveMetadataResource.java | 25 +-
.../tika/server/core/resource/TikaResource.java | 4 +-
.../org/apache/tika/server/core/CXFTestBase.java | 6 +-
.../tika/server/core/IntegrationTestBase.java | 88 +++
.../apache/tika/server/core/StackTraceOffTest.java | 7 +-
.../apache/tika/server/core/StackTraceTest.java | 3 +-
.../apache/tika/server/core/TikaEmitterTest.java | 288 +++++++++
.../core/TikaServerEmitterIntegrationTest.java | 275 +++++++++
.../server/core/TikaServerIntegrationTest.java | 75 +--
.../resources/test-documents/mock/hello_world.xml | 1 +
100 files changed, 6004 insertions(+), 614 deletions(-)
diff --git a/pom.xml b/pom.xml
index a311211..e69028a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+ <?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
@@ -37,6 +37,7 @@
<modules>
<module>tika-parent</module>
<module>tika-core</module>
+ <module>tika-pipes</module>
<module>tika-parsers</module>
<module>tika-bundles</module>
<module>tika-xmp</module>
diff --git a/tika-core/src/main/java/org/apache/tika/config/Param.java b/tika-core/src/main/java/org/apache/tika/config/Param.java
index 652fb6c..74123ac 100644
--- a/tika-core/src/main/java/org/apache/tika/config/Param.java
+++ b/tika-core/src/main/java/org/apache/tika/config/Param.java
@@ -16,6 +16,7 @@
*/
package org.apache.tika.config;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.multiple.AbstractMultipleParser;
import org.apache.tika.utils.XMLReaderUtils;
@@ -205,25 +206,28 @@ public class Param<T> implements Serializable {
return load(document.getFirstChild());
}
- public static <T> Param<T> load(Node node) {
+ public static <T> Param<T> load(Node node) throws TikaConfigException{
Node nameAttr = node.getAttributes().getNamedItem("name");
Node typeAttr = node.getAttributes().getNamedItem("type");
Node valueAttr = node.getAttributes().getNamedItem("value");
Node value = node.getFirstChild();
if (value instanceof NodeList && valueAttr != null) {
- throw new IllegalArgumentException("can't specify a value attr _and_ a node list");
+ throw new TikaConfigException("can't specify a value attr _and_ a node list");
}
if (valueAttr != null && (value == null || value.getTextContent() == null)) {
value = valueAttr;
}
-
+
Param<T> ret = new Param<T>();
ret.name = nameAttr.getTextContent();
if (typeAttr != null) {
ret.setTypeString(typeAttr.getTextContent());
} else {
ret.type = (Class<T>)wellKnownMap.get(ret.name);
+ if (ret.type == null) {
+ throw new TikaConfigException("Must specify a \"type\" in: " + node.getLocalName());
+ }
}
if (List.class.isAssignableFrom(ret.type)) {
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index d7fb8b3..f852c57 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -46,8 +46,14 @@ import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.emitter.EmitterManager;
+import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetchiterator.EmptyFetchIterator;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.language.translate.DefaultTranslator;
import org.apache.tika.language.translate.Translator;
import org.apache.tika.metadata.filter.CompositeMetadataFilter;
@@ -112,6 +118,18 @@ public class TikaConfig {
return new DefaultMetadataFilter(loader);
}
+ private static FetcherManager getDefaultFetcherManager(ServiceLoader loader) {
+ return new FetcherManager(Collections.EMPTY_LIST);
+ }
+
+ private static EmitterManager getDefaultEmitterManager(ServiceLoader loader) {
+ return new EmitterManager(Collections.EMPTY_LIST);
+ }
+
+ private static FetchIterator getDefaultFetchIterator(ServiceLoader loader) {
+ return new EmptyFetchIterator();
+ }
+
//use this to look for unneeded instantiations of TikaConfig
protected static AtomicInteger TIMES_INSTANTIATED = new AtomicInteger();
@@ -124,6 +142,9 @@ public class TikaConfig {
private final ExecutorService executorService;
private final EncodingDetector encodingDetector;
private final MetadataFilter metadataFilter;
+ private final FetcherManager fetcherManager;
+ private final EmitterManager emitterManager;
+ private final FetchIterator fetchIterator;
public TikaConfig(String file)
throws TikaException, IOException, SAXException {
@@ -134,6 +155,7 @@ public class TikaConfig {
throws TikaException, IOException, SAXException {
this(XMLReaderUtils.buildDOM(path));
}
+
public TikaConfig(Path path, ServiceLoader loader)
throws TikaException, IOException, SAXException {
this(XMLReaderUtils.buildDOM(path), loader);
@@ -153,10 +175,12 @@ public class TikaConfig {
throws TikaException, IOException, SAXException {
this(url, ServiceLoader.getContextClassLoader());
}
+
public TikaConfig(URL url, ClassLoader loader)
throws TikaException, IOException, SAXException {
this(XMLReaderUtils.buildDOM(url.toString()).getDocumentElement(), loader);
}
+
public TikaConfig(URL url, ServiceLoader loader)
throws TikaException, IOException, SAXException {
this(XMLReaderUtils.buildDOM(url.toString()).getDocumentElement(), loader);
@@ -170,6 +194,7 @@ public class TikaConfig {
public TikaConfig(Document document) throws TikaException, IOException {
this(document.getDocumentElement());
}
+
public TikaConfig(Document document, ServiceLoader loader) throws TikaException, IOException {
this(document.getDocumentElement(), loader);
}
@@ -190,6 +215,9 @@ public class TikaConfig {
ExecutorServiceXmlLoader executorLoader = new ExecutorServiceXmlLoader();
EncodingDetectorXmlLoader encodingDetectorXmlLoader = new EncodingDetectorXmlLoader();
MetadataFilterXmlLoader metadataFilterXmlLoader = new MetadataFilterXmlLoader();
+ FetcherXmlLoader fetcherXmlLoader = new FetcherXmlLoader();
+ EmitterXmlLoader emitterXmlLoader = new EmitterXmlLoader();
+ FetchIteratorXmlLoader fetchIteratorXmlLoader = new FetchIteratorXmlLoader();
updateXMLReaderUtils(element);
this.mimeTypes = typesFromDomElement(element);
this.detector = detectorLoader.loadOverall(element, mimeTypes, loader);
@@ -200,6 +228,9 @@ public class TikaConfig {
this.translator = translatorLoader.loadOverall(element, mimeTypes, loader);
this.executorService = executorLoader.loadOverall(element, mimeTypes, loader);
this.metadataFilter = metadataFilterXmlLoader.loadOverall(element, mimeTypes, loader);
+ this.fetcherManager = fetcherXmlLoader.loadOverall(element, mimeTypes, loader);
+ this.emitterManager = emitterXmlLoader.loadOverall(element, mimeTypes, loader);
+ this.fetchIterator = fetchIteratorXmlLoader.loadOverall(element, mimeTypes, loader);
this.serviceLoader = loader;
TIMES_INSTANTIATED.incrementAndGet();
}
@@ -210,11 +241,11 @@ public class TikaConfig {
* {@link ServiceRegistry service provider mechanism} in the given
* class loader.
*
- * @since Apache Tika 0.8
* @param loader the class loader through which parser implementations
* are loaded, or <code>null</code> for no parsers
* @throws MimeTypeException if the built-in media type rules are broken
- * @throws IOException if the built-in media type rules can not be read
+ * @throws IOException if the built-in media type rules can not be read
+ * @since Apache Tika 0.8
*/
public TikaConfig(ClassLoader loader)
throws MimeTypeException, IOException {
@@ -226,6 +257,9 @@ public class TikaConfig {
this.translator = getDefaultTranslator(serviceLoader);
this.executorService = getDefaultExecutorService();
this.metadataFilter = getDefaultMetadataFilter(serviceLoader);
+ this.fetcherManager = getDefaultFetcherManager(serviceLoader);
+ this.emitterManager = getDefaultEmitterManager(serviceLoader);
+ this.fetchIterator = getDefaultFetchIterator(serviceLoader);
TIMES_INSTANTIATED.incrementAndGet();
}
@@ -243,7 +277,7 @@ public class TikaConfig {
* the {@link ServiceRegistry service provider mechanism} in the context
* class loader of the current thread.</p>
*
- * @throws IOException if the configuration can not be read
+ * @throws IOException if the configuration can not be read
* @throws TikaException if problem with MimeTypes or parsing XML config
*/
public TikaConfig() throws TikaException, IOException {
@@ -262,6 +296,9 @@ public class TikaConfig {
this.translator = getDefaultTranslator(serviceLoader);
this.executorService = getDefaultExecutorService();
this.metadataFilter = getDefaultMetadataFilter(serviceLoader);
+ this.fetcherManager = getDefaultFetcherManager(serviceLoader);
+ this.emitterManager = getDefaultEmitterManager(serviceLoader);
+ this.fetchIterator = getDefaultFetchIterator(serviceLoader);
} else {
ServiceLoader tmpServiceLoader = new ServiceLoader();
try (InputStream stream = getConfigInputStream(config, tmpServiceLoader)) {
@@ -273,6 +310,9 @@ public class TikaConfig {
TranslatorXmlLoader translatorLoader = new TranslatorXmlLoader();
ExecutorServiceXmlLoader executorLoader = new ExecutorServiceXmlLoader();
MetadataFilterXmlLoader metadataFilterXmlLoader = new MetadataFilterXmlLoader();
+ FetcherXmlLoader fetcherXmlLoader = new FetcherXmlLoader();
+ EmitterXmlLoader emitterXmlLoader = new EmitterXmlLoader();
+ FetchIteratorXmlLoader fetchIteratorXmlLoader = new FetchIteratorXmlLoader();
this.mimeTypes = typesFromDomElement(element);
this.encodingDetector = encodingDetectorLoader.loadOverall(element, mimeTypes, serviceLoader);
@@ -284,6 +324,9 @@ public class TikaConfig {
this.translator = translatorLoader.loadOverall(element, mimeTypes, serviceLoader);
this.executorService = executorLoader.loadOverall(element, mimeTypes, serviceLoader);
this.metadataFilter = metadataFilterXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
+ this.fetcherManager = fetcherXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
+ this.emitterManager = emitterXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
+ this.fetchIterator = fetchIteratorXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
} catch (SAXException e) {
throw new TikaException(
"Specified Tika configuration has syntax errors: "
@@ -377,6 +420,7 @@ public class TikaConfig {
/**
* Returns the configured encoding detector instance
+ *
* @return configured encoding detector
*/
public EncodingDetector getEncodingDetector() {
@@ -391,19 +435,19 @@ public class TikaConfig {
public Translator getTranslator() {
return translator;
}
-
+
public ExecutorService getExecutorService() {
return executorService;
}
- public MimeTypes getMimeRepository(){
+ public MimeTypes getMimeRepository() {
return mimeTypes;
}
public MediaTypeRegistry getMediaTypeRegistry() {
return mimeTypes.getMediaTypeRegistry();
}
-
+
public ServiceLoader getServiceLoader() {
return serviceLoader;
}
@@ -411,6 +455,19 @@ public class TikaConfig {
public MetadataFilter getMetadataFilter() {
return metadataFilter;
}
+
+ public FetcherManager getFetcherManager() {
+ return fetcherManager;
+ }
+
+ public EmitterManager getEmitterManager() {
+ return emitterManager;
+ }
+
+ public FetchIterator getFetchIterator() {
+ return fetchIterator;
+ }
+
/**
* Provides a default configuration (TikaConfig). Currently creates a
* new instance each time it's called; we may be able to have it
@@ -441,23 +498,23 @@ public class TikaConfig {
}
return null;
}
- private static List<Element> getTopLevelElementChildren(Element element,
- String parentName, String childrenName) throws TikaException {
+
+ private static List<Element> getTopLevelElementChildren(Element element,
+ String parentName, String childrenName) throws TikaException {
Node parentNode = null;
if (parentName != null) {
// Should be only zero or one <parsers> / <detectors> etc tag
NodeList nodes = element.getElementsByTagName(parentName);
if (nodes.getLength() > 1) {
- throw new TikaException("Properties may not contain multiple "+parentName+" entries");
- }
- else if (nodes.getLength() == 1) {
+ throw new TikaException("Properties may not contain multiple " + parentName + " entries");
+ } else if (nodes.getLength() == 1) {
parentNode = nodes.item(0);
}
} else {
// All children directly on the master element
parentNode = element;
}
-
+
if (parentNode != null) {
// Find only the direct child parser/detector objects
NodeList nodes = parentNode.getChildNodes();
@@ -465,7 +522,7 @@ public class TikaConfig {
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
if (node instanceof Element) {
- Element nodeE = (Element)node;
+ Element nodeE = (Element) node;
if (childrenName.equals(nodeE.getTagName())) {
elements.add(nodeE);
}
@@ -487,16 +544,16 @@ public class TikaConfig {
return getDefaultMimeTypes(null);
}
}
-
+
private static Set<MediaType> mediaTypesListFromDomElement(
- Element node, String tag)
+ Element node, String tag)
throws TikaException, IOException {
Set<MediaType> types = null;
NodeList children = node.getChildNodes();
- for (int i=0; i<children.getLength(); i++) {
+ for (int i = 0; i < children.getLength(); i++) {
Node cNode = children.item(i);
if (cNode instanceof Element) {
- Element cElement = (Element)cNode;
+ Element cElement = (Element) cNode;
if (tag.equals(cElement.getTagName())) {
String mime = getText(cElement);
MediaType type = MediaType.parse(mime);
@@ -513,7 +570,7 @@ public class TikaConfig {
if (types != null) return types;
return Collections.emptySet();
}
-
+
private static ServiceLoader serviceLoaderFromDomElement(Element element, ClassLoader loader) throws TikaConfigException {
Element serviceLoaderElement = getChild(element, "service-loader");
ServiceLoader serviceLoader;
@@ -522,9 +579,9 @@ public class TikaConfig {
boolean dynamic = Boolean.parseBoolean(serviceLoaderElement.getAttribute("dynamic"));
LoadErrorHandler loadErrorHandler = LoadErrorHandler.IGNORE;
String loadErrorHandleConfig = serviceLoaderElement.getAttribute("loadErrorHandler");
- if(LoadErrorHandler.WARN.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
+ if (LoadErrorHandler.WARN.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
loadErrorHandler = LoadErrorHandler.WARN;
- } else if(LoadErrorHandler.THROW.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
+ } else if (LoadErrorHandler.THROW.toString().equalsIgnoreCase(loadErrorHandleConfig)) {
loadErrorHandler = LoadErrorHandler.THROW;
}
InitializableProblemHandler initializableProblemHandler = getInitializableProblemHandler(serviceLoaderElement.getAttribute("initializableProblemHandler"));
@@ -533,7 +590,7 @@ public class TikaConfig {
loader = ServiceLoader.getContextClassLoader();
}
serviceLoader = new ServiceLoader(loader, loadErrorHandler, initializableProblemHandler, dynamic);
- } else if(loader != null) {
+ } else if (loader != null) {
serviceLoader = new ServiceLoader(loader);
} else {
serviceLoader = new ServiceLoader();
@@ -542,7 +599,7 @@ public class TikaConfig {
}
private static InitializableProblemHandler getInitializableProblemHandler(String initializableProblemHandler)
- throws TikaConfigException {
+ throws TikaConfigException {
if (initializableProblemHandler == null || initializableProblemHandler.length() == 0) {
return InitializableProblemHandler.DEFAULT;
}
@@ -561,36 +618,46 @@ public class TikaConfig {
}
- private static abstract class XmlLoader<CT,T> {
+ private static abstract class XmlLoader<CT, T> {
protected static final String PARAMS_TAG_NAME = "params";
+
abstract boolean supportsComposite();
+
abstract String getParentTagName(); // eg parsers
+
abstract String getLoaderTagName(); // eg parser
+
abstract Class<? extends T> getLoaderClass(); // Generics workaround
+
abstract boolean isComposite(T loaded);
+
abstract boolean isComposite(Class<? extends T> loadedClass);
- abstract T preLoadOne(Class<? extends T> loadedClass, String classname,
- MimeTypes mimeTypes) throws TikaException;
+
+ abstract T preLoadOne(Class<? extends T> loadedClass, String classname,
+ MimeTypes mimeTypes) throws TikaException;
+
abstract CT createDefault(MimeTypes mimeTypes, ServiceLoader loader);
+
abstract CT createComposite(List<T> loaded, MimeTypes mimeTypes, ServiceLoader loader);
- abstract T createComposite(Class<? extends T> compositeClass,
- List<T> children, Set<Class<? extends T>> excludeChildren,
- Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+
+ abstract T createComposite(Class<? extends T> compositeClass,
+ List<T> children, Set<Class<? extends T>> excludeChildren,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
throws InvocationTargetException, IllegalAccessException, InstantiationException;
- abstract T decorate(T created, Element element)
+
+ abstract T decorate(T created, Element element)
throws IOException, TikaException; // eg explicit mime types
-
+
@SuppressWarnings("unchecked")
- CT loadOverall(Element element, MimeTypes mimeTypes,
- ServiceLoader loader) throws TikaException, IOException {
+ CT loadOverall(Element element, MimeTypes mimeTypes,
+ ServiceLoader loader) throws TikaException, IOException {
List<T> loaded = new ArrayList<T>();
-
+
// Find the children of the parent tag, if any
- for (Element le : getTopLevelElementChildren(element, getParentTagName(), getLoaderTagName())) {
- T loadedChild = loadOne(le, mimeTypes, loader);
+ for (Element le : getTopLevelElementChildren(element, getParentTagName(), getLoaderTagName())) {T loadedChild = loadOne(le, mimeTypes, loader);
if (loadedChild != null) loaded.add(loadedChild);
}
-
+
// Build the classes, and wrap as needed
if (loaded.isEmpty()) {
// Nothing defined, create a Default
@@ -599,26 +666,36 @@ public class TikaConfig {
T single = loaded.get(0);
if (isComposite(single)) {
// Single Composite defined, use that
- return (CT)single;
+ return (CT) single;
}
- } else if (! supportsComposite()) {
+ } else if (!supportsComposite()) {
// No composite support, just return the first one
- return (CT)loaded.get(0);
+ if (loaded.size() == 1) {
+ return (CT) loaded.get(0);
+ } else if (loaded.size() > 1) {
+ throw new TikaConfigException("Composite not supported for " +
+ getParentTagName() +
+ ". Must specify only one child!");
+ } else {
+ //throw exception if empty?
+ }
}
// Wrap the defined parsers/detectors up in a Composite
return createComposite(loaded, mimeTypes, loader);
}
- T loadOne(Element element, MimeTypes mimeTypes, ServiceLoader loader)
+ T loadOne(Element element, MimeTypes mimeTypes, ServiceLoader loader)
throws TikaException, IOException {
String name = element.getAttribute("class");
-
+ if (name == null) {
+ throw new TikaConfigException("class attribute must not be null: " + element);
+ }
String initProbHandler = element.getAttribute("initializableProblemHandler");
InitializableProblemHandler initializableProblemHandler;
if (initProbHandler == null || initProbHandler.length() == 0) {
initializableProblemHandler = loader.getInitializableProblemHandler();
} else {
- initializableProblemHandler =
+ initializableProblemHandler =
getInitializableProblemHandler(initProbHandler);
}
@@ -632,7 +709,7 @@ public class TikaConfig {
//TODO : allow duplicate instances with different configurations
loaded = preLoadOne(loadedClass, name, mimeTypes);
if (loaded != null) return loaded;
-
+
// Get any parameters / settings for the parser
Map<String, Param> params = null;
try {
@@ -640,7 +717,7 @@ public class TikaConfig {
} catch (Exception e) {
throw new TikaConfigException(e.getMessage(), e);
}
-
+
// Is this a composite or decorated class? If so, support recursion
if (isComposite(loadedClass)) {
// Get the child objects for it
@@ -648,28 +725,28 @@ public class TikaConfig {
NodeList childNodes = element.getElementsByTagName(getLoaderTagName());
if (childNodes.getLength() > 0) {
for (int i = 0; i < childNodes.getLength(); i++) {
- T loadedChild = loadOne((Element)childNodes.item(i),
- mimeTypes, loader);
+ T loadedChild = loadOne((Element) childNodes.item(i),
+ mimeTypes, loader);
if (loadedChild != null) children.add(loadedChild);
}
}
-
+
// Get the list of children to exclude
Set<Class<? extends T>> excludeChildren = new HashSet<Class<? extends T>>();
- NodeList excludeChildNodes = element.getElementsByTagName(getLoaderTagName()+"-exclude");
+ NodeList excludeChildNodes = element.getElementsByTagName(getLoaderTagName() + "-exclude");
if (excludeChildNodes.getLength() > 0) {
for (int i = 0; i < excludeChildNodes.getLength(); i++) {
- Element excl = (Element)excludeChildNodes.item(i);
+ Element excl = (Element) excludeChildNodes.item(i);
String exclName = excl.getAttribute("class");
try {
excludeChildren.add(loader.getServiceClass(getLoaderClass(), exclName));
} catch (ClassNotFoundException e) {
//TIKA-3268 -- This should stop the world.
- throw new TikaConfigException("Class now found in -exclude list: " +exclName);
+ throw new TikaConfigException("Class now found in -exclude list: " + exclName);
}
}
}
-
+
// Create the Composite
loaded = createComposite(loadedClass, children, excludeChildren, params, mimeTypes, loader);
@@ -697,24 +774,24 @@ public class TikaConfig {
} catch (ClassNotFoundException e) {
if (loader.getLoadErrorHandler() == LoadErrorHandler.THROW) {
// Use a different exception signature here
- throw new TikaException(
- "Unable to find a "+getLoaderTagName()+" class: " + name, e);
+ throw new TikaConfigException(
+ "Unable to find a " + getLoaderTagName() + " class: " + name, e);
}
// Report the problem
loader.getLoadErrorHandler().handleLoadError(name, e);
return null;
} catch (IllegalAccessException e) {
throw new TikaException(
- "Unable to access a "+getLoaderTagName()+" class: " + name, e);
+ "Unable to access a " + getLoaderTagName() + " class: " + name, e);
} catch (InvocationTargetException e) {
throw new TikaException(
- "Unable to create a "+getLoaderTagName()+" class: " + name, e);
+ "Unable to create a " + getLoaderTagName() + " class: " + name, e);
} catch (InstantiationException e) {
throw new TikaException(
- "Unable to instantiate a "+getLoaderTagName()+" class: " + name, e);
+ "Unable to instantiate a " + getLoaderTagName() + " class: " + name, e);
} catch (NoSuchMethodException e) {
throw new TikaException(
- "Unable to find the right constructor for "+getLoaderTagName()+" class: " + name, e);
+ "Unable to find the right constructor for " + getLoaderTagName() + " class: " + name, e);
}
}
@@ -727,19 +804,20 @@ public class TikaConfig {
/**
* Gets parameters from a given
+ *
* @param el xml node which has {@link #PARAMS_TAG_NAME} child
* @return Map of key values read from xml
*/
- Map<String, Param> getParams(Element el){
+ Map<String, Param> getParams(Element el) throws TikaException {
Map<String, Param> params = new HashMap<>();
for (Node child = el.getFirstChild(); child != null;
- child = child.getNextSibling()){
- if (PARAMS_TAG_NAME.equals(child.getNodeName())){ //found the node
+ child = child.getNextSibling()) {
+ if (PARAMS_TAG_NAME.equals(child.getNodeName())) { //found the node
if (child.hasChildNodes()) {//it has children
NodeList childNodes = child.getChildNodes();
for (int i = 0; i < childNodes.getLength(); i++) {
Node item = childNodes.item(i);
- if (item.getNodeType() == Node.ELEMENT_NODE){
+ if (item.getNodeType() == Node.ELEMENT_NODE) {
Param<?> param = Param.load(item);
params.put(param.getName(), param);
}
@@ -752,113 +830,133 @@ public class TikaConfig {
}
}
- private static class ParserXmlLoader extends XmlLoader<CompositeParser,Parser> {
+
+ private static class ParserXmlLoader extends XmlLoader<CompositeParser, Parser> {
private final EncodingDetector encodingDetector;
- boolean supportsComposite() { return true; }
- String getParentTagName() { return "parsers"; }
- String getLoaderTagName() { return "parser"; }
+ boolean supportsComposite() {
+ return true;
+ }
+
+ String getParentTagName() {
+ return "parsers";
+ }
+
+ String getLoaderTagName() {
+ return "parser";
+ }
private ParserXmlLoader(EncodingDetector encodingDetector) {
this.encodingDetector = encodingDetector;
}
+
@Override
Class<? extends Parser> getLoaderClass() {
return Parser.class;
}
+
@Override
- Parser preLoadOne(Class<? extends Parser> loadedClass, String classname,
+ Parser preLoadOne(Class<? extends Parser> loadedClass, String classname,
MimeTypes mimeTypes) throws TikaException {
// Check for classes which can't be set in config
if (AutoDetectParser.class.isAssignableFrom(loadedClass)) {
// https://issues.apache.org/jira/browse/TIKA-866
throw new TikaException(
"AutoDetectParser not supported in a <parser>"
- + " configuration element: " + classname);
+ + " configuration element: " + classname);
}
// Continue with normal loading
return null;
}
+
@Override
boolean isComposite(Parser loaded) {
return loaded instanceof CompositeParser;
}
+
@Override
boolean isComposite(Class<? extends Parser> loadedClass) {
if (CompositeParser.class.isAssignableFrom(loadedClass) ||
- AbstractMultipleParser.class.isAssignableFrom(loadedClass) ||
- ParserDecorator.class.isAssignableFrom(loadedClass)) {
+ AbstractMultipleParser.class.isAssignableFrom(loadedClass) ||
+ ParserDecorator.class.isAssignableFrom(loadedClass)) {
return true;
}
return false;
}
+
@Override
CompositeParser createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
return getDefaultParser(mimeTypes, loader, encodingDetector);
}
+
@Override
CompositeParser createComposite(List<Parser> parsers, MimeTypes mimeTypes, ServiceLoader loader) {
MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();
return new CompositeParser(registry, parsers);
}
+
@Override
Parser createComposite(Class<? extends Parser> parserClass,
- List<Parser> childParsers, Set<Class<? extends Parser>> excludeParsers,
- Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ List<Parser> childParsers, Set<Class<? extends Parser>> excludeParsers,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
throws InvocationTargetException, IllegalAccessException, InstantiationException {
Parser parser = null;
Constructor<? extends Parser> c = null;
MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();
-
+
// Try the possible default and composite parser constructors
if (parser == null) {
try {
c = parserClass.getConstructor(MediaTypeRegistry.class,
ServiceLoader.class, Collection.class, EncodingDetector.class);
parser = c.newInstance(registry, loader, excludeParsers, encodingDetector);
+ } catch (NoSuchMethodException me) {
}
- catch (NoSuchMethodException me) {}
}
if (parser == null) {
try {
c = parserClass.getConstructor(MediaTypeRegistry.class, ServiceLoader.class, Collection.class);
parser = c.newInstance(registry, loader, excludeParsers);
- }
- catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (parser == null) {
try {
c = parserClass.getConstructor(MediaTypeRegistry.class, List.class, Collection.class);
parser = c.newInstance(registry, childParsers, excludeParsers);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (parser == null) {
try {
c = parserClass.getConstructor(MediaTypeRegistry.class, Collection.class, Map.class);
parser = c.newInstance(registry, childParsers, params);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (parser == null) {
try {
c = parserClass.getConstructor(MediaTypeRegistry.class, List.class);
parser = c.newInstance(registry, childParsers);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
-
+
// Create as a Parser Decorator
if (parser == null && ParserDecorator.class.isAssignableFrom(parserClass)) {
try {
CompositeParser cp = null;
if (childParsers.size() == 1 && excludeParsers.size() == 0 &&
childParsers.get(0) instanceof CompositeParser) {
- cp = (CompositeParser)childParsers.get(0);
+ cp = (CompositeParser) childParsers.get(0);
} else {
cp = new CompositeParser(registry, childParsers, excludeParsers);
}
c = parserClass.getConstructor(Parser.class);
parser = c.newInstance(cp);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
return parser;
}
@@ -876,34 +974,44 @@ public class TikaConfig {
@Override
Parser decorate(Parser created, Element element) throws IOException, TikaException {
Parser parser = created;
-
+
// Is there an explicit list of mime types for this to handle?
Set<MediaType> parserTypes = mediaTypesListFromDomElement(element, "mime");
- if (! parserTypes.isEmpty()) {
+ if (!parserTypes.isEmpty()) {
parser = ParserDecorator.withTypes(parser, parserTypes);
}
// Is there an explicit list of mime types this shouldn't handle?
Set<MediaType> parserExclTypes = mediaTypesListFromDomElement(element, "mime-exclude");
- if (! parserExclTypes.isEmpty()) {
+ if (!parserExclTypes.isEmpty()) {
parser = ParserDecorator.withoutTypes(parser, parserExclTypes);
}
-
+
// All done with decoration
return parser;
}
}
- private static class DetectorXmlLoader extends XmlLoader<CompositeDetector,Detector> {
- boolean supportsComposite() { return true; }
- String getParentTagName() { return "detectors"; }
- String getLoaderTagName() { return "detector"; }
-
+
+ private static class DetectorXmlLoader extends XmlLoader<CompositeDetector, Detector> {
+ boolean supportsComposite() {
+ return true;
+ }
+
+ String getParentTagName() {
+ return "detectors";
+ }
+
+ String getLoaderTagName() {
+ return "detector";
+ }
+
@Override
Class<? extends Detector> getLoaderClass() {
return Detector.class;
}
+
@Override
- Detector preLoadOne(Class<? extends Detector> loadedClass, String classname,
+ Detector preLoadOne(Class<? extends Detector> loadedClass, String classname,
MimeTypes mimeTypes) throws TikaException {
// If they asked for the mime types as a detector, give
// them the one we've already created. TIKA-1708
@@ -913,112 +1021,141 @@ public class TikaConfig {
// Continue with normal loading
return null;
}
+
@Override
boolean isComposite(Detector loaded) {
return loaded instanceof CompositeDetector;
}
+
@Override
boolean isComposite(Class<? extends Detector> loadedClass) {
return CompositeDetector.class.isAssignableFrom(loadedClass);
}
+
@Override
CompositeDetector createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
return getDefaultDetector(mimeTypes, loader);
}
+
@Override
CompositeDetector createComposite(List<Detector> detectors, MimeTypes mimeTypes, ServiceLoader loader) {
MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();
return new CompositeDetector(registry, detectors);
}
+
@Override
Detector createComposite(Class<? extends Detector> detectorClass,
- List<Detector> childDetectors,
- Set<Class<? extends Detector>> excludeDetectors,
- Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ List<Detector> childDetectors,
+ Set<Class<? extends Detector>> excludeDetectors,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
throws InvocationTargetException, IllegalAccessException,
InstantiationException {
Detector detector = null;
Constructor<? extends Detector> c;
MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();
-
+
// Try the possible default and composite detector constructors
if (detector == null) {
try {
c = detectorClass.getConstructor(MimeTypes.class, ServiceLoader.class, Collection.class);
detector = c.newInstance(mimeTypes, loader, excludeDetectors);
- }
- catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (detector == null) {
try {
c = detectorClass.getConstructor(MediaTypeRegistry.class, List.class, Collection.class);
detector = c.newInstance(registry, childDetectors, excludeDetectors);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (detector == null) {
try {
c = detectorClass.getConstructor(MediaTypeRegistry.class, List.class);
detector = c.newInstance(registry, childDetectors);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
if (detector == null) {
try {
c = detectorClass.getConstructor(List.class);
detector = c.newInstance(childDetectors);
- } catch (NoSuchMethodException me) {}
+ } catch (NoSuchMethodException me) {
+ }
}
-
+
return detector;
}
+
@Override
Detector decorate(Detector created, Element element) {
return created; // No decoration of Detectors
}
}
- private static class TranslatorXmlLoader extends XmlLoader<Translator,Translator> {
- boolean supportsComposite() { return false; }
- String getParentTagName() { return null; }
- String getLoaderTagName() { return "translator"; }
-
+
+ private static class TranslatorXmlLoader extends XmlLoader<Translator, Translator> {
+ boolean supportsComposite() {
+ return false;
+ }
+
+ String getParentTagName() {
+ return null;
+ }
+
+ String getLoaderTagName() {
+ return "translator";
+ }
+
@Override
Class<? extends Translator> getLoaderClass() {
return Translator.class;
}
+
@Override
- Translator preLoadOne(Class<? extends Translator> loadedClass, String classname,
+ Translator preLoadOne(Class<? extends Translator> loadedClass, String classname,
MimeTypes mimeTypes) throws TikaException {
// Continue with normal loading
return null;
}
+
@Override
- boolean isComposite(Translator loaded) { return false; }
+ boolean isComposite(Translator loaded) {
+ return false;
+ }
+
@Override
- boolean isComposite(Class<? extends Translator> loadedClass) { return false; }
+ boolean isComposite(Class<? extends Translator> loadedClass) {
+ return false;
+ }
+
@Override
Translator createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
return getDefaultTranslator(loader);
}
+
@Override
Translator createComposite(List<Translator> loaded,
- MimeTypes mimeTypes, ServiceLoader loader) {
+ MimeTypes mimeTypes, ServiceLoader loader) {
return loaded.get(0);
}
+
@Override
Translator createComposite(Class<? extends Translator> compositeClass,
- List<Translator> children,
- Set<Class<? extends Translator>> excludeChildren,
- Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ List<Translator> children,
+ Set<Class<? extends Translator>> excludeChildren,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
throws InvocationTargetException, IllegalAccessException,
InstantiationException {
throw new InstantiationException("Only one translator supported");
}
+
@Override
Translator decorate(Translator created, Element element) {
return created; // No decoration of Translators
- }
+ }
}
-
- private static class ExecutorServiceXmlLoader extends XmlLoader<ConfigurableThreadPoolExecutor,ConfigurableThreadPoolExecutor> {
+
+ private static class ExecutorServiceXmlLoader extends XmlLoader<ConfigurableThreadPoolExecutor, ConfigurableThreadPoolExecutor> {
@Override
ConfigurableThreadPoolExecutor createComposite(
Class<? extends ConfigurableThreadPoolExecutor> compositeClass,
@@ -1029,61 +1166,69 @@ public class TikaConfig {
InstantiationException {
throw new InstantiationException("Only one executor service supported");
}
-
+
@Override
ConfigurableThreadPoolExecutor createComposite(List<ConfigurableThreadPoolExecutor> loaded,
- MimeTypes mimeTypes, ServiceLoader loader) {
+ MimeTypes mimeTypes, ServiceLoader loader) {
return loaded.get(0);
}
-
+
@Override
ConfigurableThreadPoolExecutor createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
return getDefaultExecutorService();
}
-
+
@Override
ConfigurableThreadPoolExecutor decorate(ConfigurableThreadPoolExecutor created, Element element)
throws IOException, TikaException {
-
+
Element maxThreadElement = getChild(element, "max-threads");
- if(maxThreadElement != null)
- {
+ if (maxThreadElement != null) {
created.setMaximumPoolSize(Integer.parseInt(getText(maxThreadElement)));
}
-
+
Element coreThreadElement = getChild(element, "core-threads");
- if(coreThreadElement != null)
- {
+ if (coreThreadElement != null) {
created.setCorePoolSize(Integer.parseInt(getText(coreThreadElement)));
}
return created;
}
-
+
@Override
Class<? extends ConfigurableThreadPoolExecutor> getLoaderClass() {
return ConfigurableThreadPoolExecutor.class;
}
-
+
@Override
ConfigurableThreadPoolExecutor loadOne(Element element, MimeTypes mimeTypes,
- ServiceLoader loader) throws TikaException, IOException {
+ ServiceLoader loader) throws TikaException, IOException {
return super.loadOne(element, mimeTypes, loader);
}
@Override
- boolean supportsComposite() {return false;}
+ boolean supportsComposite() {
+ return false;
+ }
@Override
- String getParentTagName() {return null;}
+ String getParentTagName() {
+ return null;
+ }
@Override
- String getLoaderTagName() {return "executor-service";}
+ String getLoaderTagName() {
+ return "executor-service";
+ }
@Override
- boolean isComposite(ConfigurableThreadPoolExecutor loaded) {return false;}
+ boolean isComposite(ConfigurableThreadPoolExecutor loaded) {
+ return false;
+ }
@Override
- boolean isComposite(Class<? extends ConfigurableThreadPoolExecutor> loadedClass) {return false;}
+ boolean isComposite(Class<? extends ConfigurableThreadPoolExecutor> loadedClass) {
+ return false;
+ }
@Override
ConfigurableThreadPoolExecutor preLoadOne(
@@ -1213,7 +1358,7 @@ public class TikaConfig {
@Override
MetadataFilter preLoadOne(Class<? extends MetadataFilter> loadedClass,
- String classname, MimeTypes mimeTypes) throws TikaException {
+ String classname, MimeTypes mimeTypes) throws TikaException {
// Check for classes which can't be set in config
// Continue with normal loading
return null;
@@ -1232,9 +1377,9 @@ public class TikaConfig {
@Override
MetadataFilter createComposite(Class<? extends MetadataFilter> metadataFilterClass,
- List<MetadataFilter> childMetadataFilters,
- Set<Class<? extends MetadataFilter>> excludeFilters,
- Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ List<MetadataFilter> childMetadataFilters,
+ Set<Class<? extends MetadataFilter>> excludeFilters,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
throws InvocationTargetException, IllegalAccessException,
InstantiationException {
MetadataFilter metadataFilter = null;
@@ -1267,4 +1412,234 @@ public class TikaConfig {
}
}
+ private static class FetcherXmlLoader extends
+ XmlLoader<FetcherManager, Fetcher> {
+
+ //FetcherManager is composite
+ boolean supportsComposite() {
+ return true;
+ }
+
+ String getParentTagName() {
+ return "fetchers";
+ }
+
+ String getLoaderTagName() {
+ return "fetcher";
+ }
+
+ @Override
+ Class<? extends Fetcher> getLoaderClass() {
+ return Fetcher.class;
+ }
+
+
+ @Override
+ boolean isComposite(Fetcher loaded) {
+ return false;
+ }
+
+ @Override
+ boolean isComposite(Class<? extends Fetcher> loadedClass) {
+ return false;
+ }
+
+ @Override
+ Fetcher preLoadOne(Class<? extends Fetcher> loadedClass,
+ String classname, MimeTypes mimeTypes) throws TikaException {
+ // Check for classes which can't be set in config
+ // Continue with normal loading
+ return null;
+ }
+
+ @Override
+ FetcherManager createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
+ return getDefaultFetcherManager(loader);
+ }
+
+ //this ignores the service loader
+ @Override
+ FetcherManager createComposite(List<Fetcher> loaded, MimeTypes mimeTypes, ServiceLoader loader) {
+ return new FetcherManager(loaded);
+ }
+
+ @Override
+ Fetcher createComposite(Class<? extends Fetcher> fetcherClass,
+ List<Fetcher> childFetchers,
+ Set<Class<? extends Fetcher>> excludeFilters,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ throws InvocationTargetException, IllegalAccessException,
+ InstantiationException {
+ throw new InstantiationException("Only one fetcher manager supported");
+
+ }
+
+ @Override
+ Fetcher decorate(Fetcher created, Element element) {
+ return created; // No decoration of MetadataFilters
+ }
+ }
+
+ private static class EmitterXmlLoader extends
+ XmlLoader<EmitterManager, Emitter> {
+
+ boolean supportsComposite() {
+ return true;
+ }
+
+ String getParentTagName() {
+ return "emitters";
+ }
+
+ String getLoaderTagName() {
+ return "emitter";
+ }
+
+ @Override
+ Class<? extends Emitter> getLoaderClass() {
+ return Emitter.class;
+ }
+
+
+ @Override
+ boolean isComposite(Emitter loaded) {
+ return false;
+ }
+
+ @Override
+ boolean isComposite(Class<? extends Emitter> loadedClass) {
+ return false;
+ }
+
+ @Override
+ Emitter preLoadOne(Class<? extends Emitter> loadedClass,
+ String classname, MimeTypes mimeTypes) throws TikaException {
+ // Check for classes which can't be set in config
+ // Continue with normal loading
+ return null;
+ }
+
+ @Override
+ EmitterManager createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
+ return getDefaultEmitter(loader);
+ }
+
+ private EmitterManager getDefaultEmitter(ServiceLoader loader) {
+ return new EmitterManager(Collections.EMPTY_LIST);
+ }
+
+ //this ignores the service loader
+ @Override
+ EmitterManager createComposite(List<Emitter> loaded, MimeTypes mimeTypes, ServiceLoader loader) {
+ return new EmitterManager(loaded);
+ }
+
+ @Override
+ Emitter createComposite(Class<? extends Emitter> emitterClass,
+ List<Emitter> childEmitters,
+ Set<Class<? extends Emitter>> excludeFilters,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ throws InvocationTargetException, IllegalAccessException,
+ InstantiationException {
+ Emitter emitter = null;
+ Constructor<? extends Emitter> c;
+
+ // Try the possible default and composite detector constructors
+ if (emitter == null) {
+ try {
+ c = emitterClass.getConstructor(ServiceLoader.class, Collection.class);
+ emitter = c.newInstance(loader, excludeFilters);
+ } catch (NoSuchMethodException me) {
+ me.printStackTrace();
+ }
+ }
+ if (emitter == null) {
+ try {
+ c = emitterClass.getConstructor(List.class);
+ emitter = c.newInstance(childEmitters);
+ } catch (NoSuchMethodException me) {
+ me.printStackTrace();
+ }
+ }
+
+ return emitter;
+ }
+
+ @Override
+ Emitter decorate(Emitter created, Element element) {
+ return created; // No decoration of emitters yet
+ }
+ }
+
+ private static class FetchIteratorXmlLoader extends
+ XmlLoader<FetchIterator, FetchIterator> {
+
+ boolean supportsComposite() {
+ return false;
+ }
+
+ String getParentTagName() {
+ return "fetchIterators";
+ }
+
+ String getLoaderTagName() {
+ return "fetchIterator";
+ }
+
+ @Override
+ Class<? extends FetchIterator> getLoaderClass() {
+ return FetchIterator.class;
+ }
+
+ @Override
+ boolean isComposite(FetchIterator loaded) {
+ return false;
+ }
+
+ @Override
+ boolean isComposite(Class<? extends FetchIterator> loadedClass) {
+ return false;
+ }
+
+ @Override
+ FetchIterator preLoadOne(Class<? extends FetchIterator> loadedClass,
+ String classname, MimeTypes mimeTypes) throws TikaException {
+ // Check for classes which can't be set in config
+ // Continue with normal loading
+ return null;
+ }
+
+ @Override
+ FetchIterator createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
+ return new EmptyFetchIterator();
+ }
+
+ //this ignores the service loader
+ @Override
+ FetchIterator createComposite(List<FetchIterator> loaded,
+ MimeTypes mimeTypes, ServiceLoader loader) {
+ if (loaded.size() > 1) {
+ throw new IllegalArgumentException(
+ "Can only specify one fetch iterator per config file");
+ }
+ return loaded.get(0);
+ }
+
+ @Override
+ FetchIterator createComposite(Class<? extends FetchIterator> fetchIteratorClass,
+ List<FetchIterator> fetchIteratorChildren,
+ Set<Class<? extends FetchIterator>> excludeFetchIterators,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ throws InvocationTargetException, IllegalAccessException,
+ InstantiationException {
+ return null;
+ }
+
+ @Override
+ FetchIterator decorate(FetchIterator created, Element element) {
+ return created; // No decoration of FetchIterators yet
+ }
+ }
+
+
}
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
index 29e80fd..a8e804a 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
@@ -113,12 +113,22 @@ public interface TikaCoreProperties {
/**
* Some file formats can store information about their original
- * file name/location or about their attachment's original file name/location.
+ * file name/location or about their attachment's original file name/location
+ * within the file.
*/
- public static final Property ORIGINAL_RESOURCE_NAME =
+ Property ORIGINAL_RESOURCE_NAME =
Property.internalTextBag(TIKA_META_PREFIX+"origResourceName");
/**
+ * This should be used to store the path (relative or full)
+ * of the source file, including the file name,
+ * e.g. doc/path/to/my_pdf.pdf
+ *
+ * This can also be used for a primary key within a database.
+ */
+ Property SOURCE_PATH =
+ Property.internalText(TIKA_META_PREFIX+"sourcePath");
+ /**
* This is currently used to identify Content-Type that may be
* included within a document, such as in html documents
* (e.g. <meta http-equiv="content-type" content="text/html; charset=UTF-8">)
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java
new file mode 100644
index 0000000..c64d784
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java
@@ -0,0 +1,82 @@
+package org.apache.tika.metadata.filter;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FieldNameMappingFilter implements MetadataFilter {
+ private static String MAPPING_OPERATOR = "->";
+
+ Map<String, String> mapping = new HashMap<>();
+
+ boolean excludeUnmapped = true;
+
+ @Override
+ public void filter(Metadata metadata) throws TikaException {
+ if (excludeUnmapped) {
+ for (String n : metadata.names()) {
+ if (mapping.containsKey(n)) {
+ String[] vals = metadata.getValues(n);
+ metadata.remove(n);
+ for (String val : vals) {
+ metadata.add(mapping.get(n), val);
+ }
+ } else {
+ mapping.remove(n);
+ }
+ }
+ } else {
+ for (String n : metadata.names()) {
+ if (mapping.containsKey(n)) {
+ String[] vals = metadata.getValues(n);
+ metadata.remove(n);
+ for (String val : vals) {
+ metadata.add(mapping.get(n), val);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * If this is <code>true</code> (default), this means that only the fields that
+ * have a "from" value in the mapper will be passed through. Otherwise,
+ * this will pass through all keys/values and mutate the keys
+ * that exist in the mappings.
+ * @param excludeUnmapped
+ */
+ @Field
+ public void setExcludeUnmapped(boolean excludeUnmapped) {
+ this.excludeUnmapped = excludeUnmapped;
+ }
+
+ @Field
+ public void setMappings(List<String> mappings) {
+ for (String m : mappings) {
+ String[] args = m.split(MAPPING_OPERATOR);
+ if (args.length == 0 || args.length == 1) {
+ throw new IllegalArgumentException(
+ "Can't find mapping operator '->' in: " + m);
+ } else if (args.length > 2) {
+ throw new IllegalArgumentException(
+ "Must have only one mapping operator. I found more than one: " + m
+ );
+ }
+ String from = args[0].trim();
+ if (from.length() == 0) {
+ throw new IllegalArgumentException("Must contain content before the "+
+ "mapping operator '->'");
+ }
+ String to = args[1].trim();
+ if (to.length() == 0) {
+ throw new IllegalArgumentException("Must contain content after the "+
+ "mapping operator '->'");
+ }
+ mapping.put(from, to);
+ }
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
similarity index 59%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
index c09b139..0aa9e7f 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
@@ -14,27 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.emitter;
-package org.apache.tika.server.core;
+import org.apache.tika.config.Field;
-import org.apache.tika.metadata.Metadata;
+public abstract class AbstractEmitter implements Emitter {
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.io.InputStream;
+ private String name;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
-
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
+ @Field
+ public void setName(String name) {
+ this.name = name;
}
-
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public String getName() {
+ return name;
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
similarity index 62%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
index c09b139..8f31311 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
@@ -14,27 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.emitter;
-package org.apache.tika.server.core;
-
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
-import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+public interface Emitter {
+
+ String getName();
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+ void emit(List<Metadata> metadataList) throws IOException, TikaException;
+ //TODO we can add this later?
+ //void emit(String txt, Metadata metadata) throws IOException, TikaException;
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitterManager.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitterManager.java
new file mode 100644
index 0000000..3c6ec70
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitterManager.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Utility class that will apply the appropriate fetcher
+ * to the fetcherString based on the prefix.
+ *
+ * This does not allow multiple fetchers supporting the same prefix.
+ */
+public class EmitterManager {
+
+ private final Map<String, Emitter> emitterMap = new ConcurrentHashMap<>();
+
+
+ public EmitterManager(List<Emitter> emitters) {
+ for (Emitter emitter : emitters) {
+ if (emitterMap.containsKey(emitter.getName())) {
+ throw new IllegalArgumentException(
+ "Multiple emitters cannot support the same name: "
+ + emitter.getName());
+ }
+ emitterMap.put(emitter.getName(), emitter);
+
+ }
+ }
+
+ public Set<String> getSupported() {
+ return emitterMap.keySet();
+ }
+
+
+ public Emitter getEmitter(String emitterName) {
+ Emitter emitter = emitterMap.get(emitterName);
+ if (emitter == null) {
+ throw new IllegalArgumentException("Can't find emitter for prefix: "+
+ emitterName);
+ }
+ return emitter;
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
similarity index 65%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
index c09b139..9a1a10a 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
@@ -14,27 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.emitter;
-package org.apache.tika.server.core;
-
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+import java.util.List;
+public class EmptyEmitter implements Emitter {
@Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public String getName() {
+ return "empty";
}
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public void emit(List<Metadata> metadataList) throws IOException, TikaException {
+
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/TikaEmitterException.java
similarity index 58%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/emitter/TikaEmitterException.java
index c09b139..00a5197 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/TikaEmitterException.java
@@ -14,27 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.emitter;
-package org.apache.tika.server.core;
+import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
-
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
-
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+public class TikaEmitterException extends TikaException {
+ public TikaEmitterException(String msg) {
+ super(msg);
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/AbstractFetcher.java
similarity index 59%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/fetcher/AbstractFetcher.java
index c09b139..0b417e3 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/AbstractFetcher.java
@@ -14,27 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.fetcher;
-package org.apache.tika.server.core;
+import org.apache.tika.config.Field;
-import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.io.InputStream;
+public abstract class AbstractFetcher implements Fetcher {
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+ private String name;
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public AbstractFetcher() {
+
+ }
+
+ public AbstractFetcher(String name) {
+ this.name = name;
}
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public String getName() {
+ return name;
+ }
+
+ @Field
+ public void setName(String name) {
+ this.name = name;
}
+
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
similarity index 67%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
index c09b139..5bc84b0 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
@@ -14,27 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.fetcher;
-package org.apache.tika.server.core;
-
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+public class EmptyFetcher implements Fetcher {
@Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public String getName() {
+ return "empty";
}
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
+ return null;
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchId.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchId.java
new file mode 100644
index 0000000..a327fc6
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchId.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+/**
+ * Pair of fetcherName (which fetcher to call) and the key
+ * to send to that fetcher to retrieve a specific file.
+ */
+public class FetchId {
+ private final String fetcherName;
+ private final String fetchKey;
+
+ public FetchId(String fetcherName, String fetchKey) {
+ this.fetcherName = fetcherName;
+ this.fetchKey = fetchKey;
+ }
+
+ public String getFetcherName() {
+ return fetcherName;
+ }
+
+ public String getFetchKey() {
+ return fetchKey;
+ }
+
+ @Override
+ public String toString() {
+ return "FetcherKeyPair{" +
+ "fetcherName='" + fetcherName + '\'' +
+ ", fetchKey='" + fetchKey + '\'' +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FetchId fetchId = (FetchId) o;
+
+ if (fetcherName != null ? !fetcherName.equals(fetchId.fetcherName) : fetchId.fetcherName != null) return false;
+ return fetchKey != null ? fetchKey.equals(fetchId.fetchKey) : fetchId.fetchKey == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = fetcherName != null ? fetcherName.hashCode() : 0;
+ result = 31 * result + (fetchKey != null ? fetchKey.hashCode() : 0);
+ return result;
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchIdMetadataPair.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchIdMetadataPair.java
new file mode 100644
index 0000000..25d89db
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchIdMetadataPair.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import org.apache.tika.metadata.Metadata;
+
+public class FetchIdMetadataPair {
+
+ private final FetchId fetchId;
+ private final Metadata metadata;
+
+ public FetchIdMetadataPair(FetchId fetchId, Metadata metadata) {
+ this.fetchId = fetchId;
+ this.metadata = metadata;
+ }
+
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ public FetchId getFetchId() {
+ return fetchId;
+ }
+
+ @Override
+ public String toString() {
+ return "FetchIdMetadataPair{" +
+ "fetchId=" + fetchId +
+ ", metadata=" + metadata +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FetchIdMetadataPair that = (FetchIdMetadataPair) o;
+
+ if (fetchId != null ? !fetchId.equals(that.fetchId) : that.fetchId != null) return false;
+ return metadata != null ? metadata.equals(that.metadata) : that.metadata == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = fetchId != null ? fetchId.hashCode() : 0;
+ result = 31 * result + (metadata != null ? metadata.hashCode() : 0);
+ return result;
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
similarity index 65%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
index c09b139..0450737 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
@@ -14,27 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.fetcher;
-package org.apache.tika.server.core;
-
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Set;
/**
- * Passthrough -- returns InputStream as is
+ * Interface for an object that will fetch an InputStream given
+ * a fetch string. This will also update the metadata object
+ * based on the fetch.
+ *
+ * Implementations of Fetcher must be thread safe.
+ *
*/
-public class DefaultInputStreamFactory implements InputStreamFactory {
+public interface Fetcher {
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+ String getName();
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+ InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException;
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherManager.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherManager.java
new file mode 100644
index 0000000..1ea46fd
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherManager.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Utility class to hold multiple fetchers.
+ * <p>
+ * This forbids multiple fetchers supporting the same prefix.
+ */
+public class FetcherManager {
+
+ private final Map<String, Fetcher> fetcherMap = new ConcurrentHashMap<>();
+
+ public FetcherManager(List<Fetcher> fetchers) {
+ for (Fetcher fetcher : fetchers) {
+ String name = fetcher.getName();
+ if (name == null || name.trim().length() == 0) {
+ throw new IllegalArgumentException("fetcher name must not be blank");
+ }
+ if (fetcherMap.containsKey(fetcher.getName())) {
+ throw new IllegalArgumentException(
+ "Multiple fetchers cannot support the same prefix: "
+ + fetcher.getName());
+ }
+ fetcherMap.put(fetcher.getName(), fetcher);
+ }
+ }
+
+ public Fetcher getFetcher(String fetcherName)
+ throws IOException, TikaException {
+ Fetcher fetcher = fetcherMap.get(fetcherName);
+ if (fetcher == null) {
+ throw new IllegalArgumentException("Can't find fetcher for fetcherName: " +
+ fetcherName +
+ ". I've loaded: "+fetcherMap.keySet());
+ }
+ return fetcher;
+ }
+
+ public Set<String> getSupported() {
+ return fetcherMap.keySet();
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherStringException.java
similarity index 58%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherStringException.java
index c09b139..a07439a 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetcherStringException.java
@@ -14,27 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.fetcher;
-package org.apache.tika.server.core;
-
-import org.apache.tika.metadata.Metadata;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.tika.exception.TikaException;
/**
- * Passthrough -- returns InputStream as is
+ * If something goes wrong in parsing the fetcher string
*/
-public class DefaultInputStreamFactory implements InputStreamFactory {
-
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+public class FetcherStringException extends TikaException {
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ public FetcherStringException(String msg) {
+ super(msg);
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FileSystemFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FileSystemFetcher.java
new file mode 100644
index 0000000..7bdf56d
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FileSystemFetcher.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+public class FileSystemFetcher extends AbstractFetcher implements Initializable {
+
+ private Path basePath = null;
+
+ @Override
+ public InputStream fetch(String fetchKey, Metadata metadata)
+ throws IOException, TikaException {
+ if (basePath == null) {
+ throw new IllegalStateException("must set 'basePath' before calling fetch");
+
+ }
+ if (fetchKey.contains("\u0000")) {
+ throw new IllegalArgumentException("Path must not contain \u0000. " +
+ "Please review the life decisions that led you to requesting " +
+ "a file name with this character in it.");
+ }
+ Path p = basePath.resolve(fetchKey);
+ if (!p.toRealPath().startsWith(basePath.toRealPath())) {
+ throw new IllegalArgumentException("fetchKey must resolve to be a" +
+ " descendant of the 'basePath'");
+ }
+
+ metadata.set(TikaCoreProperties.SOURCE_PATH, fetchKey);
+
+ if (!Files.isRegularFile(p)) {
+ if (!Files.isDirectory(basePath)) {
+ throw new IOException("BasePath is not a directory: " + basePath);
+ } else {
+ throw new FileNotFoundException(p.toAbsolutePath().toString());
+ }
+ }
+
+ return TikaInputStream.get(p, metadata);
+ }
+
+ /**
+ * If clients will send in relative paths, this
+ * must be set to allow this fetcher to fetch the
+ * full path.
+ *
+ * @param basePath
+ */
+ @Field
+ public void setBasePath(String basePath) {
+ this.basePath = Paths.get(basePath);
+ }
+
+ static boolean isDescendant(Path root, Path descendant) {
+ return descendant.toAbsolutePath().normalize().startsWith(
+ root.toAbsolutePath().normalize());
+ }
+
+ public Path getBasePath() {
+ return basePath;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //no-op
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ if (basePath == null || basePath.toString().trim().length() == 0) {
+ throw new IllegalArgumentException("'basePath' must be specified");
+ }
+ if (basePath.toAbsolutePath().toString().contains("\u0000")) {
+ throw new IllegalArgumentException("base path must not contain \u0000. " +
+ "Seriously, what were you thinking?");
+ }
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/SimpleUrlFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/SimpleUrlFetcher.java
new file mode 100644
index 0000000..40faadd
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/SimpleUrlFetcher.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * This is a lightweight fetcher that uses Java's
+ * {@link URL#openStream()}. Please consider a more
+ * robust way to fetch URLs, e.g. Apache httpcomponents,
+ * curl or wget...
+ *
+ * This is limited to http: and https: urls. This does
+ * not support the file:/// protocol. See {@link FileSystemFetcher}.
+ */
+public class SimpleUrlFetcher extends AbstractFetcher {
+
+ private static String NAME = "url";
+
+ public SimpleUrlFetcher() {
+ super(NAME);
+ }
+
+
+ @Override
+ public InputStream fetch(String fetchKey, Metadata metadata)
+ throws IOException, TikaException {
+ URL url = new URL(fetchKey);
+ if (! url.getProtocol().equals("http") &&
+ ! url.getProtocol().equals("https") &&
+ ! url.getProtocol().equals("ftp")) {
+ throw new TikaException("This fetcher only handles: http, https; NOT: "
+ + url.getProtocol());
+ }
+ return TikaInputStream.get(url, metadata);
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/EmptyFetchIterator.java
similarity index 61%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/EmptyFetchIterator.java
index c09b139..1a2d40e 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/EmptyFetchIterator.java
@@ -14,27 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.fetchiterator;
-package org.apache.tika.server.core;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
-import java.io.InputStream;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+public class EmptyFetchIterator extends FetchIterator {
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ protected void enqueue() throws IOException, TimeoutException, InterruptedException {
+
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
new file mode 100644
index 0000000..b11615f
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Abstract class that handles the testing for timeouts/thread safety
+ * issues. Concrete classes implement the blocking {@link #enqueue()}.
+ * <p>
+ * This must be "called", obviously...
+ */
+public abstract class FetchIterator implements Callable<Integer>, Initializable {
+
+ public static final long DEFAULT_MAX_WAIT_MS = 300_000;
+ public static final int DEFAULT_QUEUE_SIZE = 1000;
+ public static final FetchIdMetadataPair COMPLETED_SEMAPHORE =
+ new FetchIdMetadataPair(null, null);
+
+ private long maxWaitMs = DEFAULT_MAX_WAIT_MS;
+ private int numConsumers = -1;
+ private ArrayBlockingQueue<FetchIdMetadataPair> queue = null;
+ private int queueSize = DEFAULT_QUEUE_SIZE;
+ private String fetcherName;
+ private int added = 0;
+ public FetchIterator() {
+
+ }
+
+ public FetchIterator(String fetcherName) {
+ this.fetcherName = fetcherName;
+ }
+
+ /**
+ * This must be called before 'calling' this object.
+ * @param numConsumers
+ */
+ public ArrayBlockingQueue<FetchIdMetadataPair> init(int numConsumers) {
+ this.queue = new ArrayBlockingQueue<>(queueSize);
+ this.numConsumers = numConsumers;
+ return queue;
+ }
+
+ @Field
+ public void setFetcherName(String fetcherName) {
+ this.fetcherName = fetcherName;
+ }
+
+ public String getFetcherName() {
+ return fetcherName;
+ }
+
+
+ @Field
+ public void setMaxWaitMs(long maxWaitMs) {
+ this.maxWaitMs = maxWaitMs;
+ }
+
+ @Field
+ public void setQueueSize(int queueSize) {
+ this.queueSize = queueSize;
+ }
+ @Override
+ public Integer call() throws Exception {
+ if (queue == null || numConsumers < 0) {
+ throw new IllegalStateException("Must call 'init' before calling this object");
+ }
+
+ enqueue();
+ for (int i = 0; i < numConsumers; i++) {
+ try {
+ tryToAdd(COMPLETED_SEMAPHORE);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return added;
+ }
+
+ protected abstract void enqueue() throws IOException, TimeoutException, InterruptedException;
+
+ protected void tryToAdd(FetchIdMetadataPair p) throws InterruptedException, TimeoutException {
+ if (p != COMPLETED_SEMAPHORE) {
+ added++;
+ }
+ boolean offered = queue.offer(p, maxWaitMs, TimeUnit.MILLISECONDS);
+ if (!offered) {
+ throw new TimeoutException("timed out while offering");
+ }
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //no-op
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+ //no-op
+ }
+
+ protected static void mustNotBeEmpty(String paramName, String paramValue) {
+ if (paramValue == null || paramValue.trim().equals("")) {
+ throw new IllegalArgumentException("parameter '"+
+ paramName + "' must be set in the config file");
+ }
+ }
+
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
new file mode 100644
index 0000000..35e2ea8
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIterator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+public class FileSystemFetchIterator
+ extends FetchIterator implements Initializable {
+
+ private Path basePath;
+
+ public FileSystemFetchIterator() {
+
+ }
+
+ public FileSystemFetchIterator(String fetcherName, Path basePath) {
+ super(fetcherName);
+ this.basePath = basePath;
+ }
+
+ @Field
+ public void setBasePath(String basePath) {
+ this.basePath = Paths.get(basePath);
+ }
+
+ @Override
+ protected void enqueue() throws InterruptedException, IOException, TimeoutException {
+ if (! Files.isDirectory(basePath)) {
+ throw new IllegalArgumentException("\"basePath\" directory does not exist: " +
+ basePath.toAbsolutePath());
+ }
+ String fetcherName = getFetcherName();
+ try {
+ Files.walkFileTree(basePath, new FSFileVisitor(fetcherName));
+ } catch (IOException e) {
+ Throwable cause = e.getCause();
+ if (cause != null && cause instanceof TimeoutException) {
+ throw (TimeoutException) cause;
+ }
+ throw e;
+ }
+ }
+
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+
+ //ignore problem handler. These should all be fatal.
+ if (basePath == null) {
+ throw new TikaConfigException("Must specify a \"basePath\"");
+ }
+ if (getFetcherName() == null || getFetcherName().trim().length() == 0) {
+ throw new TikaConfigException("\"fetcherName\" must be specified and must be not blank");
+ }
+ }
+
+
+ private class FSFileVisitor implements FileVisitor<Path> {
+
+ private final String fetcherName;
+ private FSFileVisitor(String fetcherName) {
+ this.fetcherName = fetcherName;
+ }
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ String relPath = basePath.relativize(file).toString();
+
+ try {
+ tryToAdd(new FetchIdMetadataPair(
+ new FetchId(fetcherName, relPath), new Metadata()));
+ } catch (TimeoutException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ return FileVisitResult.TERMINATE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+ }
+
+
+}
diff --git a/tika-core/src/test/java/org/apache/tika/config/TikaConfigTest.java b/tika-core/src/test/java/org/apache/tika/config/TikaConfigTest.java
index 5ebfdf5..f2cbf73 100644
--- a/tika-core/src/test/java/org/apache/tika/config/TikaConfigTest.java
+++ b/tika-core/src/test/java/org/apache/tika/config/TikaConfigTest.java
@@ -43,6 +43,13 @@ import org.apache.tika.parser.ErrorParser;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.multiple.FallbackParser;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.EmitterManager;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetcher.FileSystemFetcher;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator;
import org.apache.tika.utils.XMLReaderUtils;
import org.junit.Test;
@@ -355,4 +362,64 @@ public class TikaConfigTest extends AbstractTikaConfigTest {
}
+ @Test
+ public void testFetchers() throws Exception {
+ TikaConfig config = getConfig("fetchers-config.xml");
+ FetcherManager m = config.getFetcherManager();
+ Fetcher f1 = m.getFetcher("fs1");
+ assertEquals(Paths.get("/my/base/path1"),
+ ((FileSystemFetcher)f1).getBasePath()
+ );
+
+ Fetcher f2 = m.getFetcher("fs2");
+ assertEquals(Paths.get("/my/base/path2"),
+ ((FileSystemFetcher)f2).getBasePath()
+ );
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDuplicateFetchers() throws Exception {
+ //can't have two fetchers with the same name
+ TikaConfig config = getConfig("fetchers-duplicate-config.xml");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNoNameFetchers() throws Exception {
+ //can't have two fetchers with the same name
+ TikaConfig config = getConfig("fetchers-noname-config.xml");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNoBasePathFetchers() throws Exception {
+ //can't have an fs fetcher with no basepath specified
+ TikaConfig config = getConfig("fetchers-nobasepath-config.xml");
+ }
+
+ @Test
+ public void testEmitters() throws Exception {
+ EmitterManager emitterManager = getConfig("emitters-config.xml").getEmitterManager();
+ Emitter em1 = emitterManager.getEmitter("em1");
+ assertNotNull(em1);
+ Emitter em2 = emitterManager.getEmitter("em2");
+ assertNotNull(em2);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDuplicateEmitters() throws Exception {
+ getConfig("emitters-duplicate-config.xml").getEmitterManager();
+ }
+
+ @Test
+ public void testFetchIterator() throws Exception {
+ FetchIterator f = getConfig("fetch-iterator-config.xml")
+ .getFetchIterator();
+ assertEquals("fs1", f.getFetcherName());
+ }
+
+ @Test(expected = TikaConfigException.class)
+ public void testMultipleFetchIterators() throws Exception {
+ FetchIterator f = getConfig("fetch-iterator-multiple-config.xml")
+ .getFetchIterator();
+ assertEquals("fs1", f.getFetcherName());
+ }
}
\ No newline at end of file
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-core/src/test/java/org/apache/tika/pipes/emitter/MockEmitter.java
similarity index 64%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-core/src/test/java/org/apache/tika/pipes/emitter/MockEmitter.java
index c09b139..f00d815 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/emitter/MockEmitter.java
@@ -14,27 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.pipes.emitter;
-package org.apache.tika.server.core;
-
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
-import java.io.InputStream;
+import java.util.List;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
+public class MockEmitter extends AbstractEmitter {
@Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+ public void emit(List<Metadata> metadataList) throws IOException, TikaException {
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
}
}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/fetcher/FileSystemFetcherTest.java b/tika-core/src/test/java/org/apache/tika/pipes/fetcher/FileSystemFetcherTest.java
new file mode 100644
index 0000000..60e14e3
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/pipes/fetcher/FileSystemFetcherTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import org.apache.tika.config.InitializableProblemHandler;
+import org.junit.Test;
+
+
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class FileSystemFetcherTest {
+
+ @Test
+ public void testDescendant() throws Exception {
+
+ Path root = Paths.get("/ab/cd/");
+ Path descendant = root.resolve("ef/gh/ij.pdf");
+ assertTrue(FileSystemFetcher.isDescendant(root, descendant));
+
+ descendant = Paths.get("/cd/ef.pdf");
+ assertFalse(FileSystemFetcher.isDescendant(root, descendant));
+
+ descendant = root.resolve("../../ij.pdf");
+ assertFalse(FileSystemFetcher.isDescendant(root, descendant));
+ }
+
+ @Test(expected = InvalidPathException.class)
+ public void testNullByte() throws Exception {
+ FileSystemFetcher f = new FileSystemFetcher();
+ f.setBasePath("bad\u0000path");
+ f.setName("fs");
+ f.checkInitialization(InitializableProblemHandler.IGNORE);
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
new file mode 100644
index 0000000..f23489a
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/pipes/fetchiterator/FileSystemFetchIteratorTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator;
+
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class FileSystemFetchIteratorTest {
+
+ @Test(timeout = 30000)
+ public void testBasic() throws Exception {
+ Path root = Paths.get(".");
+
+ List<Path> files = listFiles(root);
+ Set<String> truthSet = new HashSet<>();
+ for (Path p : files) {
+ String fetchString = root.relativize(p).toString();
+ truthSet.add(fetchString);
+ }
+
+ String fetcherName = "fs";
+ ExecutorService es = Executors.newFixedThreadPool(1);
+ ExecutorCompletionService cs = new ExecutorCompletionService(es);
+ FetchIterator it = new FileSystemFetchIterator(fetcherName, root);
+ it.setQueueSize(20000);
+ ArrayBlockingQueue<FetchIdMetadataPair> q = it.init(1);
+
+ cs.submit(it);
+
+
+ Future f = cs.take();
+ f.get();
+
+ Set<String> iteratorSet = new HashSet<>();
+ for (FetchIdMetadataPair p : q) {
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ break;
+ }
+ iteratorSet.add(p.getFetchId().getFetchKey());
+ }
+
+ assertEquals(truthSet, iteratorSet);
+ }
+
+ public static List<Path> listFiles(Path path) throws IOException {
+
+ List<Path> result;
+ try (Stream<Path> walk = Files.walk(path)) {
+ result = walk.filter(Files::isRegularFile)
+ .collect(Collectors.toList());
+ }
+ return result;
+
+ }
+}
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/emitters-config.xml
similarity index 60%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/emitters-config.xml
index 00284a5..73218c4 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/emitters-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,17 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <emitters>
+ <emitter class="org.apache.tika.pipes.emitter.MockEmitter">
+ <params>
+ <param name="name" type="string">em1</param>
+ </params>
+ </emitter>
+ <emitter class="org.apache.tika.pipes.emitter.MockEmitter">
+ <params>
+ <param name="name" type="string">em2</param>
+ </params>
+ </emitter>
+ </emitters>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/emitters-duplicate-config.xml
similarity index 60%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/emitters-duplicate-config.xml
index 00284a5..25dd7ce 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/emitters-duplicate-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,17 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <emitters>
+ <emitter class="org.apache.tika.pipes.emitter.MockEmitter">
+ <params>
+ <param name="name" type="string">em1</param>
+ </params>
+ </emitter>
+ <emitter class="org.apache.tika.pipes.emitter.MockEmitter">
+ <params>
+ <param name="name" type="string">em1</param>
+ </params>
+ </emitter>
+ </emitters>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-config.xml
similarity index 60%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-config.xml
index 00284a5..7f3dd41 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,13 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchIterators>
+ <fetchIterator class="org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator">
+ <params>
+ <param name="fetcherName" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetchIterator>
+ </fetchIterators>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-multiple-config.xml
similarity index 51%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-multiple-config.xml
index 00284a5..4417fbc 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetch-iterator-multiple-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,19 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchIterators>
+ <fetchIterator class="org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator">
+ <params>
+ <param name="fetcherName" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetchIterator>
+ <fetchIterator class="org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator">
+ <params>
+ <param name="fetcherName" type="string">fs2</param>
+ <param name="basePath" type="string">/my/base/path2</param>
+ </params>
+ </fetchIterator>
+ </fetchIterators>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetchers-config.xml
similarity index 54%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetchers-config.xml
index 00284a5..647c676 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetchers-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,19 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetcher>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs2</param>
+ <param name="basePath" type="string">/my/base/path2</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetchers-duplicate-config.xml
similarity index 54%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetchers-duplicate-config.xml
index 00284a5..314ed35 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetchers-duplicate-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,19 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetcher>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path2</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetchers-nobasepath-config.xml
similarity index 57%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetchers-nobasepath-config.xml
index 00284a5..de954cd 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetchers-nobasepath-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,18 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetcher>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs2</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
diff --git a/tika-server/tika-server-client/pom.xml b/tika-core/src/test/resources/org/apache/tika/config/fetchers-noname-config.xml
similarity index 56%
copy from tika-server/tika-server-client/pom.xml
copy to tika-core/src/test/resources/org/apache/tika/config/fetchers-noname-config.xml
index 00284a5..bb92c42 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/fetchers-noname-config.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,18 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs1</param>
+ <param name="basePath" type="string">/my/base/path1</param>
+ </params>
+ </fetcher>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="basePath" type="string">/my/base/path2</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 8fad29a..73eb653 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -274,6 +274,7 @@
<rat.version>0.13</rat.version>
<!-- dependency versions -->
+ <aws.version>1.11.937</aws.version>
<boilerpipe.version>1.1.0</boilerpipe.version>
<!-- used by POI, PDFBox and Jackcess ...try to sync -->
<bouncycastle.version>1.68</bouncycastle.version>
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/config/TikaTranslatorConfigTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/config/TikaTranslatorConfigTest.java
index 71af206..3c7e227 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/config/TikaTranslatorConfigTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/config/TikaTranslatorConfigTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.config;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.language.translate.DefaultTranslator;
import org.apache.tika.language.translate.EmptyTranslator;
import org.junit.Test;
@@ -58,9 +59,9 @@ public class TikaTranslatorConfigTest extends AbstractTikaConfigTest {
/**
* Currently, Translators don't support Composites, so
- * if multiple translators are given, only the first wins
+ * if multiple translators are given, throw a TikaConfigException
*/
- @Test
+ @Test(expected = TikaConfigException.class)
public void testRequestsMultiple() throws Exception {
TikaConfig config = getConfig("TIKA-1702-translator-empty-default.xml");
assertNotNull(config.getParser());
diff --git a/tika-server/pom.xml b/tika-pipes/pom.xml
similarity index 69%
copy from tika-server/pom.xml
copy to tika-pipes/pom.xml
index 6262b2f..b90d3c3 100644
--- a/tika-server/pom.xml
+++ b/tika-pipes/pom.xml
@@ -17,11 +17,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
- <modules>
- <module>tika-server-core</module>
- <module>tika-server-classic</module>
- <module>tika-server-client</module>
- </modules>
+
<parent>
<groupId>org.apache.tika</groupId>
@@ -30,22 +26,16 @@
<relativePath>../tika-parent/pom.xml</relativePath>
</parent>
- <artifactId>tika-server</artifactId>
- <name>Apache Tika server</name>
+ <artifactId>tika-pipes</artifactId>
+ <name>Apache Tika pipes</name>
<url>http://tika.apache.org/</url>
+ <modules>
+ <module>tika-httpclient-commons</module>
+ <module>tika-fetchers</module>
+ <module>tika-emitters</module>
+ <module>tika-fetch-iterators</module>
+ <module>tika-pipes-integration-tests</module>
+ </modules>
-
- <organization>
- <name>The Apache Software Foundation</name>
- <url>http://www.apache.org</url>
- </organization>
- <issueManagement>
- <system>JIRA</system>
- <url>https://issues.apache.org/jira/browse/TIKA</url>
- </issueManagement>
- <ciManagement>
- <system>Jenkins</system>
- <url>https://builds.apache.org/job/Tika-trunk/</url>
- </ciManagement>
</project>
diff --git a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml b/tika-pipes/tika-emitters/pom.xml
similarity index 50%
copy from tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
copy to tika-pipes/tika-emitters/pom.xml
index e1cb32b..bfb55c8 100644
--- a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
+++ b/tika-pipes/tika-emitters/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -17,9 +17,26 @@
specific language governing permissions and limitations
under the License.
-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-pipes</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
-<mock>
- <metadata action="add" name="author">Nikolai Lobachevsky</metadata>
- <metadata action="add" name="title">你好,世界</metadata>
- <write element="p">hello world</write>
-</mock>
\ No newline at end of file
+ <artifactId>tika-emitters</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Tika emitters</name>
+ <url>http://tika.apache.org/</url>
+
+ <modules>
+ <module>tika-emitter-fs</module>
+ <module>tika-emitter-solr</module>
+ </modules>
+
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
new file mode 100644
index 0000000..6c5c2cf
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-emitters</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-emitter-fs</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- should serialization be provided or bundled? -->
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-serialization</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/emitter/fs/FileSystemEmitter.java b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/emitter/fs/FileSystemEmitter.java
new file mode 100644
index 0000000..e42b57c
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/emitter/fs/FileSystemEmitter.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.emitter.fs;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+public class FileSystemEmitter implements Emitter {
+
+ private String name = "fs";
+ private Path basePath = null;
+ private String fileExtension = "json";
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void emit(List<Metadata> metadataList) throws IOException, TikaException {
+ Path output;
+ if (metadataList == null || metadataList.size() == 0) {
+ throw new TikaEmitterException("metadata list must not be null or of size 0");
+ }
+
+ String relPath = metadataList.get(0)
+ .get(TikaCoreProperties.SOURCE_PATH);
+ if (relPath == null) {
+ throw new TikaEmitterException("Must specify a "+TikaCoreProperties.SOURCE_PATH.getName() +
+ " in the metadata in order for this emitter to generate the output file path.");
+ }
+ if (fileExtension != null && fileExtension.length() > 0) {
+ relPath += "." + fileExtension;
+ }
+ if (basePath != null) {
+ output = basePath.resolve(relPath);
+ } else {
+ output = Paths.get(relPath);
+ }
+
+ if (!Files.isDirectory(output.getParent())) {
+ Files.createDirectories(output.getParent());
+ }
+ try (Writer writer = Files.newBufferedWriter(output, StandardCharsets.UTF_8)) {
+ JsonMetadataList.toJson(metadataList, writer);
+ }
+ }
+
+ @Field
+ public void setBasePath(String basePath) {
+ this.basePath = Paths.get(basePath);
+ }
+
+ /**
+ * If you want to customize the output file's file extension.
+ * Do not include the "."
+ * @param fileExtension
+ */
+ @Field
+ public void setFileExtension(String fileExtension) {
+ this.fileExtension = fileExtension;
+ }
+
+ /**
+ * Set this so to uniquely identify this emitter if
+ * there might be others available. The default is "fs"
+ * @param name
+ */
+ @Field
+ public void setName(String name) {
+ this.name = name;
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
new file mode 100644
index 0000000..6aefb89
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-emitters</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-emitter-solr</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-httpclient-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/emitter/solr/SolrEmitter.java
new file mode 100644
index 0000000..df78bb7
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/emitter/solr/SolrEmitter.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.emitter.solr;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import org.apache.tika.client.HttpClientUtil;
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public class SolrEmitter implements Emitter, Initializable {
+
+ enum AttachmentStrategy {
+ SKIP,
+ CONCATENATE_CONTENT,
+ PARENT_CHILD,
+ //anything else?
+ }
+ private static final Gson GSON = new Gson();
+ private static final String ATTACHMENTS = "attachments";
+ private static final String UPDATE_PATH = "/update";
+ private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
+
+ private String name = "solr";
+ private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
+ private String url;
+ private String contentField = "content";
+ private String idField = "id";
+ private int commitWithin = 100;
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void emit(List<Metadata> metadataList) throws IOException,
+ TikaException {
+ if (metadataList == null || metadataList.size() == 0) {
+ LOG.warn("metadataList is null or empty");
+ return;
+ }
+ String json = jsonify(metadataList);
+ LOG.debug("emitting json:"+json);
+ HttpClientUtil.postJson(url+UPDATE_PATH+"?commitWithin="+getCommitWithin(), json);
+ }
+
+ private String jsonify(List<Metadata> metadataList) {
+ if (attachmentStrategy == AttachmentStrategy.SKIP) {
+ return toJsonString(jsonify(metadataList.get(0)));
+ } else if (attachmentStrategy == AttachmentStrategy.CONCATENATE_CONTENT) {
+ //this only handles text for now, not xhtml
+ StringBuilder sb = new StringBuilder();
+ for (Metadata metadata : metadataList) {
+ String content = metadata.get(getContentField());
+ if (content != null) {
+ sb.append(content).append("\n");
+ }
+ }
+ Metadata parent = metadataList.get(0);
+ parent.set(getContentField(), sb.toString());
+ return toJsonString(jsonify(parent));
+ } else if (attachmentStrategy == AttachmentStrategy.PARENT_CHILD) {
+ if (metadataList.size() == 1) {
+ JsonObject obj = jsonify(metadataList.get(0));
+ return toJsonString(obj);
+ }
+ JsonObject parent = jsonify(metadataList.get(0));
+ JsonArray children = new JsonArray();
+ for (int i = 1; i < metadataList.size(); i++) {
+ Metadata m = metadataList.get(i);
+ m.set(idField, UUID.randomUUID().toString());
+ children.add(jsonify(m));
+ }
+ parent.add(ATTACHMENTS, children);
+ return toJsonString(parent);
+ } else {
+ throw new IllegalArgumentException("I don't yet support this attachment strategy: "
+ + attachmentStrategy);
+ }
+ }
+
+ private String toJsonString(JsonObject obj) {
+ //wrap the document into an array
+ //so that Solr correctly interprets this as
+ //upload docs vs a command.
+ JsonArray docs = new JsonArray();
+ docs.add(obj);
+ return GSON.toJson(docs);
+ }
+
+ private JsonObject jsonify(Metadata metadata) {
+ JsonObject obj = new JsonObject();
+ for (String n : metadata.names()) {
+ String[] vals = metadata.getValues(n);
+ if (vals.length == 0) {
+ continue;
+ } else if (vals.length == 1) {
+ obj.addProperty(n, vals[0]);
+ } else if (vals.length > 1) {
+ JsonArray valArr = new JsonArray();
+ for (int i = 0; i < vals.length; i++) {
+ valArr.add(vals[i]);
+ }
+ obj.add(n, valArr);
+ }
+ }
+ return obj;
+ }
+
+ /**
+ * Options: "skip", "concatenate-content", "parent-child". Default is "parent-child".
+ * If set to "skip", this will index only the main file and ignore all info
+ * in the attachments. If set to "concatenate", this will concatenate the
+ * content extracted from the attachments into the main document and
+ * then index the main document with the concatenated content _and_ the
+ * main document's metadata (metadata from attachments will be thrown away).
+ * If set to "parent-child", this will index the attachments as children
+ * of the parent document via Solr's parent-child relationship.
+ *
+ * @param attachmentStrategy
+ */
+ @Field
+ public void setAttachmentStrategy(String attachmentStrategy) {
+ if (attachmentStrategy.equals("skip")) {
+ this.attachmentStrategy = AttachmentStrategy.SKIP;
+ } else if (attachmentStrategy.equals("concatenate-content")) {
+ this.attachmentStrategy = AttachmentStrategy.CONCATENATE_CONTENT;
+ } else if (attachmentStrategy.equals("parent-child")) {
+ this.attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
+ } else {
+ throw new IllegalArgumentException("Expected 'skip', 'concatenate-content' or "+
+ "'parent-child'. I regret I do not recognize: " + attachmentStrategy);
+ }
+ }
+
+ @Field
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Specify the url for Solr
+ * @param url
+ */
+ @Field
+ public void setUrl(String url) {
+ if (url.endsWith("/")) {
+ url = url.substring(0, url.length()-1);
+ }
+ this.url = url;
+ }
+
+ /**
+ * This is the field _after_ metadata mappings have been applied
+ * that contains the "content" for each metadata object.
+ *
+ * This is the field that is used if {@link #attachmentStrategy}
+ * is {@link AttachmentStrategy#CONCATENATE_CONTENT}.
+ * @param contentField
+ */
+ @Field
+ public void setContentField(String contentField) {
+ this.contentField = contentField;
+ }
+
+ public String getContentField() {
+ return contentField;
+ }
+
+ @Field
+ public void setCommitWithin(int commitWithin) {
+ this.commitWithin = commitWithin;
+ }
+
+ public int getCommitWithin() {
+ return commitWithin;
+ }
+ //TODO: add username/password for authentication?
+
+ /**
+ * Specify the field in the first Metadata that should be
+ * used as the id field for the document.
+ *
+ * @param idField
+ */
+ @Field
+ public void setIdField(String idField) {
+ this.idField = idField;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //TODO: build the client here?
+
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+
+ }
+
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/emitter/solr/TestBasic.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/emitter/solr/TestBasic.java
new file mode 100644
index 0000000..3df4c4a
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/emitter/solr/TestBasic.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.emitter.solr;
+
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.filter.MetadataFilter;
+import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Ignore("requires solr to be up and running")
+public class TestBasic {
+
+ @Test
+ public void testBasic() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ TestBasic.class.getResourceAsStream("/tika-config-simple-emitter.xml"));
+ Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr1");
+ List<Metadata> metadataList = new ArrayList<>();
+ Metadata m1 = new Metadata();
+ m1.set("id", "1");
+ m1.set(Metadata.CONTENT_LENGTH, "314159");
+ m1.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, "the quick brown");
+ m1.set(TikaCoreProperties.TITLE, "this is the first title");
+ m1.add(TikaCoreProperties.CREATOR, "firstAuthor");
+ m1.add(TikaCoreProperties.CREATOR, "secondAuthor");
+
+ Metadata m2 = new Metadata();
+ m2.set(AbstractRecursiveParserWrapperHandler.EMBEDDED_RESOURCE_PATH, "/path_to_this.txt");
+ m2.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, "fox jumped over the lazy");
+ MetadataFilter filter = tikaConfig.getMetadataFilter();
+ filter.filter(m1);
+ filter.filter(m2);
+ metadataList.add(m1);
+ metadataList.add(m2);
+
+ emitter.emit(metadataList);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
new file mode 100644
index 0000000..92b6d56
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=debug,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
new file mode 100644
index 0000000..18c65c6
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<properties>
+ <metadataFilters>
+ <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
+ <params>
+ <param name="mappings" type="list">
+ <string>X-TIKA:content->content</string>
+ <string>X-TIKA:embedded_resource_path->embedded_path</string>
+ <string>Content-Length->length</string>
+ <string>dc:creator->creators</string>
+ <string>dc:title->title</string>
+ </param>
+ </params>
+ </metadataFilter>
+ </metadataFilters>
+ <emitters>
+ <emitter class="org.apache.tika.emitter.solr.SolrEmitter">
+ <params>
+ <param name="name" type="string">solr1</param>
+ <param name="url" type="string">http://localhost:8983/solr/tika-test</param>
+ <param name="attachmentStrategy" type="string">concatenate-content</param>
+ <param name="contentField" type="string">content</param>
+ <param name="commitWithin" type="int">10</param>
+ </params>
+ </emitter>
+ <emitter class="org.apache.tika.emitter.solr.SolrEmitter">
+ <params>
+ <param name="name" type="string">solr2</param>
+ <param name="url" type="string">http://localhost:8983/solr/tika-test</param>
+ <param name="attachmentStrategy" type="string">parent-child</param>
+ <param name="contentField" type="string">content</param>
+ <param name="commitWithin" type="int">10</param>
+ </params>
+ </emitter>
+ </emitters>
+</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetch-iterators/pom.xml b/tika-pipes/tika-fetch-iterators/pom.xml
new file mode 100644
index 0000000..38603af
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-pipes</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetch-iterators</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Tika fetch iterators</name>
+ <url>http://tika.apache.org/</url>
+
+ <!-- see also org.apache.tika.pipes.fetchiterator.FileSystemIterator
+ in tika-core if you want a file system directory crawler -->
+ <modules>
+ <module>tika-fetch-iterator-csv</module>
+ <module>tika-fetch-iterator-jdbc</module>
+ <module>tika-fetch-iterator-s3</module>
+ </modules>
+</project>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/pom.xml
new file mode 100644
index 0000000..bac6ace
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-fetch-iterators</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetch-iterator-csv</artifactId>
+
+ <name>Apache Tika Fetch Iterator - CSV</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>${commons.csv.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
new file mode 100644
index 0000000..e87f730
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator.csv;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+public class CSVFetchIterator extends FetchIterator implements Initializable {
+
+ private Charset charset = StandardCharsets.UTF_8;
+ private Path csvPath;
+ private String fetchKeyColumn;
+
+
+ @Field
+ public void setCsvPath(String csvPath) {
+ setCsvPath(Paths.get(csvPath));
+ }
+
+ @Field
+ public void setFetchKeyColumn(String fetchKeyColumn) {
+ this.fetchKeyColumn = fetchKeyColumn;
+ }
+
+ public void setCsvPath(Path csvPath) {
+ this.csvPath = csvPath;
+ }
+
+ @Override
+ protected void enqueue() throws InterruptedException, IOException, TimeoutException {
+ String fetcherName = getFetcherName();
+ try (Reader reader = Files.newBufferedReader(csvPath, charset)) {
+ Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(reader);
+ int fetchKeyIndex = -1;
+ List<String> headers = new ArrayList<>();
+ for (CSVRecord record : records) {
+ fetchKeyIndex = loadHeaders(record, headers);
+ break;
+ }
+ //check that if a user set a fetchKeyColumn, but we didn't find it
+ if (fetchKeyIndex < 0 && (fetchKeyColumn != null)) {
+ throw new IllegalArgumentException("Can't find " + fetchKeyColumn +
+ " in the csv. I see:"+
+ headers);
+ }
+ for (CSVRecord record : records) {
+ String fetchKey = "";
+ if (fetchKeyIndex > -1) {
+ fetchKey = record.get(fetchKeyIndex);
+ }
+ Metadata metadata = loadMetadata(fetchKeyIndex, headers, record);
+ tryToAdd(new FetchIdMetadataPair(new FetchId(fetcherName, fetchKey), metadata));
+ }
+ }
+ }
+
+ private Metadata loadMetadata(int fetchKeyIndex, List<String> headers, CSVRecord record) {
+ Metadata metadata = new Metadata();
+ for (int i = 0; i < record.size(); i++) {
+ if (fetchKeyIndex == i) {
+ continue;
+ }
+ metadata.set(headers.get(i), record.get(i));
+ }
+ return metadata;
+ }
+
+
+ private int loadHeaders(CSVRecord record, List<String> headers) {
+ int fetchKeyColumnIndex = -1;
+
+ for (int col = 0; col < record.size(); col++) {
+ String header = record.get(col);
+ headers.add(header);
+ if (header.equals(fetchKeyColumn)) {
+ fetchKeyColumnIndex = col;
+ }
+ }
+ return fetchKeyColumnIndex;
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
+ super.checkInitialization(problemHandler);
+ mustNotBeEmpty("csvPath", this.csvPath.toString());
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
new file mode 100644
index 0000000..2a6ab52
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/java/TestCSVFetchIterator.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.pipes.fetchiterator.csv.CSVFetchIterator;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestCSVFetchIterator {
+
+
+ @Test
+ public void testSimple() throws Exception {
+ Path p = get("test-simple.csv");
+ CSVFetchIterator it = new CSVFetchIterator();
+ it.setFetcherName("fs");
+ it.setCsvPath(p);
+ it.setFetchKeyColumn("fetchKey");
+ int numConsumers = 2;
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
+ ArrayBlockingQueue<FetchIdMetadataPair> queue = it.init(numConsumers);
+ ExecutorCompletionService c = new ExecutorCompletionService(es);
+ c.submit(it);
+ List<MockFetcher> fetchers = new ArrayList<>();
+ for (int i = 0; i < numConsumers; i++) {
+ MockFetcher f = new MockFetcher(queue);
+ fetchers.add(f);
+ c.submit(f);
+ }
+ int finished = 0;
+ int completed = 0;
+ try {
+ while (finished++ < numConsumers+1) {
+ Future<Integer> f = c.take();
+ completed += f.get();
+ }
+ } finally {
+ es.shutdownNow();
+ }
+ assertEquals(10, completed);
+ for (MockFetcher f : fetchers) {
+ for (FetchIdMetadataPair pair : f.pairs) {
+ String id = pair.getMetadata().get("id");
+ assertEquals("path/to/my/file"+id,
+ pair.getFetchId().getFetchKey());
+ assertEquals("project"+
+ (Integer.parseInt(id) % 2 == 1 ? "a" : "b"),
+ pair.getMetadata().get("project"));
+ }
+ }
+ }
+
+ @Test(expected = ExecutionException.class)
+ public void testBadFetchKeyCol() throws Exception {
+ Path p = get("test-simple.csv");
+ CSVFetchIterator it = new CSVFetchIterator();
+ it.setFetcherName("fs");
+ it.setCsvPath(p);
+ it.setFetchKeyColumn("fetchKeyDoesntExist");
+ ExecutorService es = Executors.newFixedThreadPool(2);
+ ExecutorCompletionService c = new ExecutorCompletionService(es);
+ c.submit(it);
+ c.submit(new MockFetcher(it.init(1)));
+ int finished = 0;
+ try {
+ while (finished++ < 2) {
+ Future f = c.take();
+ f.get();
+ }
+ } finally {
+ es.shutdownNow();
+ }
+
+ }
+
+ private Path get(String testFileName) throws Exception {
+ return Paths.get(TestCSVFetchIterator.class.getResource("/"+testFileName).toURI());
+ }
+
+ private static class MockFetcher implements Callable<Integer> {
+ private final ArrayBlockingQueue<FetchIdMetadataPair> queue;
+ private final List<FetchIdMetadataPair> pairs = new ArrayList<>();
+ private MockFetcher(ArrayBlockingQueue<FetchIdMetadataPair> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ while (true) {
+ FetchIdMetadataPair p = queue.poll(1, TimeUnit.HOURS);
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ return pairs.size();
+ }
+ pairs.add(p);
+ }
+ }
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/resources/test-simple.csv b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/resources/test-simple.csv
new file mode 100644
index 0000000..22630ef
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/test/resources/test-simple.csv
@@ -0,0 +1,6 @@
+id,project,fetchKey
+1,projecta,path/to/my/file1
+2,projectb,path/to/my/file2
+3,projecta,path/to/my/file3
+4,projectb,path/to/my/file4
+5,projecta,path/to/my/file5
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
new file mode 100644
index 0000000..14cd6f3
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-fetch-iterators</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetch-iterator-jdbc</artifactId>
+
+ <name>Apache Tika Fetch Iterator - jdbc</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>${h2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
new file mode 100644
index 0000000..9908681
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator.jdbc;
+
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+public class JDBCFetchIterator extends FetchIterator implements Initializable {
+
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(JDBCFetchIterator.class);
+
+ private String fetchKeyColumn;
+ private String connection;
+ private String select;
+
+ private Connection db;
+
+ @Field
+ public void setFetchKeyColumn(String fetchKeyColumn) {
+ this.fetchKeyColumn = fetchKeyColumn;
+ }
+
+ @Field
+ public void setConnection(String connection) {
+ this.connection = connection;
+ }
+
+ @Field
+ public void setSelect(String select) {
+ this.select = select;
+ }
+
+ @Override
+ protected void enqueue() throws InterruptedException, IOException, TimeoutException {
+ String fetcherName = getFetcherName();
+ int fetchKeyIndex = -1;
+ List<String> headers = new ArrayList<>();
+ int rowCount = 0;
+ LOGGER.debug("select: {}", select);
+ try (Statement st = db.createStatement()) {
+ try (ResultSet rs = st.executeQuery(select)) {
+ while (rs.next()) {
+ if (headers.size() == 0) {
+ fetchKeyIndex = loadHeaders(rs.getMetaData(), headers);
+ }
+ try {
+ processRow(fetcherName, headers, fetchKeyIndex, rs);
+ } catch (SQLException e) {
+ LOGGER.warn("Failed to insert: "+rs, e);
+ }
+ rowCount++;
+ if (rowCount % 1000 == 0) {
+ LOGGER.info("added "+rowCount + " rows to the queue");
+ }
+ }
+ }
+ } catch (SQLException e) {
+ LOGGER.warn("problem initializing connection and selecting", e);
+ throw new IOException(e);
+ } finally {
+ try {
+ db.close();
+ } catch (SQLException e){
+ LOGGER.warn("failed to close connection", e);
+ }
+ }
+ }
+
+ private void processRow(String fetcherName, List<String> headers,
+ int fetchKeyIndex, ResultSet rs)
+ throws SQLException, TimeoutException, InterruptedException {
+ Metadata metadata = new Metadata();
+ String fetchKey = null;
+ for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
+ if (i == fetchKeyIndex) {
+ fetchKey = getString(i, rs);
+ fetchKey = (fetchKey == null) ? "" : fetchKey;
+ continue;
+ }
+ String val = getString(i, rs);
+ if (val != null) {
+ metadata.set(headers.get(i - 1), val);
+ }
+ }
+
+ tryToAdd(new FetchIdMetadataPair(new FetchId(fetcherName, fetchKey), metadata));
+ }
+
+ private String getString(int i, ResultSet rs) throws SQLException {
+ //TODO: improve this later with special handling for numerals/dates/timestamps, etc
+ String val = rs.getString(i);
+ if (rs.wasNull()) {
+ return null;
+ }
+ return val;
+ }
+
+
+ private int loadHeaders(ResultSetMetaData metaData, List<String> headers) throws SQLException {
+ int fetchKeyIndex = -1;
+ for (int i = 1; i <= metaData.getColumnCount(); i++) {
+ if (metaData.getColumnLabel(i).equalsIgnoreCase(fetchKeyColumn)) {
+ fetchKeyIndex = i;
+ }
+ headers.add(metaData.getColumnLabel(i));
+ }
+ return fetchKeyIndex;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ try {
+ db = DriverManager.getConnection(connection);
+ } catch (SQLException e) {
+ throw new TikaConfigException("couldn't connect to db", e);
+ }
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
+ super.checkInitialization(problemHandler);
+ mustNotBeEmpty("connection", this.connection);
+ mustNotBeEmpty("select", this.select);
+ if (StringUtils.isEmpty(fetchKeyColumn)) {
+ LOGGER.info("no fetch key column has been specified");
+ }
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
new file mode 100644
index 0000000..fb0277b
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/java/org/apache/tika/pipes/fetchiterator/jdbc/TestJDBCFetchIterator.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator.jdbc;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+public class TestJDBCFetchIterator {
+
+ static final String TABLE = "fetchkeys";
+ static Connection CONNECTION;
+ static Path DB_DIR;
+ static final String db = "mydb";
+ private static final int NUM_ROWS = 1000;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ DB_DIR = Files.createTempDirectory("tika-jdbc-fetchiterator-test-");
+
+ CONNECTION = DriverManager.getConnection("jdbc:h2:file:"+DB_DIR.toAbsolutePath()+"/"+db);
+ String sql = "create table "+TABLE +
+ " (id varchar(128), " +
+ "project varchar(128), " +
+ "fetchKey varchar(128))";
+ CONNECTION.createStatement().execute(sql);
+
+ for (int i = 0; i < NUM_ROWS; i++) {
+ sql = "insert into "+TABLE + " (id, project, fetchKey) values ('id"+i+"','project"+
+ (i%2 == 0 ? "a" : "b") +"','fk"+i+"')";
+ CONNECTION.createStatement().execute(sql);
+ }
+ sql = "select count(1) from "+ TABLE;
+ ResultSet rs = CONNECTION.createStatement().executeQuery(sql);
+ while (rs.next()) {
+ int cnt = rs.getInt(1);
+ assertEquals(NUM_ROWS, cnt);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ FileUtils.deleteDirectory(DB_DIR.toFile());
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ TikaConfig tk = getConfig();
+ int numConsumers = 5;
+ FetchIterator fetchIterator = tk.getFetchIterator();
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
+ ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
+ ArrayBlockingQueue<FetchIdMetadataPair> queue = fetchIterator.init(numConsumers);
+ completionService.submit(fetchIterator);
+ List<MockFetcher> fetchers = new ArrayList<>();
+ for (int i = 0; i < numConsumers; i++) {
+ MockFetcher mockFetcher = new MockFetcher(queue);
+ fetchers.add(mockFetcher);
+ completionService.submit(mockFetcher);
+ }
+ int processed = 0;
+ int completed = 0;
+ while (completed < numConsumers+1) {
+ Future<Integer> f = completionService.take();
+ processed += f.get();
+ completed++;
+ }
+ //fetchiterator added + MockFetcher yields 2x
+ assertEquals(NUM_ROWS * 2, processed);
+ int cnt = 0;
+ Matcher m = Pattern.compile("fk(\\d+)").matcher("");
+ for (MockFetcher f : fetchers) {
+ for (FetchIdMetadataPair p : f.pairs) {
+ String k = p.getFetchId().getFetchKey();
+ String num = "";
+ if (m.reset(k).find()) {
+ num = m.group(1);
+ } else {
+ fail("failed to find key pattern: "+k);
+ }
+ String aOrB = Integer.parseInt(num) % 2 == 0 ? "a" : "b";
+ assertEquals("id"+num, p.getMetadata().get("MY_ID"));
+ assertEquals("project"+aOrB, p.getMetadata().get("MY_PROJECT"));
+ assertNull(p.getMetadata().get("fetchKey"));
+ assertNull(p.getMetadata().get("MY_FETCHKEY"));
+ cnt++;
+ }
+ }
+ assertEquals(NUM_ROWS, cnt);
+ }
+
+ private TikaConfig getConfig() throws Exception {
+ String config = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?><properties>\n" +
+ " <fetchIterators>\n" +
+ " <fetchIterator class=\"org.apache.tika.pipes.fetchiterator.jdbc.JDBCFetchIterator\">\n" +
+ " <params>\n" +
+ " <param name=\"fetcherName\" type=\"string\">s3</param>\n" +
+ " <param name=\"queueSize\" type=\"int\">57</param>\n" +
+ " <param name=\"fetchKeyColumn\" type=\"string\">my_fetchkey</param>\n" +
+ " <param name=\"select\" type=\"string\">" +
+ "select id as my_id, project as my_project, fetchKey as my_fetchKey from fetchkeys</param>\n" +
+ " <param name=\"connection\" type=\"string\">jdbc:h2:file:"+
+ DB_DIR.toAbsolutePath()+"/"+db +"</param>\n" +
+ " </params>\n" +
+ " </fetchIterator>\n" +
+ " </fetchIterators>\n" +
+ "</properties>";
+ return new TikaConfig(new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static class MockFetcher implements Callable<Integer> {
+ private final ArrayBlockingQueue<FetchIdMetadataPair> queue;
+ private final List<FetchIdMetadataPair> pairs = new ArrayList<>();
+ private MockFetcher(ArrayBlockingQueue<FetchIdMetadataPair> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ while (true) {
+ FetchIdMetadataPair p = queue.poll(1, TimeUnit.HOURS);
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ return pairs.size();
+ }
+ pairs.add(p);
+ }
+ }
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
new file mode 100644
index 0000000..1bee240
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=info,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
new file mode 100644
index 0000000..b0cce91
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/pom.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-fetch-iterators</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetch-iterator-s3</artifactId>
+
+ <name>Apache Tika Fetch Iterator - S3</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons.logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
new file mode 100644
index 0000000..0fd88b6
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator.s3;
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.iterable.S3Objects;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+public class S3FetchIterator extends FetchIterator implements Initializable {
+
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(S3FetchIterator.class);
+ private String s3PathPrefix = "";
+ private String region;
+ private String profile;
+ private String bucket;
+
+ private AmazonS3 s3Client;
+
+ @Field
+ public void setBucket(String bucket) {
+ this.bucket = bucket;
+ }
+
+ @Field
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ @Field
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ @Field
+ public void setS3PathPrefix(String s3PathPrefix) {
+ this.s3PathPrefix = s3PathPrefix;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //params have already been set
+ //ignore them
+
+ s3Client = AmazonS3ClientBuilder.standard()
+ .withRegion(region)
+ .withCredentials(new ProfileCredentialsProvider(profile))
+ .build();
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler)
+ throws TikaConfigException {
+ super.checkInitialization(problemHandler);
+ mustNotBeEmpty("bucket", this.bucket);
+ mustNotBeEmpty("profile", this.profile);
+ mustNotBeEmpty("region", this.region);
+ }
+
+ @Override
+ protected void enqueue() throws InterruptedException, IOException, TimeoutException {
+ String fetcherName = getFetcherName();
+ long start = System.currentTimeMillis();
+ int count = 0;
+ for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, s3PathPrefix)) {
+ long elapsed = System.currentTimeMillis() - start;
+ LOGGER.debug("adding ({}) {} in {} ms", count, bucket+"/"+summary.getKey(),
+ elapsed);
+ tryToAdd( new FetchIdMetadataPair(
+ new FetchId(fetcherName, bucket+"/"+summary.getKey()),
+ new Metadata()));
+ count++;
+ }
+ long elapsed = System.currentTimeMillis() - start;
+ LOGGER.info("finished enqueuing {} files in {} ms", count, elapsed);
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
new file mode 100644
index 0000000..1984577
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/java/org/apache/tika/pipes/fetchiterator/s3/TestS3FetchIterator.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchiterator.s3;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+@Ignore("turn into an actual unit test")
+public class TestS3FetchIterator {
+
+
+ @Test
+ public void testSimple() throws Exception {
+ S3FetchIterator it = new S3FetchIterator();
+ it.setFetcherName("s3");
+ it.setBucket("");//find one
+ it.setProfile("");//use one
+ it.setRegion("");//select one
+ it.initialize(Collections.EMPTY_MAP);
+ int numConsumers = 6;
+ ArrayBlockingQueue<FetchIdMetadataPair> queue = it.init(numConsumers);
+
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers+1);
+ ExecutorCompletionService c = new ExecutorCompletionService(es);
+ c.submit(it);
+ List<MockFetcher> fetchers = new ArrayList<>();
+ for (int i = 0; i < numConsumers; i++) {
+ MockFetcher fetcher = new MockFetcher(queue);
+ fetchers.add(fetcher);
+ c.submit(fetcher);
+ }
+ int finished = 0;
+ int completed = 0;
+ try {
+ while (finished++ < numConsumers+1) {
+ Future<Integer> f = c.take();
+ completed += f.get();
+ }
+ } finally {
+ es.shutdownNow();
+ }
+ assertEquals(20000, completed);
+
+ }
+
+ private static class MockFetcher implements Callable<Integer> {
+ private final ArrayBlockingQueue<FetchIdMetadataPair> queue;
+ private final List<FetchIdMetadataPair> pairs = new ArrayList<>();
+ private MockFetcher(ArrayBlockingQueue<FetchIdMetadataPair> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ while (true) {
+ FetchIdMetadataPair p = queue.poll(1, TimeUnit.HOURS);
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ return pairs.size();
+ }
+ pairs.add(p);
+ }
+ }
+ }
+}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
new file mode 100644
index 0000000..1bee240
--- /dev/null
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=info,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml b/tika-pipes/tika-fetchers/pom.xml
similarity index 51%
copy from tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
copy to tika-pipes/tika-fetchers/pom.xml
index e1cb32b..597c76b 100644
--- a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
+++ b/tika-pipes/tika-fetchers/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -17,9 +17,24 @@
specific language governing permissions and limitations
under the License.
-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-pipes</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
-<mock>
- <metadata action="add" name="author">Nikolai Lobachevsky</metadata>
- <metadata action="add" name="title">你好,世界</metadata>
- <write element="p">hello world</write>
-</mock>
\ No newline at end of file
+ <artifactId>tika-fetchers</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Tika fetchers</name>
+ <url>http://tika.apache.org/</url>
+
+ <modules>
+ <module>tika-fetcher-s3</module>
+ </modules>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
new file mode 100644
index 0000000..1d561aa
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-fetchers</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetcher-s3</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons.logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Automatic-Module-Name>org.apache.tika.fetcher.s3</Automatic-Module-Name>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.version}</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>
+ false
+ </createDependencyReducedPom>
+ <!-- <filters> -->
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*</exclude>
+ <exclude>LICENSE.txt</exclude>
+ <exclude>NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.tika.eval.app.TikaEvalCLI</mainClass>
+ </transformer>
+
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/LICENSE</resource>
+ <file>target/classes/META-INF/LICENSE</file>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/NOTICE</resource>
+ <file>target/classes/META-INF/NOTICE</file>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/DEPENDENCIES</resource>
+ <file>target/classes/META-INF/DEPENDENCIES</file>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
new file mode 100644
index 0000000..126f8fa
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.s3;
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import org.apache.commons.io.IOUtils;
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.pipes.fetcher.AbstractFetcher;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.pipes.fetcher.FetcherStringException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.CopyOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
+ * This will parse the bucket out of that string and retrieve the path.
+ */
+public class S3Fetcher extends AbstractFetcher implements Initializable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(S3Fetcher.class);
+ private static final String PREFIX = "s3";
+ private String region;
+ private String profile;
+ private boolean extractUserMetadata = true;
+ private AmazonS3 s3Client;
+ private boolean spoolToTemp = true;
+
+ @Override
+ public InputStream fetch(String fetchKey, Metadata metadata)
+ throws TikaException, IOException {
+
+ LOGGER.debug("about to fetch fetchkey={}", fetchKey);
+ final String origFetchKey = fetchKey;
+ if (fetchKey.startsWith("//")) {
+ fetchKey = fetchKey.substring(2);
+ } else if (fetchKey.startsWith("/")) {
+ fetchKey = fetchKey.substring(1);
+ }
+ int i = fetchKey.indexOf("/");
+ if (i < 0) {
+ throw new FetcherStringException("Couldn't find bucket:" +
+ origFetchKey);
+ }
+ String bucket = fetchKey.substring(0, i);
+ String key = fetchKey.substring(i + 1);
+ LOGGER.debug("about to fetch bucket: ({}); key: ({})", bucket, key);
+ S3Object fullObject = s3Client.getObject(new GetObjectRequest(bucket, key));
+ if (extractUserMetadata) {
+ for (Map.Entry<String, String> e :
+ fullObject.getObjectMetadata().getUserMetadata().entrySet()) {
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ }
+ }
+ if (! spoolToTemp) {
+ return TikaInputStream.get(
+ fullObject.getObjectContent());
+ } else {
+ long start = System.currentTimeMillis();
+ TikaInputStream tis = TikaInputStream.get(
+ fullObject.getObjectContent());
+ tis.getPath();
+ long elapsed = System.currentTimeMillis()-start;
+ LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
+ return tis;
+ }
+ }
+
+ @Field
+ public void setSpoolToTemp(boolean spoolToTemp) {
+ this.spoolToTemp = spoolToTemp;
+ }
+
+ @Field
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ @Field
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Whether or not to extract user metadata from the S3Object
+ *
+ * @param extractUserMetadata
+ */
+ @Field
+ public void setExtractUserMetadata(boolean extractUserMetadata) {
+ this.extractUserMetadata = extractUserMetadata;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //params have already been set
+ //ignore them
+ s3Client = AmazonS3ClientBuilder.standard()
+ .withRegion(region)
+ .withCredentials(new ProfileCredentialsProvider(profile))
+ .build();
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
new file mode 100644
index 0000000..450d933
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.s3;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.metadata.Metadata;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+
+@Ignore("write actual unit tests")
+public class TestS3Fetcher {
+ private static final String FETCH_STRING = "";
+ private Path outputFile = Paths.get("");
+ private String region = "us-east-1";
+ private String profile = "";
+
+ @Test
+ public void testBasic() throws Exception {
+ S3Fetcher fetcher = new S3Fetcher();
+ fetcher.setProfile(profile);
+ fetcher.setRegion(region);
+ fetcher.initialize(Collections.EMPTY_MAP);
+
+ Metadata metadata = new Metadata();
+ try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
+ Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ TikaConfig config = new TikaConfig(
+ this.getClass().getResourceAsStream("/tika-config-s3.xml")
+ );
+ Fetcher fetcher = config.getFetcherManager().getFetcher("s3");
+ Metadata metadata = new Metadata();
+ try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
+ Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+}
diff --git a/tika-server/tika-server-client/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
similarity index 60%
copy from tika-server/tika-server-client/pom.xml
copy to tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
index 00284a5..5c1de6c 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/resources/tika-config-s3.xml
@@ -7,7 +7,7 @@
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,15 +15,14 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>2.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
-
-
-</project>
\ No newline at end of file
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.fetcher.s3.S3Fetcher">
+ <params>
+ <param name="name" type="string">s3</param>
+ <param name="region" type="string">us-east-1</param>
+ <param name="profile" type="string">my_profile</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-httpclient-commons/pom.xml b/tika-pipes/tika-httpclient-commons/pom.xml
new file mode 100644
index 0000000..ef7797a
--- /dev/null
+++ b/tika-pipes/tika-httpclient-commons/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-pipes</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-httpclient-commons</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
new file mode 100644
index 0000000..041e7af
--- /dev/null
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
@@ -0,0 +1,38 @@
+package org.apache.tika.client;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.BasicHttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class HttpClientUtil {
+
+ private static HttpClient CLIENT = HttpClients.createDefault();
+
+ public static boolean postJson(String url, String json) throws IOException,
+ TikaClientException {
+ HttpPost post = new HttpPost(url);
+ ByteArrayEntity entity = new ByteArrayEntity(json.getBytes(StandardCharsets.UTF_8));
+ post.setEntity(entity);
+ post.setHeader("Content-Type", "application/json");
+ HttpResponse response = CLIENT.execute(post);
+
+
+ if (response.getStatusLine().getStatusCode() != 200) {
+ String msg = EntityUtils.toString(response.getEntity());
+ throw new TikaClientException("Bad status: " +
+ response.getStatusLine().getStatusCode() + " : "+
+ msg);
+ } else {
+ String msg = EntityUtils.toString(response.getEntity());
+ System.out.println("httputil: " + msg);
+ }
+ return true;
+ }
+}
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/TikaClientException.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/TikaClientException.java
new file mode 100644
index 0000000..1827f3e
--- /dev/null
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/TikaClientException.java
@@ -0,0 +1,14 @@
+package org.apache.tika.client;
+
+
+import org.apache.tika.exception.TikaException;
+
+public class TikaClientException extends TikaException {
+ public TikaClientException(String msg) {
+ super(msg);
+ }
+
+ public TikaClientException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
diff --git a/tika-pipes/tika-pipes-integration-tests/pom.xml b/tika-pipes/tika-pipes-integration-tests/pom.xml
new file mode 100644
index 0000000..ccb138c
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-pipes</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-pipes-integration-tests</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-fetcher-s3</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-fetch-iterator-s3</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
new file mode 100644
index 0000000..590a125
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.pipes;
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.iterable.S3Objects;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Ignore("turn these into actual tests")
+public class PipeIntegrationTests {
+
+ private static final Path OUTDIR = Paths.get("");
+
+ @Test
+ public void testBruteForce() throws Exception {
+ String region = "";
+ String profile = "";
+ String bucket = "";
+ AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
+ .withRegion(region)
+ .withCredentials(new ProfileCredentialsProvider(profile))
+ .build();
+ s3Client.listObjects(bucket);
+ int cnt = 0;
+ long sz = 0;
+
+ for (S3ObjectSummary summary : S3Objects.withPrefix(s3Client, bucket, "")) {
+ Path targ = OUTDIR.resolve(summary.getKey());
+ if (Files.isRegularFile(targ)) {
+ continue;
+ }
+ if (! Files.isDirectory(targ.getParent())) {
+ Files.createDirectories(targ.getParent());
+ }
+ System.out.println("id: " + cnt + " :: " + summary.getKey() + " : " + summary.getSize());
+ S3Object s3Object = s3Client.getObject(bucket, summary.getKey());
+ Files.copy(s3Object.getObjectContent(), targ);
+ summary.getSize();
+ cnt++;
+ sz += summary.getSize();
+ }
+ System.out.println("iterated: "+cnt + " sz: "+sz);
+ }
+
+ @Test
+ public void testS3ToFS() throws Exception {
+ TikaConfig tikaConfig = getConfig("tika-config-s3ToFs.xml");
+ FetchIterator it = tikaConfig.getFetchIterator();
+ int numConsumers = 1;
+ ExecutorService es = Executors.newFixedThreadPool(numConsumers + 1);
+ ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(es);
+ ArrayBlockingQueue<FetchIdMetadataPair> queue = it.init(numConsumers);
+ completionService.submit(it);
+ for (int i = 0; i < numConsumers; i++) {
+ completionService.submit(new FetcherEmitter(
+ queue, tikaConfig.getFetcherManager().getFetcher("s3"), null));
+ }
+ int finished = 0;
+ try {
+ while (finished++ < numConsumers+1) {
+ Future<Integer> future = completionService.take();
+ future.get();
+ }
+ } finally {
+ es.shutdownNow();
+ }
+
+
+ }
+
+ private TikaConfig getConfig(String fileName) throws Exception {
+ try (InputStream is =
+ PipeIntegrationTests.class.getResourceAsStream("/"+fileName)) {
+ return new TikaConfig(is);
+ }
+ }
+
+
+ private static class FetcherEmitter implements Callable<Integer> {
+ private static final AtomicInteger counter = new AtomicInteger(0);
+
+ private final Fetcher fetcher;
+ private final Emitter emitter;
+ private final ArrayBlockingQueue<FetchIdMetadataPair> queue;
+
+ FetcherEmitter(ArrayBlockingQueue<FetchIdMetadataPair> queue, Fetcher
+ fetcher, Emitter emitter) {
+ this.queue = queue;
+ this.fetcher = fetcher;
+ this.emitter = emitter;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+
+ while (true) {
+ FetchIdMetadataPair p = queue.poll(5, TimeUnit.MINUTES);
+ if (p == null) {
+ throw new TimeoutException("");
+ }
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ return 1;
+ }
+ process(p);
+ }
+ }
+
+ private void process(FetchIdMetadataPair p) throws IOException, TikaException {
+ Path targ = OUTDIR.resolve(p.getFetchId().getFetchKey());
+ if (Files.isRegularFile(targ)) {
+ return;
+ }
+ try (InputStream is = fetcher.fetch(p.getFetchId().getFetchKey(), p.getMetadata())) {
+ System.out.println(counter.getAndIncrement() + " : "+p );
+ Files.createDirectories(targ.getParent());
+ Files.copy(is, targ);
+ }
+ }
+ }
+}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..1bee240
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=info,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern=%-5p [%t]: %m%n
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/tika-config-s3ToFs.xml b/tika-pipes/tika-pipes-integration-tests/src/test/resources/tika-config-s3ToFs.xml
new file mode 100644
index 0000000..3e5bbc5
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/tika-config-s3ToFs.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.s3.S3Fetcher">
+ <params>
+ <param name="name" type="string">s3</param>
+ <param name="region" type="string">us-east-1</param>
+ <param name="profile" type="string"><!-- fill in here --></param>
+ </params>
+ </fetcher>
+ </fetchers>
+ <fetchIterators>
+ <fetchIterator class="org.apache.tika.pipes.fetchiterator.s3.S3FetchIterator">
+ <params>
+ <param name="fetcherName" type="string">s3</param>
+ <param name="bucket" type="string"><!-- fill in here --></param>
+ <param name="region" type="string">us-east-1</param>
+ <param name="profile" type="string"><!-- fill in here --></param>
+ </params>
+ </fetchIterator>
+ </fetchIterators>
+</properties>
\ No newline at end of file
diff --git a/tika-server/pom.xml b/tika-server/pom.xml
index 6262b2f..f35927b 100644
--- a/tika-server/pom.xml
+++ b/tika-server/pom.xml
@@ -34,18 +34,4 @@
<name>Apache Tika server</name>
<url>http://tika.apache.org/</url>
-
-
- <organization>
- <name>The Apache Software Foundation</name>
- <url>http://www.apache.org</url>
- </organization>
- <issueManagement>
- <system>JIRA</system>
- <url>https://issues.apache.org/jira/browse/TIKA</url>
- </issueManagement>
- <ciManagement>
- <system>Jenkins</system>
- <url>https://builds.apache.org/job/Tika-trunk/</url>
- </ciManagement>
</project>
diff --git a/tika-server/tika-server-classic/pom.xml b/tika-server/tika-server-classic/pom.xml
index fbd9071..a212671 100644
--- a/tika-server/tika-server-classic/pom.xml
+++ b/tika-server/tika-server-classic/pom.xml
@@ -53,7 +53,11 @@
<artifactId>tika-xmp</artifactId>
<version>${project.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-emitter-fs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- test jars -->
<dependency>
<groupId>${project.groupId}</groupId>
@@ -100,7 +104,6 @@
<artifactSet>
<excludes>
<exclude>org.apache.tika:tika-parsers-classic-package:jar:</exclude>
- <exclude>org.apache.tika:tika-server-core:jar:</exclude>
</excludes>
</artifactSet>
<filters>
diff --git a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/FetcherTest.java b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/FetcherTest.java
new file mode 100644
index 0000000..2343d4a
--- /dev/null
+++ b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/FetcherTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.server.classic;
+
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.apache.tika.TikaTest;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
+import org.apache.tika.server.core.CXFTestBase;
+import org.apache.tika.server.core.FetcherStreamFactory;
+import org.apache.tika.server.core.InputStreamFactory;
+import org.apache.tika.server.core.resource.RecursiveMetadataResource;
+import org.apache.tika.server.core.writer.MetadataListMessageBodyWriter;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+
+@Ignore("turn into actual unit tests -- this relies on network connectivity...bad")
+public class FetcherTest extends CXFTestBase {
+
+ private static final String META_PATH = "/rmeta";
+ private static final String TEXT_PATH = "/text";
+
+ private static final String TEST_RECURSIVE_DOC = "test-documents/test_recursive_embedded.docx";
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ sf.setResourceClasses(RecursiveMetadataResource.class);
+ sf.setResourceProvider(RecursiveMetadataResource.class,
+ new SingletonResourceProvider(new RecursiveMetadataResource()));
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new MetadataListMessageBodyWriter());
+ sf.setProviders(providers);
+ }
+
+ @Override
+ protected InputStream getTikaConfigInputStream() {
+ return getClass().getResourceAsStream("/config/tika-config-url-fetcher.xml");
+ }
+
+ @Override
+ protected InputStreamFactory getInputStreamFactory(TikaConfig tikaConfig) {
+ return new FetcherStreamFactory(tikaConfig.getFetcherManager());
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ Response response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .acceptEncoding("gzip")
+ .header("fetcherName", "url")
+ .header("fetchKey", "https://tika.apache.org").put("");
+
+ Reader reader = new InputStreamReader(new GzipCompressorInputStream((InputStream) response.getEntity()), UTF_8);
+ List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
+ Metadata parent = metadataList.get(0);
+ String txt = parent.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT);
+ assertContains("toolkit detects and extracts metadata", txt);
+ assertEquals("Apache Tika – Apache Tika", parent.get(TikaCoreProperties.TITLE));
+ }
+
+}
diff --git a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml b/tika-server/tika-server-classic/src/test/resources/config/tika-config-url-fetcher.xml
similarity index 70%
copy from tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
copy to tika-server/tika-server-classic/src/test/resources/config/tika-config-url-fetcher.xml
index e1cb32b..d3aaff0 100644
--- a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
+++ b/tika-server/tika-server-classic/src/test/resources/config/tika-config-url-fetcher.xml
@@ -17,9 +17,15 @@
specific language governing permissions and limitations
under the License.
-->
-
-<mock>
- <metadata action="add" name="author">Nikolai Lobachevsky</metadata>
- <metadata action="add" name="title">你好,世界</metadata>
- <write element="p">hello world</write>
-</mock>
\ No newline at end of file
+<properties>
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser"/>
+ </parsers>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.SimpleUrlFetcher">
+ <params>
+ <param name="name" type="string">url</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
\ No newline at end of file
diff --git a/tika-server/tika-server-client/pom.xml b/tika-server/tika-server-client/pom.xml
index 00284a5..23e304f 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-server/tika-server-client/pom.xml
@@ -25,5 +25,37 @@
<artifactId>tika-server-client</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-emitter-fs</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
\ No newline at end of file
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
new file mode 100644
index 0000000..d7ebcf0
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.client;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetchId;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class TikaClient {
+
+ private static final Gson GSON = new GsonBuilder().create();
+
+ private final Random random = new Random();
+ private final TikaConfig tikaConfig;
+ private List<TikaHttpClient> clients;
+
+
+ public static TikaClient get(TikaConfig tikaConfig, List<String> tikaServers) throws TikaClientConfigException {
+ List clients = new ArrayList<>();
+ for (String url : tikaServers) {
+ clients.add(TikaHttpClient.get(url));
+ }
+ return new TikaClient(tikaConfig, clients);
+ }
+
+ private TikaClient(TikaConfig tikaConfig, List<TikaHttpClient> clients) {
+ this.tikaConfig = tikaConfig;
+ this.clients = clients;
+ }
+
+ /*public List<Metadata> parse(InputStream is, Metadata metadata) throws IOException, TikaException {
+
+ }*/
+
+ public TikaEmitterResult parse(FetchId fetchId, Metadata metadata, String emitter)
+ throws IOException, TikaException {
+ TikaHttpClient client = getHttpClient();
+ String jsonRequest = jsonifyRequest(fetchId, metadata, emitter);
+ return client.postJson(jsonRequest);
+
+ }
+
+ private String jsonifyRequest(FetchId fetchId, Metadata metadata, String emitter) {
+ JsonObject root = new JsonObject();
+ root.add("fetcherName", new JsonPrimitive(fetchId.getFetcherName()));
+ root.add("fetchKey", new JsonPrimitive(fetchId.getFetchKey()));
+ root.add("emitter", new JsonPrimitive(emitter));
+ if (metadata.size() > 0) {
+ JsonObject m = new JsonObject();
+ for (String n : metadata.names()) {
+ String[] vals = metadata.getValues(n);
+ if (vals.length == 1) {
+ m.add(n, new JsonPrimitive(vals[0]));
+ } else if (vals.length > 1) {
+ JsonArray arr = new JsonArray();
+ for (int i = 0; i < vals.length; i++) {
+ arr.add(vals[i]);
+ }
+ m.add(n, arr);
+ }
+ }
+ root.add("metadata", m);
+ }
+ return GSON.toJson(root);
+ }
+
+ private TikaHttpClient getHttpClient() {
+ if (clients.size() == 1) {
+ return clients.get(0);
+ }
+ int index = random.nextInt(clients.size());
+ return clients.get(index);
+ }
+}
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
new file mode 100644
index 0000000..2ca9f97
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.client;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.fetchiterator.FetchIterator;
+import org.apache.tika.pipes.fetcher.FetchIdMetadataPair;
+import org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class TikaClientCLI {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TikaClientCLI.class);
+
+ //make these configurable
+ private int numThreads = 1;
+ private long maxWaitMs = 300000;
+
+ public static void main(String[] args) throws Exception {
+ //TODO -- add an actual commandline
+ Path tikaConfigPath = Paths.get(args[0]);
+ List<String> tikaServerUrls = Arrays.asList(args[1].split(","));
+ String fetcherString = args[2];
+
+ TikaClientCLI cli = new TikaClientCLI();
+ cli.execute(tikaConfigPath, tikaServerUrls, fetcherString);
+ }
+
+ private void execute(Path tikaConfigPath, List<String> tikaServerUrls, String fetcherString)
+ throws TikaException, IOException, SAXException {
+ TikaConfig config = new TikaConfig(tikaConfigPath);
+
+ ExecutorService executorService = Executors.newFixedThreadPool(numThreads+1);
+ ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(executorService);
+ //TODO: fix this!
+ final FetchIterator fetchIterator = config.getFetchIterator();
+ ArrayBlockingQueue<FetchIdMetadataPair> queue = fetchIterator.init(numThreads);
+
+ completionService.submit(fetchIterator);
+ if (tikaServerUrls.size() == numThreads) {
+ logDiffSizes(tikaServerUrls.size(), numThreads);
+ for (int i = 0; i < numThreads; i++) {
+ TikaClient client = TikaClient.get(config,
+ Collections.singletonList(tikaServerUrls.get(i)));
+ completionService.submit(new FetchWorker(queue, client, fetcherString));
+ }
+ } else {
+ for (int i = 0; i < numThreads; i++) {
+ TikaClient client = TikaClient.get(config, tikaServerUrls);
+ completionService.submit(new FetchWorker(queue, client, fetcherString));
+ }
+ }
+
+ int finished = 0;
+ while (finished < numThreads+1) {
+ Future<Integer> future = null;
+ try {
+ future = completionService.poll(maxWaitMs, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ //stop the world
+ LOGGER.error("", e);
+ throw new RuntimeException(e);
+ }
+ if (future != null) {
+ finished++;
+ try {
+ future.get();
+ } catch (InterruptedException|ExecutionException e) {
+ //stop the world
+ LOGGER.error("", e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ private void logDiffSizes(int servers, int numThreads) {
+ LOGGER.info("tika server count ({}) != numThreads ({}). " +
+ "Each client will randomly select a server from this list",
+ servers, numThreads);
+ }
+
+ private class FetchWorker implements Callable<Integer> {
+ private final ArrayBlockingQueue<FetchIdMetadataPair> queue;
+ private final TikaClient client;
+ private final String emitterString;
+ public FetchWorker(ArrayBlockingQueue<FetchIdMetadataPair> queue, TikaClient client,
+ String emitterString) {
+ this.queue = queue;
+ this.client = client;
+ this.emitterString = emitterString;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+
+ while (true) {
+
+ FetchIdMetadataPair p = queue.poll(maxWaitMs, TimeUnit.MILLISECONDS);
+ if (p == null) {
+ throw new TimeoutException("exceeded maxWaitMs");
+ }
+ if (p == FetchIterator.COMPLETED_SEMAPHORE) {
+ return 1;
+ }
+ try {
+ LOGGER.debug("about to parse: {}", p.getFetchId());
+ client.parse(p.getFetchId(), p.getMetadata(), emitterString);
+ } catch (IOException e) {
+ LOGGER.warn(p.getFetchId().toString(), e);
+ } catch (TikaException e) {
+ LOGGER.warn(p.getFetchId().toString(), e);
+ }
+ }
+ }
+ }
+}
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java
new file mode 100644
index 0000000..9d5b51a
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java
@@ -0,0 +1,13 @@
+package org.apache.tika.server.client;
+
+import org.apache.tika.exception.TikaException;
+
+public class TikaClientConfigException extends TikaException {
+ public TikaClientConfigException(String msg) {
+ super(msg);
+ }
+
+ public TikaClientConfigException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
new file mode 100644
index 0000000..9bed7a6
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.client;
+
+public class TikaEmitterResult {
+
+
+ enum STATUS {
+ OK,
+ NOT_OK,
+ EXCEEDED_MAX_RETRIES,
+ TIMED_OUT_WAITING_FOR_TIKA
+ }
+
+ private STATUS status;
+ private String msg;//used for exceptions. will be null for status ok
+ private long timeElapsed;
+
+ public TikaEmitterResult(STATUS status, long timeElapsed, String msg) {
+ this.status = status;
+ this.timeElapsed = timeElapsed;
+ this.msg = msg;
+ }
+
+ @Override
+ public String toString() {
+ return "TikaEmitterResult{" +
+ "status=" + status +
+ ", msg='" + msg + '\'' +
+ ", timeElapsed=" + timeElapsed +
+ '}';
+ }
+
+ public STATUS getStatus() {
+ return status;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public long getTimeElapsed() {
+ return timeElapsed;
+ }
+
+
+}
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
new file mode 100644
index 0000000..621f5db
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.client;
+
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.tika.exception.TikaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Low-level class to handle the http layer.
+ */
+class TikaHttpClient {
+
+ private static final String ENDPOINT = "emit";
+ private static final String TIKA_ENDPOINT = "tika";
+ private static final Logger LOGGER = LoggerFactory.getLogger(TikaHttpClient.class);
+ private final HttpHost httpHost;
+ private final HttpClient httpClient;
+ private final String endPointUrl;
+ private final String tikaUrl;
+ private int maxRetries = 3;
+ //if can't make contact with Tika server, max wait time in ms
+ private long maxWaitForTikaMs = 120000;
+ //how often to ping /tika (in ms) to see if the server is up and running
+ private long pulseWaitForTikaMs = 1000;
+
+ static TikaHttpClient get(String baseUrl) throws TikaClientConfigException {
+ String endPointUrl = baseUrl.endsWith("/") ? baseUrl+ENDPOINT : baseUrl+"/"+ENDPOINT;
+ String tikaUrl = baseUrl.endsWith("/") ? baseUrl+TIKA_ENDPOINT : baseUrl+"/"+TIKA_ENDPOINT;
+ URI uri;
+ try {
+ uri = new URI(endPointUrl);
+ } catch (URISyntaxException e) {
+ throw new TikaClientConfigException("bad URI", e);
+ }
+ HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
+ //TODO: need to add other configuration stuff? proxy, username, password, timeouts...
+ HttpClient client = HttpClients.createDefault();
+ return new TikaHttpClient(endPointUrl, tikaUrl, httpHost, client);
+ }
+
+ /**
+ *
+ * @param endPointUrl full url to the tika-server including endpoint
+ * @param tikaUrl url to /tika endpoint to use to check on server status
+ * @param httpHost
+ * @param httpClient
+ */
+ private TikaHttpClient(String endPointUrl, String tikaUrl, HttpHost httpHost, HttpClient httpClient) {
+ this.endPointUrl = endPointUrl;
+ this.tikaUrl = tikaUrl;
+ this.httpHost = httpHost;
+ this.httpClient = httpClient;
+ }
+
+
+ public TikaEmitterResult postJson(String jsonRequest) {
+ System.out.println("NED:"+endPointUrl);
+ HttpPost post = new HttpPost(endPointUrl);
+ ByteArrayEntity entity = new ByteArrayEntity(jsonRequest.getBytes(StandardCharsets.UTF_8));
+ post.setEntity(entity);
+ post.setHeader("Content-Type", "application/json");
+
+ int tries = 0;
+ long start = System.currentTimeMillis();
+ try {
+ while (tries++ < maxRetries) {
+ HttpResponse response = null;
+ try {
+ response = httpClient.execute(httpHost, post);
+ } catch (IOException e) {
+ LOGGER.warn("Exception trying to parse", e);
+ waitForServer();
+ continue;
+ }
+ String msg = "";
+ try {
+ msg = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ }
+ long elapsed = System.currentTimeMillis() - start;
+ TikaEmitterResult.STATUS status = TikaEmitterResult.STATUS.OK;
+ if (response.getStatusLine().getStatusCode() != 200) {
+ status = TikaEmitterResult.STATUS.NOT_OK;
+ } else {
+ //pull out stacktrace from parse exception?
+ }
+ return new TikaEmitterResult(status, elapsed, msg);
+ }
+ } catch (TimeoutWaitingForTikaException e) {
+ long elapsed = System.currentTimeMillis() - start;
+ return new TikaEmitterResult(
+ TikaEmitterResult.STATUS.TIMED_OUT_WAITING_FOR_TIKA,
+ elapsed, ""
+ );
+ }
+ long elapsed = System.currentTimeMillis() - start;
+ return new TikaEmitterResult(
+ TikaEmitterResult.STATUS.EXCEEDED_MAX_RETRIES,
+ elapsed, ""
+ );
+ }
+
+
+ private void waitForServer() throws TimeoutWaitingForTikaException {
+ long start = System.currentTimeMillis();
+ long elapsed = System.currentTimeMillis() - start;
+ LOGGER.info("server unreachable; waiting for it to restart");
+ while (elapsed < maxWaitForTikaMs) {
+ try {
+ Thread.sleep(pulseWaitForTikaMs);
+ } catch (InterruptedException e) {
+
+ }
+
+ HttpGet get = new HttpGet(tikaUrl);
+ try {
+ HttpResponse response = httpClient.execute(httpHost, get);
+ if (response.getStatusLine().getStatusCode() == 200) {
+ LOGGER.debug("server back up");
+ return;
+ }
+ } catch (IOException e) {
+ elapsed = System.currentTimeMillis()-start;
+ LOGGER.debug("waiting for server; failed to reach it: {} ms",
+ elapsed);
+ }
+
+ elapsed = System.currentTimeMillis()-start;
+ }
+
+ LOGGER.warn("Timeout waiting for tika server {} in {} ms", tikaUrl,
+ elapsed);
+ throw new TimeoutWaitingForTikaException("");
+ }
+
+ private class TimeoutWaitingForTikaException extends TikaException {
+ public TimeoutWaitingForTikaException(String msg) {
+ super(msg);
+ }
+ }
+}
\ No newline at end of file
diff --git a/tika-server/tika-server-client/src/main/resources/log4j.properties b/tika-server/tika-server-client/src/main/resources/log4j.properties
new file mode 100644
index 0000000..7d3b372
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=info,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern= %-5p %m%n
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-server/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java
similarity index 54%
copy from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
copy to tika-server/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java
index c09b139..1cee50f 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-server/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java
@@ -14,27 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.server.client;
-package org.apache.tika.server.core;
+import org.junit.Ignore;
+import org.junit.Test;
-import org.apache.tika.metadata.Metadata;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.io.InputStream;
+import static org.junit.Assert.assertTrue;
-/**
- * Passthrough -- returns InputStream as is
- */
-public class DefaultInputStreamFactory implements InputStreamFactory {
-
- @Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
+@Ignore("turn into actual unit test")
+public class TestBasic {
- @Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- return is;
+ @Test
+ public void testBasic() throws Exception {
+ Path p = Paths.get(TestBasic.class.getResource("/tika-config-simple-fs-emitter.xml").toURI());
+ assertTrue(Files.isRegularFile(p));
+ String[] args = new String[]{
+ p.toAbsolutePath().toString(),
+ "http://localhost:9998/",
+ "fs"
+ };
+ TikaClientCLI.main(args);
}
}
diff --git a/tika-server/tika-server-client/src/test/resources/log4j.properties b/tika-server/tika-server-client/src/test/resources/log4j.properties
new file mode 100644
index 0000000..afc3ec5
--- /dev/null
+++ b/tika-server/tika-server-client/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#info,debug, error,fatal ...
+log4j.rootLogger=debug,stderr
+
+#console
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.Target=System.err
+
+log4j.appender.stderr.layout.ConversionPattern= %-5p %d [%t] (%F:%L) - %m%n
diff --git a/tika-server/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml b/tika-server/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml
new file mode 100644
index 0000000..bfa3716
--- /dev/null
+++ b/tika-server/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<properties>
+ <service-loader initializableProblemHandler="throw"/>
+ <fetchIterators>
+ <fetchIterator class="org.apache.tika.pipes.fetchiterator.FileSystemFetchIterator">
+ <params>
+ <param name="fetcherName" type="string">fs</param>
+ <param name="basePath" type="string">fix</param>
+ </params>
+ </fetchIterator>
+ </fetchIterators>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.FileSystemFetcher">
+ <params>
+ <param name="name" type="string">fs</param>
+ <param name="basePath" type="string">fix</param>
+ </params>
+ </fetcher>
+ </fetchers>
+ <metadataFilters>
+ <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
+ <params>
+ <param name="mappings" type="list">
+ <string>X-TIKA:content->content</string>
+ <string>X-TIKA:embedded_resource_path->embedded_path</string>
+ <string>Content-Length->length</string>
+ <string>dc:creator->creators</string>
+ <string>dc:title->title</string>
+ </param>
+ </params>
+ </metadataFilter>
+ </metadataFilters>
+ <emitters>
+ <emitter class="org.apache.tika.emitter.fs.FileSystemEmitter">
+ <params>
+ <param name="name" type="string">fs</param>
+ <param name="basePath" type="string">fix</param>
+ </params>
+ </emitter>
+ <!--
+ <emitter class="org.apache.tika.emitter.solr.SolrEmitter">
+ <params>
+ <param name="name" type="string">solr1</param>
+ <param name="url" type="string">http://localhost:8983/solr/tika-test</param>
+ <param name="attachmentStrategy" type="string">concatenate-content</param>
+ <param name="contentField" type="string">content</param>
+ <param name="commitWithin" type="int">10</param>
+ </params>
+ </emitter>
+ <emitter class="org.apache.tika.emitter.solr.SolrEmitter">
+ <params>
+ <param name="name" type="string">solr2</param>
+ <param name="url" type="string">http://localhost:8983/solr/tika-test</param>
+ <param name="attachmentStrategy" type="string">parent-child</param>
+ <param name="contentField" type="string">content</param>
+ <param name="commitWithin" type="int">10</param>
+ </params>
+ </emitter>-->
+ </emitters>
+</properties>
\ No newline at end of file
diff --git a/tika-server/tika-server-core/pom.xml b/tika-server/tika-server-core/pom.xml
index 11e6664..740debe 100644
--- a/tika-server/tika-server-core/pom.xml
+++ b/tika-server/tika-server-core/pom.xml
@@ -185,6 +185,12 @@
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-emitter-fs</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
index c09b139..022d178 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/DefaultInputStreamFactory.java
@@ -29,11 +29,6 @@ import java.io.InputStream;
public class DefaultInputStreamFactory implements InputStreamFactory {
@Override
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- return is;
- }
-
- @Override
public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
return is;
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/URLEnabledInputStreamFactory.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
similarity index 62%
rename from tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/URLEnabledInputStreamFactory.java
rename to tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
index bdb71c6..949ab57 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/URLEnabledInputStreamFactory.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
@@ -19,10 +19,12 @@ package org.apache.tika.server.core;
import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
-import java.net.URL;
-import org.apache.tika.io.TikaInputStream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.FetcherManager;
/**
* This class looks for "fileUrl" in the http header. If it is not null
@@ -40,30 +42,30 @@ import org.apache.tika.metadata.Metadata;
* See <a href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-3271">CVE-2015-3271</a>
*
*/
-public class URLEnabledInputStreamFactory implements InputStreamFactory {
+public class FetcherStreamFactory implements InputStreamFactory {
- /**
- * @deprecated use {@link #getInputSteam(InputStream, Metadata, HttpHeaders)}
- * @param is
- * @param httpHeaders
- * @return
- * @throws IOException
- */
- @Override
- @Deprecated
- public InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException {
- String fileUrl = httpHeaders.getHeaderString("fileUrl");
- if(fileUrl != null && !"".equals(fileUrl)){
- return TikaInputStream.get(new URL(fileUrl));
- }
- return is;
+ private final FetcherManager fetcherManager;
+
+ public FetcherStreamFactory(FetcherManager fetcherManager) {
+ this.fetcherManager = fetcherManager;
}
@Override
- public InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException {
- String fileUrl = httpHeaders.getHeaderString("fileUrl");
- if(fileUrl != null && !"".equals(fileUrl)){
- return TikaInputStream.get(new URL(fileUrl), metadata);
+ public InputStream getInputSteam(InputStream is, Metadata metadata,
+ HttpHeaders httpHeaders) throws IOException {
+ String fetcherName= httpHeaders.getHeaderString("fetcherName");
+ String fetchKey = httpHeaders.getHeaderString("fetchKey");
+ if (StringUtils.isBlank(fetcherName) != StringUtils.isBlank(fetchKey)) {
+ throw new IOException("Must specify both a 'fetcherName' and a 'fetchKey'. I see: "+
+ " fetcherName:"+fetcherName+" and fetchKey:"+fetchKey);
+ }
+
+ if (!StringUtils.isBlank(fetcherName)){
+ try {
+ return fetcherManager.getFetcher(fetcherName).fetch(fetchKey, metadata);
+ } catch (TikaException e) {
+ throw new IOException(e);
+ }
}
return is;
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java
index 4d293c6..66ec45a 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java
@@ -31,7 +31,6 @@ import java.io.InputStream;
*/
public interface InputStreamFactory {
- InputStream getInputSteam(InputStream is, HttpHeaders httpHeaders) throws IOException;
InputStream getInputSteam(InputStream is, Metadata metadata, HttpHeaders httpHeaders) throws IOException;
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index d4773ba..6680751 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -49,6 +49,7 @@ import org.apache.tika.parser.DigestingParser;
import org.apache.tika.parser.digestutils.BouncyCastleDigester;
import org.apache.tika.parser.digestutils.CommonsDigester;
import org.apache.tika.server.core.resource.DetectorResource;
+import org.apache.tika.server.core.resource.EmitterResource;
import org.apache.tika.server.core.resource.LanguageResource;
import org.apache.tika.server.core.resource.MetadataResource;
import org.apache.tika.server.core.resource.RecursiveMetadataResource;
@@ -85,11 +86,13 @@ public class TikaServerCli {
public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
private static final Logger LOG = LoggerFactory.getLogger(TikaServerCli.class);
- private static final String FILE_URL_WARNING =
- "WARNING: You have chosen to run tika-server with fileUrl enabled.\n"+
+ private static final String UNSECURE_WARNING =
+ "WARNING: You have chosen to run tika-server with unsecure features enabled.\n"+
"Whoever has access to your service now has the same read permissions\n"+
- "as tika-server. Users could request and receive a sensitive file from your\n" +
- "drive or a webpage from your intranet. See CVE-2015-3271.\n"+
+ "as you've given your fetchers and the same write permissions as your emitters.\n" +
+ "Users could request and receive a sensitive file from your\n" +
+ "drive or a webpage from your intranet and/or send malicious content to\n" +
+ " your emitter endpoints. See CVE-2015-3271.\n"+
"Please make sure you know what you are doing.";
private static final List<String> ONLY_IN_FORK_MODE =
@@ -111,8 +114,8 @@ public class TikaServerCli {
options.addOption("i", "id", true, "id to use for server in server status endpoint");
options.addOption("status", false, "enable the status endpoint");
options.addOption("?", "help", false, "this help message");
- options.addOption("enableUnsecureFeatures", false, "this is required to enable fileUrl.");
- options.addOption("enableFileUrl", false, "allows user to pass in fileUrl instead of InputStream.");
+ options.addOption("enableUnsecureFeatures", false, "this is required to enable fetchers and emitters. "+
+ " The user acknowledges that fetchers and emitters introduce potential security vulnerabilities.");
options.addOption("noFork", false, "legacy mode, less robust -- this starts up tika-server" +
" without forking a process.");
options.addOption("taskTimeoutMillis", true,
@@ -272,21 +275,14 @@ public class TikaServerCli {
}
}
- if (line.hasOption("enableFileUrl") &&
- !line.hasOption("enableUnsecureFeatures")) {
- System.err.println("If you want to enable fileUrl, you must also acknowledge the security risks\n"+
- "by including --enableUnsecureFeatures. See CVE-2015-3271.");
- System.exit(-1);
- }
InputStreamFactory inputStreamFactory = null;
- if (line.hasOption("enableFileUrl") &&
- line.hasOption("enableUnsecureFeatures")) {
- inputStreamFactory = new URLEnabledInputStreamFactory();
- System.out.println(FILE_URL_WARNING);
+ if (line.hasOption("enableUnsecureFeatures")) {
+ inputStreamFactory = new FetcherStreamFactory(tika.getFetcherManager());
+ LOG.info(UNSECURE_WARNING);
} else {
inputStreamFactory = new DefaultInputStreamFactory();
}
-
+ logFetchersAndEmitters(line.hasOption("enableUnsecureFeatures"), tika);
String serverId = line.hasOption("i") ? line.getOptionValue("i") : UUID.randomUUID().toString();
LOG.debug("SERVER ID:" +serverId);
ServerStatus serverStatus;
@@ -329,6 +325,9 @@ public class TikaServerCli {
rCoreProviders.add(new SingletonResourceProvider(new TikaDetectors()));
rCoreProviders.add(new SingletonResourceProvider(new TikaParsers()));
rCoreProviders.add(new SingletonResourceProvider(new TikaVersion()));
+ if (line.hasOption("enableUnsecureFeatures")) {
+ rCoreProviders.add(new SingletonResourceProvider(new EmitterResource()));
+ }
rCoreProviders.addAll(loadResourceServices());
if (line.hasOption("status")) {
rCoreProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
@@ -346,9 +345,8 @@ public class TikaServerCli {
providers.add(new TextMessageBodyWriter());
providers.addAll(loadWriterServices());
providers.add(new TikaServerParseExceptionMapper(returnStackTrace));
- if (line.hasOption("status")) {
- providers.add(new JSONObjWriter());
- }
+ providers.add(new JSONObjWriter());
+
if (logFilter != null) {
providers.add(logFilter);
}
@@ -374,6 +372,47 @@ public class TikaServerCli {
LOG.info("Started Apache Tika server at {}", url);
}
+ private static void logFetchersAndEmitters(boolean enableUnsecureFeatures, TikaConfig tika) {
+ if (enableUnsecureFeatures) {
+ StringBuilder sb = new StringBuilder();
+ Set<String> supportedFetchers = tika.getFetcherManager().getSupported();
+ sb.append("enableSecureFeatures has been selected.\n");
+ if (supportedFetchers.size() == 0) {
+ sb.append("There are no fetchers specified in the TikaConfig");
+ } else {
+ sb.append("The following fetchers are available to whomever has access to this server:\n");
+ for (String p : supportedFetchers) {
+ sb.append(p).append("\n");
+ }
+ }
+ Set<String> emitters = tika.getEmitterManager().getSupported();
+ if (supportedFetchers.size() == 0) {
+ sb.append("There are no emitters specified in the TikaConfig");
+ } else {
+ sb.append("The following emitters are available to whomever has access to this server:\n");
+ for (String e : emitters) {
+ sb.append(e).append("\n");
+ }
+ }
+ LOG.info(sb.toString());
+ } else {
+ if (tika.getEmitterManager().getSupported().size() > 0) {
+ String warn = "-enableUnsecureFeatures has not been specified on the commandline.\n"+
+ "The "+tika.getEmitterManager().getSupported().size() + " emitter(s) that you've\n"+
+ "specified in TikaConfig will not be available on the /emit endpoint\n"+
+ "To enable your emitters, start tika-server with the -enableUnsecureFeatures flag\n\n";
+ LOG.warn(warn);
+ }
+ if (tika.getFetcherManager().getSupported().size() > 0) {
+ String warn = "-enableUnsecureFeatures has not been specified on the commandline.\n"+
+ "The "+tika.getFetcherManager().getSupported().size() + " fetcher(s) that you've\n"+
+ "specified in TikaConfig will not be available\n"+
+ "To enable your fetchers, start tika-server with the -enableUnsecureFeatures flag\n\n";
+ LOG.warn(warn);
+ }
+ }
+ }
+
private static Collection<? extends ResourceProvider> loadResourceServices() {
List<TikaServerResource> resources = new ServiceLoader(TikaServerCli.class.getClassLoader())
.loadServiceProviders(TikaServerResource.class);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
new file mode 100644
index 0000000..7f116fa
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.server.core.resource;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.fetcher.FetchId;
+import org.apache.tika.pipes.fetcher.Fetcher;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
+import org.apache.tika.utils.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Path("/emit")
+public class EmitterResource {
+
+ private static final String EMITTER_PARAM = "emitter";
+ private static final String FETCHER_NAME_ABBREV = "fn";
+ private static final String FETCH_KEY_ABBREV = "fk";
+
+ /**
+ * key that is safe to pass through http header.
+ * The user _must_ specify this for the fsemitter if calling 'put'
+ */
+ public static final String PATH_KEY_FOR_HTTP_HEADER = TikaCoreProperties.SOURCE_PATH.getName().replaceAll(":", "-");
+ private static final Logger LOG = LoggerFactory.getLogger(EmitterResource.class);
+
+
+ /**
+ * @param is input stream is ignored in 'get'
+ * @param httpHeaders
+ * @param info
+ * @param emitterName
+ * @param fetcherName specify the fetcherName in the url's query section
+ * @param fetchKey specify the fetch key in the url's query section
+ * @return
+ * @throws Exception
+ */
+ @GET
+ @Produces("application/json")
+ @Path("{" + EMITTER_PARAM + " : (\\w+)?}")
+ public Map<String, String> getRmeta(InputStream is, @Context HttpHeaders httpHeaders,
+ @Context UriInfo info,
+ @PathParam(EMITTER_PARAM) String emitterName,
+ @QueryParam(FETCHER_NAME_ABBREV) String fetcherName,
+ @QueryParam(FETCH_KEY_ABBREV) String fetchKey) throws Exception {
+ Metadata metadata = new Metadata();
+ Fetcher fetcher = TikaResource.getConfig().getFetcherManager().getFetcher(fetcherName);
+ List<Metadata> metadataList;
+ try (InputStream fetchedIs = fetcher.fetch(fetchKey, metadata)) {
+ for (String n : metadata.names()) {
+ System.out.println(n + " ; "+metadata.get(n));
+ }
+ metadataList =
+ RecursiveMetadataResource.parseMetadata(fetchedIs,
+ metadata,
+ httpHeaders.getRequestHeaders(), info, "text");
+ }
+ return emit(emitterName, metadataList);
+ }
+
+ /**
+ * The user puts the raw bytes of the file and specifies the emitter
+ * as elsewhere. This will not trigger a fetcher. If you want a
+ * fetcher, use the get or post options.
+ * <p>
+ * The extracted text content is stored with the key
+ * {@link org.apache.tika.sax.AbstractRecursiveParserWrapperHandler#TIKA_CONTENT}
+ * <p>
+ * Must specify an emitter in the path, e.g. /emit/solr
+ * @param info uri info
+ * @param emitterName which emitter to use; emitters must be configured in
+ * the TikaConfig file.
+ * @return InputStream that can be deserialized as a list of {@link Metadata} objects
+ * @throws Exception
+ */
+ @PUT
+ @Produces("application/json")
+ @Path("{" + EMITTER_PARAM + " : (\\w+)?}")
+ public Map<String, String> putRmeta(InputStream is,
+ @Context HttpHeaders httpHeaders,
+ @Context UriInfo info,
+ @PathParam(EMITTER_PARAM) String emitterName
+ ) throws Exception {
+
+ Metadata metadata = new Metadata();
+ String path = httpHeaders.getHeaderString(PATH_KEY_FOR_HTTP_HEADER);
+ metadata.set(TikaCoreProperties.SOURCE_PATH, path);
+ List<Metadata> metadataList =
+ RecursiveMetadataResource.parseMetadata(is,
+ metadata,
+ httpHeaders.getRequestHeaders(), info, "text");
+ return emit(emitterName, metadataList);
+ }
+
+ /**
+ * The client posts a json request. At a minimum, this must be a
+ * json object that contains an emitter and a fetcherString key with
+ * the key to fetch the inputStream. Optionally, it may contain a metadata
+ * object that will be used to populate the metadata key for pass
+ * through of metadata from the client.
+ * <p>
+ * The extracted text content is stored with the key
+ * {@link AbstractRecursiveParserWrapperHandler#TIKA_CONTENT}
+ * <p>
+ * Must specify a fetcherString and an emitter in the posted json.
+ * @param info uri info
+ * @return InputStream that can be deserialized as a list of {@link Metadata} objects
+ * @throws Exception
+ */
+ @POST
+ @Produces("application/json")
+ public Map<String, String> postRmeta(InputStream is,
+ @Context HttpHeaders httpHeaders,
+ @Context UriInfo info
+ ) throws Exception {
+ JsonElement root = null;
+ try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
+ root = JsonParser.parseReader(reader);
+ }
+ String fetcherName = root.getAsJsonObject().get("fetcherName").getAsString();
+ String fetchKey = root.getAsJsonObject().get("fetchKey").getAsString();
+ String emitterName = root.getAsJsonObject().get("emitter").getAsString();
+ Metadata metadata = new Metadata();
+
+
+ List<Metadata> metadataList = null;
+ try (InputStream stream =
+ TikaResource.getConfig().getFetcherManager()
+ .getFetcher(fetcherName).fetch(fetchKey, metadata)) {
+
+ metadataList = RecursiveMetadataResource.parseMetadata(
+ stream,
+ metadata,
+ httpHeaders.getRequestHeaders(), info, "text");
+ } catch (Error error) {
+ return returnError(emitterName, error);
+ }
+
+ injectUserMetadata(metadataList.get(0), root.getAsJsonObject());
+
+ for (String n : metadataList.get(0).names()) {
+ LOG.debug("post parse/pre emit metadata {}: {}",
+ n, metadataList.get(0).get(n));
+ }
+ return emit(emitterName, metadataList);
+ }
+
+ private void injectUserMetadata(Metadata metadata, JsonObject root) {
+ if (root.getAsJsonObject().has("metadata")) {
+ JsonObject meta = root.getAsJsonObject().getAsJsonObject("metadata");
+ for (String k : meta.keySet()) {
+ JsonElement val = meta.get(k);
+ if (val.isJsonArray()) {
+ for (JsonElement v : val.getAsJsonArray()) {
+ metadata.add(k, v.getAsString());
+ }
+ } else {
+ metadata.set(k, val.getAsString());
+ }
+ }
+ }
+ }
+
+ private Map<String, String> returnError(String emitterName, Error error) {
+ Map<String, String> statusMap = new HashMap<>();
+ statusMap.put("status", "parse_error");
+ statusMap.put("emitter", emitterName);
+ String msg = ExceptionUtils.getStackTrace(error);
+ statusMap.put("parse_error", msg);
+ return statusMap;
+ }
+
+ private Map<String, String> emit(String emitterName, List<Metadata> metadataList) throws TikaException {
+ Emitter emitter = TikaResource.getConfig().getEmitterManager().getEmitter(emitterName);
+ String status = "ok";
+ String exceptionMsg = "";
+ try {
+ emitter.emit(metadataList);
+ } catch (IOException|TikaEmitterException e) {
+ LOG.warn("problem with emitting", e);
+ status = "emitter_exception";
+ exceptionMsg = ExceptionUtils.getStackTrace(e);
+ }
+ Map<String, String> statusMap = new HashMap<>();
+ statusMap.put("status", status);
+ statusMap.put("emitter", emitterName);
+ if (exceptionMsg.length() > 0) {
+ statusMap.put("emitter_exception", exceptionMsg);
+ }
+ String parseStackTrace = metadataList.get(0).get(
+ AbstractRecursiveParserWrapperHandler.CONTAINER_EXCEPTION);
+ if (parseStackTrace != null) {
+ statusMap.put("parse_exception", parseStackTrace);
+ }
+ return statusMap;
+ }
+
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
index e0bc1ca..298d848 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
@@ -29,6 +29,7 @@ import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import java.io.InputStream;
+import java.util.List;
import org.apache.cxf.jaxrs.ext.multipart.Attachment;
import org.apache.tika.metadata.Metadata;
@@ -84,7 +85,7 @@ public class RecursiveMetadataResource {
@PathParam(HANDLER_TYPE_PARAM) String handlerTypeName)
throws Exception {
return Response.ok(
- parseMetadata(att.getObject(InputStream.class), new Metadata(),
+ parseMetadataToMetadataList(att.getObject(InputStream.class), new Metadata(),
att.getHeaders(), info, handlerTypeName)).build();
}
@@ -122,14 +123,17 @@ public class RecursiveMetadataResource {
) throws Exception {
Metadata metadata = new Metadata();
return Response.ok(
- parseMetadata(TikaResource.getInputStream(is, metadata, httpHeaders),
+ parseMetadataToMetadataList(TikaResource.getInputStream(is, metadata, httpHeaders),
metadata,
httpHeaders.getRequestHeaders(), info, handlerTypeName)).build();
}
- private MetadataList parseMetadata(InputStream is, Metadata metadata,
- MultivaluedMap<String, String> httpHeaders, UriInfo info, String handlerTypeName)
- throws Exception {
+ public static List<Metadata> parseMetadata(InputStream is,
+ Metadata metadata,
+ MultivaluedMap<String, String> httpHeaders,
+ UriInfo info, String handlerTypeName)
+ throws Exception {
+
final ParseContext context = new ParseContext();
Parser parser = TikaResource.createParser();
// TODO: parameterize choice of max chars/max embedded attachments
@@ -147,7 +151,7 @@ public class RecursiveMetadataResource {
int maxEmbeddedResources = -1;
if (httpHeaders.containsKey("maxEmbeddedResources")) {
- maxEmbeddedResources = Integer.parseInt(httpHeaders.getFirst("maxEmbeddedResources"));
+ maxEmbeddedResources = Integer.parseInt(httpHeaders.getFirst("maxEmbeddedResources"));
}
BasicContentHandlerFactory.HANDLER_TYPE type =
@@ -170,7 +174,14 @@ public class RecursiveMetadataResource {
}
},
*/
- return new MetadataList(handler.getMetadataList());
+ return handler.getMetadataList();
}
+
+ private MetadataList parseMetadataToMetadataList(InputStream is, Metadata metadata,
+ MultivaluedMap<String, String> httpHeaders,
+ UriInfo info, String handlerTypeName)
+ throws Exception {
+ return new MetadataList(parseMetadata(is, metadata, httpHeaders, info, handlerTypeName));
+ }
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index 896f50d..46cf093 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -279,7 +279,9 @@ public class TikaResource {
}
String contentTypeHeader = httpHeaders.getFirst(HttpHeaders.CONTENT_TYPE);
- javax.ws.rs.core.MediaType mediaType = contentTypeHeader == null ? null
+ javax.ws.rs.core.MediaType mediaType =
+ (contentTypeHeader == null || "*/*".equals(contentTypeHeader))
+ ? null
: javax.ws.rs.core.MediaType.valueOf(contentTypeHeader);
if (mediaType != null && "xml".equals(mediaType.getSubtype())) {
mediaType = null;
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
index 5a9afbe..47ada93 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
@@ -93,7 +93,7 @@ public abstract class CXFTestBase {
this.tika = new TikaConfig(getTikaConfigInputStream());
TikaResource.init(tika,
new CommonsDigester(DIGESTER_READ_LIMIT, "md5,sha1:32"),
- new DefaultInputStreamFactory(), new ServerStatus("", 0,true));
+ getInputStreamFactory(tika), new ServerStatus("", 0,true));
JAXRSServerFactoryBean sf = new JAXRSServerFactoryBean();
//set compression interceptors
sf.setOutInterceptors(
@@ -120,6 +120,10 @@ public abstract class CXFTestBase {
server = sf.create();
}
+ protected InputStreamFactory getInputStreamFactory(TikaConfig tikaConfig) {
+ return new DefaultInputStreamFactory();
+ }
+
protected InputStream getTikaConfigInputStream() {
return new ByteArrayInputStream(
new String("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"+
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java
new file mode 100644
index 0000000..5144798
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java
@@ -0,0 +1,88 @@
+package org.apache.tika.server.core;
+
+import org.apache.cxf.common.logging.LogUtils;
+import org.apache.tika.TikaTest;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.security.Permission;
+
+public class IntegrationTestBase extends TikaTest {
+
+ static final String TEST_HELLO_WORLD = "test-documents/mock/hello_world.xml";
+ static final String TEST_OOM = "test-documents/mock/fake_oom.xml";
+ static final String TEST_SYSTEM_EXIT = "test-documents/mock/system_exit.xml";
+ static final String TEST_HEAVY_HANG = "test-documents/mock/heavy_hang_30000.xml";
+ static final String TEST_HEAVY_HANG_SHORT = "test-documents/mock/heavy_hang_100.xml";
+ static final String TEST_STDOUT_STDERR = "test-documents/mock/testStdOutErr.xml";
+ static final String TEST_STATIC_STDOUT_STDERR = "test-documents/mock/testStaticStdOutErr.xml";
+ static final String META_PATH = "/rmeta";
+ static final String STATUS_PATH = "/status";
+
+ static final long MAX_WAIT_MS = 60000;
+
+ //running into conflicts on 9998 with the CXFTestBase tests
+ //TODO: figure out why?!
+ static final String INTEGRATION_TEST_PORT = "9999";
+
+ protected static final String endPoint =
+ "http://localhost:" + INTEGRATION_TEST_PORT;
+
+ private SecurityManager existingSecurityManager = null;
+ static Path LOG_FILE;
+
+
+ @BeforeClass
+ public static void staticSetup() throws Exception {
+ LogUtils.setLoggerClass(NullWebClientLogger.class);
+ LOG_FILE = Files.createTempFile("tika-server-integration", ".xml");
+ Files.copy(TikaServerIntegrationTest.class.getResourceAsStream("/logging/log4j_forked.xml"),
+ LOG_FILE, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ existingSecurityManager = System.getSecurityManager();
+ System.setSecurityManager(new SecurityManager() {
+ @Override
+ public void checkExit(int status) {
+ super.checkExit(status);
+ throw new MyExitException(status);
+ }
+ @Override
+ public void checkPermission(Permission perm) {
+ // all ok
+ }
+ @Override
+ public void checkPermission(Permission perm, Object context) {
+ // all ok
+ }
+ });
+ }
+
+ @AfterClass
+ public static void staticTearDown() throws Exception {
+ Files.delete(LOG_FILE);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.setSecurityManager(existingSecurityManager);
+ }
+
+ static class MyExitException extends RuntimeException {
+ private final int status;
+ MyExitException(int status) {
+ this.status = status;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+ }
+}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceOffTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceOffTest.java
index 2c418f0..9dadd63 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceOffTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceOffTest.java
@@ -1,6 +1,4 @@
-package org.apache.tika.server.core;
-
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -16,6 +14,7 @@ package org.apache.tika.server.core;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.server.core;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -146,7 +145,7 @@ public class StackTraceOffTest extends CXFTestBase {
}
//For now, make sure that non-complete document
- //still returtestXDP.xdpns BAD_REQUEST. We may want to
+ //still returns BAD_REQUEST. We may want to
//make MetadataResource return the same types of parse
//exceptions as the others...
@Test
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
index 90c9a6d..0df6a78 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
@@ -1,4 +1,3 @@
-package org.apache.tika.server.core;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,6 +14,8 @@ package org.apache.tika.server.core;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.server.core;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java
new file mode 100644
index 0000000..0631eef
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java
@@ -0,0 +1,288 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.apache.tika.TikaTest;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
+import org.apache.tika.server.core.resource.EmitterResource;
+import org.apache.tika.server.core.writer.JSONObjWriter;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * This offers basic integration tests with fetchers and emitters.
+ * We use file system fetchers and emitters.
+ */
+public class TikaEmitterTest extends CXFTestBase {
+
+ private static final String EMITTER_PATH = "/emit";
+ private static final String EMITTER_PATH_AND_FS = "/emit/fse";
+ private static Path TMP_DIR;
+ private static Path TMP_OUTPUT_DIR;
+ private static Path TMP_OUTPUT_FILE;
+ private static String TIKA_CONFIG_XML;
+ private static String HELLO_WORLD = "hello_world.xml";
+ private static String HELLO_WORLD_JSON = "hello_world.xml.json";
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TMP_DIR = Files.createTempDirectory("tika-emitter-test-");
+ Path inputDir = TMP_DIR.resolve("input");
+ TMP_OUTPUT_DIR = TMP_DIR.resolve("output");
+ TMP_OUTPUT_FILE = TMP_OUTPUT_DIR.resolve(HELLO_WORLD_JSON);
+ Files.createDirectories(inputDir);
+ Files.createDirectories(TMP_OUTPUT_DIR);
+
+ for (String mockFile : new String[]{
+ "hello_world.xml", "null_pointer.xml"}) {
+ Files.copy(TikaEmitterTest.class.getResourceAsStream(
+ "/test-documents/mock/"+mockFile),
+ inputDir.resolve(mockFile));
+ }
+
+ TIKA_CONFIG_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+
+ "<properties>"+
+ "<fetchers>"+
+ "<fetcher class=\"org.apache.tika.pipes.fetcher.FileSystemFetcher\">"+
+ "<params>"+
+ "<param name=\"name\" type=\"string\">fsf</param>"+
+ "<param name=\"basePath\" type=\"string\">"+inputDir.toAbsolutePath()+"</param>"+
+ "</params>"+
+ "</fetcher>"+
+ "</fetchers>"+
+ "<emitters>"+
+ "<emitter class=\"org.apache.tika.emitter.fs.FileSystemEmitter\">"+
+ "<params>"+
+ "<param name=\"name\" type=\"string\">fse</param>"+
+ "<param name=\"basePath\" type=\"string\">"+ TMP_OUTPUT_DIR.toAbsolutePath()+"</param>"+
+ "</params>"+
+ "</emitter>"+
+ "</emitters>"+
+ "</properties>";
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ FileUtils.deleteDirectory(TMP_DIR.toFile());
+ }
+
+ @Before
+ public void setUpEachTest() throws Exception {
+ if (Files.exists(TMP_OUTPUT_FILE)) {
+ Files.delete(TMP_OUTPUT_FILE);
+ }
+ assertFalse(Files.isRegularFile(TMP_OUTPUT_FILE));
+ }
+
+ @Override
+ protected void setUpResources(JAXRSServerFactoryBean sf) {
+ List<ResourceProvider> rCoreProviders = new ArrayList<ResourceProvider>();
+ rCoreProviders.add(new SingletonResourceProvider(new EmitterResource()));
+ sf.setResourceProviders(rCoreProviders);
+ }
+
+ @Override
+ protected void setUpProviders(JAXRSServerFactoryBean sf) {
+ List<Object> providers = new ArrayList<>();
+ providers.add(new JSONObjWriter());
+ sf.setProviders(providers);
+ }
+
+ @Override
+ protected InputStream getTikaConfigInputStream() {
+ return new ByteArrayInputStream(TIKA_CONFIG_XML.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ protected InputStreamFactory getInputStreamFactory(TikaConfig tikaConfig) {
+ return new FetcherStreamFactory(tikaConfig.getFetcherManager());
+ }
+
+ @Test
+ public void testGet() throws Exception {
+
+ String q = "?fn=fsf&fk=hello_world.xml";
+ String getUrl = endPoint+EMITTER_PATH_AND_FS+q;
+ Response response = WebClient
+ .create(getUrl)
+ .accept("application/json").get();
+ assertEquals(200, response.getStatus());
+ List<Metadata> metadataList = null;
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ }
+ assertEquals(1, metadataList.size());
+ Metadata metadata = metadataList.get(0);
+ assertEquals("hello world",
+ metadata.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT).trim());
+ assertEquals("Nikolai Lobachevsky", metadata.get("author"));
+ assertEquals("你好,世界", metadata.get("title"));
+ assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
+ }
+
+ @Test
+ public void testPost() throws Exception {
+
+ JsonObject root = new JsonObject();
+ root.add("fetcherName", new JsonPrimitive("fsf"));
+ root.add("fetchKey", new JsonPrimitive("hello_world.xml"));
+ root.add("emitter", new JsonPrimitive("fse"));
+ JsonObject userMetadata = new JsonObject();
+ String[] valueArray = new String[] {"my-value-1", "my-value-2", "my-value-3"};
+ JsonArray arr = new JsonArray();
+ for (int i = 0; i < valueArray.length; i++) {
+ arr.add(valueArray[i]);
+ }
+
+ userMetadata.add("my-key", new JsonPrimitive("my-value"));
+ userMetadata.add("my-key-multi", arr);
+ root.add("metadata", userMetadata);
+ String jsonPost = new Gson().toJson(root);
+
+ String getUrl = endPoint+EMITTER_PATH;
+ Response response = WebClient
+ .create(getUrl)
+ .accept("application/json")
+ .post(jsonPost);
+ assertEquals(200, response.getStatus());
+
+ List<Metadata> metadataList = null;
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ }
+ assertEquals(1, metadataList.size());
+ Metadata metadata = metadataList.get(0);
+ assertEquals("hello world",
+ metadata.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT).trim());
+ assertEquals("Nikolai Lobachevsky", metadata.get("author"));
+ assertEquals("你好,世界", metadata.get("title"));
+ assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("my-value", metadata.get("my-key"));
+ assertArrayEquals(valueArray, metadata.getValues("my-key-multi"));
+ }
+
+ @Test
+ public void testPut() throws Exception {
+
+ String getUrl = endPoint+EMITTER_PATH_AND_FS;
+ String metaPathKey = EmitterResource.PATH_KEY_FOR_HTTP_HEADER;
+
+ Response response = WebClient
+ .create(getUrl)
+ .accept("application/json")
+ .header(metaPathKey, "hello_world.xml")
+ .put(
+ ClassLoader
+ .getSystemResourceAsStream("test-documents/mock/hello_world.xml")
+ );
+ assertEquals(200, response.getStatus());
+ List<Metadata> metadataList = null;
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ }
+ assertEquals(1, metadataList.size());
+ Metadata metadata = metadataList.get(0);
+ assertEquals("hello world",
+ metadata.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT).trim());
+ assertEquals("Nikolai Lobachevsky", metadata.get("author"));
+ assertEquals("你好,世界", metadata.get("title"));
+ assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
+ }
+
+ @Test
+ public void testPostNPE() throws Exception {
+
+ JsonObject root = new JsonObject();
+ root.add("fetcherName", new JsonPrimitive("fsf"));
+ root.add("fetchKey", new JsonPrimitive("null_pointer.xml"));
+ root.add("emitter", new JsonPrimitive("fse"));
+ JsonObject userMetadata = new JsonObject();
+ String[] valueArray = new String[] {"my-value-1", "my-value-2", "my-value-3"};
+ JsonArray arr = new JsonArray();
+ for (int i = 0; i < valueArray.length; i++) {
+ arr.add(valueArray[i]);
+ }
+
+ userMetadata.add("my-key", new JsonPrimitive("my-value"));
+ userMetadata.add("my-key-multi", arr);
+ root.add("metadata", userMetadata);
+ String jsonPost = new Gson().toJson(root);
+
+ String getUrl = endPoint+EMITTER_PATH;
+ Response response = WebClient
+ .create(getUrl)
+ .accept("application/json")
+ .post(jsonPost);
+ assertEquals(200, response.getStatus());
+
+ JsonObject jsonResponse;
+ try (Reader reader = new InputStreamReader(
+ (InputStream)response.getEntity(), StandardCharsets.UTF_8)) {
+ jsonResponse = JsonParser.parseReader(reader).getAsJsonObject();
+ };
+ String parseException = jsonResponse.get("parse_exception").getAsString();
+ assertNotNull(parseException);
+ assertContains("NullPointerException", parseException);
+
+ List<Metadata> metadataList = null;
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_DIR.resolve("null_pointer.xml.json"))) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ }
+ assertEquals(1, metadataList.size());
+ Metadata metadata = metadataList.get(0);
+ assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("my-value", metadata.get("my-key"));
+ assertArrayEquals(valueArray, metadata.getValues("my-key-multi"));
+ assertContains("NullPointerException", metadata.get(AbstractRecursiveParserWrapperHandler.CONTAINER_EXCEPTION));
+ }
+
+ //can't test system_exit here because server is in same process
+}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
new file mode 100644
index 0000000..3fa10f5
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import org.apache.commons.io.FileUtils;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.core.Response;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TikaServerEmitterIntegrationTest.class);
+
+ private static Path TMP_DIR;
+ private static Path TMP_OUTPUT_DIR;
+ private static String TIKA_CONFIG_XML;
+ private static Path TIKA_CONFIG;
+ private static Gson GSON = new GsonBuilder().create();
+
+ private static final String EMITTER_NAME = "fse";
+ private static final String FETCHER_NAME = "fsf";
+
+ private static String[] FILES = new String[] {
+ "hello_world.xml",
+ "heavy_hang_30000.xml", "real_oom.xml", "system_exit.xml"
+ };
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TMP_DIR = Files.createTempDirectory("tika-emitter-test-");
+ Path inputDir = TMP_DIR.resolve("input");
+ TMP_OUTPUT_DIR = TMP_DIR.resolve("output");
+ Files.createDirectories(inputDir);
+ Files.createDirectories(TMP_OUTPUT_DIR);
+
+ for (String mockFile : FILES) {
+ Files.copy(TikaEmitterTest.class.getResourceAsStream(
+ "/test-documents/mock/"+mockFile),
+ inputDir.resolve(mockFile));
+ }
+ TIKA_CONFIG = TMP_DIR.resolve("tika-config.xml");
+
+ TIKA_CONFIG_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+
+ "<properties>"+
+ "<fetchers>"+
+ "<fetcher class=\"org.apache.tika.pipes.fetcher.FileSystemFetcher\">"+
+ "<params>"+
+ "<param name=\"name\" type=\"string\">"+FETCHER_NAME+"</param>"+
+ "<param name=\"basePath\" type=\"string\">"+inputDir.toAbsolutePath()+"</param>"+
+ "</params>"+
+ "</fetcher>"+
+ "</fetchers>"+
+ "<emitters>"+
+ "<emitter class=\"org.apache.tika.emitter.fs.FileSystemEmitter\">"+
+ "<params>"+
+ "<param name=\"name\" type=\"string\">"+EMITTER_NAME+"</param>"+
+
+ "<param name=\"basePath\" type=\"string\">"+ TMP_OUTPUT_DIR.toAbsolutePath()+"</param>"+
+ "</params>"+
+ "</emitter>"+
+ "</emitters>"+
+ "</properties>";
+
+ FileUtils.write(TIKA_CONFIG.toFile(), TIKA_CONFIG_XML, UTF_8);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ FileUtils.deleteDirectory(TMP_DIR.toFile());
+ }
+
+ @Before
+ public void setUpEachTest() throws Exception {
+ for (String problemFile : FILES) {
+ Path targ = TMP_OUTPUT_DIR.resolve(problemFile + ".json");
+
+ if (Files.exists(targ)) {
+ Files.delete(targ);
+ assertFalse(Files.isRegularFile(targ));
+ }
+ }
+ }
+
+
+ @Test
+ public void testBasic() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-maxFiles", "2000",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ testOne("hello_world.xml", true);
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ @Test(expected = ProcessingException.class)
+ public void testSystemExit() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-maxFiles", "2000",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ testOne("system_exit.xml", false);
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ @Test
+ public void testOOM() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-JXmx128m",
+ "-maxFiles", "2000",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ JsonObject response = testOne("real_oom.xml", false);
+ assertContains("heap space", response.get("parse_error").getAsString());
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ @Test(expected = ProcessingException.class)
+ public void testTimeout() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-JXmx128m",
+ "-taskTimeoutMillis", "2000", "-taskPulseMillis", "100",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ JsonObject response = testOne("heavy_hang_30000.xml", false);
+ assertContains("heap space", response.get("parse_error").getAsString());
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ private void awaitServerStartup() throws Exception {
+ Instant started = Instant.now();
+ long elapsed = Duration.between(started, Instant.now()).toMillis();
+ WebClient client = WebClient.create(endPoint+"/tika").accept("text/plain");
+ while (elapsed < MAX_WAIT_MS) {
+ try {
+ Response response = client.get();
+ if (response.getStatus() == 200) {
+ elapsed = Duration.between(started, Instant.now()).toMillis();
+ LOG.info("client observes server successfully started after " +
+ elapsed+ " ms");
+ return;
+ }
+ LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
+
+ } catch (javax.ws.rs.ProcessingException e) {
+ LOG.debug("tika test client failed to connect to server", e);
+ }
+
+ Thread.sleep(100);
+ elapsed = Duration.between(started, Instant.now()).toMillis();
+ }
+ throw new TimeoutException("couldn't connect to server after " +
+ elapsed + " ms");
+ }
+
+ private JsonObject testOne(String fileName, boolean shouldFileExist) throws Exception {
+ awaitServerStartup();
+ Response response = WebClient
+ .create(endPoint + "/emit")
+ .accept("application/json")
+ .post(getJsonString(fileName));
+ if (shouldFileExist) {
+ Path targFile = TMP_OUTPUT_DIR.resolve(fileName + ".json");
+ assertTrue(Files.size(targFile) > 1);
+ }
+ Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
+ return JsonParser.parseReader(reader).getAsJsonObject();
+ }
+
+ private String getJsonString(String fileName) {
+ JsonObject root = new JsonObject();
+ root.add("fetcherName", new JsonPrimitive(FETCHER_NAME));
+ root.add("fetchKey", new JsonPrimitive(fileName));
+ root.add("emitter", new JsonPrimitive(EMITTER_NAME));
+ return GSON.toJson(root);
+ }
+}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
index 74b4f24..8d25d83 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
@@ -50,79 +50,11 @@ import java.util.concurrent.atomic.AtomicInteger;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
-public class TikaServerIntegrationTest extends TikaTest {
+public class TikaServerIntegrationTest extends IntegrationTestBase {
private static final Logger LOG = LoggerFactory.getLogger(TikaServerIntegrationTest.class);
- private static final String TEST_HELLO_WORLD = "test-documents/mock/hello_world.xml";
- private static final String TEST_OOM = "test-documents/mock/fake_oom.xml";
- private static final String TEST_SYSTEM_EXIT = "test-documents/mock/system_exit.xml";
- private static final String TEST_HEAVY_HANG = "test-documents/mock/heavy_hang_30000.xml";
- private static final String TEST_HEAVY_HANG_SHORT = "test-documents/mock/heavy_hang_100.xml";
- private static final String TEST_STDOUT_STDERR = "test-documents/mock/testStdOutErr.xml";
- private static final String TEST_STATIC_STDOUT_STDERR = "test-documents/mock/testStaticStdOutErr.xml";
- private static final String META_PATH = "/rmeta";
- private static final String STATUS_PATH = "/status";
- private static final long MAX_WAIT_MS = 60000;
-
- //running into conflicts on 9998 with the CXFTestBase tests
- //TODO: figure out why?!
- private static final String INTEGRATION_TEST_PORT = "9999";
-
- protected static final String endPoint =
- "http://localhost:" + INTEGRATION_TEST_PORT;
-
- private SecurityManager existingSecurityManager = null;
- private static Path LOG_FILE;
-
- private static class MyExitException extends RuntimeException {
- private final int status;
- MyExitException(int status) {
- this.status = status;
- }
-
- public int getStatus() {
- return status;
- }
- }
- @BeforeClass
- public static void staticSetup() throws Exception {
- LogUtils.setLoggerClass(NullWebClientLogger.class);
- LOG_FILE = Files.createTempFile("tika-server-integration", ".xml");
- Files.copy(TikaServerIntegrationTest.class.getResourceAsStream("/logging/log4j_forked.xml"),
- LOG_FILE, StandardCopyOption.REPLACE_EXISTING);
- }
-
- @Before
- public void setUp() throws Exception {
- existingSecurityManager = System.getSecurityManager();
- System.setSecurityManager(new SecurityManager() {
- @Override
- public void checkExit(int status) {
- super.checkExit(status);
- throw new MyExitException(status);
- }
- @Override
- public void checkPermission(Permission perm) {
- // all ok
- }
- @Override
- public void checkPermission(Permission perm, Object context) {
- // all ok
- }
- });
- }
-
- @AfterClass
- public static void staticTearDown() throws Exception {
- Files.delete(LOG_FILE);
- }
-
- @After
- public void tearDown() throws Exception {
- System.setSecurityManager(existingSecurityManager);
- }
@Test
public void testBasic() throws Exception {
@@ -529,6 +461,11 @@ public class TikaServerIntegrationTest extends TikaTest {
}
}
+ @Test
+ public void testEmitterSysExit() throws Exception {
+
+ }
+
private void awaitServerStartup() throws Exception {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
diff --git a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml b/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
index e1cb32b..27cd62a 100644
--- a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
+++ b/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world.xml
@@ -21,5 +21,6 @@
<mock>
<metadata action="add" name="author">Nikolai Lobachevsky</metadata>
<metadata action="add" name="title">你好,世界</metadata>
+ <metadata action="add" name="my-key">parsers-value</metadata>
<write element="p">hello world</write>
</mock>
\ No newline at end of file