You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/01/14 19:39:43 UTC
[tika] 01/01: TIKA-3226 -- initial fetcher with s3 example
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-3226
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 3d0692adfe0f17f698ba298cc5c7f1a01af32bf7
Author: tballison <ta...@apache.org>
AuthorDate: Thu Jan 14 14:39:29 2021 -0500
TIKA-3226 -- initial fetcher with s3 example
---
pom.xml | 1 +
.../java/org/apache/tika/config/TikaConfig.java | 104 +++++++++++++
.../org/apache/tika/fetcher/DefaultFetcher.java | 80 ++++++++++
.../apache/tika/fetcher/FetchPrefixKeyPair.java | 53 +++++++
.../main/java/org/apache/tika/fetcher/Fetcher.java | 41 ++++++
.../tika/fetcher/FetcherStringException.java | 29 ++++
.../org/apache/tika/fetcher/FileSystemFetcher.java | 55 +++++++
tika-fetchers/pom.xml | 40 +++++
tika-fetchers/s3-fetcher/pom.xml | 161 +++++++++++++++++++++
.../java/org/apache/tika/fetcher/s3/S3Fetcher.java | 124 ++++++++++++++++
.../org/apache/tika/fetcher/s3/TestS3Fetcher.java | 63 ++++++++
.../src/test/resources/tika-config-s3.xml | 27 ++++
tika-parent/pom.xml | 1 +
13 files changed, 779 insertions(+)
diff --git a/pom.xml b/pom.xml
index 6a9ef15..aa1e6b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,6 +37,7 @@
<modules>
<module>tika-parent</module>
<module>tika-core</module>
+ <module>tika-fetchers</module>
<module>tika-parsers</module>
<module>tika-bundles</module>
<module>tika-xmp</module>
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index d7fb8b3..20742d8 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -48,6 +48,8 @@ import org.apache.tika.detect.Detector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.fetcher.DefaultFetcher;
+import org.apache.tika.fetcher.Fetcher;
import org.apache.tika.language.translate.DefaultTranslator;
import org.apache.tika.language.translate.Translator;
import org.apache.tika.metadata.filter.CompositeMetadataFilter;
@@ -112,6 +114,10 @@ public class TikaConfig {
return new DefaultMetadataFilter(loader);
}
+ private static Fetcher getDefaultFetcher(ServiceLoader loader) {
+ return new DefaultFetcher(loader);
+ }
+
//use this to look for unneeded instantiations of TikaConfig
protected static AtomicInteger TIMES_INSTANTIATED = new AtomicInteger();
@@ -124,6 +130,7 @@ public class TikaConfig {
private final ExecutorService executorService;
private final EncodingDetector encodingDetector;
private final MetadataFilter metadataFilter;
+ private final Fetcher fetcher;
public TikaConfig(String file)
throws TikaException, IOException, SAXException {
@@ -190,6 +197,7 @@ public class TikaConfig {
ExecutorServiceXmlLoader executorLoader = new ExecutorServiceXmlLoader();
EncodingDetectorXmlLoader encodingDetectorXmlLoader = new EncodingDetectorXmlLoader();
MetadataFilterXmlLoader metadataFilterXmlLoader = new MetadataFilterXmlLoader();
+ FetcherXmlLoader fetcherXmlLoader = new FetcherXmlLoader();
updateXMLReaderUtils(element);
this.mimeTypes = typesFromDomElement(element);
this.detector = detectorLoader.loadOverall(element, mimeTypes, loader);
@@ -200,6 +208,7 @@ public class TikaConfig {
this.translator = translatorLoader.loadOverall(element, mimeTypes, loader);
this.executorService = executorLoader.loadOverall(element, mimeTypes, loader);
this.metadataFilter = metadataFilterXmlLoader.loadOverall(element, mimeTypes, loader);
+ this.fetcher = fetcherXmlLoader.loadOverall(element, mimeTypes, loader);
this.serviceLoader = loader;
TIMES_INSTANTIATED.incrementAndGet();
}
@@ -226,6 +235,7 @@ public class TikaConfig {
this.translator = getDefaultTranslator(serviceLoader);
this.executorService = getDefaultExecutorService();
this.metadataFilter = getDefaultMetadataFilter(serviceLoader);
+ this.fetcher = getDefaultFetcher(serviceLoader);
TIMES_INSTANTIATED.incrementAndGet();
}
@@ -262,6 +272,7 @@ public class TikaConfig {
this.translator = getDefaultTranslator(serviceLoader);
this.executorService = getDefaultExecutorService();
this.metadataFilter = getDefaultMetadataFilter(serviceLoader);
+ this.fetcher = getDefaultFetcher(serviceLoader);
} else {
ServiceLoader tmpServiceLoader = new ServiceLoader();
try (InputStream stream = getConfigInputStream(config, tmpServiceLoader)) {
@@ -273,6 +284,7 @@ public class TikaConfig {
TranslatorXmlLoader translatorLoader = new TranslatorXmlLoader();
ExecutorServiceXmlLoader executorLoader = new ExecutorServiceXmlLoader();
MetadataFilterXmlLoader metadataFilterXmlLoader = new MetadataFilterXmlLoader();
+ FetcherXmlLoader fetcherXmlLoader = new FetcherXmlLoader();
this.mimeTypes = typesFromDomElement(element);
this.encodingDetector = encodingDetectorLoader.loadOverall(element, mimeTypes, serviceLoader);
@@ -284,6 +296,7 @@ public class TikaConfig {
this.translator = translatorLoader.loadOverall(element, mimeTypes, serviceLoader);
this.executorService = executorLoader.loadOverall(element, mimeTypes, serviceLoader);
this.metadataFilter = metadataFilterXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
+ this.fetcher = fetcherXmlLoader.loadOverall(element, mimeTypes, serviceLoader);
} catch (SAXException e) {
throw new TikaException(
"Specified Tika configuration has syntax errors: "
@@ -411,6 +424,11 @@ public class TikaConfig {
public MetadataFilter getMetadataFilter() {
return metadataFilter;
}
+
+ public Fetcher getFetcher() {
+ return fetcher;
+ }
+
/**
* Provides a default configuration (TikaConfig). Currently creates a
* new instance each time it's called; we may be able to have it
@@ -1267,4 +1285,90 @@ public class TikaConfig {
}
}
+ private static class FetcherXmlLoader extends
+ XmlLoader<Fetcher, Fetcher> {
+
+ boolean supportsComposite() {
+ return true;
+ }
+
+ String getParentTagName() {
+ return "fetchers";
+ }
+
+ String getLoaderTagName() {
+ return "fetcher";
+ }
+
+ @Override
+ Class<? extends Fetcher> getLoaderClass() {
+ return Fetcher.class;
+ }
+
+
+ @Override
+ boolean isComposite(Fetcher loaded) {
+ return loaded instanceof DefaultFetcher;
+ }
+
+ @Override
+ boolean isComposite(Class<? extends Fetcher> loadedClass) {
+ return DefaultFetcher.class.isAssignableFrom(loadedClass);
+ }
+
+ @Override
+ Fetcher preLoadOne(Class<? extends Fetcher> loadedClass,
+ String classname, MimeTypes mimeTypes) throws TikaException {
+ // Check for classes which can't be set in config
+ // Continue with normal loading
+ return null;
+ }
+
+ @Override
+ Fetcher createDefault(MimeTypes mimeTypes, ServiceLoader loader) {
+ return getDefaultFetcher(loader);
+ }
+
+ //this ignores the service loader
+ @Override
+ Fetcher createComposite(List<Fetcher> loaded, MimeTypes mimeTypes, ServiceLoader loader) {
+ return new DefaultFetcher(loaded);
+ }
+
+ @Override
+ Fetcher createComposite(Class<? extends Fetcher> fetcherClass,
+ List<Fetcher> childFetchers,
+ Set<Class<? extends Fetcher>> excludeFilters,
+ Map<String, Param> params, MimeTypes mimeTypes, ServiceLoader loader)
+ throws InvocationTargetException, IllegalAccessException,
+ InstantiationException {
+ Fetcher fetcher = null;
+ Constructor<? extends Fetcher> c;
+
+ // Try the possible default and composite detector constructors
+ if (fetcher == null) {
+ try {
+ c = fetcherClass.getConstructor(ServiceLoader.class, Collection.class);
+ fetcher = c.newInstance(loader, excludeFilters);
+ } catch (NoSuchMethodException me) {
+ me.printStackTrace();
+ }
+ }
+ if (fetcher == null) {
+ try {
+ c = fetcherClass.getConstructor(List.class);
+ fetcher = c.newInstance(childFetchers);
+ } catch (NoSuchMethodException me) {
+ me.printStackTrace();
+ }
+ }
+
+ return fetcher;
+ }
+
+ @Override
+ Fetcher decorate(Fetcher created, Element element) {
+ return created; // No decoration of MetadataFilters
+ }
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/fetcher/DefaultFetcher.java b/tika-core/src/main/java/org/apache/tika/fetcher/DefaultFetcher.java
new file mode 100644
index 0000000..77e77d7
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/fetcher/DefaultFetcher.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher;
+
+import org.apache.tika.config.ServiceLoader;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Utility class that will apply the appropriate fetcher
+ * to the fetchString based on the prefix.
+ *
+ * This does not allow multiple fetchers supporting the same prefix.
+ */
+public class DefaultFetcher implements Fetcher {
+
+ private final Map<String, Fetcher> fetcherMap = new ConcurrentHashMap<>();
+
+ private static List<Fetcher> getDefaultFilters(
+ ServiceLoader loader) {
+ return loader.loadStaticServiceProviders(Fetcher.class);
+ }
+
+
+ public DefaultFetcher(ServiceLoader serviceLoader) {
+ this(getDefaultFilters(serviceLoader));
+ }
+
+ public DefaultFetcher(List<Fetcher> fetchers) {
+ for (Fetcher fetcher : fetchers) {
+ for (String supportedPrefix : fetcher.getSupportedPrefixes()) {
+ if (fetcherMap.containsKey(supportedPrefix)) {
+ throw new IllegalArgumentException(
+ "Multiple fetchers cannot support the same prefix: "
+ + supportedPrefix);
+ }
+ fetcherMap.put(supportedPrefix, fetcher);
+ }
+ }
+ }
+
+ @Override
+ public Set<String> getSupportedPrefixes() {
+ return fetcherMap.keySet();
+ }
+
+ @Override
+ public InputStream fetch(String fetchString, Metadata metadata)
+ throws IOException, TikaException {
+ FetchPrefixKeyPair fetchPrefixKeyPair = FetchPrefixKeyPair.create(fetchString);
+
+ Fetcher fetcher = fetcherMap.get(fetchPrefixKeyPair.getPrefix());
+ if (fetcher == null) {
+ throw new IllegalArgumentException("Can't find fetcher for prefix: "+
+ fetchPrefixKeyPair.getPrefix());
+ }
+ return fetcher.fetch(fetchString, metadata);
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/fetcher/FetchPrefixKeyPair.java b/tika-core/src/main/java/org/apache/tika/fetcher/FetchPrefixKeyPair.java
new file mode 100644
index 0000000..9b263ae
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/fetcher/FetchPrefixKeyPair.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher;
+
+public class FetchPrefixKeyPair {
+ private final String prefix;
+ private final String key;
+
+ private FetchPrefixKeyPair(String prefix, String key) {
+ this.prefix = prefix;
+ this.key = key;
+ }
+
+ public static FetchPrefixKeyPair create(String fetchString) throws FetcherStringException {
+ int prefixIndex = fetchString.indexOf(":");
+ if (prefixIndex < 0) {
+ throw new FetcherStringException("Can't find fetcher prefix, e.g. the 's3' in s3:/myfile");
+ }
+ String prefix = fetchString.substring(0, prefixIndex);
+ String key = fetchString.substring(prefixIndex+1);
+ return new FetchPrefixKeyPair(prefix, key);
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public String toString() {
+ return "FetchPrefixKeyPair{" +
+ "prefix='" + prefix + '\'' +
+ ", key='" + key + '\'' +
+ '}';
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/fetcher/Fetcher.java b/tika-core/src/main/java/org/apache/tika/fetcher/Fetcher.java
new file mode 100644
index 0000000..12c6a5b
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/fetcher/Fetcher.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+/**
+ * Interface for an object that will fetch an InputStream given
+ * a fetch string. This will also update the metadata object
+ * based on the fetch.
+ *
+ * Implementations of Fetcher must be thread safe.
+ *
+ * The fetchString must start with a prefix that can be
+ * used to uniquely select the fetcher, e.g. file:my_file.pdf, s3:bucket/path/to/my_file
+ *
+ * Each fetcher must specify which prefixes it can handle.
+ */
+public interface Fetcher {
+ Set<String> getSupportedPrefixes();
+ InputStream fetch(String fetchString, Metadata metadata) throws TikaException, IOException;
+}
diff --git a/tika-core/src/main/java/org/apache/tika/fetcher/FetcherStringException.java b/tika-core/src/main/java/org/apache/tika/fetcher/FetcherStringException.java
new file mode 100644
index 0000000..9f1e93b
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/fetcher/FetcherStringException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher;
+
+import org.apache.tika.exception.TikaException;
+
+/**
+ * If something goes wrong in parsing the fetcher string
+ */
+public class FetcherStringException extends TikaException {
+
+ public FetcherStringException(String msg) {
+ super(msg);
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/fetcher/FileSystemFetcher.java b/tika-core/src/main/java/org/apache/tika/fetcher/FileSystemFetcher.java
new file mode 100644
index 0000000..5060d35
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/fetcher/FileSystemFetcher.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Set;
+
+public class FileSystemFetcher implements Fetcher {
+
+ private static String PREFIX = "file";
+ public static Property PATH_BASE = Property.externalText(PREFIX + ":base");
+ private static final Set<String> SUPPORTED = Collections.singleton(PREFIX);
+
+ @Override
+ public Set<String> getSupportedPrefixes() {
+ return SUPPORTED;
+ }
+
+ @Override
+ public InputStream fetch(String fetchString, Metadata metadata)
+ throws IOException, TikaException {
+ FetchPrefixKeyPair fetchPrefixKeyPair = FetchPrefixKeyPair.create(fetchString);
+ String base = metadata.get(PATH_BASE);
+ Path p = null;
+ if (base != null) {
+ p = Paths.get(base).resolve(fetchPrefixKeyPair.getKey());
+ } else {
+ p = Paths.get(fetchPrefixKeyPair.getKey());
+ }
+ return TikaInputStream.get(p, metadata);
+ }
+}
diff --git a/tika-fetchers/pom.xml b/tika-fetchers/pom.xml
new file mode 100644
index 0000000..61e0963
--- /dev/null
+++ b/tika-fetchers/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parent</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../tika-parent/pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tika-fetchers</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Tika fetchers</name>
+ <url>http://tika.apache.org/</url>
+
+ <modules>
+ <module>s3-fetcher</module>
+ </modules>
+
+</project>
\ No newline at end of file
diff --git a/tika-fetchers/s3-fetcher/pom.xml b/tika-fetchers/s3-fetcher/pom.xml
new file mode 100644
index 0000000..0247262
--- /dev/null
+++ b/tika-fetchers/s3-fetcher/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tika-fetchers</artifactId>
+ <groupId>org.apache.tika</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>s3-fetcher</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-simpleworkflow</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-simpleworkflow</artifactId>
+ <version>${aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons.logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Automatic-Module-Name>org.apache.tika.fetcher.s3</Automatic-Module-Name>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.version}</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>
+ false
+ </createDependencyReducedPom>
+ <!-- <filters> -->
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*</exclude>
+ <exclude>LICENSE.txt</exclude>
+ <exclude>NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.tika.eval.app.TikaEvalCLI</mainClass>
+ </transformer>
+
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/LICENSE</resource>
+ <file>target/classes/META-INF/LICENSE</file>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/NOTICE</resource>
+ <file>target/classes/META-INF/NOTICE</file>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/DEPENDENCIES</resource>
+ <file>target/classes/META-INF/DEPENDENCIES</file>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/tika-fetchers/s3-fetcher/src/main/java/org/apache/tika/fetcher/s3/S3Fetcher.java b/tika-fetchers/s3-fetcher/src/main/java/org/apache/tika/fetcher/s3/S3Fetcher.java
new file mode 100644
index 0000000..4e592c7
--- /dev/null
+++ b/tika-fetchers/s3-fetcher/src/main/java/org/apache/tika/fetcher/s3/S3Fetcher.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher.s3;
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import org.apache.tika.config.Field;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.InitializableProblemHandler;
+import org.apache.tika.config.Param;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.fetcher.FetchPrefixKeyPair;
+import org.apache.tika.fetcher.Fetcher;
+import org.apache.tika.fetcher.FetcherStringException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
+ * This will parse the bucket out of that string and retrieve the path.
+ */
+public class S3Fetcher implements Fetcher, Initializable {
+
+ private static final String PREFIX = "s3";
+ private static final Set<String> SUPPORTED = Collections.singleton(PREFIX);
+
+ private String region;
+ private String profile;
+ private boolean extractUserMetadata = true;
+ private AmazonS3 s3Client;
+
+ @Override
+ public Set<String> getSupportedPrefixes() {
+ return SUPPORTED;
+ }
+
+ @Override
+ public InputStream fetch(String fetchString, Metadata metadata)
+ throws TikaException, IOException {
+ FetchPrefixKeyPair fetchPrefixKeyPair = FetchPrefixKeyPair.create(fetchString);
+ String bucketKey = fetchPrefixKeyPair.getKey();
+ if (bucketKey.startsWith("//")) {
+ bucketKey = bucketKey.substring(2);
+ } else if (bucketKey.startsWith("/")) {
+ bucketKey = bucketKey.substring(1);
+ }
+ int i = bucketKey.indexOf("/");
+ if (i < 0) {
+ throw new FetcherStringException("Couldn't find bucket:" + fetchPrefixKeyPair.getKey());
+ }
+ String bucket = bucketKey.substring(0, i);
+ String key = bucketKey.substring(i + 1);
+ //should we cache this to a local file so that
+ //we can close the s3Object?
+ S3Object fullObject = s3Client.getObject(new GetObjectRequest(bucket, key));
+ if (extractUserMetadata) {
+ for (Map.Entry<String, String> e :
+ fullObject.getObjectMetadata().getUserMetadata().entrySet()) {
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ }
+ }
+ return TikaInputStream.get(
+ fullObject.getObjectContent());
+ }
+
+ @Field
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ @Field
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Whether or not to extract user metadata from the S3Object
+ *
+ * @param extractUserMetadata
+ */
+ @Field
+ public void setExtractUserMetadata(boolean extractUserMetadata) {
+ this.extractUserMetadata = extractUserMetadata;
+ }
+
+ @Override
+ public void initialize(Map<String, Param> params) throws TikaConfigException {
+ //params have already been set
+ //ignore them
+ s3Client = AmazonS3ClientBuilder.standard()
+ .withRegion(region)
+ .withCredentials(new ProfileCredentialsProvider(profile))
+ .build();
+ }
+
+ @Override
+ public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
+
+ }
+}
diff --git a/tika-fetchers/s3-fetcher/src/test/java/org/apache/tika/fetcher/s3/TestS3Fetcher.java b/tika-fetchers/s3-fetcher/src/test/java/org/apache/tika/fetcher/s3/TestS3Fetcher.java
new file mode 100644
index 0000000..1e352f9
--- /dev/null
+++ b/tika-fetchers/s3-fetcher/src/test/java/org/apache/tika/fetcher/s3/TestS3Fetcher.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.fetcher.s3;
+import com.amazonaws.regions.Regions;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.fetcher.Fetcher;
+import org.apache.tika.metadata.Metadata;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+
+@Ignore("write actual unit tests")
+public class TestS3Fetcher {
+ private static final String FETCH_STRING = "";
+ private Path outputFile = Paths.get("");
+ private String region = "us-east-1";
+ private String profile = "";
+
+ @Test
+ public void testBasic() throws Exception {
+ S3Fetcher fetcher = new S3Fetcher();
+ fetcher.setProfile(profile);
+ fetcher.setRegion(region);
+ fetcher.initialize(Collections.EMPTY_MAP);
+
+ Metadata metadata = new Metadata();
+ try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
+ Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ TikaConfig config = new TikaConfig(
+ this.getClass().getResourceAsStream("/tika-config-s3.xml")
+ );
+ Fetcher fetcher = config.getFetcher();
+ Metadata metadata = new Metadata();
+ try (InputStream is = fetcher.fetch(FETCH_STRING, metadata)) {
+ Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+}
diff --git a/tika-fetchers/s3-fetcher/src/test/resources/tika-config-s3.xml b/tika-fetchers/s3-fetcher/src/test/resources/tika-config-s3.xml
new file mode 100644
index 0000000..39a2ce8
--- /dev/null
+++ b/tika-fetchers/s3-fetcher/src/test/resources/tika-config-s3.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.fetcher.s3.S3Fetcher">
+ <params>
+ <param name="region" type="string">us-east-1</param>
+ <param name="profile" type="string">my_profile</param>
+ </params>
+ </fetcher>
+ </fetchers>
+</properties>
\ No newline at end of file
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index b219b02..d19598f 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -274,6 +274,7 @@
<rat.version>0.13</rat.version>
<!-- dependency versions -->
+ <aws.version>1.11.937</aws.version>
<boilerpipe.version>1.1.0</boilerpipe.version>
<!-- used by POI, PDFBox and Jackcess ...try to sync -->
<bouncycastle.version>1.68</bouncycastle.version>