You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/08/30 18:46:58 UTC
[tika] branch main updated: TIKA-3541 -- add RangeFetcher interface
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new c18cc6d TIKA-3541 -- add RangeFetcher interface
c18cc6d is described below
commit c18cc6d08df26bd935e5611cd0ce60593a9b9ed1
Author: tallison <ta...@apache.org>
AuthorDate: Mon Aug 30 14:40:19 2021 -0400
TIKA-3541 -- add RangeFetcher interface
---
.../apache/tika/pipes/fetcher/RangeFetcher.java | 34 ++++++++++++++++++++++
.../tika/pipes/fetcher/http/HttpFetcher.java | 4 ++-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 4 ++-
.../pipes/pipesiterator/s3/S3PipesIterator.java | 2 +-
4 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java
new file mode 100644
index 0000000..0a3ceae
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * This class extracts a range of bytes from a given fetch key.
+ */
+public interface RangeFetcher extends Fetcher {
+ //At some point, Tika 3.x?, we may want to add optional ranges to the fetchKey?
+
+ InputStream fetch(String fetchKey, long startOffset, long endOffset, Metadata metadata)
+ throws TikaException, IOException;
+
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index be39765..55d9f18 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -40,11 +40,12 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
+import org.apache.tika.pipes.fetcher.RangeFetcher;
/**
* Based on Apache httpclient
*/
-public class HttpFetcher extends AbstractFetcher implements Initializable {
+public class HttpFetcher extends AbstractFetcher implements Initializable, RangeFetcher {
Logger LOG = LoggerFactory.getLogger(HttpFetcher.class);
private HttpClientFactory httpClientFactory;
@@ -60,6 +61,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable {
return get(get);
}
+ @Override
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index a6b34dd..66fae7f 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -43,12 +43,13 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
+import org.apache.tika.pipes.fetcher.RangeFetcher;
/**
* Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
* This will parse the bucket out of that string and retrieve the path.
*/
-public class S3Fetcher extends AbstractFetcher implements Initializable {
+public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFetcher {
private static final Logger LOGGER = LoggerFactory.getLogger(S3Fetcher.class);
private static final String PREFIX = "s3";
@@ -89,6 +90,7 @@ public class S3Fetcher extends AbstractFetcher implements Initializable {
}
}
+ @Override
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws TikaException, IOException {
//TODO -- figure out how to integrate this
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIterator.java
index 69c089a..ab55291 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIterator.java
@@ -109,7 +109,7 @@ public class S3PipesIterator extends PipesIterator implements Initializable {
s3Client = AmazonS3ClientBuilder.standard().withRegion(region).withCredentials(provider)
.build();
} catch (AmazonClientException e) {
- throw new TikaConfigException("can't initialize s3 pipesiterator");
+ throw new TikaConfigException("can't initialize s3 pipesiterator", e);
}
}