You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2024/05/11 14:06:11 UTC
(tika) 01/02: Revert "TIKA-4252: add request metadata (#1753)"
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch revert-bad-commits
in repository https://gitbox.apache.org/repos/asf/tika.git
commit f4e2a65b7a3bbb3b21fd673723ccdb3dc74734ea
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Sat May 11 09:05:48 2024 -0500
Revert "TIKA-4252: add request metadata (#1753)"
This reverts commit b068e4290ad311b1e5f1ddaa6afa40be9e7bd797.
---
.../java/org/apache/tika/pipes/PipesServer.java | 24 +++++++++++-----------
.../apache/tika/pipes/fetcher/EmptyFetcher.java | 2 +-
.../org/apache/tika/pipes/fetcher/Fetcher.java | 8 +-------
.../apache/tika/pipes/fetcher/RangeFetcher.java | 8 ++------
.../tika/pipes/fetcher/fs/FileSystemFetcher.java | 8 ++++----
.../apache/tika/pipes/fetcher/url/UrlFetcher.java | 4 ++--
.../org/apache/tika/pipes/async/MockFetcher.java | 2 +-
.../org/apache/tika/pipes/fetcher/MockFetcher.java | 2 +-
.../tika/pipes/fetcher/azblob/AZBlobFetcher.java | 6 +++---
.../apache/tika/pipes/fetcher/gcs/GCSFetcher.java | 6 +++---
.../tika/pipes/fetcher/http/HttpFetcher.java | 16 +++++++--------
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 8 ++++----
12 files changed, 42 insertions(+), 52 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 1562a8d83..20a5def59 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -455,33 +455,33 @@ public class PipesServer implements Runnable {
}
}
- protected MetadataListAndEmbeddedBytes parseFromTuple(FetchEmitTuple fetchEmitTuple, Fetcher fetcher) {
- FetchKey fetchKey = fetchEmitTuple.getFetchKey();
- Metadata fetchResponseMetadata = new Metadata();
- Metadata fetchRequestMetadata = fetchEmitTuple.getMetadata();
+ protected MetadataListAndEmbeddedBytes parseFromTuple(FetchEmitTuple t, Fetcher fetcher) {
+ FetchKey fetchKey = t.getFetchKey();
if (fetchKey.hasRange()) {
if (!(fetcher instanceof RangeFetcher)) {
throw new IllegalArgumentException(
"fetch key has a range, but the fetcher is not a range fetcher");
}
+ Metadata metadata = t.getMetadata() == null ? new Metadata() : t.getMetadata();
try (InputStream stream = ((RangeFetcher) fetcher).fetch(fetchKey.getFetchKey(),
- fetchKey.getRangeStart(), fetchKey.getRangeEnd(), fetchRequestMetadata, fetchResponseMetadata)) {
- return parseWithStream(fetchEmitTuple, stream, fetchResponseMetadata);
+ fetchKey.getRangeStart(), fetchKey.getRangeEnd(), metadata)) {
+ return parseWithStream(t, stream, metadata);
} catch (SecurityException e) {
- LOG.error("security exception " + fetchEmitTuple.getId(), e);
+ LOG.error("security exception " + t.getId(), e);
throw e;
} catch (TikaException | IOException e) {
- LOG.warn("fetch exception " + fetchEmitTuple.getId(), e);
+ LOG.warn("fetch exception " + t.getId(), e);
write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
}
} else {
- try (InputStream stream = fetcher.fetch(fetchEmitTuple.getFetchKey().getFetchKey(), fetchRequestMetadata, fetchResponseMetadata)) {
- return parseWithStream(fetchEmitTuple, stream, fetchResponseMetadata);
+ Metadata metadata = t.getMetadata() == null ? new Metadata() : t.getMetadata();
+ try (InputStream stream = fetcher.fetch(t.getFetchKey().getFetchKey(), metadata)) {
+ return parseWithStream(t, stream, metadata);
} catch (SecurityException e) {
- LOG.error("security exception " + fetchEmitTuple.getId(), e);
+ LOG.error("security exception " + t.getId(), e);
throw e;
} catch (TikaException | IOException e) {
- LOG.warn("fetch exception " + fetchEmitTuple.getId(), e);
+ LOG.warn("fetch exception " + t.getId(), e);
write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
index 925839694..022d00a8c 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/EmptyFetcher.java
@@ -30,7 +30,7 @@ public class EmptyFetcher implements Fetcher {
}
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
return null;
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
index 4bdb69cd8..1b3fa2a24 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/Fetcher.java
@@ -33,11 +33,5 @@ public interface Fetcher {
String getName();
- default InputStream fetch(String fetchKey, Metadata fetchResponseMetadata)
- throws TikaException, IOException {
- return fetch(fetchKey, new Metadata(), fetchResponseMetadata);
- }
-
- InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata)
- throws TikaException, IOException;
+ InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException;
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java
index ed9227eeb..0a3ceae7f 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/RangeFetcher.java
@@ -28,11 +28,7 @@ import org.apache.tika.metadata.Metadata;
public interface RangeFetcher extends Fetcher {
//At some point, Tika 3.x?, we may want to add optional ranges to the fetchKey?
- default InputStream fetch(String fetchKey, long startOffset, long endOffset, Metadata fetchResponseMetadata)
- throws TikaException, IOException {
- return fetch(fetchKey, startOffset, endOffset, new Metadata(), fetchResponseMetadata);
- }
-
- InputStream fetch(String fetchKey, long startOffset, long endOffset, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata)
+ InputStream fetch(String fetchKey, long startOffset, long endOffset, Metadata metadata)
throws TikaException, IOException;
+
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
index 31a8b2c81..d926e3ca6 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
@@ -58,7 +58,7 @@ public class FileSystemFetcher extends AbstractFetcher implements Initializable
}
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws IOException, TikaException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
if (fetchKey.contains("\u0000")) {
throw new IllegalArgumentException("Path must not contain \u0000. " +
@@ -76,8 +76,8 @@ public class FileSystemFetcher extends AbstractFetcher implements Initializable
p = Paths.get(fetchKey);
}
- fetchRequestMetadata.set(TikaCoreProperties.SOURCE_PATH, fetchKey);
- updateFileSystemMetadata(p, fetchRequestMetadata);
+ metadata.set(TikaCoreProperties.SOURCE_PATH, fetchKey);
+ updateFileSystemMetadata(p, metadata);
if (!Files.isRegularFile(p)) {
if (basePath != null && !Files.isDirectory(basePath)) {
@@ -87,7 +87,7 @@ public class FileSystemFetcher extends AbstractFetcher implements Initializable
}
}
- return TikaInputStream.get(p, fetchRequestMetadata);
+ return TikaInputStream.get(p, metadata);
}
private void updateFileSystemMetadata(Path p, Metadata metadata) throws IOException {
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/url/UrlFetcher.java b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/url/UrlFetcher.java
index 8c571b258..f415a3560 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/url/UrlFetcher.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/url/UrlFetcher.java
@@ -35,7 +35,7 @@ import org.apache.tika.pipes.fetcher.AbstractFetcher;
public class UrlFetcher extends AbstractFetcher {
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws IOException, TikaException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
if (fetchKey.contains("\u0000")) {
throw new IllegalArgumentException("URL must not contain \u0000. " +
"Please review the life decisions that led you to requesting " +
@@ -46,7 +46,7 @@ public class UrlFetcher extends AbstractFetcher {
"The UrlFetcher does not fetch from file shares; " +
"please use the FileSystemFetcher");
}
- return TikaInputStream.get(new URL(fetchKey), fetchRequestMetadata);
+ return TikaInputStream.get(new URL(fetchKey), metadata);
}
}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/async/MockFetcher.java b/tika-core/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
index 1a731f74a..10af275e3 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/async/MockFetcher.java
@@ -37,7 +37,7 @@ public class MockFetcher implements Fetcher {
}
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
return new ByteArrayInputStream(BYTES);
}
}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/fetcher/MockFetcher.java b/tika-core/src/test/java/org/apache/tika/pipes/fetcher/MockFetcher.java
index df23708eb..060432724 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/fetcher/MockFetcher.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/fetcher/MockFetcher.java
@@ -64,7 +64,7 @@ public class MockFetcher extends AbstractFetcher implements Initializable {
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
return byteString == null ? new ByteArrayInputStream(new byte[0]) :
new ByteArrayInputStream(byteString.getBytes(StandardCharsets.UTF_8));
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
index c25b85552..dee903040 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
@@ -70,7 +70,7 @@ public class AZBlobFetcher extends AbstractFetcher implements Initializable {
private boolean spoolToTemp = true;
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
LOGGER.debug("about to fetch fetchkey={} from endpoint ({})", fetchKey, endpoint);
@@ -81,7 +81,7 @@ public class AZBlobFetcher extends AbstractFetcher implements Initializable {
BlobProperties properties = blobClient.getProperties();
if (properties.getMetadata() != null) {
for (Map.Entry<String, String> e : properties.getMetadata().entrySet()) {
- fetchRequestMetadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
}
}
}
@@ -94,7 +94,7 @@ public class AZBlobFetcher extends AbstractFetcher implements Initializable {
try (OutputStream os = Files.newOutputStream(tmp)) {
blobClient.download(os);
}
- TikaInputStream tis = TikaInputStream.get(tmp, fetchRequestMetadata, tmpResources);
+ TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
long elapsed = System.currentTimeMillis() - start;
LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
return tis;
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
index 868c1b8fa..6881c5a66 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
@@ -55,7 +55,7 @@ public class GCSFetcher extends AbstractFetcher implements Initializable {
private boolean spoolToTemp = true;
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
@@ -65,7 +65,7 @@ public class GCSFetcher extends AbstractFetcher implements Initializable {
if (extractUserMetadata) {
if (blob.getMetadata() != null) {
for (Map.Entry<String, String> e : blob.getMetadata().entrySet()) {
- fetchRequestMetadata.add(PREFIX + ":" + e.getKey(), e.getValue());
+ metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
}
}
}
@@ -76,7 +76,7 @@ public class GCSFetcher extends AbstractFetcher implements Initializable {
TemporaryResources tmpResources = new TemporaryResources();
Path tmp = tmpResources.createTempFile();
blob.downloadTo(tmp);
- TikaInputStream tis = TikaInputStream.get(tmp, fetchRequestMetadata, tmpResources);
+ TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
long elapsed = System.currentTimeMillis() - start;
LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
return tis;
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index 6d4ddab28..26b45f8bf 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -136,7 +136,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws IOException, TikaException {
+ public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
RequestConfig requestConfig =
RequestConfig.custom()
@@ -146,21 +146,21 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
if (! StringUtils.isBlank(userAgent)) {
get.setHeader(USER_AGENT, userAgent);
}
- return execute(get, fetchResponseMetadata, httpClient, true);
+ return execute(get, metadata, httpClient, true);
}
@Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata)
+ public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws IOException {
HttpGet get = new HttpGet(fetchKey);
if (! StringUtils.isBlank(userAgent)) {
get.setHeader(USER_AGENT, userAgent);
}
get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
- return execute(get, fetchResponseMetadata, httpClient, true);
+ return execute(get, metadata, httpClient, true);
}
- private InputStream execute(HttpGet get, Metadata fetchRequestMetadata, HttpClient client,
+ private InputStream execute(HttpGet get, Metadata metadata, HttpClient client,
boolean retryOnBadLength) throws IOException {
HttpClientContext context = HttpClientContext.create();
HttpResponse response = null;
@@ -183,7 +183,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
}
response = client.execute(get, context);
- updateMetadata(get.getURI().toString(), response, context, fetchRequestMetadata);
+ updateMetadata(get.getURI().toString(), response, context, metadata);
int code = response.getStatusLine().getStatusCode();
if (code < 200 || code > 299) {
@@ -191,7 +191,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
responseToString(response));
}
try (InputStream is = response.getEntity().getContent()) {
- return spool(is, fetchRequestMetadata);
+ return spool(is, metadata);
}
} catch (ConnectionClosedException e) {
@@ -202,7 +202,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
//and then compresses the stream. See HTTPCLIENT-2176
LOG.warn("premature end of content-length delimited message; retrying with " +
"content compression disabled for {}", get.getURI());
- return execute(get, fetchRequestMetadata, noCompressHttpClient, false);
+ return execute(get, metadata, noCompressHttpClient, false);
}
throw e;
} catch (IOException e) {
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index 38161a1b3..b57c361b9 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -106,12 +106,12 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
private boolean pathStyleAccessEnabled = false;
@Override
- public InputStream fetch(String fetchKey, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata) throws TikaException, IOException {
- return fetch(fetchKey, -1, -1, fetchRequestMetadata);
+ public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
+ return fetch(fetchKey, -1, -1, metadata);
}
@Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata fetchRequestMetadata, Metadata fetchResponseMetadata)
+ public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws TikaException, IOException {
String theFetchKey = StringUtils.isBlank(prefix) ? fetchKey : prefix + fetchKey;
@@ -129,7 +129,7 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
do {
try {
long start = System.currentTimeMillis();
- InputStream is = _fetch(theFetchKey, fetchResponseMetadata, startRange, endRange);
+ InputStream is = _fetch(theFetchKey, metadata, startRange, endRange);
long elapsed = System.currentTimeMillis() - start;
LOGGER.debug("total to fetch {}", elapsed);
return is;