You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/08/05 18:03:19 UTC
[tika] branch main updated: TIKA-3514 -- add parse success with
exception
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new d9b7b63 TIKA-3514 -- add parse success with exception
d9b7b63 is described below
commit d9b7b63e08405fdea40a83444aa96064ba1c6b9e
Author: tballison <ta...@apache.org>
AuthorDate: Thu Aug 5 14:03:09 2021 -0400
TIKA-3514 -- add parse success with exception
---
.../java/org/apache/tika/pipes/PipesClient.java | 21 ++++++++++++++++++++-
.../java/org/apache/tika/pipes/PipesResult.java | 20 ++++++++++++++++++--
.../java/org/apache/tika/pipes/PipesServer.java | 4 ++--
.../org/apache/tika/pipes/async/AsyncProcessor.java | 3 ++-
.../tika/server/core/resource/PipesResource.java | 1 +
5 files changed, 43 insertions(+), 6 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesClient.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesClient.java
index 273ed12..52999f8 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesClient.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesClient.java
@@ -44,8 +44,10 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.pipes.emitter.EmitData;
import org.apache.tika.utils.ProcessUtils;
+import org.apache.tika.utils.StringUtils;
public class PipesClient implements Closeable {
@@ -220,6 +222,7 @@ public class PipesClient implements Closeable {
LOG.warn("pipesClientId={} fetch exception: {} in {} ms", pipesClientId, t.getId(), millis);
return readMessage(PipesResult.STATUS.FETCH_EXCEPTION);
case PARSE_SUCCESS:
+ //there may have been a parse exception, but the parse didn't crash
LOG.info("pipesClientId={} parse success: {} in {} ms", pipesClientId, t.getId(), millis);
return deserializeEmitData();
case PARSE_EXCEPTION_NO_EMIT:
@@ -257,7 +260,15 @@ public class PipesClient implements Closeable {
input.readFully(bytes);
try (ObjectInputStream objectInputStream =
new ObjectInputStream(new ByteArrayInputStream(bytes))) {
- return new PipesResult((EmitData)objectInputStream.readObject());
+ EmitData emitData = (EmitData)objectInputStream.readObject();
+
+ String stack = getStack(emitData);
+ if (StringUtils.isBlank(stack)) {
+ return new PipesResult(emitData);
+ } else {
+ return new PipesResult(
+ emitData, stack);
+ }
} catch (ClassNotFoundException e) {
LOG.error("class not found exception deserializing data", e);
//this should be catastrophic
@@ -266,6 +277,14 @@ public class PipesClient implements Closeable {
}
+ private String getStack(EmitData emitData) {
+ if (emitData.getMetadataList() == null ||
+ emitData.getMetadataList().size() < 1) {
+ return StringUtils.EMPTY;
+ }
+ return emitData.getMetadataList().get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ }
+
private void restart() throws IOException {
if (process != null) {
process.destroyForcibly();
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesResult.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesResult.java
index 39c32e3..5c8c998 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesResult.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesResult.java
@@ -25,8 +25,10 @@ public class PipesResult {
FETCHER_INITIALIZATION_EXCEPTION,
FETCH_EXCEPTION,
EMPTY_OUTPUT,
- PARSE_EXCEPTION_NO_EMIT,
- PARSE_EXCEPTION_EMIT, PARSE_SUCCESS,
+ PARSE_EXCEPTION_NO_EMIT, //within the pipes server
+ PARSE_EXCEPTION_EMIT, //within the pipes server
+ PARSE_SUCCESS, //when passed back to the async processor for emit
+ PARSE_SUCCESS_WITH_EXCEPTION,//when passed back to the async processor for emit
OOM, TIMEOUT, UNSPECIFIED_CRASH,
NO_EMITTER_FOUND,
EMIT_SUCCESS, EMIT_SUCCESS_PARSE_EXCEPTION, EMIT_EXCEPTION,
@@ -60,10 +62,24 @@ public class PipesResult {
this(status, null, message);
}
+ /**
+ * This assumes parse success with no parse exception
+ * @param emitData
+ */
public PipesResult(EmitData emitData) {
this(STATUS.PARSE_SUCCESS, emitData, null);
}
+ /**
+ * This assumes that the message is a stack trace (container
+ * parse exception).
+ * @param emitData
+ * @param message
+ */
+ public PipesResult(EmitData emitData, String message) {
+ this(STATUS.PARSE_SUCCESS_WITH_EXCEPTION, emitData, message);
+ }
+
public STATUS getStatus() {
return status;
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 8f3ff09..daf1a4c 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -466,7 +466,7 @@ public class PipesServer implements Runnable {
throw e;
} catch (Exception e) {
containerException = ExceptionUtils.getStackTrace(e);
- LOG.warn("exception: " + fetchEmitTuple.getId(), e);
+ LOG.warn("parse exception: " + fetchEmitTuple.getId(), e);
} finally {
metadata.add(TikaCoreProperties.TIKA_CONTENT, handler.toString());
if (containerException != null) {
@@ -502,7 +502,7 @@ public class PipesServer implements Runnable {
LOG.warn("security exception:" + fetchEmitTuple.getId(), e);
throw e;
} catch (Exception e) {
- LOG.warn("exception: " + fetchEmitTuple.getId(), e);
+ LOG.warn("parse exception: " + fetchEmitTuple.getId(), e);
} finally {
if (LOG.isTraceEnabled()) {
LOG.trace("timer -- parse only time: {} ms", System.currentTimeMillis() - start);
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
index dd749c3..93001fa 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
@@ -230,7 +230,8 @@ public class AsyncProcessor implements Closeable {
System.currentTimeMillis() - start);
}
long offerStart = System.currentTimeMillis();
- if (result.getStatus() == PipesResult.STATUS.PARSE_SUCCESS) {
+ if (result.getStatus() == PipesResult.STATUS.PARSE_SUCCESS ||
+ result.getStatus() == PipesResult.STATUS.PARSE_SUCCESS_WITH_EXCEPTION) {
boolean offered = emitDataQueue.offer(result.getEmitData(),
MAX_OFFER_WAIT_MS,
TimeUnit.MILLISECONDS);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java
index a2bf880..d7589d8 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java
@@ -103,6 +103,7 @@ public class PipesResource {
case EMIT_EXCEPTION:
return returnEmitException(pipesResult.getMessage());
case PARSE_SUCCESS:
+ case PARSE_SUCCESS_WITH_EXCEPTION:
throw new IllegalArgumentException("Should have emitted in forked process?!");
case EMIT_SUCCESS:
return returnSuccess();