You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2022/02/22 17:00:11 UTC
[tika] branch main updated (e83195e -> 4ac0c3a)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git.
from e83195e TIKA-3679 -- prefer commons-io IOUtils skipFully and readFully to trigger EOF automatically where possible.
new 1338262 improve 3d detection comment
new 641aa97 TIKA-3680 -- remove printStackTrace() in favor of actual logging everywhere except the ForkParser (for now).
new 4ac0c3a remove debugging test that shouldn't have been part of earlier commit.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/main/java/org/apache/tika/config/TikaConfig.java | 11 +++++++++--
.../java/org/apache/tika/parser/external/ExternalParser.java | 6 +++++-
.../src/main/java/org/apache/tika/pipes/PipesServer.java | 1 -
.../main/java/org/apache/tika/pipes/async/AsyncProcessor.java | 2 +-
.../java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java | 2 ++
.../java/org/apache/tika/server/core/ServerStatusWatcher.java | 2 +-
.../main/java/org/apache/tika/server/core/TikaServerCli.java | 3 +--
.../java/org/apache/tika/server/core/TikaServerProcess.java | 4 ++--
.../tika/server/core/resource/RecursiveMetadataResource.java | 3 ++-
9 files changed, 23 insertions(+), 11 deletions(-)
[tika] 01/03: improve 3d detection comment
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 1338262da11a5b1183b939b7812ce436d4e26348
Author: tallison <ta...@apache.org>
AuthorDate: Fri Feb 11 14:33:26 2022 -0500
improve 3d detection comment
---
.../java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java | 2 ++
.../java/org/apache/tika/parser/pdf/PDFParserTest.java | 14 ++++++++++++++
2 files changed, 16 insertions(+)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 0be6122..a35cb48 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -555,6 +555,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
annotationSubtype = "unknown";
} else if (annotationSubtype.equals(THREE_D) ||
annotation.getCOSObject().containsKey(THREE_DD)) {
+ //To make this stricter, we could get the 3DD stream object and see if the
+ //subtype is U3D or PRC or model/ (prefix for model mime type)
metadata.set(PDF.HAS_3D, true);
}
for (COSDictionary fileSpec :
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index a20111f..b5a792d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
@@ -1359,6 +1360,19 @@ public class PDFParserTest extends TikaTest {
assertEquals("RM1", metadata.get(0).getValues(PDF.ANNOTATION_TYPES)[0]);
}
+ @Test
+ public void test3d() throws Exception {
+ File dir = new File("/home/tallison/Downloads/3d_pdfs");
+ for (File f : dir.listFiles()) {
+ List<Metadata> metadataList = getRecursiveMetadata(f.toPath());
+ String[] vlas = metadataList.get(0).getValues(PDF.HAS_3D);
+ if (vlas != null && vlas.length > 0) {
+ System.out.println("vlas: " + vlas[0]);
+ } else {
+ System.out.println("no: " + f);
+ }
+ }
+ }
/**
@Test
public void testWriteLimit() throws Exception {
[tika] 02/03: TIKA-3680 -- remove printStackTrace() in favor of actual logging everywhere except the ForkParser (for now).
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 641aa9764bc2d75137e13c6a221a10ba55b22660
Author: tallison <ta...@apache.org>
AuthorDate: Tue Feb 22 11:56:29 2022 -0500
TIKA-3680 -- remove printStackTrace() in favor of actual logging everywhere except the ForkParser (for now).
---
.../src/main/java/org/apache/tika/config/TikaConfig.java | 11 +++++++++--
.../java/org/apache/tika/parser/external/ExternalParser.java | 6 +++++-
.../src/main/java/org/apache/tika/pipes/PipesServer.java | 1 -
.../main/java/org/apache/tika/pipes/async/AsyncProcessor.java | 2 +-
.../java/org/apache/tika/server/core/ServerStatusWatcher.java | 2 +-
.../main/java/org/apache/tika/server/core/TikaServerCli.java | 3 +--
.../java/org/apache/tika/server/core/TikaServerProcess.java | 4 ++--
.../tika/server/core/resource/RecursiveMetadataResource.java | 3 ++-
8 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index ad51ee0..1606262 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -40,6 +40,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import javax.imageio.spi.ServiceRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -83,6 +85,9 @@ public class TikaConfig {
//use this to look for unneeded instantiations of TikaConfig
protected static final AtomicInteger TIMES_INSTANTIATED = new AtomicInteger();
+
+ private static final Logger LOG = LoggerFactory.getLogger(TikaConfig.class);
+
private final ServiceLoader serviceLoader;
private final CompositeParser parser;
private final CompositeDetector detector;
@@ -1292,7 +1297,8 @@ public class TikaConfig {
c = encodingDetectorClass.getConstructor(ServiceLoader.class, Collection.class);
encodingDetector = c.newInstance(loader, excludeDetectors);
} catch (NoSuchMethodException me) {
- me.printStackTrace();
+ LOG.debug("couldn't find constructor for service loader + collection for {}",
+ encodingDetectorClass);
}
}
if (encodingDetector == null) {
@@ -1300,7 +1306,8 @@ public class TikaConfig {
c = encodingDetectorClass.getConstructor(List.class);
encodingDetector = c.newInstance(childEncodingDetectors);
} catch (NoSuchMethodException me) {
- me.printStackTrace();
+ LOG.debug("couldn't find constructor for EncodingDetecto(List) for {}",
+ encodingDetectorClass);
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
index b217d4f..7c76db3 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
@@ -38,6 +38,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -56,6 +58,8 @@ import org.apache.tika.sax.XHTMLContentHandler;
*/
public class ExternalParser extends AbstractParser {
+ private static final Logger LOG = LoggerFactory.getLogger(ExternalParser.class);
+
/**
* The token, which if present in the Command string, will
* be replaced with the input filename.
@@ -307,7 +311,7 @@ public class ExternalParser extends AbstractParser {
process = Runtime.getRuntime().exec(cmd);
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.warn("problem with process exec", e);
}
try {
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 52c620f..481f1d2 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -204,7 +204,6 @@ public class PipesServer implements Runnable {
}
LOG.debug("pipes server initialized");
} catch (Throwable t) {
- t.printStackTrace();
LOG.error("couldn't initialize parser", t);
try {
output.writeByte(STATUS.FAILED_TO_START.getByte());
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
index b46e742..410950a 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
@@ -124,9 +124,9 @@ public class AsyncProcessor implements Closeable {
fetchEmitTuples.addAll(newFetchEmitTuples);
return true;
} catch (IllegalStateException e) {
- e.printStackTrace();
//this means that the add all failed because the queue couldn't
//take the full list
+ LOG.debug("couldn't add full list", e);
}
}
Thread.sleep(100);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
index adfc90e..beabe1b 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
@@ -201,7 +201,7 @@ public class ServerStatusWatcher implements Runnable {
writeStatus(false);
lastWrite = Instant.now();
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.debug("interrupted", e);
}
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index c8c04b3..7e4d5cc 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -65,7 +65,6 @@ public class TikaServerCli {
try {
execute(args);
} catch (Exception e) {
- e.printStackTrace();
LOG.error("Can't start: ", e);
System.exit(-1);
}
@@ -87,8 +86,8 @@ public class TikaServerCli {
try {
mainLoop(tikaServerConfig);
} catch (InterruptedException e) {
- e.printStackTrace();
//swallow
+ LOG.debug("interrupted", e);
}
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
index ccb3933..d8c1305 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
@@ -387,7 +387,7 @@ public class TikaServerProcess {
try {
localAsyncResource.shutdownNow();
} catch (Exception e) {
- e.printStackTrace();
+ LOG.warn("problem shutting down local async resource", e);
}
}));
resourceProviders.add(new SingletonResourceProvider(localAsyncResource));
@@ -399,7 +399,7 @@ public class TikaServerProcess {
try {
localPipesResource.close();
} catch (Exception e) {
- e.printStackTrace();
+ LOG.warn("exception closing local pipes resource", e);
}
}));
resourceProviders.add(new SingletonResourceProvider(localPipesResource));
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
index c001f0b..648baf0 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
@@ -78,11 +78,12 @@ public class RecursiveMetadataResource {
TikaResource.parse(wrapper, LOG, "/rmeta", is, handler, metadata, context);
} catch (TikaServerParseException e) {
//do nothing
+ LOG.debug("server parse exception", e);
} catch (SecurityException | WebApplicationException e) {
throw e;
} catch (Exception e) {
//we shouldn't get here?
- e.printStackTrace();
+ LOG.error("something went seriously wrong", e);
}
return handler.getMetadataList();
[tika] 03/03: remove debugging test that shouldn't have been part of earlier commit.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4ac0c3aa31f09bccffb1dc8d4623e59223b91e14
Author: tallison <ta...@apache.org>
AuthorDate: Tue Feb 22 11:59:58 2022 -0500
remove debugging test that shouldn't have been part of earlier commit.
---
.../java/org/apache/tika/parser/pdf/PDFParserTest.java | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index b5a792d..a20111f 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -23,7 +23,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
@@ -1360,19 +1359,6 @@ public class PDFParserTest extends TikaTest {
assertEquals("RM1", metadata.get(0).getValues(PDF.ANNOTATION_TYPES)[0]);
}
- @Test
- public void test3d() throws Exception {
- File dir = new File("/home/tallison/Downloads/3d_pdfs");
- for (File f : dir.listFiles()) {
- List<Metadata> metadataList = getRecursiveMetadata(f.toPath());
- String[] vlas = metadataList.get(0).getValues(PDF.HAS_3D);
- if (vlas != null && vlas.length > 0) {
- System.out.println("vlas: " + vlas[0]);
- } else {
- System.out.println("no: " + f);
- }
- }
- }
/**
@Test
public void testWriteLimit() throws Exception {