You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/09/21 20:50:31 UTC
[tika] branch master updated (015c695 -> ac25932)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 015c695 TIKA-2429 -- upgrade to POI 3.17, and get it right in tika-eval
new 384e971 make strawman app driver actually work. Add ability to specify a list of files.
new ac25932 TIKA-2466 Remove JAXB for easier use with Java 9 via Robert Munteanu.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 2 +
.../batch/fs/strawman/StrawManTikaAppDriver.java | 139 +++++++++++----------
.../main/java/org/apache/tika/config/Param.java | 135 ++++++++++++--------
.../java/org/apache/tika/config/TikaConfig.java | 9 +-
.../java/org/apache/tika/parser/ParseContext.java | 31 +++++
.../java/org/apache/tika/utils/XMLReaderUtils.java | 25 ++++
6 files changed, 215 insertions(+), 126 deletions(-)
--
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].
[tika] 02/02: TIKA-2466 Remove JAXB for easier use with Java 9 via
Robert Munteanu.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ac259320057d8b7a8937dc5ee183b3a5c346a3bc
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Sep 21 16:50:21 2017 -0400
TIKA-2466 Remove JAXB for easier use with Java 9 via Robert Munteanu.
---
CHANGES.txt | 2 +
.../main/java/org/apache/tika/config/Param.java | 135 ++++++++++++---------
.../java/org/apache/tika/config/TikaConfig.java | 9 +-
.../java/org/apache/tika/parser/ParseContext.java | 31 +++++
.../java/org/apache/tika/utils/XMLReaderUtils.java | 25 ++++
5 files changed, 141 insertions(+), 61 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 7ad0f99..b6eb6ab 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,7 @@
Release 1.17 - ???
+ * Remove JAXB for easier use with Java 9 via Robert Munteanu (TIKA-2466).
+
* Upgrade to POI 3.17 (TIKA-2429).
* Enabling extraction of standard references from text (TIKA-2449).
diff --git a/tika-core/src/main/java/org/apache/tika/config/Param.java b/tika-core/src/main/java/org/apache/tika/config/Param.java
index b54f6be..112955b 100644
--- a/tika-core/src/main/java/org/apache/tika/config/Param.java
+++ b/tika-core/src/main/java/org/apache/tika/config/Param.java
@@ -16,15 +16,21 @@
*/
package org.apache.tika.config;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.utils.XMLReaderUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
import org.w3c.dom.Node;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.bind.annotation.*;
-import javax.xml.bind.helpers.DefaultValidationEventHandler;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
@@ -38,18 +44,13 @@ import java.util.Map;
/**
- * This is a JAXB serializable model class for parameters from configuration file.
+ * This is a serializable model class for parameters from configuration file.
*
* @param <T> value type. Should be serializable to string and have a constructor with string param
* @since Apache Tika 1.14
*/
-@XmlRootElement()
-@XmlAccessorType(XmlAccessType.NONE)
public class Param<T> implements Serializable {
- private static final JAXBContext JAXB_CTX;
- private static final Marshaller MARSHALLER;
- private static final Unmarshaller UNMARSHALLER;
private static final Map<Class<?>, String> map = new HashMap<>();
private static final Map<String, Class<?>> reverseMap = new HashMap<>();
@@ -69,27 +70,14 @@ public class Param<T> implements Serializable {
for (Map.Entry<Class<?>, String> entry : map.entrySet()) {
reverseMap.put(entry.getValue(), entry.getKey());
}
- try {
- JAXB_CTX = JAXBContext.newInstance(Param.class);
- MARSHALLER = JAXB_CTX.createMarshaller();
- MARSHALLER.setEventHandler(new DefaultValidationEventHandler());
- UNMARSHALLER = JAXB_CTX.createUnmarshaller();
- UNMARSHALLER.setEventHandler(new DefaultValidationEventHandler());
- } catch (JAXBException e) {
- throw new RuntimeException(e);
- }
}
- @XmlTransient
private Class<T> type;
- @XmlAttribute(name = "name")
private String name;
- @XmlValue()
private String value;
- @XmlTransient
private T actualValue;
public Param(){
@@ -113,7 +101,6 @@ public class Param<T> implements Serializable {
this.name = name;
}
- @XmlTransient
public Class<T> getType() {
return type;
}
@@ -122,7 +109,6 @@ public class Param<T> implements Serializable {
this.type = type;
}
- @XmlAttribute(name = "type")
public String getTypeString(){
if (type == null) {
return null;
@@ -137,28 +123,14 @@ public class Param<T> implements Serializable {
if (type == null || type.isEmpty()){
return;
}
- if (reverseMap.containsKey(type)){
- this.type = (Class<T>) reverseMap.get(type);
- } else try {
- this.type = (Class<T>) Class.forName(type);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
+
+ this.type = classFromType(type);
this.actualValue = null;
}
- @XmlTransient
public T getValue(){
if (actualValue == null) {
- try {
- Constructor<T> constructor = type.getConstructor(String.class);
- constructor.setAccessible(true);
- this.actualValue = constructor.newInstance(value);
- } catch (NoSuchMethodException e) {
- throw new RuntimeException(type + " doesnt have a constructor that takes String arg", e);
- } catch (IllegalAccessException | InstantiationException | InvocationTargetException e) {
- throw new RuntimeException(e);
- }
+ actualValue = getTypedValue(type, value);
}
return actualValue;
}
@@ -172,20 +144,75 @@ public class Param<T> implements Serializable {
'}';
}
- public void save(OutputStream stream) throws JAXBException {
- MARSHALLER.marshal(this, stream);
+ public void save(OutputStream stream) throws TransformerException, TikaException {
+
+
+ DocumentBuilder builder = XMLReaderUtils.getDocumentBuilder();
+ Document doc = builder.newDocument();
+ Element paramEl = doc.createElement("param");
+ doc.appendChild(paramEl);
+
+ save(paramEl);
+
+ Transformer transformer = XMLReaderUtils.getTransformer();
+ transformer.transform(new DOMSource(paramEl), new StreamResult(stream));
}
- public void save(Node node) throws JAXBException {
- MARSHALLER.marshal(this, node);
- }
+ public void save(Node node) {
- public static <T> Param<T> load(InputStream stream) throws JAXBException {
- return (Param<T>) UNMARSHALLER.unmarshal(stream);
+ if ( !(node instanceof Element) ) {
+ throw new IllegalArgumentException("Not an Element : " + node);
+ }
+
+ Element el = (Element) node;
+
+ el.setAttribute("name", getName());
+ el.setAttribute("type", getTypeString());
+ el.setTextContent(value);
+ }
+
+ public static <T> Param<T> load(InputStream stream) throws SAXException, IOException, TikaException {
+
+ DocumentBuilder db = XMLReaderUtils.getDocumentBuilder();
+ Document document = db.parse(stream);
+
+ return load(document.getFirstChild());
+ }
+
+ public static <T> Param<T> load(Node node) {
+
+ Node nameAttr = node.getAttributes().getNamedItem("name");
+ Node typeAttr = node.getAttributes().getNamedItem("type");
+ Node value = node.getFirstChild();
+
+ Param<T> ret = new Param<T>();
+ ret.name = nameAttr.getTextContent();
+ ret.setTypeString(typeAttr.getTextContent());
+ ret.value = value.getTextContent();
+
+ return ret;
+ }
+
+ private static <T> Class<T> classFromType(String type) {
+ if (reverseMap.containsKey(type)){
+ return (Class<T>) reverseMap.get(type);
+ } else try {
+ return (Class<T>) Class.forName(type);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
-
- public static <T> Param<T> load(Node node) throws JAXBException {
- return (Param<T>) UNMARSHALLER.unmarshal(node);
+
+ private static <T> T getTypedValue(Class<T> type, String value) {
+ try {
+ Constructor<T> constructor = type.getConstructor(String.class);
+ constructor.setAccessible(true);
+ return constructor.newInstance(value);
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException(type + " doesnt have a constructor that takes String arg", e);
+ } catch (IllegalAccessException | InstantiationException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index 28ac235..9518042 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -17,7 +17,6 @@
package org.apache.tika.config;
import javax.imageio.spi.ServiceRegistry;
-import javax.xml.bind.JAXBException;
import javax.xml.parsers.DocumentBuilder;
import java.io.File;
import java.io.IOException;
@@ -685,12 +684,8 @@ public class TikaConfig {
for (int i = 0; i < childNodes.getLength(); i++) {
Node item = childNodes.item(i);
if (item.getNodeType() == Node.ELEMENT_NODE){
- try {
- Param<?> param = Param.load(item);
- params.put(param.getName(), param);
- } catch (JAXBException e) {
- throw new RuntimeException(e);
- }
+ Param<?> param = Param.load(item);
+ params.put(param.getName(), param);
}
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 332efcd..68df47b 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -23,6 +23,14 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLResolver;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.TransformerFactoryConfigurationError;
+
+import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -217,5 +225,28 @@ public class ParseContext implements Serializable {
}
return XMLReaderUtils.getXMLInputFactory();
}
+
+
+ /**
+ * Returns the transformer specified in this parsing context.
+ *
+ * If a transformer is not explicitly specified, then a default transformer
+ * instance is created and returned. The default transformer instance is
+ * configured to to use
+ * {@link XMLConstants#FEATURE_SECURE_PROCESSING secure XML processing}.
+ *
+ * @since Apache Tika 1.17
+ * @return Transformer
+ * @throws TikaException when the transformer can not be created
+ */
+ public Transformer getTransformer() throws TikaException {
+
+ Transformer transformer = get(Transformer.class);
+ if ( transformer != null ) {
+ return transformer;
+ }
+
+ return XMLReaderUtils.getTransformer();
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
index 6f24708..a326f14 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
@@ -34,6 +34,11 @@ import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLResolver;
import javax.xml.stream.XMLStreamException;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.TransformerFactoryConfigurationError;
+
import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Method;
@@ -236,4 +241,24 @@ public class XMLReaderUtils {
}
}
+ /**
+ * Returns a new transformer
+ *
+ * The transformer instance is configured to to use
+ * {@link XMLConstants#FEATURE_SECURE_PROCESSING secure XML processing}.
+ *
+ * @since Apache Tika 1.17
+ * @return Transformer
+ * @throws TikaException when the transformer can not be created
+ */
+ public static Transformer getTransformer() throws TikaException {
+ try {
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ return transformerFactory.newTransformer();
+ } catch (TransformerConfigurationException | TransformerFactoryConfigurationError e) {
+ throw new TikaException("Transformer not available", e);
+ }
+ }
+
}
--
To stop receiving notification emails like this one, please contact
"commits@tika.apache.org" <co...@tika.apache.org>.
[tika] 01/02: make strawman app driver actually work. Add ability
to specify a list of files.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 384e97156c5c5bf54d54b452c1783b7c4e5df068
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Sep 21 16:20:58 2017 -0400
make strawman app driver actually work. Add ability to specify a list of files.
---
.../batch/fs/strawman/StrawManTikaAppDriver.java | 139 +++++++++++----------
1 file changed, 74 insertions(+), 65 deletions(-)
diff --git a/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java b/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
index 8523d5c..4fef6c8 100644
--- a/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
+++ b/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
@@ -17,9 +17,9 @@ package org.apache.tika.batch.fs.strawman;
* limitations under the License.
*/
+import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -38,7 +38,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MarkerFactory;
@@ -60,12 +59,14 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
private final int threadNum;
private Path inputRoot = null;
private Path outputRoot = null;
+ private Path fileList = null;
private String[] args = null;
public StrawManTikaAppDriver(Path inputRoot, Path outputRoot,
- int totalThreads, String[] args) {
+ int totalThreads, Path fileList, String[] args) {
this.inputRoot = inputRoot;
this.outputRoot = outputRoot;
+ this.fileList = fileList;
this.args = args;
threadNum = threadCount.getAndIncrement();
this.totalThreads = totalThreads;
@@ -73,7 +74,7 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
private class TikaVisitor extends SimpleFileVisitor<Path> {
- private int processed = 0;
+ private volatile int processed = 0;
int getProcessed() {
return processed;
@@ -87,10 +88,31 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
return FileVisitResult.CONTINUE;
}
}
- assert(file.startsWith(inputRoot));
+ if (! file.startsWith(inputRoot)) {
+ LOG.warn("File ("+file.toAbsolutePath()+
+ ") doesn't start with input root ("+inputRoot.toAbsolutePath()+")");
+ return FileVisitResult.CONTINUE;
+ }
Path relPath = inputRoot.relativize(file);
+ String suffix = ".txt";
+ List<String> commandLine = new ArrayList<>();
+ for (String arg : args) {
+ commandLine.add(arg);
+ if (arg.equals("-J")) {
+ suffix = ".json";
+ } else if (arg.contains("-x")) {
+ suffix = ".html";
+ }
+ }
+ String fullPath = file.toAbsolutePath().toString();
+ if (fullPath.contains(" ")) {
+ fullPath = "\""+fullPath+"\"";
+ }
+ commandLine.add(fullPath);
+
+
Path outputFile = Paths.get(outputRoot.toAbsolutePath().toString(),
- relPath.toString() + ".txt");
+ relPath.toString() + suffix);
try {
Files.createDirectories(outputFile.getParent());
} catch (IOException e) {
@@ -98,23 +120,15 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
"parent directory for {} was not made!", outputFile);
throw new RuntimeException("couldn't make parent file for " + outputFile);
}
- List<String> commandLine = new ArrayList<>();
- for (String arg : args) {
- commandLine.add(arg);
- }
- commandLine.add("-t");
- commandLine.add("\""+outputFile.toAbsolutePath()+"\"");
- ProcessBuilder builder = new ProcessBuilder(commandLine.toArray(new String[commandLine.size()]));
+ ProcessBuilder builder = new ProcessBuilder();
+ builder.command(commandLine);
LOG.info("about to process: {}", file.toAbsolutePath());
+ builder.redirectOutput(outputFile.toFile());
+ builder.redirectError(ProcessBuilder.Redirect.INHERIT);
+
Process proc = null;
- RedirectGobbler gobbler = null;
- Thread gobblerThread = null;
try {
- OutputStream os = Files.newOutputStream(outputFile);
proc = builder.start();
- gobbler = new RedirectGobbler(proc.getInputStream(), os);
- gobblerThread = new Thread(gobbler);
- gobblerThread.start();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
return FileVisitResult.CONTINUE;
@@ -141,8 +155,12 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
LOG.warn("Had to kill process working on: {}", file.toAbsolutePath());
proc.destroy();
}
- gobbler.close();
- gobblerThread.interrupt();
+ try {
+ proc.getOutputStream().flush();
+ proc.getOutputStream().close();
+ } catch (IOException e) {
+ LOG.warn("couldn't close process outputstream", e);
+ }
processed++;
return FileVisitResult.CONTINUE;
}
@@ -155,53 +173,33 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
public Integer call() throws Exception {
long start = new Date().getTime();
TikaVisitor v = new TikaVisitor();
- Files.walkFileTree(inputRoot, v);
+ if (fileList != null) {
+ TikaVisitor tikaVisitor = new TikaVisitor();
+ try (BufferedReader reader = Files.newBufferedReader(fileList, StandardCharsets.UTF_8)) {
+ String line = reader.readLine();
+ while (line != null) {
+ Path inputFile = inputRoot.resolve(line.trim());
+ if (Files.isReadable(inputFile)) {
+ try {
+ tikaVisitor.visitFile(inputFile, Files.readAttributes(inputFile, BasicFileAttributes.class));
+ } catch (IOException e) {
+ LOG.warn("Problem with: "+inputFile, e);
+ }
+ } else {
+ LOG.warn("Not readable: "+inputFile);
+ }
+ line = reader.readLine();
+ }
+ }
+ } else {
+ Files.walkFileTree(inputRoot, v);
+ }
int processed = v.getProcessed();
double elapsedSecs = ((double)new Date().getTime()-(double)start)/(double)1000;
LOG.info("Finished processing {} files in {} seconds.", processed, elapsedSecs);
return processed;
}
- private class RedirectGobbler implements Runnable {
- private OutputStream redirectOs = null;
- private InputStream redirectIs = null;
-
- private RedirectGobbler(InputStream is, OutputStream os) {
- this.redirectIs = is;
- this.redirectOs = os;
- }
-
- private void close() {
- if (redirectOs != null) {
- try {
- redirectOs.flush();
- } catch (IOException e) {
- LOG.error("can't flush");
- }
- try {
- redirectIs.close();
- } catch (IOException e) {
- LOG.error("can't close input in redirect gobbler");
- }
- try {
- redirectOs.close();
- } catch (IOException e) {
- LOG.error("can't close output in redirect gobbler");
- }
- }
- }
-
- @Override
- public void run() {
- try {
- IOUtils.copy(redirectIs, redirectOs);
- } catch (IOException e) {
- LOG.error("IOException while gobbling");
- }
- }
- }
-
-
public static String usage() {
StringBuilder sb = new StringBuilder();
@@ -220,9 +218,18 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
Path inputDir = Paths.get(args[0]);
Path outputDir = Paths.get(args[1]);
int totalThreads = Integer.parseInt(args[2]);
+ Path fileList = null;
+ if (args.length > 3) {
+ fileList = Paths.get(args[3]);
+ if (! Files.isReadable(fileList)) {
+ fileList = null;
+ }
+ }
List<String> commandLine = new ArrayList<>();
- commandLine.addAll(Arrays.asList(args).subList(3, args.length));
+
+ int initialParams = (fileList == null) ? 3 : 4;
+ commandLine.addAll(Arrays.asList(args).subList(initialParams, args.length));
totalThreads = (totalThreads < 1) ? 1 : totalThreads;
ExecutorService ex = Executors.newFixedThreadPool(totalThreads);
ExecutorCompletionService<Integer> completionService =
@@ -230,7 +237,8 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
for (int i = 0; i < totalThreads; i++) {
StrawManTikaAppDriver driver =
- new StrawManTikaAppDriver(inputDir, outputDir, totalThreads, commandLine.toArray(new String[commandLine.size()]));
+ new StrawManTikaAppDriver(inputDir, outputDir, totalThreads, fileList,
+ commandLine.toArray(new String[commandLine.size()]));
completionService.submit(driver);
}
@@ -247,5 +255,6 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
}
double elapsedSeconds = (double)(new Date().getTime() - start) / (double)1000;
LOG.info("Processed {} in {} seconds", totalFilesProcessed, elapsedSeconds);
+ ex.shutdownNow();
}
}
--
To stop receiving notification emails like this one, please contact
"commits@tika.apache.org" <co...@tika.apache.org>.