You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/04/16 19:38:42 UTC
[tika] branch branch_1x updated (5b12e7f -> 5d983aa)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 5b12e7f [maven-release-plugin] prepare for next development iteration
new e82c2ef fix potential resource leak
new b2d3932 fix potential resource leak
new 4fdc51a followup fix
new ffb48dd fix chm parser
new 302f22a fix readUE7
new 5d983aa fix chm; remove println
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/tika/io/EndianUtils.java | 3 +
.../chm/accessor/ChmDirectoryListingSet.java | 12 ++-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 83 +++++++++++-----
.../apache/tika/parser/ocr/TesseractOCRParser.java | 76 +++++++++++----
.../tika/parser/ocr/TesseractOCRConfigTest.java | 51 ++++++++++
.../apache/tika/server/resource/TikaResource.java | 105 ++++++++++++++++-----
.../org/apache/tika/server/TikaResourceTest.java | 35 +++++++
7 files changed, 299 insertions(+), 66 deletions(-)
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 03/06: followup fix
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4fdc51a40bf9532d7db57d0b08c1aec3931468ad
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 12:28:34 2018 -0400
followup fix
---
.../apache/tika/server/resource/TikaResource.java | 22 ++++++++++++++--------
.../org/apache/tika/server/TikaResourceTest.java | 13 +++++++++++++
2 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
index 2becf07..d8795f7 100644
--- a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
+++ b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
@@ -198,20 +198,26 @@ public class TikaResource {
try {
String property = StringUtils.removeStart(key, prefix);
+ Field field = null;
+ try {
+ object.getClass().getDeclaredField(StringUtils.uncapitalize(property));
+ } catch (NoSuchFieldException e) {
+ //swallow
+ }
String setter = property;
setter = "set"+setter.substring(0,1).toUpperCase(Locale.US)+setter.substring(1);
- Field field = object.getClass().getDeclaredField(StringUtils.uncapitalize(property));
//default assume string class
//if there's a more specific type, e.g. double, int, boolean
//try that.
Class clazz = String.class;
-
- if (field.getType() == int.class) {
- clazz = int.class;
- } else if (field.getType() == double.class) {
- clazz = double.class;
- } else if (field.getType() == boolean.class) {
- clazz = boolean.class;
+ if (field != null) {
+ if (field.getType() == int.class) {
+ clazz = int.class;
+ } else if (field.getType() == double.class) {
+ clazz = double.class;
+ } else if (field.getType() == boolean.class) {
+ clazz = boolean.class;
+ }
}
Method m = tryToGetMethod(object, setter, clazz);
diff --git a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
index 9e624f8..bd86fd8 100644
--- a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
+++ b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
@@ -301,4 +301,17 @@ public class TikaResourceTest extends CXFTestBase {
.put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
assertEquals(200, response.getStatus());
}
+
+ @Test
+ public void testTrustedMethodPrevention() {
+ Response response = WebClient.create(endPoint + TIKA_PATH)
+ .type("application/pdf")
+ .accept("text/plain")
+ .header(TikaResource.X_TIKA_OCR_HEADER_PREFIX +
+ "trustedPageSeparator",
+ "\u0010")
+ .put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
+ assertEquals(500, response.getStatus());
+
+ }
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 01/06: fix potential resource leak
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit e82c2efd2b1ac731b6954634741b70ecf0ed6f01
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 10:44:24 2018 -0400
fix potential resource leak
---
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 83 ++++++++++++++-----
.../apache/tika/parser/ocr/TesseractOCRParser.java | 76 ++++++++++++-----
.../tika/parser/ocr/TesseractOCRConfigTest.java | 51 ++++++++++++
.../apache/tika/server/resource/TikaResource.java | 95 +++++++++++++++++-----
.../org/apache/tika/server/TikaResourceTest.java | 22 +++++
5 files changed, 264 insertions(+), 63 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
index 07bb7f8..c4d8958 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
@@ -16,7 +16,8 @@
*/
package org.apache.tika.parser.ocr;
-import java.io.File;
+import org.apache.commons.io.FilenameUtils;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
@@ -24,6 +25,8 @@ import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Configuration for TesseractOCRParser.
@@ -43,6 +46,12 @@ public class TesseractOCRConfig implements Serializable {
private static final long serialVersionUID = -4861942486845757891L;
+ private static Pattern ALLOWABLE_PAGE_SEPARATORS_PATTERN =
+ Pattern.compile("(?i)^[-_<>A-Z0-9 \n\r]+$");
+
+ private static Pattern ALLOWABLE_OTHER_PARAMS_PATTERN =
+ Pattern.compile("(?i)^[-/_<>A-Z0-9]+$");
+
public enum OUTPUT_TYPE {
TXT,
HOCR
@@ -157,12 +166,7 @@ public class TesseractOCRConfig implements Serializable {
getProp(props, "maxFileSizeToOcr", getMaxFileSizeToOcr()));
setTimeout(
getProp(props, "timeout", getTimeout()));
- String outputTypeString = props.getProperty("outputType");
- if ("txt".equals(outputTypeString)) {
- setOutputType(OUTPUT_TYPE.TXT);
- } else if ("hocr".equals(outputTypeString)) {
- setOutputType(OUTPUT_TYPE.HOCR);
- }
+ setOutputType(getProp(props, "outputType", getOutputType().toString()));
setPreserveInterwordSpacing(getProp(props, "preserveInterwordSpacing", false));
// set parameters for ImageMagick
@@ -201,10 +205,7 @@ public class TesseractOCRConfig implements Serializable {
* </p>
*/
public void setTesseractPath(String tesseractPath) {
- if (!tesseractPath.isEmpty() && !tesseractPath.endsWith(File.separator))
- tesseractPath += File.separator;
-
- this.tesseractPath = tesseractPath;
+ this.tesseractPath = FilenameUtils.normalize(tesseractPath);
}
/**
@@ -220,10 +221,7 @@ public class TesseractOCRConfig implements Serializable {
* (such as when Tesseract is built from source), it may be located elsewhere.
*/
public void setTessdataPath(String tessdataPath) {
- if (!tessdataPath.isEmpty() && !tessdataPath.endsWith(File.separator))
- tessdataPath += File.separator;
-
- this.tessdataPath = tessdataPath;
+ this.tessdataPath = FilenameUtils.normalize(tessdataPath);
}
/**
@@ -280,6 +278,20 @@ public class TesseractOCRConfig implements Serializable {
* @param pageSeparator
*/
public void setPageSeparator(String pageSeparator) {
+ Matcher m = ALLOWABLE_PAGE_SEPARATORS_PATTERN.matcher(pageSeparator);
+ if (! m.find()) {
+ throw new IllegalArgumentException(pageSeparator + " contains illegal characters.\n"+
+ "If you trust this value, set it with setTrustedPageSeparator");
+ }
+ this.pageSeparator = pageSeparator;
+ }
+
+ /**
+ * Same as {@link #setPageSeparator(String)} but does not perform
+ * any checks on the string.
+ * @param pageSeparator
+ */
+ public void setTrustedPageSeparator(String pageSeparator) {
this.pageSeparator = pageSeparator;
}
@@ -353,6 +365,22 @@ public class TesseractOCRConfig implements Serializable {
this.outputType = outputType;
}
+ public void setOutputType(String outputType) {
+ if (outputType == null) {
+ throw new IllegalArgumentException("outputType must not be null");
+ }
+ String lc = outputType.toLowerCase(Locale.US);
+ if ("txt".equals(lc)) {
+ setOutputType(OUTPUT_TYPE.TXT);
+ } else if ("hocr".equals(lc)) {
+ setOutputType(OUTPUT_TYPE.HOCR);
+ } else {
+ throw new IllegalArgumentException("outputType must be either 'txt' or 'hocr'");
+ }
+
+
+ }
+
/**
* @see #setOutputType(OUTPUT_TYPE outputType)
*/
@@ -499,10 +527,7 @@ public class TesseractOCRConfig implements Serializable {
* @param ImageMagickPath to ImageMagick file.
*/
public void setImageMagickPath(String ImageMagickPath) {
- if (!ImageMagickPath.isEmpty() && !ImageMagickPath.endsWith(File.separator))
- ImageMagickPath += File.separator;
-
- this.ImageMagickPath = ImageMagickPath;
+ this.ImageMagickPath = FilenameUtils.normalize(ImageMagickPath);
}
/**
@@ -541,7 +566,23 @@ public class TesseractOCRConfig implements Serializable {
* @param value
*/
public void addOtherTesseractConfig(String key, String value) {
- otherTesseractConfig.put(key, value);
+ if (key == null) {
+ throw new IllegalArgumentException("key must not be null");
+ }
+ if (value == null) {
+ throw new IllegalArgumentException("value must not be null");
+ }
+
+ Matcher m = ALLOWABLE_OTHER_PARAMS_PATTERN.matcher(key);
+ if (! m.find()) {
+ throw new IllegalArgumentException("Value contains illegal characters: "+key);
+ }
+ m.reset(value);
+ if (! m.find()) {
+ throw new IllegalArgumentException("Value contains illegal characters: "+value);
+ }
+
+ otherTesseractConfig.put(key.trim(), value.trim());
}
/**
@@ -603,7 +644,7 @@ public class TesseractOCRConfig implements Serializable {
private void loadOtherTesseractConfig(Properties properties) {
for (String k : properties.stringPropertyNames()) {
if (k.contains("_")) {
- otherTesseractConfig.put(k, properties.getProperty(k));
+ addOtherTesseractConfig(k, properties.getProperty(k));
}
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index f274ce1..5bd2bad 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -33,6 +33,7 @@ import java.io.OutputStreamWriter;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
@@ -111,6 +112,7 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
MediaType.image("jpx"), MediaType.image("x-portable-pixmap")
})));
private static Map<String,Boolean> TESSERACT_PRESENT = new HashMap<>();
+ private static Map<String,Boolean> IMAGE_MAGICK_PRESENT = new HashMap<>();
@Override
@@ -145,6 +147,16 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
if (TESSERACT_PRESENT.containsKey(tesseract)) {
return TESSERACT_PRESENT.get(tesseract);
}
+ //prevent memory bloat
+ if (TESSERACT_PRESENT.size() > 100) {
+ TESSERACT_PRESENT.clear();
+ }
+ //check that the parent directory exists
+ if (! Files.isDirectory(Paths.get(config.getTesseractPath()))) {
+ TESSERACT_PRESENT.put(tesseract, false);
+ return false;
+ }
+
// Try running Tesseract from there, and see if it exists + works
String[] checkCmd = { tesseract };
boolean hasTesseract = ExternalParser.check(checkCmd);
@@ -158,14 +170,22 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
String ImageMagick = getImageMagickPath(config);
// Have we already checked for a copy of ImageMagick Program there?
- if (TESSERACT_PRESENT.containsKey(ImageMagick)) {
- return TESSERACT_PRESENT.get(ImageMagick);
+ if (IMAGE_MAGICK_PRESENT.containsKey(ImageMagick)) {
+ return IMAGE_MAGICK_PRESENT.get(ImageMagick);
+ }
+ //prevent memory bloat
+ if (IMAGE_MAGICK_PRESENT.size() > 100) {
+ IMAGE_MAGICK_PRESENT.clear();
+ }
+ //check that directory exists
+ if (! Files.isDirectory(Paths.get(config.getImageMagickPath()))) {
+ IMAGE_MAGICK_PRESENT.put(ImageMagick, false);
+ return false;
}
-
// Try running ImageMagick program from there, and see if it exists + works
String[] checkCmd = { ImageMagick };
boolean hasImageMagick = ExternalParser.check(checkCmd);
- TESSERACT_PRESENT.put(ImageMagick, hasImageMagick);
+ IMAGE_MAGICK_PRESENT.put(ImageMagick, hasImageMagick);
return hasImageMagick;
@@ -178,9 +198,9 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
static boolean hasPython() {
// check if python is installed and it has the required dependencies for the rotation program to run
boolean hasPython = false;
-
+ TemporaryResources tmp = null;
try {
- TemporaryResources tmp = new TemporaryResources();
+ tmp = new TemporaryResources();
File importCheck = tmp.createTemporaryFile();
String prg = "import numpy, matplotlib, skimage";
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(importCheck), Charset.forName("UTF-8"));
@@ -192,10 +212,11 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
hasPython = true;
}
- tmp.close();
} catch (Exception e) {
+ } finally {
+ IOUtils.closeQuietly(tmp);
}
return hasPython;
@@ -311,20 +332,26 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
/**
* This method is used to process the image to an OCR-friendly format.
- * @param streamingObject input image to be processed
+ * @param scratchFile input image to be processed
* @param config TesseractOCRconfig class to get ImageMagick properties
* @throws IOException if an input error occurred
* @throws TikaException if an exception timed out
*/
- private void processImage(File streamingObject, TesseractOCRConfig config) throws IOException, TikaException {
+ private void processImage(File scratchFile, TesseractOCRConfig config) throws IOException, TikaException {
// fetch rotation script from resources
InputStream in = getClass().getResourceAsStream("rotation.py");
TemporaryResources tmp = new TemporaryResources();
File rotationScript = tmp.createTemporaryFile();
Files.copy(in, rotationScript.toPath(), StandardCopyOption.REPLACE_EXISTING);
-
- String cmd = "python -W ignore " + rotationScript.getAbsolutePath() + " -f " + streamingObject.getAbsolutePath();
+
+ CommandLine commandLine = new CommandLine("python");
+ String[] args = {"-W",
+ "ignore",
+ rotationScript.getAbsolutePath(),
+ "-f",
+ scratchFile.getAbsolutePath()};
+ commandLine.addArguments(args, true);
String angle = "0";
DefaultExecutor executor = new DefaultExecutor();
@@ -333,24 +360,33 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
executor.setStreamHandler(streamHandler);
// determine the angle of rotation required to make the text horizontal
- CommandLine cmdLine = CommandLine.parse(cmd);
if(config.getApplyRotation() && hasPython()) {
try {
- executor.execute(cmdLine);
- angle = outputStream.toString("UTF-8").trim();
+ executor.execute(commandLine);
+ String tmpAngle = outputStream.toString("UTF-8").trim();
+ //verify that you've gotten a numeric value out
+ Double.parseDouble(tmpAngle);
+ angle = tmpAngle;
} catch(Exception e) {
}
}
// process the image - parameter values can be set in TesseractOCRConfig.properties
- String line = getImageMagickPath(config) + " -density " + config.getDensity() + " -depth " + config.getDepth() +
- " -colorspace " + config.getColorspace() + " -filter " + config.getFilter() +
- " -resize " + config.getResize() + "% -rotate "+ angle + " " + streamingObject.getAbsolutePath() +
- " " + streamingObject.getAbsolutePath();
- cmdLine = CommandLine.parse(line);
+ commandLine = new CommandLine(getImageMagickPath(config));
+ args = new String[]{
+ "-density", Integer.toString(config.getDensity()),
+ "-depth ", Integer.toString(config.getDepth()),
+ "-colorspace", config.getColorspace(),
+ " -filter ", config.getFilter(),
+ "-resize", config.getResize() + "%",
+ "-rotate", angle,
+ scratchFile.getAbsolutePath(),
+ scratchFile.getAbsolutePath()
+ };
+ commandLine.addArguments(args, true);
try {
- executor.execute(cmdLine);
+ executor.execute(commandLine);
} catch(Exception e) {
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTest.java
index 8e22f21..e201aa2 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTest.java
@@ -171,4 +171,55 @@ public class TesseractOCRConfigTest extends TikaTest {
config.setResize(1000);
}
+ @Test(expected=IllegalArgumentException.class)
+ public void testDataPathCheck() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.setTessdataPath("blah\u0000deblah");
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testPathCheck() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.setTesseractPath("blah\u0000deblah");
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testBadOtherKey() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.addOtherTesseractConfig("bad bad", "bad");
+
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testBadOtherValue() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.addOtherTesseractConfig("bad", "bad bad");
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testBadOtherValueSlash() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.addOtherTesseractConfig("bad", "bad\\bad");
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testBadOtherValueControl() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.addOtherTesseractConfig("bad", "bad\u0001bad");
+ }
+
+ @Test
+ public void testGoodOtherParameters() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.addOtherTesseractConfig("good", "good");
+ }
+
+ @Test
+ public void testBogusPathCheck() {
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ config.setTesseractPath("blahdeblahblah");
+ assertEquals("blahdeblahblah", config.getTesseractPath());
+ }
+
+
}
diff --git a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
index 77afed9..ba042a9 100644
--- a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
+++ b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
@@ -47,8 +47,9 @@ import java.io.Writer;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Locale;
-import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.cxf.jaxrs.ext.multipart.Attachment;
@@ -67,7 +68,6 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.html.BoilerpipeContentHandler;
-import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.parser.ocr.TesseractOCRConfig;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.sax.BodyContentHandler;
@@ -82,6 +82,9 @@ import org.xml.sax.SAXException;
@Path("/tika")
public class TikaResource {
+
+ private static Pattern ALLOWABLE_HEADER_CHARS = Pattern.compile("(?i)^[-_<>A-Z0-9 ]+$");
+
public static final String GREETING = "This is Tika Server (" + new Tika().toString() + "). Please PUT\n";
public static final String X_TIKA_OCR_HEADER_PREFIX = "X-Tika-OCR";
public static final String X_TIKA_PDF_HEADER_PREFIX = "X-Tika-PDF";
@@ -190,40 +193,88 @@ public class TikaResource {
* @throws WebApplicationException thrown when field cannot be found.
*/
private static void processHeaderConfig(MultivaluedMap<String, String> httpHeaders, Object object, String key, String prefix) {
+
try {
String property = StringUtils.removeStart(key, prefix);
+ String setter = property;
+ setter = "set"+setter.substring(0,1).toUpperCase(Locale.US)+setter.substring(1);
Field field = object.getClass().getDeclaredField(StringUtils.uncapitalize(property));
+ //default assume string class
+ //if there's a more specific type, e.g. double, int, boolean
+ //try that.
+ Class clazz = String.class;
- field.setAccessible(true);
- if (field.getType() == String.class) {
- field.set(object, httpHeaders.getFirst(key));
- } else if (field.getType() == int.class) {
- field.setInt(object, Integer.parseInt(httpHeaders.getFirst(key)));
+ if (field.getType() == int.class) {
+ clazz = int.class;
} else if (field.getType() == double.class) {
- field.setDouble(object, Double.parseDouble(httpHeaders.getFirst(key)));
+ clazz = double.class;
} else if (field.getType() == boolean.class) {
- field.setBoolean(object, Boolean.parseBoolean(httpHeaders.getFirst(key)));
- } else {
- //couldn't find a directly accessible field
- //try for setX(String s)
- String setter = StringUtils.uncapitalize(property);
- setter = "set"+setter.substring(0,1).toUpperCase(Locale.US)+setter.substring(1);
- Method m = null;
- try {
- m = object.getClass().getMethod(setter, String.class);
- } catch (NoSuchMethodException e) {
- //swallow
- }
- if (m != null) {
- m.invoke(object, httpHeaders.getFirst(key));
+ clazz = boolean.class;
+ }
+
+ Method m = tryToGetMethod(object, setter, clazz);
+ //if you couldn't find more specific setter, back off
+ //to string setter and try that.
+ if (m == null && clazz != String.class) {
+ m = tryToGetMethod(object, setter, String.class);
+ }
+
+ if (m != null) {
+ String val = httpHeaders.getFirst(key);
+ val = val.trim();
+ if (clazz == String.class) {
+ checkTrustWorthy(setter, val);
+ m.invoke(object, val);
+ } else if (clazz == int.class) {
+ m.invoke(object, Integer.parseInt(val));
+ } else if (clazz == double.class) {
+ m.invoke(object, Double.parseDouble(val));
+ } else if (clazz == boolean.class) {
+ m.invoke(object, Boolean.parseBoolean(val));
+ } else {
+ throw new IllegalArgumentException("setter must be String, int, double or boolean...for now");
}
+ } else {
+ throw new NoSuchMethodException("Couldn't find: "+setter);
}
+
} catch (Throwable ex) {
throw new WebApplicationException(String.format(Locale.ROOT,
"%s is an invalid %s header", key, X_TIKA_OCR_HEADER_PREFIX));
}
}
+ private static void checkTrustWorthy(String setter, String val) {
+ if (setter == null || val == null) {
+ throw new IllegalArgumentException("setter and val must not be null");
+ }
+ if (setter.toLowerCase(Locale.US).contains("trusted")) {
+ throw new IllegalArgumentException("Can't call a trusted method via tika-server headers");
+ }
+ Matcher m = ALLOWABLE_HEADER_CHARS.matcher(val);
+ if (! m.find()) {
+ throw new IllegalArgumentException("Header val: "+val +" contains illegal characters. " +
+ "Must contain: TikaResource.ALLOWABLE_HEADER_CHARS");
+ }
+ }
+
+ /**
+ * Tries to get method. Silently swallows NoMethodException and returns
+ * <code>null</code> if not found.
+ * @param object
+ * @param method
+ * @param clazz
+ * @return
+ */
+ private static Method tryToGetMethod(Object object, String method, Class clazz) {
+ try {
+ return object.getClass().getMethod(method, clazz);
+ } catch (NoSuchMethodException e) {
+ //swallow
+ }
+ return null;
+ }
+
@SuppressWarnings("serial")
public static void fillMetadata(Parser parser, Metadata metadata, ParseContext context, MultivaluedMap<String, String> httpHeaders) {
String fileName = detectFilename(httpHeaders);
diff --git a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
index 5d112ff..9e624f8 100644
--- a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
+++ b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
@@ -279,4 +279,26 @@ public class TikaResourceTest extends CXFTestBase {
responseMsg
);
}
+
+ @Test
+ public void testDataIntegrityCheck() throws Exception {
+ Response response = WebClient.create(endPoint + TIKA_PATH)
+ .type("application/pdf")
+ .accept("text/plain")
+ .header(TikaResource.X_TIKA_OCR_HEADER_PREFIX +
+ "tesseractPath",
+
+ "C://tmp//hello.bat\u0000")
+ .put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
+ assertEquals(500, response.getStatus());
+
+ response = WebClient.create(endPoint + TIKA_PATH)
+ .type("application/pdf")
+ .accept("text/plain")
+ .header(TikaResource.X_TIKA_OCR_HEADER_PREFIX +
+ "tesseractPath",
+ "bogus path")
+ .put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
+ assertEquals(200, response.getStatus());
+ }
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 06/06: fix chm; remove println
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 5d983aad0b68a228f180686a4135ed8c7cd589f1
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 14:14:38 2018 -0400
fix chm; remove println
---
.../org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java b/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
index b379142..5b3d3f6 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
@@ -158,7 +158,7 @@ public class ChmDirectoryListingSet {
i=nextBlock;
dir_chunk = null;
}
- System.out.println("done");
+
} catch (ChmParsingException e) {
LOG.warn("Chm parse exception", e);
} finally {
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 02/06: fix potential resource leak
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b2d3932b847a171a85e356aa230af461a0f80d91
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 15:36:46 2018 -0400
fix potential resource leak
---
.../src/main/java/org/apache/tika/server/resource/TikaResource.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
index ba042a9..2becf07 100644
--- a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
+++ b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
@@ -47,6 +47,7 @@ import java.io.Writer;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -68,6 +69,7 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.html.BoilerpipeContentHandler;
+import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.parser.ocr.TesseractOCRConfig;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.sax.BodyContentHandler;
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 05/06: fix readUE7
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 302f22aff7a836868b270038e1d66002a2004869
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 14:05:20 2018 -0400
fix readUE7
---
tika-core/src/main/java/org/apache/tika/io/EndianUtils.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/tika-core/src/main/java/org/apache/tika/io/EndianUtils.java b/tika-core/src/main/java/org/apache/tika/io/EndianUtils.java
index 9f8aca3..96ba1ae 100644
--- a/tika-core/src/main/java/org/apache/tika/io/EndianUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/io/EndianUtils.java
@@ -230,6 +230,9 @@ public class EndianUtils {
break;
}
}
+ if (i < 0) {
+ throw new IOException("Buffer underun; expected one more byte");
+ }
return v;
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 04/06: fix chm parser
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ffb48dd29d0c2009490caefda75e5b57c7958c51
Author: tballison <ta...@mitre.org>
AuthorDate: Mon Apr 16 14:03:15 2018 -0400
fix chm parser
---
.../tika/parser/chm/accessor/ChmDirectoryListingSet.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java b/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
index ccb3286..b379142 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/chm/accessor/ChmDirectoryListingSet.java
@@ -20,7 +20,9 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.math.BigInteger;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.chm.core.ChmCommons;
@@ -137,6 +139,7 @@ public class ChmDirectoryListingSet {
/* loops over all pmgls */
byte[] dir_chunk = null;
+ Set<Integer> processed = new HashSet<>();
for (int i = startPmgl; i>=0; ) {
dir_chunk = new byte[(int) chmItspHeader.getBlock_len()];
int start = i * (int) chmItspHeader.getBlock_len() + dir_offset;
@@ -147,10 +150,15 @@ public class ChmDirectoryListingSet {
PMGLheader = new ChmPmglHeader();
PMGLheader.parse(dir_chunk, PMGLheader);
enumerateOneSegment(dir_chunk);
-
- i=PMGLheader.getBlockNext();
+ int nextBlock = PMGLheader.getBlockNext();
+ processed.add(i);
+ if (processed.contains(nextBlock)) {
+ throw new ChmParsingException("already processed block; avoiding cycle");
+ }
+ i=nextBlock;
dir_chunk = null;
}
+ System.out.println("done");
} catch (ChmParsingException e) {
LOG.warn("Chm parse exception", e);
} finally {
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.