You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by mi...@apache.org on 2011/09/05 20:45:38 UTC
svn commit: r1165382 - in /tika/trunk:
tika-core/src/main/java/org/apache/tika/sax/
tika-core/src/test/java/org/apache/tika/sax/
tika-parsers/src/test/java/org/apache/tika/
tika-parsers/src/test/java/org/apache/tika/parser/prt/
tika-parsers/src/test/ja...
Author: mikemccand
Date: Mon Sep 5 18:45:37 2011
New Revision: 1165382
URL: http://svn.apache.org/viewvc?rev=1165382&view=rev
Log:
TIKA-698: use the unicode replacement char (U+fffd) when replacing invalid XML chars in SafeContentHandler
Modified:
tika/trunk/tika-core/src/main/java/org/apache/tika/sax/SafeContentHandler.java
tika/trunk/tika-core/src/test/java/org/apache/tika/sax/SafeContentHandlerTest.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/TestParsers.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/prt/PRTParserTest.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java
Modified: tika/trunk/tika-core/src/main/java/org/apache/tika/sax/SafeContentHandler.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/sax/SafeContentHandler.java?rev=1165382&r1=1165381&r2=1165382&view=diff
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/sax/SafeContentHandler.java (original)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/sax/SafeContentHandler.java Mon Sep 5 18:45:37 2011
@@ -43,7 +43,7 @@ public class SafeContentHandler extends
/**
* Replacement for invalid characters.
*/
- private static final char[] REPLACEMENT = new char[] { ' ' };
+ private static final char[] REPLACEMENT = new char[] { '\ufffd' };
/**
* Internal interface that allows both character and
Modified: tika/trunk/tika-core/src/test/java/org/apache/tika/sax/SafeContentHandlerTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/java/org/apache/tika/sax/SafeContentHandlerTest.java?rev=1165382&r1=1165381&r2=1165382&view=diff
==============================================================================
--- tika/trunk/tika-core/src/test/java/org/apache/tika/sax/SafeContentHandlerTest.java (original)
+++ tika/trunk/tika-core/src/test/java/org/apache/tika/sax/SafeContentHandlerTest.java Mon Sep 5 18:45:37 2011
@@ -55,19 +55,19 @@ public class SafeContentHandlerTest exte
safe.characters("ab\u0007".toCharArray(), 0, 3);
safe.characters("a\u000Bc".toCharArray(), 0, 3);
safe.characters("\u0019bc".toCharArray(), 0, 3);
- assertEquals("ab a c bc", output.toString());
+ assertEquals("ab\ufffda\ufffdc\ufffdbc", output.toString());
}
public void testInvalidWhitespace() throws SAXException {
safe.ignorableWhitespace("ab\u0000".toCharArray(), 0, 3);
safe.ignorableWhitespace("a\u0001c".toCharArray(), 0, 3);
safe.ignorableWhitespace("\u0002bc".toCharArray(), 0, 3);
- assertEquals("ab a c bc", output.toString());
+ assertEquals("ab\ufffda\ufffdc\ufffdbc", output.toString());
}
public void testInvalidSurrogates() throws SAXException {
safe.ignorableWhitespace("\udb00\ubfff".toCharArray(), 0, 2);
- assertEquals(" \ubfff", output.toString());
+ assertEquals("\ufffd\ubfff", output.toString());
}
}
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/TestParsers.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/TestParsers.java?rev=1165382&r1=1165381&r2=1165382&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/TestParsers.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/TestParsers.java Mon Sep 5 18:45:37 2011
@@ -102,7 +102,9 @@ public class TestParsers extends TikaTes
String s1 = ParseUtils.getStringContent(file, tc);
String s2 = ParseUtils.getStringContent(file, tc, "application/rtf");
assertEquals(s1, s2);
- String content = s1.replaceAll("\\s+"," ");
+ String content = s1;
+ content = content.replaceAll("\ufffd", " ");
+ content = content.replaceAll("\\s+"," ");
assertTrue(content.contains("a b c d \u00E4 \u00EB \u00F6 \u00FC"));
}
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/prt/PRTParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/prt/PRTParserTest.java?rev=1165382&r1=1165381&r2=1165382&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/prt/PRTParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/prt/PRTParserTest.java Mon Sep 5 18:45:37 2011
@@ -103,7 +103,7 @@ public class PRTParserTest extends TikaT
assertContains("112"+'\u00b0', contents); // Degrees
assertContains("TIKA TEST LETTER", contents);
assertContains("17.11", contents);
- assertContains('\u00d8'+" 2.000", contents); // Diameter
+ assertContains('\u00d8'+"\ufffd2.000", contents); // Diameter
assertContains("Diameter", contents);
assertContains("The Apache Tika toolkit", contents);
} finally {
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java?rev=1165382&r1=1165381&r2=1165382&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java Mon Sep 5 18:45:37 2011
@@ -21,7 +21,6 @@ import java.io.FileInputStream;
import java.io.StringWriter;
import org.apache.tika.TikaTest;
-import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.WriteOutContentHandler;
@@ -30,11 +29,9 @@ import org.apache.tika.sax.WriteOutConte
* Junit test class for the Tika {@link RTFParser}
*/
public class RTFParserTest extends TikaTest {
- private TikaConfig tc;
private RTFParser parser;
public void setUp() throws Exception {
- tc = TikaConfig.getDefaultConfig();
parser = new RTFParser();
}
@@ -96,6 +93,7 @@ public class RTFParserTest extends TikaT
public void testRTFTableCellSeparation() throws Exception {
String content = getText("testRTFTableCellSeparation.rtf");
// TODO: why do we insert extra whitespace...?
+ content = content.replaceAll("\ufffd", " ");
content = content.replaceAll("\\s+"," ");
assertContains("a b c d \u00E4 \u00EB \u00F6 \u00FC", content);
}
@@ -103,6 +101,7 @@ public class RTFParserTest extends TikaT
public void testRTFTableCellSeparation2() throws Exception {
String content = getText("testRTFTableCellSeparation2.rtf");
// TODO: why do we insert extra whitespace...?
+ content = content.replaceAll("\ufffd", " ");
content = content.replaceAll("\\s+"," ");
assertContains("Station Fax", content);
}