You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2010/05/12 17:30:42 UTC
svn commit: r943554 - in /tika/trunk/tika-core/src:
main/resources/org/apache/tika/mime/tika-mimetypes.xml
test/java/org/apache/tika/mime/MimeDetectionTest.java
Author: jukka
Date: Wed May 12 15:30:42 2010
New Revision: 943554
URL: http://svn.apache.org/viewvc?rev=943554&view=rev
Log:
TIKA-417: Unable to parse the content for UCS2 Litte Endian encoded file
Disabled the MP3 magic pattern that conflicts with the UTF-16 LE byte order mark. It's much more likely that a document starting with this byte pattern is text instead of this specific MP3 variant.
TODO: Is there a more specific MP3 byte pattern we could use for this case?
Modified:
tika/trunk/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
tika/trunk/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
Modified: tika/trunk/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml?rev=943554&r1=943553&r2=943554&view=diff
==============================================================================
--- tika/trunk/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml (original)
+++ tika/trunk/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml Wed May 12 15:30:42 2010
@@ -2818,7 +2818,8 @@
<match value="0xfffb" type="string" offset="0"/> <!-- V1, L3 -->
<match value="0xfffc" type="string" offset="0"/> <!-- V1, L2, CRC -->
<match value="0xfffd" type="string" offset="0"/> <!-- V1, L2 -->
- <match value="0xfffe" type="string" offset="0"/> <!-- V1, L1, CRC -->
+ <!-- TIKA-417: This is the UTF-16 LE byte order mark! -->
+ <!-- match value="0xfffe" type="string" offset="0"/ --> <!-- V1, L1, CRC -->
<match value="0xffff" type="string" offset="0"/> <!-- V1, L1 -->
<match value="ID3" type="string" offset="0"/>
</magic>
Modified: tika/trunk/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java?rev=943554&r1=943553&r2=943554&view=diff
==============================================================================
--- tika/trunk/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java (original)
+++ tika/trunk/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java Wed May 12 15:30:42 2010
@@ -16,6 +16,7 @@
*/
package org.apache.tika.mime;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -60,7 +61,19 @@ public class MimeDetectionTest extends T
// add another evil html test from TIKA-357
testFile("text/html", "testlargerbuffer.html");
}
-
+
+ public void testByteOrderMark() throws Exception {
+ assertEquals(MediaType.TEXT_PLAIN, mimeTypes.detect(
+ new ByteArrayInputStream("\ufffetest".getBytes("UTF-16LE")),
+ new Metadata()));
+ assertEquals(MediaType.TEXT_PLAIN, mimeTypes.detect(
+ new ByteArrayInputStream("\ufffetest".getBytes("UTF-16BE")),
+ new Metadata()));
+ assertEquals(MediaType.TEXT_PLAIN, mimeTypes.detect(
+ new ByteArrayInputStream("\ufffetest".getBytes("UTF-8")),
+ new Metadata()));
+ }
+
public void testAutosetSupertype() throws MimeTypeException {
MimeTypes types = new MimeTypes();
MimeType type = types.forName("application/something+xml");