You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ni...@apache.org on 2016/05/16 00:13:25 UTC
[1/2] tika git commit: TIKA-1928 More tests for files with # in them
Repository: tika
Updated Branches:
refs/heads/2.x a882a3242 -> 2a69db7bb
TIKA-1928 More tests for files with # in them
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/398e73de
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/398e73de
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/398e73de
Branch: refs/heads/2.x
Commit: 398e73dee3e6ece5a0af376abe954b75bce0cfda
Parents: a882a32
Author: Nick Burch <ni...@gagravarr.org>
Authored: Mon May 16 01:04:03 2016 +0100
Committer: Nick Burch <ni...@gagravarr.org>
Committed: Mon May 16 01:04:03 2016 +0100
----------------------------------------------------------------------
.../test/java/org/apache/tika/mime/MimeDetectionTest.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/398e73de/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index 31df3ec..482aae1 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -77,6 +77,9 @@ public class MimeDetectionTest extends TikaTest {
testFile("image/cgm", "plotutils-bin-cgm-v3.cgm");
// test HTML detection of malformed file, previously identified as image/cgm (TIKA-1170)
testFile("text/html", "test-malformed-header.html.bin");
+ // file contains a # in the name (TIKA-1928)
+ testFile("application/xml", "test-utf8.xml", "test#.xml");
+ testFile("application/xml", "test-utf8.xml", "test#123.xml");
//test GCMD Directory Interchange Format (.dif) TIKA-1561
testFile("application/dif+xml", "brwNIMS_2014.dif");
@@ -142,7 +145,10 @@ public class MimeDetectionTest extends TikaTest {
}
private void testFile(String expected, String filename) throws IOException {
- InputStream in = getTestDocumentAsStream(filename);
+ testFile(expected, filename, filename);
+ }
+ private void testFile(String expected, String resourcename, String filename) throws IOException {
+ InputStream in = getTestDocumentAsStream(resourcename);
testStream(expected, filename, in);
}
[2/2] tika git commit: TIKA-1928 Fix detection for filenames
containing a #, avoid mis-detecting that part as a page anchor
Posted by ni...@apache.org.
TIKA-1928 Fix detection for filenames containing a #, avoid mis-detecting that part as a page anchor
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/2a69db7b
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/2a69db7b
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/2a69db7b
Branch: refs/heads/2.x
Commit: 2a69db7bbad17f235e66274186f5f3814f32a56e
Parents: 398e73d
Author: Nick Burch <ni...@gagravarr.org>
Authored: Mon May 16 01:02:16 2016 +0100
Committer: Nick Burch <ni...@gagravarr.org>
Committed: Mon May 16 01:06:06 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/tika/detect/NameDetector.java | 15 ++++++++++-----
.../org/apache/tika/detect/NameDetectorTest.java | 10 ++++++++++
2 files changed, 20 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/2a69db7b/tika-core/src/main/java/org/apache/tika/detect/NameDetector.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/detect/NameDetector.java b/tika-core/src/main/java/org/apache/tika/detect/NameDetector.java
index 1638d50..7135493 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/NameDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/NameDetector.java
@@ -97,15 +97,11 @@ public class NameDetector implements Detector {
// Look for a resource name in the input metadata
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (name != null) {
- // If the name is a URL, skip the trailing query and fragment parts
+ // If the name is a URL, skip the trailing query
int question = name.indexOf('?');
if (question != -1) {
name = name.substring(0, question);
}
- int hash = name.indexOf('#');
- if (hash != -1) {
- name = name.substring(0, hash);
- }
// If the name is a URL or a path, skip all but the last component
int slash = name.lastIndexOf('/');
@@ -117,6 +113,15 @@ public class NameDetector implements Detector {
name = name.substring(backslash + 1);
}
+ // Strip any fragments from the end, but only ones after the extension
+ int hash = name.lastIndexOf('#');
+ int dot = name.indexOf('.');
+ if (hash != -1) {
+ if (dot == -1 || hash > dot) {
+ name = name.substring(0, hash);
+ }
+ }
+
// Decode any potential URL encoding
int percent = name.indexOf('%');
if (percent != -1) {
http://git-wip-us.apache.org/repos/asf/tika/blob/2a69db7b/tika-core/src/test/java/org/apache/tika/detect/NameDetectorTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/detect/NameDetectorTest.java b/tika-core/src/test/java/org/apache/tika/detect/NameDetectorTest.java
index 24d4d42..23fbb15 100644
--- a/tika-core/src/test/java/org/apache/tika/detect/NameDetectorTest.java
+++ b/tika-core/src/test/java/org/apache/tika/detect/NameDetectorTest.java
@@ -65,6 +65,16 @@ public class NameDetectorTest {
assertDetect(MediaType.OCTET_STREAM, "ReadMe"); // case sensitive
assertDetect(MediaType.OCTET_STREAM, "README.NOW");
+ // TIKA-1928 # in the filename
+ assertDetect(MediaType.TEXT_PLAIN, "text.txt");
+ assertDetect(MediaType.TEXT_PLAIN, "text#.txt"); // # before extension
+ assertDetect(MediaType.TEXT_PLAIN, "text#123.txt");// # before extension
+ assertDetect(MediaType.TEXT_PLAIN, "text.txt#pdf");// # after extension
+
+ // Check # as URL fragment too
+ assertDetect(MediaType.TEXT_PLAIN, "http://foo/test.txt?1=2#pdf");
+ assertDetect(MediaType.TEXT_PLAIN, "http://foo/test.txt#pdf");
+
// tough one
assertDetect(
MediaType.TEXT_PLAIN,