You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2013/08/09 15:27:53 UTC
svn commit: r1512297 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/contrib/
solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/
solr/contrib/extraction/src/test-files/extraction/
solr/contrib/extraction/src/test/org/apache/solr/ha...
Author: uschindler
Date: Fri Aug 9 13:27:53 2013
New Revision: 1512297
URL: http://svn.apache.org/r1512297
Log:
Merged revision(s) 1512296 from lucene/dev/trunk:
SOLR-4679, SOLR-4908, SOLR-5124: Text extracted from HTML or PDF files using Solr Cell was missing ignorable whitespace, which is inserted by TIKA for convenience to support plain text extraction without using the HTML elements. This bug resulted in glued words.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test-files/extraction/simple.html
lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1512297&r1=1512296&r2=1512297&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Fri Aug 9 13:27:53 2013
@@ -81,6 +81,11 @@ Bug Fixes
* SOLR-5107: Fixed NPE when using numTerms=0 in LukeRequestHandler
(Ahmet Arslan, hossman)
+* SOLR-4679, SOLR-4908, SOLR-5124: Text extracted from HTML or PDF files
+ using Solr Cell was missing ignorable whitespace, which is inserted by
+ TIKA for convenience to support plain text extraction without using the
+ HTML elements. This bug resulted in glued words. (hossman, Uwe Schindler)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java?rev=1512297&r1=1512296&r2=1512297&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java Fri Aug 9 13:27:53 2013
@@ -303,6 +303,13 @@ public class SolrContentHandler extends
bldrStack.getLast().append(chars, offset, length);
}
+ /**
+ * Treat the same as any other characters
+ */
+ @Override
+ public void ignorableWhitespace(char[] chars, int offset, int length) throws SAXException {
+ characters(chars, offset, length);
+ }
/**
* Can be used to transform input values based on their {@link org.apache.solr.schema.SchemaField}
Modified: lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test-files/extraction/simple.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test-files/extraction/simple.html?rev=1512297&r1=1512296&r2=1512297&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test-files/extraction/simple.html (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test-files/extraction/simple.html Fri Aug 9 13:27:53 2013
@@ -6,6 +6,7 @@
<p>
Here is some text
</p>
+<p>distinct<br/>words</p>
<div>Here is some text in a div</div>
<div>This has a <a href="http://www.apache.org">link</a>.</div>
</body>
Modified: lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1512297&r1=1512296&r2=1512297&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Fri Aug 9 13:27:53 2013
@@ -88,6 +88,10 @@ public class ExtractingRequestHandlerTes
assertU(commit());
assertQ(req("title:Welcome"), "//*[@numFound='1']");
+ assertQ(req("extractedContent:distinctwords"), "//*[@numFound='0']");
+ assertQ(req("extractedContent:distinct"), "//*[@numFound='1']");
+ assertQ(req("extractedContent:words"), "//*[@numFound='2']");
+ assertQ(req("extractedContent:\"distinct words\""), "//*[@numFound='1']");
loadLocal("extraction/simple.html",
"literal.id","simple2",