You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2015/01/24 00:51:33 UTC
svn commit: r1654433 - in /lucene/dev/branches/lucene_solr_5_0: ./ solr/
solr/contrib/
solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/
solr/contrib/extraction/src/test-files/extraction/
solr/contrib/extraction/src/test/org/apache/s...
Author: sarowe
Date: Fri Jan 23 23:51:32 2015
New Revision: 1654433
URL: http://svn.apache.org/r1654433
Log:
SOLR-6856: Restore ExtractingRequestHandler's ability to capture all HTML tags when parsing (X)HTML. (merged trunk r1654431)
Modified:
lucene/dev/branches/lucene_solr_5_0/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_5_0/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test-files/extraction/simple.html
lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
Modified: lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt?rev=1654433&r1=1654432&r2=1654433&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt Fri Jan 23 23:51:32 2015
@@ -456,6 +456,9 @@ Bug Fixes
* SOLR-6847: LeaderInitiatedRecoveryThread compares wrong replica's state with lirState.
(shalin)
+
+* SOLR-6856: Restore ExtractingRequestHandler's ability to capture all HTML tags when
+ parsing (X)HTML. (hossman, ehatcher, Steve Rowe)
Optimizations
----------------------
Modified: lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1654433&r1=1654432&r2=1654433&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Fri Jan 23 23:51:32 2015
@@ -45,6 +45,7 @@ import org.apache.tika.parser.DefaultPar
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
+import org.apache.tika.parser.html.HtmlMapper;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.sax.xpath.Matcher;
import org.apache.tika.sax.xpath.MatchingContentHandler;
@@ -199,6 +200,7 @@ public class ExtractingDocumentLoader ex
try{
//potentially use a wrapper handler for parsing, but we still need the SolrContentHandler for getting the document.
ParseContext context = new ParseContext();//TODO: should we design a way to pass in parse context?
+ context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE);
// Password handling
RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();
@@ -250,4 +252,34 @@ public class ExtractingDocumentLoader ex
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stream type of " + streamType + " didn't match any known parsers. Please supply the " + ExtractingParams.STREAM_TYPE + " parameter.");
}
}
-}
+
+ public static class MostlyPassthroughHtmlMapper implements HtmlMapper {
+ public static final HtmlMapper INSTANCE = new MostlyPassthroughHtmlMapper();
+
+ /**
+ * Keep all elements and their content.
+ *
+ * Apparently <SCRIPT> and <STYLE> elements are blocked elsewhere
+ */
+ @Override
+ public boolean isDiscardElement(String name) {
+ return false;
+ }
+
+ /** Lowercases the attribute name */
+ @Override
+ public String mapSafeAttribute(String elementName, String attributeName) {
+ return attributeName.toLowerCase(Locale.ENGLISH);
+ }
+
+ /**
+ * Lowercases the element name, but returns null for <BR>,
+ * which suppresses the start-element event for lt;BR> tags.
+ */
+ @Override
+ public String mapSafeElement(String name) {
+ String lowerName = name.toLowerCase(Locale.ROOT);
+ return lowerName.equals("br") ? null : lowerName;
+ }
+ }
+ }
\ No newline at end of file
Modified: lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test-files/extraction/simple.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test-files/extraction/simple.html?rev=1654433&r1=1654432&r2=1654433&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test-files/extraction/simple.html (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test-files/extraction/simple.html Fri Jan 23 23:51:32 2015
@@ -1,6 +1,9 @@
<html>
<head>
<title>Welcome to Solr</title>
+ <style type="text/css">
+ body { font-family: serif; }
+ </style>
</head>
<body>
<p>
@@ -10,4 +13,7 @@
<div>Here is some text in a div</div>
<div>This has a <a href="http://www.apache.org">link</a>.</div>
</body>
+<script>
+ document.getElementById("div").blur();
+</script>
</html>
Modified: lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1654433&r1=1654432&r2=1654433&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Fri Jan 23 23:51:32 2015
@@ -111,6 +111,8 @@ public class ExtractingRequestHandlerTes
//assertQ(req("+id:simple2 +t_content_type:[* TO *]"), "//*[@numFound='1']");
assertQ(req("+id:simple2 +t_href:[* TO *]"), "//*[@numFound='1']");
assertQ(req("+id:simple2 +t_abcxyz:[* TO *]"), "//*[@numFound='1']");
+ assertQ(req("+id:simple2 +t_content:serif"), "//*[@numFound='0']"); // make sure <style> content is excluded
+ assertQ(req("+id:simple2 +t_content:blur"), "//*[@numFound='0']"); // make sure <script> content is excluded
// load again in the exact same way, but boost one field
loadLocal("extraction/simple.html",
@@ -127,16 +129,6 @@ public class ExtractingRequestHandlerTes
assertQ(req("t_href:http"), "//doc[1]/str[.='simple3']");
assertQ(req("+id:simple3 +t_content_type:[* TO *]"), "//*[@numFound='1']");//test lowercase and then uprefix
- // test capture
- loadLocal("extraction/simple.html",
- "literal.id","simple4",
- "uprefix", "t_",
- "capture","p", // capture only what is in the title element
- "commit", "true"
- );
- assertQ(req("+id:simple4 +t_content:Solr"), "//*[@numFound='1']");
- assertQ(req("+id:simple4 +t_p:\"here is some text\""), "//*[@numFound='1']");
-
loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
@@ -187,6 +179,33 @@ public class ExtractingRequestHandlerTes
}
+ @Test
+ public void testCapture() throws Exception {
+ loadLocal("extraction/simple.html",
+ "literal.id","capture1",
+ "uprefix","t_",
+ "capture","div",
+ "fmap.div", "foo_t",
+ "commit", "true"
+ );
+ assertQ(req("+id:capture1 +t_content:Solr"), "//*[@numFound='1']");
+ assertQ(req("+id:capture1 +foo_t:\"here is some text in a div\""), "//*[@numFound='1']");
+
+ loadLocal("extraction/simple.html",
+ "literal.id", "capture2",
+ "captureAttr", "true",
+ "defaultField", "text",
+ "fmap.div", "div_t",
+ "fmap.a", "anchor_t",
+ "capture", "div",
+ "capture", "a",
+ "commit", "true"
+ );
+ assertQ(req("+id:capture2 +text:Solr"), "//*[@numFound='1']");
+ assertQ(req("+id:capture2 +div_t:\"here is some text in a div\""), "//*[@numFound='1']");
+ assertQ(req("+id:capture2 +anchor_t:http\\://www.apache.org"), "//*[@numFound='1']");
+ assertQ(req("+id:capture2 +anchor_t:link"), "//*[@numFound='1']");
+ }
@Test
public void testDefaultField() throws Exception {
@@ -466,14 +485,25 @@ public class ExtractingRequestHandlerTes
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
SolrQueryResponse rsp = loadLocal("extraction/example.html",
- ExtractingParams.XPATH_EXPRESSION, "/xhtml:html/xhtml:body/xhtml:a/descendant:node()",
+ ExtractingParams.XPATH_EXPRESSION, "/xhtml:html/xhtml:body/xhtml:a/descendant::node()",
ExtractingParams.EXTRACT_ONLY, "true"
);
assertTrue("rsp is null and it shouldn't be", rsp != null);
NamedList list = rsp.getValues();
String val = (String) list.get("example.html");
- val = val.trim();
- assertTrue(val + " is not equal to " + "linkNews", val.equals("linkNews") == true);//there are two <a> tags, and they get collapesd
+ assertEquals("News", val.trim()); //there is only one matching <a> tag
+
+ loadLocal("extraction/example.html",
+ "literal.id", "example1",
+ "captureAttr", "true",
+ "defaultField", "text",
+ "capture", "div",
+ "fmap.div", "foo_t",
+ "boost.foo_t", "3",
+ "xpath", "/xhtml:html/xhtml:body/xhtml:div//node()",
+ "commit", "true"
+ );
+ assertQ(req("+id:example1 +foo_t:\"here is some text in a div\""), "//*[@numFound='1']");
}
/** test arabic PDF extraction is functional */