You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/04/06 13:48:48 UTC
svn commit: r1465228 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/highlight/
core/src/test-files/solr/collection1/conf/
core/src/test/org/apache/solr/highlight/
solrj/src/java/org/apache/solr/common/params/
Author: rmuir
Date: Sat Apr 6 11:48:47 2013
New Revision: 1465228
URL: http://svn.apache.org/r1465228
Log:
SOLR-4675: Improve PostingsHighlighter integration
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml
lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat Apr 6 11:48:47 2013
@@ -125,6 +125,10 @@ New Features
prevent the "best" match from being found if it appears later in the MV list than the
cutoff specified by either of these params. (Erick Erickson)
+* SOLR-4675: Improve PostingsSolrHighlighter to support per-field/query-time overrides
+ and add additional configuration parameters. See the javadocs for more details and
+ examples. (Robert Muir)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Sat Apr 6 11:48:47 2013
@@ -69,6 +69,30 @@ public class DefaultSolrHighlighter exte
this.solrCore = solrCore;
}
+ // Thread safe registry
+ protected final Map<String,SolrFormatter> formatters =
+ new HashMap<String, SolrFormatter>();
+
+ // Thread safe registry
+ protected final Map<String,SolrEncoder> encoders =
+ new HashMap<String, SolrEncoder>();
+
+ // Thread safe registry
+ protected final Map<String,SolrFragmenter> fragmenters =
+ new HashMap<String, SolrFragmenter>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrFragListBuilder> fragListBuilders =
+ new HashMap<String, SolrFragListBuilder>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrFragmentsBuilder> fragmentsBuilders =
+ new HashMap<String, SolrFragmentsBuilder>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrBoundaryScanner> boundaryScanners =
+ new HashMap<String, SolrBoundaryScanner>() ;
+
@Override
public void init(PluginInfo info) {
formatters.clear();
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Sat Apr 6 11:48:47 2013
@@ -19,9 +19,7 @@ package org.apache.solr.highlight;
import java.io.IOException;
import java.text.BreakIterator;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.Locale;
import java.util.Map;
import java.util.Set;
@@ -50,87 +48,80 @@ import org.apache.solr.util.plugin.Plugi
* <p>
* Example configuration:
* <pre class="prettyprint">
+ * <requestHandler name="standard" class="solr.StandardRequestHandler">
+ * <lst name="defaults">
+ * <int name="hl.snippets">1</int>
+ * <str name="hl.tag.pre">&lt;em&gt;</str>
+ * <str name="hl.tag.post">&lt;/em&gt;</str>
+ * <str name="hl.tag.ellipsis">... </str>
+ * <bool name="hl.defaultSummary">true</bool>
+ * <float name="hl.score.k1">1.2</float>
+ * <float name="hl.score.b">0.75</float>
+ * <float name="hl.score.pivot">87</float>
+ * <int name="hl.maxAnalyzedChars">10000</int>
+ * </lst>
+ * </requestHandler>
+ * </pre>
+ * ...
+ * <pre class="prettyprint">
* <searchComponent class="solr.HighlightComponent" name="highlight">
- * <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter"
- * preTag="&lt;em&gt;"
- * postTag="&lt;/em&gt;"
- * ellipsis="... "
- * k1="1.2"
- * b="0.75"
- * pivot="87"
- * maxLength=10000
- * summarizeEmpty=true/>
+ * <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter"/>
* </searchComponent>
* </pre>
* <p>
* Notes:
* <ul>
* <li>fields to highlight must be configured with storeOffsetsWithPositions="true"
- * <li>hl.fl specifies the field list.
- * <li>hl.snippets specifies how many underlying sentence fragments form the resulting snippet.
+ * <li>hl.q (string) can specify the query
+ * <li>hl.fl (string) specifies the field list.
+ * <li>hl.snippets (int) specifies how many underlying sentence fragments form the resulting snippet.
+ * <li>hl.tag.pre (string) specifies text which appears before a highlighted term.
+ * <li>hl.tag.post (string) specifies text which appears after a highlighted term.
+ * <li>hl.tag.ellipsis (string) specifies text which joins non-adjacent passages.
+ * <li>hl.defaultSummary (bool) specifies if a field should have a default summary.
+ * <li>hl.score.k1 (float) specifies bm25 scoring parameter 'k1'
+ * <li>hl.score.b (float) specifies bm25 scoring parameter 'b'
+ * <li>hl.score.pivot (float) specifies bm25 scoring parameter 'avgdl'
+ * <li>hl.maxAnalyzedChars specifies how many characters at most will be processed in a document.
+ * NOTE: currently hl.maxAnalyzedChars cannot yet be specified per-field
* </ul>
*
* @lucene.experimental
*/
public class PostingsSolrHighlighter extends SolrHighlighter implements PluginInfoInitialized {
- protected PostingsHighlighter highlighter;
@Override
public void initalize(SolrConfig config) {}
@Override
- public void init(PluginInfo info) {
- Map<String,String> attributes = info.attributes;
-
- // scorer parameters: k1/b/pivot
- String k1 = attributes.get("k1");
- if (k1 == null) {
- k1 = "1.2";
- }
-
- String b = attributes.get("b");
- if (b == null) {
- b = "0.75";
- }
-
- String pivot = attributes.get("pivot");
- if (pivot == null) {
- pivot = "87";
- }
- final PassageScorer scorer = new PassageScorer(Float.parseFloat(k1), Float.parseFloat(b), Float.parseFloat(pivot));
-
- // formatter parameters: preTag/postTag/ellipsis
- String preTag = attributes.get("preTag");
- if (preTag == null) {
- preTag = "<em>";
- }
- String postTag = attributes.get("postTag");
- if (postTag == null) {
- postTag = "</em>";
- }
- String ellipsis = attributes.get("ellipsis");
- if (ellipsis == null) {
- ellipsis = "... ";
- }
- final PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
+ public void init(PluginInfo info) {}
- String summarizeEmpty = attributes.get("summarizeEmpty");
- final boolean summarizeEmptyBoolean;
- if (summarizeEmpty == null) {
- summarizeEmptyBoolean = true;
- } else {
- summarizeEmptyBoolean = Boolean.parseBoolean(summarizeEmpty);
- }
-
- // maximum content size to process
- int maxLength = PostingsHighlighter.DEFAULT_MAX_LENGTH;
- if (attributes.containsKey("maxLength")) {
- maxLength = Integer.parseInt(attributes.get("maxLength"));
- }
- highlighter = new PostingsHighlighter(maxLength) {
+ @Override
+ public NamedList<Object> doHighlighting(DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
+ final SolrParams params = req.getParams();
+
+ // if highlighting isnt enabled, then why call doHighlighting?
+ if (isHighlightingEnabled(params)) {
+ SolrIndexSearcher searcher = req.getSearcher();
+ int[] docIDs = toDocIDs(docs);
+
+ // fetch the unique keys
+ String[] keys = getUniqueKeys(searcher, docIDs);
+
+ // query-time parameters
+ int maxLength = params.getInt(HighlightParams.MAX_CHARS, PostingsHighlighter.DEFAULT_MAX_LENGTH);
+ String[] fieldNames = getHighlightFields(query, req, defaultFields);
+
+ int maxPassages[] = new int[fieldNames.length];
+ for (int i = 0; i < fieldNames.length; i++) {
+ maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1);
+ }
+
+ PostingsHighlighter highlighter = new PostingsHighlighter(maxLength) {
@Override
protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- if (summarizeEmptyBoolean) {
+ boolean defaultSummary = params.getFieldBool(fieldName, HighlightParams.DEFAULT_SUMMARY, true);
+ if (defaultSummary) {
return super.getEmptyHighlight(fieldName, bi, maxPassages);
} else {
return new Passage[0];
@@ -139,34 +130,20 @@ public class PostingsSolrHighlighter ext
@Override
protected PassageFormatter getFormatter(String fieldName) {
- return formatter;
+ String preTag = params.getFieldParam(fieldName, HighlightParams.TAG_PRE, "<em>");
+ String postTag = params.getFieldParam(fieldName, HighlightParams.TAG_POST, "</em>");
+ String ellipsis = params.getFieldParam(fieldName, HighlightParams.TAG_ELLIPSIS, "... ");
+ return new PassageFormatter(preTag, postTag, ellipsis);
}
@Override
protected PassageScorer getScorer(String fieldName) {
- return scorer;
+ float k1 = params.getFieldFloat(fieldName, HighlightParams.SCORE_K1, 1.2f);
+ float b = params.getFieldFloat(fieldName, HighlightParams.SCORE_B, 0.75f);
+ float pivot = params.getFieldFloat(fieldName, HighlightParams.SCORE_PIVOT, 87f);
+ return new PassageScorer(k1, b, pivot);
}
};
- }
-
- @Override
- public NamedList<Object> doHighlighting(DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
- SolrParams params = req.getParams();
-
- // if highlighting isnt enabled, then why call doHighlighting?
- if (isHighlightingEnabled(params)) {
- SolrIndexSearcher searcher = req.getSearcher();
- int[] docIDs = toDocIDs(docs);
-
- // fetch the unique keys
- String[] keys = getUniqueKeys(searcher, docIDs);
-
- // query-time parameters
- String[] fieldNames = getHighlightFields(query, req, defaultFields);
- // TODO: make this per-field
- int numSnippets = params.getInt(HighlightParams.SNIPPETS, 1);
- int maxPassages[] = new int[fieldNames.length];
- Arrays.fill(maxPassages, numSnippets);
Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, maxPassages);
return encodeSnippets(keys, fieldNames, snippets);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java Sat Apr 6 11:48:47 2013
@@ -19,9 +19,7 @@ package org.apache.solr.highlight;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,34 +36,9 @@ public abstract class SolrHighlighter
{
public static Logger log = LoggerFactory.getLogger(SolrHighlighter.class);
- // Thread safe registry
- protected final Map<String,SolrFormatter> formatters =
- new HashMap<String, SolrFormatter>();
-
- // Thread safe registry
- protected final Map<String,SolrEncoder> encoders =
- new HashMap<String, SolrEncoder>();
-
- // Thread safe registry
- protected final Map<String,SolrFragmenter> fragmenters =
- new HashMap<String, SolrFragmenter>() ;
-
- // Thread safe registry
- protected final Map<String, SolrFragListBuilder> fragListBuilders =
- new HashMap<String, SolrFragListBuilder>() ;
-
- // Thread safe registry
- protected final Map<String, SolrFragmentsBuilder> fragmentsBuilders =
- new HashMap<String, SolrFragmentsBuilder>() ;
-
- // Thread safe registry
- protected final Map<String, SolrBoundaryScanner> boundaryScanners =
- new HashMap<String, SolrBoundaryScanner>() ;
-
@Deprecated
public abstract void initalize( SolrConfig config );
-
/**
* Check whether Highlighting is enabled for this request.
* @param params The params controlling Highlighting
Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml Sat Apr 6 11:48:47 2013
@@ -22,9 +22,12 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <requestHandler name="standard" class="solr.StandardRequestHandler"></requestHandler>
-
+ <requestHandler name="standard" class="solr.StandardRequestHandler">
+ <lst name="defaults">
+ <bool name="hl.defaultSummary">false</bool>
+ </lst>
+ </requestHandler>
<searchComponent class="solr.HighlightComponent" name="highlight">
- <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter" summarizeEmpty="false"/>
+ <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter"/>
</searchComponent>
</config>
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java Sat Apr 6 11:48:47 2013
@@ -34,7 +34,7 @@ public class FastVectorHighlighterTest e
@Test
public void testConfig(){
- SolrHighlighter highlighter = HighlightComponent.getHighlighter(h.getCore());
+ DefaultSolrHighlighter highlighter = (DefaultSolrHighlighter) HighlightComponent.getHighlighter(h.getCore());
// Make sure we loaded one fragListBuilder
SolrFragListBuilder solrFlbNull = highlighter.fragListBuilders.get( null );
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java Sat Apr 6 11:48:47 2013
@@ -64,7 +64,7 @@ public class HighlighterTest extends Sol
@Test
public void testConfig()
{
- SolrHighlighter highlighter = HighlightComponent.getHighlighter(h.getCore());
+ DefaultSolrHighlighter highlighter = (DefaultSolrHighlighter) HighlightComponent.getHighlighter(h.getCore());
// Make sure we loaded the one formatter
SolrFormatter fmt1 = highlighter.formatters.get( null );
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java Sat Apr 6 11:48:47 2013
@@ -71,6 +71,14 @@ public class TestPostingsSolrHighlighter
"count(//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/*)=0");
}
+ public void testDefaultSummary() {
+ assertQ("null snippet test",
+ req("q", "text:one OR *:*", "sort", "id asc", "hl", "true", "hl.defaultSummary", "true"),
+ "count(//lst[@name='highlighting']/*)=2",
+ "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document <em>one</em>'",
+ "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'");
+ }
+
public void testDifferentField() {
assertQ("highlighting text3",
req("q", "text3:document", "sort", "id asc", "hl", "true", "hl.fl", "text3"),
@@ -100,4 +108,22 @@ public class TestPostingsSolrHighlighter
}
resetExceptionIgnores();
}
+
+ public void testTags() {
+ assertQ("different pre/post tags",
+ req("q", "text:document", "sort", "id asc", "hl", "true", "hl.tag.pre", "[", "hl.tag.post", "]"),
+ "count(//lst[@name='highlighting']/*)=2",
+ "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='[document] one'",
+ "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second [document]'");
+ }
+
+ public void testTagsPerField() {
+ assertQ("highlighting text and text3",
+ req("q", "text:document text3:document", "sort", "id asc", "hl", "true", "hl.fl", "text,text3", "f.text3.hl.tag.pre", "[", "f.text3.hl.tag.post", "]"),
+ "count(//lst[@name='highlighting']/*)=2",
+ "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='<em>document</em> one'",
+ "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text3']/str='crappy [document]'",
+ "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second <em>document</em>'",
+ "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text3']/str='crappier [document]'");
+ }
}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java?rev=1465228&r1=1465227&r2=1465228&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java Sat Apr 6 11:48:47 2013
@@ -42,6 +42,7 @@ public interface HighlightParams {
public static final String BS_LANGUAGE = HIGHLIGHT+".bs.language";
public static final String BS_COUNTRY = HIGHLIGHT+".bs.country";
public static final String FIELD_MATCH = HIGHLIGHT+".requireFieldMatch";
+ public static final String DEFAULT_SUMMARY = HIGHLIGHT + ".defaultSummary";
public static final String ALTERNATE_FIELD = HIGHLIGHT+".alternateField";
public static final String ALTERNATE_FIELD_LENGTH = HIGHLIGHT+".maxAlternateFieldLength";
public static final String MAX_MULTIVALUED_TO_EXAMINE = HIGHLIGHT + ".maxMultiValuedToExamine";
@@ -55,6 +56,7 @@ public interface HighlightParams {
public static final String USE_FVH = HIGHLIGHT + ".useFastVectorHighlighter";
public static final String TAG_PRE = HIGHLIGHT + ".tag.pre";
public static final String TAG_POST = HIGHLIGHT + ".tag.post";
+ public static final String TAG_ELLIPSIS = HIGHLIGHT + ".tag.ellipsis";
public static final String PHRASE_LIMIT = HIGHLIGHT + ".phraseLimit";
public static final String MULTI_VALUED_SEPARATOR = HIGHLIGHT + ".multiValuedSeparatorChar";
@@ -68,4 +70,10 @@ public interface HighlightParams {
public static final String SLOP = HIGHLIGHT+"."+REGEX+".slop";
public static final String PATTERN = HIGHLIGHT+"."+REGEX+".pattern";
public static final String MAX_RE_CHARS = HIGHLIGHT+"."+REGEX+".maxAnalyzedChars";
+
+ // Scoring parameters
+ public static final String SCORE = "score";
+ public static final String SCORE_K1 = HIGHLIGHT +"."+SCORE+".k1";
+ public static final String SCORE_B = HIGHLIGHT +"."+SCORE+".b";
+ public static final String SCORE_PIVOT = HIGHLIGHT +"."+SCORE+".pivot";
}