You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/01/26 06:07:09 UTC
svn commit: r1561456 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/java/org/apache/lucene/search/spans/ lucene/highlighter/
lucene/highlighter/src/java/org/apache/lucene/sear...
Author: rmuir
Date: Sun Jan 26 05:07:08 2014
New Revision: 1561456
URL: http://svn.apache.org/r1561456
Log:
LUCENE-5415: add multitermquery support to PostingsHighlighter
Added:
lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
- copied, changed from r1561451, lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
- copied, changed from r1561451, lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sun Jan 26 05:07:08 2014
@@ -51,6 +51,9 @@ New Features
fixes too. More info:
https://github.com/spatial4j/spatial4j/blob/master/CHANGES.md (David Smiley)
+* LUCENE-5415: Add multitermquery (wildcards,prefix,etc) to PostingsHighlighter.
+ (Mike McCandless, Robert Muir)
+
Build
* LUCENE-5217: Maven config: get dependencies from Ant+Ivy config; disable
@@ -133,6 +136,9 @@ Bug fixes
the same Directory to multiple concurrent addIndexes calls (which is
anyways unusual). (Robert Muir, Mike McCandless)
+* LUCENE-5415: SpanMultiTermQueryWrapper didn't handle its boost in
+ hashcode/equals/tostring/rewrite. (Robert Muir)
+
API Changes
* LUCENE-5339: The facet module was simplified/reworked to make the
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java Sun Jan 26 05:07:08 2014
@@ -128,4 +128,9 @@ public class AutomatonQuery extends Mult
buffer.append(ToStringUtils.boost(getBoost()));
return buffer.toString();
}
+
+ /** Returns the automaton used to create this query */
+ public Automaton getAutomaton() {
+ return automaton;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java Sun Jan 26 05:07:08 2014
@@ -138,6 +138,14 @@ public class FuzzyQuery extends MultiTer
public int getPrefixLength() {
return prefixLength;
}
+
+ /**
+ * Returns true if transpositions should be treated as a primitive edit operation.
+ * If this is false, comparisons will implement the classic Levenshtein algorithm.
+ */
+ public boolean getTranspositions() {
+ return transpositions;
+ }
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Sun Jan 26 05:07:08 2014
@@ -100,6 +100,11 @@ public class SpanMultiTermQueryWrapper<Q
public String getField() {
return query.getField();
}
+
+ /** Returns the wrapped query */
+ public Query getWrappedQuery() {
+ return query;
+ }
@Override
public String toString(String field) {
@@ -107,6 +112,10 @@ public class SpanMultiTermQueryWrapper<Q
builder.append("SpanMultiTermQueryWrapper(");
builder.append(query.toString(field));
builder.append(")");
+ if (getBoost() != 1F) {
+ builder.append('^');
+ builder.append(getBoost());
+ }
return builder.toString();
}
@@ -115,22 +124,26 @@ public class SpanMultiTermQueryWrapper<Q
final Query q = query.rewrite(reader);
if (!(q instanceof SpanQuery))
throw new UnsupportedOperationException("You can only use SpanMultiTermQueryWrapper with a suitable SpanRewriteMethod.");
+ q.setBoost(q.getBoost() * getBoost()); // multiply boost
return q;
}
@Override
public int hashCode() {
- return 31 * query.hashCode();
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + query.hashCode();
+ return result;
}
@Override
- @SuppressWarnings({"rawtypes","unchecked"})
public boolean equals(Object obj) {
if (this == obj) return true;
- if (obj == null) return false;
+ if (!super.equals(obj)) return false;
if (getClass() != obj.getClass()) return false;
- final SpanMultiTermQueryWrapper other = (SpanMultiTermQueryWrapper) obj;
- return query.equals(other.query);
+ SpanMultiTermQueryWrapper<?> other = (SpanMultiTermQueryWrapper<?>) obj;
+ if (!query.equals(other.query)) return false;
+ return true;
}
/** Abstract class that defines how the query is rewritten. */
Copied: lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java (from r1561451, lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java?p2=lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java&p1=lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java&r1=1561451&r2=1561456&rev=1561456&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java (original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java Sun Jan 26 05:07:08 2014
@@ -60,7 +60,7 @@ class MultiTermHighlighting {
* automata that will match terms.
*/
static CharacterRunAutomaton[] extractAutomata(Query query, String field) {
- List<CharacterRunAutomaton> list = new ArrayList<>();
+ List<CharacterRunAutomaton> list = new ArrayList<CharacterRunAutomaton>();
if (query instanceof BooleanQuery) {
BooleanClause clauses[] = ((BooleanQuery) query).getClauses();
for (BooleanClause clause : clauses) {
Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java Sun Jan 26 05:07:08 2014
@@ -30,6 +30,7 @@ import java.util.PriorityQueue;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -50,6 +51,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
/**
* Simple highlighter that does not analyze fields nor use
@@ -64,6 +66,14 @@ import org.apache.lucene.util.UnicodeUti
* into a {@link Passage}, and then scores each Passage using a separate {@link PassageScorer}.
* Passages are finally formatted into highlighted snippets with a {@link PassageFormatter}.
* <p>
+ * You can customize the behavior by subclassing this highlighter, some important hooks:
+ * <ul>
+ * <li>{@link #getBreakIterator(String)}: Customize how the text is divided into passages.
+ * <li>{@link #getScorer(String)}: Customize how passages are ranked.
+ * <li>{@link #getFormatter(String)}: Customize how snippets are formatted.
+ * <li>{@link #getIndexAnalyzer(String)}: Enable highlighting of MultiTermQuerys such as {@code WildcardQuery}.
+ * </ul>
+ * <p>
* <b>WARNING</b>: The code is very new and probably still has some exciting bugs!
* <p>
* Example usage:
@@ -335,9 +345,9 @@ public class PostingsHighlighter {
throw new IllegalArgumentException("invalid number of maxPassagesIn");
}
final IndexReader reader = searcher.getIndexReader();
- query = rewrite(query);
+ Query rewritten = rewrite(query);
SortedSet<Term> queryTerms = new TreeSet<Term>();
- query.extractTerms(queryTerms);
+ rewritten.extractTerms(queryTerms);
IndexReaderContext readerContext = reader.getContext();
List<AtomicReaderContext> leaves = readerContext.leaves();
@@ -389,7 +399,7 @@ public class PostingsHighlighter {
for(Term term : fieldTerms) {
terms[termUpto++] = term.bytes();
}
- Map<Integer,Object> fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages);
+ Map<Integer,Object> fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages, query);
Object[] result = new Object[docids.length];
for (int j = 0; j < docidsIn.length; j++) {
@@ -432,8 +442,18 @@ public class PostingsHighlighter {
protected char getMultiValuedSeparator(String field) {
return ' ';
}
+
+ /**
+ * Returns the analyzer originally used to index the content for {@code field}.
+ * <p>
+ * This is used to highlight some MultiTermQueries.
+ * @return Analyzer or null (the default, meaning no special multi-term processing)
+ */
+ protected Analyzer getIndexAnalyzer(String field) {
+ return null;
+ }
- private Map<Integer,Object> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages) throws IOException {
+ private Map<Integer,Object> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages, Query query) throws IOException {
Map<Integer,Object> highlights = new HashMap<Integer,Object>();
// reuse in the real sense... for docs in same segment we just advance our old enum
@@ -445,6 +465,21 @@ public class PostingsHighlighter {
if (fieldFormatter == null) {
throw new NullPointerException("PassageFormatter cannot be null");
}
+
+ // check if we should do any multitermprocessing
+ Analyzer analyzer = getIndexAnalyzer(field);
+ CharacterRunAutomaton automata[] = new CharacterRunAutomaton[0];
+ if (analyzer != null) {
+ automata = MultiTermHighlighting.extractAutomata(query, field);
+ }
+
+ final BytesRef allTerms[];
+ if (automata.length > 0) {
+ allTerms = new BytesRef[terms.length + 1];
+ System.arraycopy(terms, 0, allTerms, 0, terms.length);
+ } else {
+ allTerms = terms;
+ }
for (int i = 0; i < docids.length; i++) {
String content = contents[i];
@@ -462,9 +497,14 @@ public class PostingsHighlighter {
}
if (leaf != lastLeaf) {
termsEnum = t.iterator(null);
- postings = new DocsAndPositionsEnum[terms.length];
+ postings = new DocsAndPositionsEnum[allTerms.length];
+ }
+ if (automata.length > 0) {
+ DocsAndPositionsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata);
+ dp.advance(doc - subContext.docBase);
+ postings[terms.length] = dp;
}
- Passage passages[] = highlightDoc(field, terms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
+ Passage passages[] = highlightDoc(field, allTerms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
if (passages.length == 0) {
passages = getEmptyHighlight(field, bi, maxPassages);
}
@@ -593,7 +633,13 @@ public class PostingsHighlighter {
int tf = 0;
while (true) {
tf++;
- current.addMatch(start, end, terms[off.id]);
+ BytesRef term = terms[off.id];
+ if (term == null) {
+ // multitermquery match, pull from payload
+ term = off.dp.getPayload();
+ assert term != null;
+ }
+ current.addMatch(start, end, term);
if (off.pos == dp.freq()) {
break; // removed from pq
} else {
Copied: lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java (from r1561451, lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java?p2=lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java&p1=lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java&r1=1561451&r2=1561456&rev=1561456&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java (original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java Sun Jan 26 05:07:08 2014
@@ -57,7 +57,7 @@ import org.apache.lucene.util.LuceneTest
* Some tests that override {@link PostingsHighlighter#getIndexAnalyzer} to
* highlight wilcard, fuzzy, etc queries.
*/
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x"})
public class TestMultiTermHighlighting extends LuceneTestCase {
public void testWildcards() throws Exception {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Sun Jan 26 05:07:08 2014
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.document.Document;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.postingshighlight.DefaultPassageFormatter;
import org.apache.lucene.search.postingshighlight.Passage;
@@ -68,6 +69,7 @@ import org.apache.solr.util.plugin.Plugi
* <str name="hl.bs.type">SENTENCE</str>
* <int name="hl.maxAnalyzedChars">10000</int>
* <str name="hl.multiValuedSeparatorChar"> </str>
+ * <bool name="hl.highlightMultiTerm">false</bool>
* </lst>
* </requestHandler>
* </pre>
@@ -98,6 +100,7 @@ import org.apache.solr.util.plugin.Plugi
* <li>hl.bs.variant (string) specifies country code for BreakIterator. default is empty string (root locale)
* <li>hl.maxAnalyzedChars specifies how many characters at most will be processed in a document.
* <li>hl.multiValuedSeparatorChar specifies the logical separator between values for multi-valued fields.
+ * <li>hl.highlightMultiTerm enables highlighting for range/wildcard/fuzzy/prefix queries.
* NOTE: currently hl.maxAnalyzedChars cannot yet be specified per-field
* </ul>
*
@@ -132,6 +135,8 @@ public class PostingsSolrHighlighter ext
maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1);
}
+ final IndexSchema schema = req.getSchema();
+
PostingsHighlighter highlighter = new PostingsHighlighter(maxLength) {
@Override
protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
@@ -178,6 +183,15 @@ public class PostingsSolrHighlighter ext
}
return sep.charAt(0);
}
+
+ @Override
+ protected Analyzer getIndexAnalyzer(String field) {
+ if (params.getFieldBool(field, HighlightParams.HIGHLIGHT_MULTI_TERM, false)) {
+ return schema.getAnalyzer();
+ } else {
+ return null;
+ }
+ }
};
Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, maxPassages);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java?rev=1561456&r1=1561455&r2=1561456&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java Sun Jan 26 05:07:08 2014
@@ -155,4 +155,12 @@ public class TestPostingsSolrHighlighter
req("q", "text:document", "sort", "id asc", "hl", "true", "hl.encoder", "html"),
"//lst[@name='highlighting']/lst[@name='103']/arr[@name='text']/str='<em>Document</em> one has a first <i>sentence</i>.'");
}
+
+ public void testWildcard() {
+ assertQ("simplest test",
+ req("q", "text:doc*ment", "sort", "id asc", "hl", "true", "hl.highlightMultiTerm", "true"),
+ "count(//lst[@name='highlighting']/*)=2",
+ "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='<em>document</em> one'",
+ "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second <em>document</em>'");
+ }
}