You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/06 00:14:39 UTC
svn commit: r1429422 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/highlighter/
lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/
lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/
lucene/sandbox/ luc...
Author: rmuir
Date: Sat Jan 5 23:14:38 2013
New Revision: 1429422
URL: http://svn.apache.org/viewvc?rev=1429422&view=rev
Log:
SOLR-4271: add support for PostingsHighlighter
Added:
lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/
- copied from r1429413, lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/
lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/
- copied from r1429413, lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
- copied, changed from r1429413, lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml
- copied unchanged from r1429413, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml
- copied unchanged from r1429413, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-postingshighlight.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
- copied, changed from r1429413, lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
Removed:
lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/postingshighlight/
lucene/dev/branches/branch_4x/lucene/sandbox/src/test/org/apache/lucene/sandbox/postingshighlight/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1429422&r1=1429421&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sat Jan 5 23:14:38 2013
@@ -147,7 +147,7 @@ New Features
extractWikipedia.alg was changed to use this task, so now it creates two
files. (Doron Cohen)
-* LUCENE-4290: Added PostingsHighlighter to the sandbox module. It uses
+* LUCENE-4290: Added PostingsHighlighter to the highlighter module. It uses
offsets from the postings lists to highlight documents. (Robert Muir)
* LUCENE-4628: Added CommonTermsQuery that executes high-frequency terms
Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java?rev=1429422&r1=1429413&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java Sat Jan 5 23:14:38 2013
@@ -42,7 +42,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x"})
public class TestPostingsHighlighter extends LuceneTestCase {
public void testBasics() throws Exception {
Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java?rev=1429422&r1=1429413&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java (original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java Sat Jan 5 23:14:38 2013
@@ -46,7 +46,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x"})
public class TestPostingsHighlighterRanking extends LuceneTestCase {
/**
* indexes a bunch of gibberish, and then highlights top(n).
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1429422&r1=1429421&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Jan 5 23:14:38 2013
@@ -169,6 +169,8 @@ New Features
rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
(Alan Woodward, Shawn Heisey, Adrien Grand, Uwe Schindler)
+* SOLR-4271: Add support for PostingsHighlighter. (Robert Muir)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java?rev=1429422&r1=1429421&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java Sat Jan 5 23:14:38 2013
@@ -24,6 +24,7 @@ import org.apache.solr.common.params.Hig
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.highlight.PostingsSolrHighlighter;
import org.apache.solr.highlight.SolrHighlighter;
import org.apache.solr.highlight.DefaultSolrHighlighter;
import org.apache.solr.request.SolrQueryRequest;
@@ -128,7 +129,7 @@ public class HighlightComponent extends
}
if(highlightQuery != null) {
- boolean rewrite = !(Boolean.valueOf(params.get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true")) &&
+ boolean rewrite = (highlighter instanceof PostingsSolrHighlighter == false) && !(Boolean.valueOf(params.get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true")) &&
Boolean.valueOf(params.get(HighlightParams.HIGHLIGHT_MULTI_TERM, "true")));
highlightQuery = rewrite ? highlightQuery.rewrite(req.getSearcher().getIndexReader()) : highlightQuery;
}
Copied: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (from r1429413, lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?p2=lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java&p1=lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java&r1=1429413&r2=1429422&rev=1429422&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Sat Jan 5 23:14:38 2013
@@ -24,7 +24,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.document.Document;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
@@ -177,7 +177,7 @@ public class PostingsSolrHighlighter ext
String uniqueKeys[] = new String[topDocs.scoreDocs.length];
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
int docid = topDocs.scoreDocs[i].doc;
- StoredDocument doc = searcher.doc(docid, selector);
+ Document doc = searcher.doc(docid, selector);
String id = schema.printableUniqueKey(doc);
uniqueKeys[i] = id;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1429422&r1=1429421&r2=1429422&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Jan 5 23:14:38 2013
@@ -109,6 +109,8 @@ public class SolrIndexSearcher extends I
private final SolrCache[] cacheList;
private static final SolrCache[] noCaches = new SolrCache[0];
+ private final FieldInfos fieldInfos;
+ // TODO: do we need this separate set of field names? we can just use the fieldinfos?
private final Collection<String> fieldNames;
private Collection<String> storedHighlightFieldNames;
private DirectoryFactory directoryFactory;
@@ -199,7 +201,8 @@ public class SolrIndexSearcher extends I
optimizer = null;
fieldNames = new HashSet<String>();
- for(FieldInfo fieldInfo : atomicReader.getFieldInfos()) {
+ fieldInfos = atomicReader.getFieldInfos();
+ for(FieldInfo fieldInfo : fieldInfos) {
fieldNames.add(fieldInfo.name);
}
@@ -509,13 +512,56 @@ public class SolrIndexSearcher extends I
}
/** Visit a document's fields using a {@link StoredFieldVisitor}
- * This method does not currently use the Solr document cache.
+ * This method does not currently add to the Solr document cache.
*
* @see IndexReader#document(int, StoredFieldVisitor) */
@Override
public void doc(int n, StoredFieldVisitor visitor) throws IOException {
+ if (documentCache != null) {
+ Document cached = documentCache.get(n);
+ if (cached != null) {
+ visitFromCached(cached, visitor);
+ return;
+ }
+ }
getIndexReader().document(n, visitor);
}
+
+ /** Executes a stored field visitor against a hit from the document cache */
+ private void visitFromCached(Document document, StoredFieldVisitor visitor) throws IOException {
+ for (IndexableField f : document) {
+ FieldInfo info = fieldInfos.fieldInfo(f.name());
+ switch(visitor.needsField(info)) {
+ case YES:
+ if (f.binaryValue() != null) {
+ BytesRef binaryValue = f.binaryValue();
+ byte copy[] = new byte[binaryValue.length];
+ System.arraycopy(binaryValue.bytes, binaryValue.offset, copy, 0, copy.length);
+ visitor.binaryField(info, copy);
+ } else if (f.numericValue() != null) {
+ Number numericValue = f.numericValue();
+ if (numericValue instanceof Double) {
+ visitor.doubleField(info, numericValue.doubleValue());
+ } else if (numericValue instanceof Integer) {
+ visitor.intField(info, numericValue.intValue());
+ } else if (numericValue instanceof Float) {
+ visitor.floatField(info, numericValue.floatValue());
+ } else if (numericValue instanceof Long) {
+ visitor.longField(info, numericValue.longValue());
+ } else {
+ throw new AssertionError();
+ }
+ } else {
+ visitor.stringField(info, f.stringValue());
+ }
+ break;
+ case NO:
+ break;
+ case STOP:
+ return;
+ }
+ }
+ }
/**
* Retrieve the {@link Document} instance corresponding to the document id.
Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java (from r1429413, lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java&r1=1429413&r2=1429422&rev=1429422&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java Sat Jan 5 23:14:38 2013
@@ -24,7 +24,7 @@ import org.apache.solr.schema.IndexSchem
import org.junit.BeforeClass;
/** simple tests for PostingsSolrHighlighter */
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x"})
public class TestPostingsSolrHighlighter extends SolrTestCaseJ4 {
@BeforeClass