You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/08/05 14:09:49 UTC
svn commit: r801160 - in /lucene/java/trunk/contrib/benchmark: ./
src/java/org/apache/lucene/benchmark/byTask/tasks/
src/test/org/apache/lucene/benchmark/byTask/tasks/
Author: mikemccand
Date: Wed Aug 5 12:09:48 2009
New Revision: 801160
URL: http://svn.apache.org/viewvc?rev=801160&view=rev
Log:
LUCENE-1773: fix highlighter test case; add contrib/memory as dependency to contrib/benchmark
Modified:
lucene/java/trunk/contrib/benchmark/CHANGES.txt
lucene/java/trunk/contrib/benchmark/build.xml
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java
lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java
Modified: lucene/java/trunk/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/CHANGES.txt?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/CHANGES.txt (original)
+++ lucene/java/trunk/contrib/benchmark/CHANGES.txt Wed Aug 5 12:09:48 2009
@@ -8,8 +8,15 @@
LUCENE-1770: Add EnwikiQueryMaker (Mark Miller)
8/04/2009
- LUCENE-1773: Add FastVectorHighlighter tasks. (Koji Sekiguchi via
- Mike McCandless)
+ LUCENE-1773: Add FastVectorHighlighter tasks. This change is a
+ non-backwards compatible change in how subclasses of ReadTask define
+ a highlighter. The methods doHighlight, isMergeContiguousFragments,
+ maxNumFragments and getHighlighter are no longer used and have been
+ mark deprecated and package protected private so there's a compile
+ time error. Instead, the new getBenchmarkHighlighter method should
+ return an appropriate highlighter for the task. The configuration of
+ the highlighter tasks (maxFrags, mergeContiguous, etc.) is now
+ accepted as params to the task. (Koji Sekiguchi via Mike McCandless)
8/03/2009
LUCENE-1778: Add support for log.step setting per task type. Perviously, if
Modified: lucene/java/trunk/contrib/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/build.xml?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/build.xml (original)
+++ lucene/java/trunk/contrib/benchmark/build.xml Wed Aug 5 12:09:48 2009
@@ -105,6 +105,7 @@
<pathelement path="${common.dir}/build/classes/java"/>
<pathelement path="${common.dir}/build/classes/demo"/>
<pathelement path="${common.dir}/build/contrib/highlighter/classes/java"/>
+ <pathelement path="${common.dir}/build/contrib/memory/classes/java"/>
<pathelement path="${common.dir}/build/contrib/fast-vector-highlighter/classes/java"/>
<fileset dir="lib">
<include name="**/*.jar"/>
@@ -149,13 +150,18 @@
<fileset dir="${common.dir}/contrib/highlighter" includes="build.xml"/>
</subant>
</target>
+ <target name="compile-memory">
+ <subant target="compile">
+ <fileset dir="${common.dir}/contrib/memory" includes="build.xml"/>
+ </subant>
+ </target>
<target name="compile-vector-highlighter">
<subant target="compile">
<fileset dir="${common.dir}/contrib/fast-vector-highlighter" includes="build.xml"/>
</subant>
</target>
- <target name="init" depends="common.init,compile-demo,compile-highlighter,compile-vector-highlighter,check-files"/>
+ <target name="init" depends="common.init,compile-demo,compile-memory,compile-highlighter,compile-vector-highlighter,check-files"/>
<!-- make sure online collections (reuters) are first downloaded -->
<target name="test" depends="init,get-files">
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java Wed Aug 5 12:09:48 2009
@@ -38,10 +38,6 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.highlight.Highlighter;
-import org.apache.lucene.search.highlight.QueryTermScorer;
-import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
-import org.apache.lucene.search.highlight.TextFragment;
-import org.apache.lucene.search.highlight.TokenSources;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.store.Directory;
@@ -122,10 +118,8 @@
int numHighlight = Math.min(numToHighlight(), scoreDocs.length);
Analyzer analyzer = getRunData().getAnalyzer();
BenchmarkHighlighter highlighter = null;
- int maxFrags = 1;
if (numHighlight > 0) {
highlighter = getBenchmarkHighlighter(q);
- maxFrags = maxNumFragments();
}
for (int m = 0; m < traversalSize; m++) {
int id = scoreDocs[m].doc;
@@ -242,37 +236,50 @@
/**
* @deprecated Use {@link #getBenchmarkHighlighter(Query)}
*/
- protected Highlighter getHighlighter(Query q){
- return new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(q));
+ final Highlighter getHighlighter(Query q) {
+ // not called
+ return null;
}
+ /**
+ * Return an appropriate highlighter to be used with
+ * highlighting tasks
+ */
protected BenchmarkHighlighter getBenchmarkHighlighter(Query q){
return null;
}
/**
- *
* @return the maxiumum number of highlighter fragments
+ * @deprecated Please define getBenchmarkHighlighter instead
*/
- public int maxNumFragments(){
+ final int maxNumFragments(){
+ // not called -- we switched this method to final to
+ // force any external subclasses to cutover to
+ // getBenchmarkHighlighter instead
return 10;
}
/**
*
* @return true if the highlighter should merge contiguous fragments
- * @deprecated
+ * @deprecated Please define getBenchmarkHighlighter instead
*/
- public boolean isMergeContiguousFragments(){
+ final boolean isMergeContiguousFragments(){
+ // not called -- we switched this method to final to
+ // force any external subclasses to cutover to
+ // getBenchmarkHighlighter instead
return false;
}
/**
- * @deprecated
+ * @deprecated Please define getBenchmarkHighlighter instead
*/
- protected int doHighlight(TokenStream ts, String text, Highlighter highlighter, boolean mergeContiguous, int maxFragments) throws IOException, InvalidTokenOffsetsException {
- TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFragments);
- return frag != null ? frag.length : 0;
+ final int doHighlight(TokenStream ts, String text, Highlighter highlighter, boolean mergeContiguous, int maxFragments) throws IOException, InvalidTokenOffsetsException {
+ // not called -- we switched this method to final to
+ // force any external subclasses to cutover to
+ // getBenchmarkHighlighter instead
+ return 0;
}
protected Sort getSort() {
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java Wed Aug 5 12:09:48 2009
@@ -24,13 +24,11 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.highlight.Highlighter;
-import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TextFragment;
import org.apache.lucene.search.highlight.TokenSources;
-import java.io.IOException;
import java.util.Set;
import java.util.Collection;
import java.util.HashSet;
@@ -103,17 +101,6 @@
};
}
- /**
- * @deprecated
- */
- public boolean isMergeContiguousFragments() {
- return mergeContiguous;
- }
-
- public int maxNumFragments() {
- return maxFrags;
- }
-
protected Collection/*<String>*/ getFieldsToHighlight(Document document) {
Collection result = super.getFieldsToHighlight(document);
//if stored is false, then result will be empty, in which case just get all the param fields
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java Wed Aug 5 12:09:48 2009
@@ -101,10 +101,6 @@
};
}
- public int maxNumFragments() {
- return maxFrags;
- }
-
protected Collection/*<String>*/ getFieldsToHighlight(Document document) {
Collection result = super.getFieldsToHighlight(document);
//if stored is false, then result will be empty, in which case just get all the param fields
Modified: lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java?rev=801160&r1=801159&r2=801160&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java Wed Aug 5 12:09:48 2009
@@ -19,9 +19,13 @@
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.TextFragment;
-import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
+import org.apache.lucene.search.highlight.QueryScorer;
+import org.apache.lucene.search.highlight.TokenSources;
+import org.apache.lucene.search.Query;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
@@ -47,9 +51,15 @@
return document;
}
- protected int doHighlight(TokenStream ts, String text, Highlighter highlighter, boolean mergeContiguous, int maxFragments) throws IOException, InvalidTokenOffsetsException {
- TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFragments);
- numHighlightedResults += frag != null ? frag.length : 0;
- return frag != null ? frag.length : 0;
+ public BenchmarkHighlighter getBenchmarkHighlighter(Query q) {
+ highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
+ return new BenchmarkHighlighter() {
+ public int doHighlight(IndexReader reader, int doc, String field, Document document, Analyzer analyzer, String text) throws Exception {
+ TokenStream ts = TokenSources.getAnyTokenStream(reader, doc, field, document, analyzer);
+ TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFrags);
+ numHighlightedResults += frag != null ? frag.length : 0;
+ return frag != null ? frag.length : 0;
+ }
+ };
}
-}
\ No newline at end of file
+}