You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by ma...@apache.org on 2009/10/29 17:34:03 UTC
svn commit: r831020 - in /lucene/java/branches/lucene_2_9/contrib: ./
highlighter/ highlighter/src/java/org/apache/lucene/search/highlight/
highlighter/src/test/org/apache/lucene/search/highlight/
Author: markrmiller
Date: Thu Oct 29 16:34:02 2009
New Revision: 831020
URL: http://svn.apache.org/viewvc?rev=831020&view=rev
Log:
LUCENE-2013: SpanRegexQuery does not work with QueryScorer.
Modified:
lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
lucene/java/branches/lucene_2_9/contrib/highlighter/build.xml
lucene/java/branches/lucene_2_9/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/java/branches/lucene_2_9/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
Modified: lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt?rev=831020&r1=831019&r2=831020&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/contrib/CHANGES.txt Thu Oct 29 16:34:02 2009
@@ -39,6 +39,9 @@
in WordTokenFilter. If enablePositionIncrements is set for StopFilter,
then this could create invalid position increments, causing IndexWriter
to crash. (Robert Muir, Uwe Schindler)
+
+ * LUCENE-2013: SpanRegexQuery does not work with QueryScorer.
+ (Benjamin Keil via Mark Miller)
======================= Release 2.9.0 2009-09-23 =======================
Modified: lucene/java/branches/lucene_2_9/contrib/highlighter/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/highlighter/build.xml?rev=831020&r1=831019&r2=831020&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/highlighter/build.xml (original)
+++ lucene/java/branches/lucene_2_9/contrib/highlighter/build.xml Thu Oct 29 16:34:02 2009
@@ -27,18 +27,26 @@
<property name="memory.jar" location="${common.dir}/build/contrib/memory/lucene-memory-${version}.jar"/>
<available property="memory.jar.present" type="file" file="${memory.jar}"/>
+
+ <property name="regex.jar" location="${common.dir}/build/contrib/regex/lucene-regex-${version}.jar"/>
+ <available property="regex.jar.present" type="file" file="${regex.jar}"/>
<path id="classpath">
<pathelement path="${lucene.jar}"/>
<pathelement path="${memory.jar}"/>
+ <pathelement path="${regex.jar}"/>
<pathelement path="${project.classpath}"/>
</path>
- <target name="compile-core" depends="build-memory, common.compile-core" />
+ <target name="compile-core" depends="build-memory, build-regex, common.compile-core" />
<target name="build-memory" unless="memory.jar.present">
<echo>Highlighter building dependency ${memory.jar}</echo>
<ant antfile="../memory/build.xml" target="default" inheritall="false" dir="../memory" />
</target>
-
+
+ <target name="build-regex" unless="regex.jar.present">
+ <echo>Highlighter building dependency ${regex.jar}</echo>
+ <ant antfile="../regex/build.xml" target="default" inheritall="false" dir="../regex" />
+ </target>
</project>
Modified: lucene/java/branches/lucene_2_9/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=831020&r1=831019&r2=831020&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Thu Oct 29 16:34:02 2009
@@ -46,7 +46,10 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
+import org.apache.lucene.search.spans.SpanFirstQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanNotQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -239,17 +242,11 @@
* @throws IOException
*/
private void extractWeightedSpanTerms(Map terms, SpanQuery spanQuery) throws IOException {
- Set nonWeightedTerms = new HashSet();
- spanQuery.extractTerms(nonWeightedTerms);
-
Set fieldNames;
-
+
if (fieldName == null) {
fieldNames = new HashSet();
- for (Iterator iter = nonWeightedTerms.iterator(); iter.hasNext();) {
- Term queryTerm = (Term) iter.next();
- fieldNames.add(queryTerm.field());
- }
+ collectSpanQueryFields(spanQuery, fieldNames);
} else {
fieldNames = new HashSet(1);
fieldNames.add(fieldName);
@@ -258,6 +255,23 @@
if (defaultField != null) {
fieldNames.add(defaultField);
}
+
+
+ Map queries = new HashMap();
+
+ Set nonWeightedTerms = new HashSet();
+ final boolean mustRewriteQuery = mustRewriteQuery(spanQuery);
+ if (mustRewriteQuery) {
+ Iterator it = fieldNames.iterator();
+ while(it.hasNext()) {
+ String field = (String) it.next();
+ final SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(getReaderForField(field));
+ queries.put(field, rewrittenQuery);
+ rewrittenQuery.extractTerms(nonWeightedTerms);
+ }
+ } else {
+ spanQuery.extractTerms(nonWeightedTerms);
+ }
Iterator it = fieldNames.iterator();
List spanPositions = new ArrayList();
@@ -266,7 +280,12 @@
String field = (String) it.next();
IndexReader reader = getReaderForField(field);
- Spans spans = spanQuery.getSpans(reader);
+ final Spans spans;
+ if (mustRewriteQuery) {
+ spans = ((SpanQuery) queries.get(field)).getSpans(reader);
+ } else {
+ spans = spanQuery.getSpans(reader);
+ }
// collect span positions
while (spans.next()) {
@@ -453,6 +472,63 @@
return terms;
}
+
+ private void collectSpanQueryFields(SpanQuery spanQuery, Set fieldNames) {
+ if (spanQuery instanceof FieldMaskingSpanQuery) {
+ collectSpanQueryFields(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery(), fieldNames);
+ } else if (spanQuery instanceof SpanFirstQuery) {
+ collectSpanQueryFields(((SpanFirstQuery)spanQuery).getMatch(), fieldNames);
+ } else if (spanQuery instanceof SpanNearQuery) {
+ SpanQuery[] clauses = ((SpanNearQuery)spanQuery).getClauses();
+ for(int i = 0; i < clauses.length; i++) {
+ collectSpanQueryFields(clauses[i], fieldNames);
+ }
+ } else if (spanQuery instanceof SpanNotQuery) {
+ collectSpanQueryFields(((SpanNotQuery)spanQuery).getInclude(), fieldNames);
+ } else if (spanQuery instanceof SpanOrQuery) {
+ SpanQuery[] clauses = ((SpanNearQuery)spanQuery).getClauses();
+ for(int i = 0; i < clauses.length; i++) {
+ collectSpanQueryFields(clauses[i], fieldNames);
+ }
+ } else {
+ fieldNames.add(spanQuery.getField());
+ }
+ }
+
+ private boolean mustRewriteQuery(SpanQuery spanQuery) {
+ if (!expandMultiTermQuery) {
+ return false; // Will throw UnsupportedOperationException in case of a SpanRegexQuery.
+ } else if (spanQuery instanceof FieldMaskingSpanQuery) {
+ return mustRewriteQuery(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery());
+ } else if (spanQuery instanceof SpanFirstQuery) {
+ return mustRewriteQuery(((SpanFirstQuery)spanQuery).getMatch());
+ } else if (spanQuery instanceof SpanNearQuery) {
+ SpanQuery[] clauses = ((SpanNearQuery)spanQuery).getClauses();
+ for(int i = 0; i < clauses.length; i++) {
+ if (mustRewriteQuery(clauses[i])) {
+ return true;
+ }
+ }
+ return false;
+ } else if (spanQuery instanceof SpanNotQuery) {
+ SpanNotQuery spanNotQuery = (SpanNotQuery)spanQuery;
+ return mustRewriteQuery(spanNotQuery.getInclude()) || mustRewriteQuery(spanNotQuery.getExclude());
+ } else if (spanQuery instanceof SpanOrQuery) {
+ SpanQuery[] clauses = ((SpanOrQuery)spanQuery).getClauses();
+ for(int i = 0; i < clauses.length; i++) {
+ if (mustRewriteQuery(clauses[i])) {
+ return true;
+ }
+ }
+ return false;
+ } else if (spanQuery instanceof SpanTermQuery) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+
/**
* This class makes sure that if both position sensitive and insensitive
* versions of the same term are added, the position insensitive one wins.
Modified: lucene/java/branches/lucene_2_9/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=831020&r1=831019&r2=831020&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Thu Oct 29 16:34:02 2009
@@ -72,8 +72,10 @@
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.highlight.SynonymTokenizer.TestHighlightRunner;
+import org.apache.lucene.search.regex.SpanRegexQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
@@ -261,6 +263,31 @@
numHighlights == 3);
}
+ public void testSpanRegexQuery() throws Exception {
+ query = new SpanOrQuery(new SpanQuery [] {
+ new SpanRegexQuery(new Term(FIELD_NAME, "ken.*")) });
+ searcher = new IndexSearcher(ramDir, true);
+ hits = searcher.search(query);
+ int maxNumFragmentsRequired = 2;
+
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this, scorer);
+
+ for (int i = 0; i < hits.length(); i++) {
+ String text = hits.doc(i).get(FIELD_NAME);
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+
+ highlighter.setTextFragmenter(new SimpleFragmenter(40));
+
+ String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
+ "...");
+ System.out.println("\t" + result);
+ }
+
+ assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
+ numHighlights == 5);
+ }
+
public void testNumericRangeQuery() throws Exception {
// doesn't currently highlight, but make sure it doesn't cause exception either
query = NumericRangeQuery.newIntRange(NUMERIC_FIELD_NAME, new Integer(2), new Integer(6), true, true);