You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by yo...@apache.org on 2006/04/04 17:26:00 UTC
svn commit: r391337 - in /lucene/java/trunk: ./
src/java/org/apache/lucene/search/ src/java/org/apache/lucene/search/spans/
src/test/org/apache/lucene/search/spans/
Author: yonik
Date: Tue Apr 4 08:25:58 2006
New Revision: 391337
URL: http://svn.apache.org/viewcvs?rev=391337&view=rev
Log:
Fix SpanScorer next()/skipTo() incompatability: LUCENE-413
Modified:
lucene/java/trunk/CHANGES.txt
lucene/java/trunk/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/CHANGES.txt?rev=391337&r1=391336&r2=391337&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Tue Apr 4 08:25:58 2006
@@ -16,6 +16,9 @@
IndexWriter. These could be set in Lucene 1.4 using a system property.
This feature had been removed without adding the corresponding
getter/setter methods. (Daniel Naber)
+
+ 4. LUCENE-413: Fixed ArrayIndexOutOfBoundsException exceptions
+ when using SpanQueries. (Paul Elschot via Yonik Seeley)
1.9.1
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/java/org/apache/lucene/search/DisjunctionSumScorer.java?rev=391337&r1=391336&r2=391337&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/DisjunctionSumScorer.java Tue Apr 4 08:25:58 2006
@@ -207,7 +207,7 @@
return false;
}
if (target <= currentDoc) {
- target = currentDoc + 1;
+ return true;
}
do {
Scorer top = (Scorer) scorerQueue.top();
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java?rev=391337&r1=391336&r2=391337&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java Tue Apr 4 08:25:58 2006
@@ -1,7 +1,7 @@
package org.apache.lucene.search.spans;
/**
- * Copyright 2004 The Apache Software Foundation
+ * Copyright 2006 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@
this.norms = norms;
this.weight = weight;
this.value = weight.getValue();
+ doc = -1;
}
public boolean next() throws IOException {
@@ -50,19 +51,35 @@
more = spans.next();
firstTime = false;
}
+ return setFreqCurrentDoc();
+ }
- if (!more) return false;
+ public boolean skipTo(int target) throws IOException {
+ if (firstTime) {
+ more = spans.skipTo(target);
+ firstTime = false;
+ }
+ if (! more) {
+ return false;
+ }
+ if (spans.doc() < target) { // setFreqCurrentDoc() leaves spans.doc() ahead
+ more = spans.skipTo(target);
+ }
+ return setFreqCurrentDoc();
+ }
- freq = 0.0f;
+ private boolean setFreqCurrentDoc() throws IOException {
+ if (! more) {
+ return false;
+ }
doc = spans.doc();
-
+ freq = 0.0f;
while (more && doc == spans.doc()) {
int matchLength = spans.end() - spans.start();
freq += getSimilarity().sloppyFreq(matchLength);
more = spans.next();
}
-
- return more || freq != 0.0f;
+ return more || (freq != 0);
}
public int doc() { return doc; }
@@ -70,22 +87,6 @@
public float score() throws IOException {
float raw = getSimilarity().tf(freq) * value; // raw score
return raw * Similarity.decodeNorm(norms[doc]); // normalize
- }
-
- public boolean skipTo(int target) throws IOException {
- more = spans.skipTo(target);
-
- if (!more) return false;
-
- freq = 0.0f;
- doc = spans.doc();
-
- while (more && spans.doc() == target) {
- freq += getSimilarity().sloppyFreq(spans.end() - spans.start());
- more = spans.next();
- }
-
- return more || freq != 0.0f;
}
public Explanation explain(final int doc) throws IOException {
Modified: lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=391337&r1=391336&r2=391337&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java Tue Apr 4 08:25:58 2006
@@ -25,11 +25,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@@ -44,6 +40,8 @@
// location to the index
protected Directory mDirectory;;
+ protected IndexSearcher searcher;
+
// field names in the index
private final static String FIELD_ID = "ID";
protected final static String FIELD_TEXT = "TEXT";
@@ -52,7 +50,6 @@
* Initializes the tests by adding 4 identical documents to the index.
*/
protected void setUp() throws Exception {
-
super.setUp();
// create test index
@@ -63,10 +60,11 @@
addDocument(writer, "3", "I think it should work.");
addDocument(writer, "4", "I think it should work.");
writer.close();
+ searcher = new IndexSearcher(mDirectory);
}
protected void tearDown() throws Exception {
-
+ searcher.close();
mDirectory.close();
mDirectory = null;
}
@@ -94,7 +92,7 @@
*/
public void testBooleanQueryWithSpanQueries() throws IOException {
- doTestBooleanQueryWithSpanQueries(0.3884282f);
+ doTestBooleanQueryWithSpanQueries(searcher,0.3884282f);
}
/**
@@ -102,57 +100,63 @@
*
* @throws IOException
*/
- protected void doTestBooleanQueryWithSpanQueries(final float expectedScore) throws IOException {
+ protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s, final float expectedScore) throws IOException {
final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
final BooleanQuery query = new BooleanQuery();
query.add(spanQuery, BooleanClause.Occur.MUST);
query.add(spanQuery, BooleanClause.Occur.MUST);
- final Hits hits = executeQuery(query);
final String[] expectedIds = new String[] { "1", "2", "3", "4" };
final float[] expectedScores = new float[] { expectedScore, expectedScore, expectedScore, expectedScore };
- assertHits(hits, "two span queries", expectedIds, expectedScores);
+ assertHits(s, query, "two span queries", expectedIds, expectedScores);
}
- /**
- * Executes the query and throws an assertion if the results don't match the
- * expectedHits.
- *
- * @param query the query to execute
- * @throws IOException
- */
- protected Hits executeQuery(final Query query) throws IOException {
-
- final IndexSearcher searcher = new IndexSearcher(mDirectory);
- final Hits hits = searcher.search(query);
- searcher.close();
- return hits;
- }
/**
* Checks to see if the hits are what we expected.
*
- * @param hits the search results
+ * @param query the query to execute
* @param description the description of the search
* @param expectedIds the expected document ids of the hits
* @param expectedScores the expected scores of the hits
*
* @throws IOException
*/
- protected void assertHits(final Hits hits, final String description, final String[] expectedIds,
+ protected static void assertHits(Searcher s, Query query, final String description, final String[] expectedIds,
final float[] expectedScores) throws IOException {
+ final float tolerance = 1e-5f;
+
+ // Hits hits = searcher.search(query);
+ // hits normalizes and throws things off if one score is greater than 1.0
+ TopDocs topdocs = s.search(query,null,10000);
+ /*****
// display the hits
- /*System.out.println(hits.length() + " hits for search: \"" + description + '\"');
+ System.out.println(hits.length() + " hits for search: \"" + description + '\"');
for (int i = 0; i < hits.length(); i++) {
System.out.println(" " + FIELD_ID + ':' + hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')');
- }*/
+ }
+ *****/
// did we get the hits we expected
- assertEquals(expectedIds.length, hits.length());
- for (int i = 0; i < hits.length(); i++) {
- assertTrue(expectedIds[i].equals(hits.doc(i).get(FIELD_ID)));
- assertEquals(expectedScores[i], hits.score(i), 0);
+ assertEquals(expectedIds.length, topdocs.totalHits);
+ for (int i = 0; i < topdocs.totalHits; i++) {
+ //System.out.println(i + " exp: " + expectedIds[i]);
+ //System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
+
+ int id = topdocs.scoreDocs[i].doc;
+ float score = topdocs.scoreDocs[i].score;
+ Document doc = s.doc(id);
+ assertEquals(expectedIds[i], doc.get(FIELD_ID));
+ boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
+ if (!scoreEq) {
+ System.out.println(i + " warning, expected score: " + expectedScores[i] + ", actual " + score);
+ System.out.println(s.explain(query,id));
+ }
+ assertEquals(expectedScores[i], score, tolerance);
+ assertEquals(s.explain(query,id).getValue(), score, tolerance);
}
}
+
+
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=391337&r1=391336&r2=391337&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java Tue Apr 4 08:25:58 2006
@@ -22,10 +22,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.search.*;
/*******************************************************************************
* Some expanded tests to make sure my patch doesn't break other SpanTermQuery
@@ -34,7 +31,7 @@
* @author Reece Wilton
*/
public class TestSpansAdvanced2 extends TestSpansAdvanced {
-
+ IndexSearcher searcher2;
/**
* Initializes the tests by adding documents to the index.
*/
@@ -48,6 +45,9 @@
addDocument(writer, "C", "It shouldn't.");
addDocument(writer, "D", "Should we, should we, should we.");
writer.close();
+
+ // re-open the searcher since we added more docs
+ searcher2 = new IndexSearcher(mDirectory);
}
/**
@@ -69,11 +69,10 @@
public void testSingleSpanQuery() throws IOException {
final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
- final Hits hits = executeQuery(spanQuery);
final String[] expectedIds = new String[] { "B", "D", "1", "2", "3", "4", "A" };
final float[] expectedScores = new float[] { 0.625f, 0.45927936f, 0.35355338f, 0.35355338f, 0.35355338f,
0.35355338f, 0.26516503f, };
- assertHits(hits, "single span query", expectedIds, expectedScores);
+ assertHits(searcher2, spanQuery, "single span query", expectedIds, expectedScores);
}
/**
@@ -88,10 +87,11 @@
final BooleanQuery query = new BooleanQuery();
query.add(spanQuery1, BooleanClause.Occur.MUST);
query.add(spanQuery2, BooleanClause.Occur.MUST);
- final Hits hits = executeQuery(query);
- final String[] expectedIds = new String[] { "A", "D" };
- final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
- assertHits(hits, "multiple different span queries", expectedIds, expectedScores);
+ final String[] expectedIds = new String[] { "D", "A" };
+ // these values were pre LUCENE-413
+ // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
+ final float[] expectedScores = new float[] { 1.0191123f, 0.93163157f };
+ assertHits(searcher2, query, "multiple different span queries", expectedIds, expectedScores);
}
/**
@@ -101,6 +101,6 @@
*/
public void testBooleanQueryWithSpanQueries() throws IOException {
- doTestBooleanQueryWithSpanQueries(0.73500174f);
+ doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
}
}