You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by gs...@apache.org on 2007/03/28 14:58:16 UTC
svn commit: r523302 - in /lucene/java/trunk: ./
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/search/
src/java/org/apache/lucene/search/payloads/
src/java/org/apache/lucene/search/spans/
src/test/org/apache/lucene/search/payloads/
Author: gsingers
Date: Wed Mar 28 05:58:15 2007
New Revision: 523302
URL: http://svn.apache.org/viewvc?view=rev&rev=523302
Log:
LUCENE-834:
Added in payloads search package, with one Query implementation: BoostingTermQuery.
Added isPayloadAvailable() method to TermPositions and implementations.
Modified access rights to some of the spans classes so that they could be accessed from the payloads package.
All tests pass.
Added:
lucene/java/trunk/src/java/org/apache/lucene/search/payloads/
lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (with props)
lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html (with props)
lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java (with props)
lucene/java/trunk/src/test/org/apache/lucene/search/payloads/
lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (with props)
Modified:
lucene/java/trunk/CHANGES.txt
lucene/java/trunk/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java
lucene/java/trunk/src/java/org/apache/lucene/index/MultipleTermPositions.java
lucene/java/trunk/src/java/org/apache/lucene/index/ParallelReader.java
lucene/java/trunk/src/java/org/apache/lucene/index/SegmentTermPositions.java
lucene/java/trunk/src/java/org/apache/lucene/index/TermPositions.java
lucene/java/trunk/src/java/org/apache/lucene/search/Similarity.java
lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanWeight.java
Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Wed Mar 28 05:58:15 2007
@@ -34,6 +34,9 @@
throw AlreadyClosedException if they are accessed after being
closed. (Mike McCandless)
+ 5. LUCENE-834: Changed some access levels for certain Span classes to allow them to be overridden. They have
+ been marked expert only and not for public consumption. (Grant Ingersoll)
+
Bug fixes
1. LUCENE-804: Fixed build.xml to pack a fully compilable src dist. (Doron Cohen)
@@ -100,6 +103,12 @@
Remark: The APIs introduced with this feature are in experimental state and thus
contain appropriate warnings in the javadocs.
(Michael Busch)
+
+ 4. LUCENE-834: Added BoostingTermQuery which can boost scores based on the values of a payload (see #3 above.) (Grant Ingersoll)
+ 5. LUCENE-834: Similarity has a new method for scoring payloads called scorePayloads that can be overridden to take advantage
+ of payload storage (see #3 above)
+ 6. LUCENE-834: Added isPayloadAvailable() onto TermPositions interface and implemented it in the appropriate places (Grant Ingersoll)
+
Optimizations
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/FilterIndexReader.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/FilterIndexReader.java Wed Mar 28 05:58:15 2007
@@ -20,7 +20,6 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
-
import java.io.IOException;
import java.util.Collection;
@@ -69,6 +68,12 @@
public byte[] getPayload(byte[] data, int offset) throws IOException {
return ((TermPositions) this.in).getPayload(data, offset);
+ }
+
+
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable() {
+ return ((TermPositions)this.in).isPayloadAvailable();
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java Wed Mar 28 05:58:15 2007
@@ -463,4 +463,10 @@
public byte[] getPayload(byte[] data, int offset) throws IOException {
return ((TermPositions)current).getPayload(data, offset);
}
+
+
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable() {
+ return ((TermPositions) current).isPayloadAvailable();
+ }
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/MultipleTermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/MultipleTermPositions.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/MultipleTermPositions.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/MultipleTermPositions.java Wed Mar 28 05:58:15 2007
@@ -17,14 +17,14 @@
* limitations under the License.
*/
+import org.apache.lucene.util.PriorityQueue;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import org.apache.lucene.util.PriorityQueue;
-
/**
* Describe class <code>MultipleTermPositions</code> here.
*
@@ -209,5 +209,12 @@
throw new UnsupportedOperationException();
}
-
+ /**
+ *
+ * @return false
+ */
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable() {
+ return false;
+ }
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/ParallelReader.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/ParallelReader.java Wed Mar 28 05:58:15 2007
@@ -18,22 +18,12 @@
*/
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.document.Fieldable;
import java.io.IOException;
-import java.util.SortedMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Enumeration;
-import java.util.Set;
-import java.util.HashSet;
+import java.util.*;
/** An IndexReader which reads multiple, parallel indexes. Each index added
@@ -425,6 +415,12 @@
public byte[] getPayload(byte[] data, int offset) throws IOException {
return ((TermPositions)termDocs).getPayload(data, offset);
+ }
+
+
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable() {
+ return ((TermPositions) termDocs).isPayloadAvailable();
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/SegmentTermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/SegmentTermPositions.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/SegmentTermPositions.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/SegmentTermPositions.java Wed Mar 28 05:58:15 2007
@@ -17,10 +17,10 @@
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.store.IndexInput;
+import java.io.IOException;
+
final class SegmentTermPositions
extends SegmentTermDocs implements TermPositions {
private IndexInput proxStream;
@@ -187,6 +187,11 @@
proxStream.readBytes(retArray, retOffset, payloadLength);
needToLoadPayload = false;
return retArray;
+ }
+
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable() {
+ return needToLoadPayload && payloadLength > 0;
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/TermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/TermPositions.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/TermPositions.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/TermPositions.java Wed Mar 28 05:58:15 2007
@@ -81,4 +81,20 @@
*/
// TODO: Remove warning after API has been finalized
byte[] getPayload(byte[] data, int offset) throws IOException;
+
+ /**
+ * Can we load the payload at this position? Payloads can only be loaded once per call
+ * to {@link #nextPosition()}
+ * @return true if there is a payload available at this position that can be loaded
+ *
+ * * <b>
+ * Warning: The status of the Payloads feature is experimental. The APIs
+ * introduced here might change in the future and will not be supported anymore
+ * in such a case. If you want to use this feature in a production environment
+ * you should wait for an official release.
+ * </b>
+ */
+ // TODO: Remove warning after API has been finalized
+ public boolean isPayloadAvailable();
+
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/Similarity.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/Similarity.java Wed Mar 28 05:58:15 2007
@@ -17,16 +17,16 @@
* limitations under the License.
*/
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Iterator;
-
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.SmallFloat;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+
/** Expert: Scoring API.
* <p>Subclasses implement search scoring.
*
@@ -503,4 +503,28 @@
* @return a score factor based on term overlap with the query
*/
public abstract float coord(int overlap, int maxOverlap);
+
+
+ /**
+ * Calculate a scoring factor based on the data in the payload. Overriding implementations
+ * are responsible for interpreting what is in the payload. Lucene makes no assumptions about
+ * what is in the byte array.
+ * <p>
+ * The default implementation returns 1.
+ *
+ * @param payload The payload byte array to be scored
+ * @return An implementation dependent float to be used as a scoring factor
+ * <b>
+ * Warning: The status of the Payloads feature is experimental. The APIs
+ * introduced here might change in the future and will not be supported anymore
+ * in such a case. If you want to use this feature in a production environment
+ * you should wait for an official release.
+ * </b>
+ */
+ // TODO: Remove warning after API has been finalized
+ public float scorePayload(byte [] payload, int offset, int length)
+ {
+ //Do nothing
+ return 1;
+ }
}
Added: lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java?view=auto&rev=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java Wed Mar 28 05:58:15 2007
@@ -0,0 +1,153 @@
+package org.apache.lucene.search.payloads;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.spans.SpanScorer;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.TermSpans;
+
+import java.io.IOException;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ * <p/>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The BoostingTermQuery is very similar to the {@link org.apache.lucene.search.spans.SpanTermQuery} except
+ * that it factors in the value of the payload located at each of the positions where the
+ * {@link org.apache.lucene.index.Term} occurs.
+ * <p>
+ * In order to take advantage of this, you must override {@link org.apache.lucene.search.Similarity#scorePayload(byte[],int,int)}
+ * which returns 1 by default.
+ *
+ *
+ * @see org.apache.lucene.search.Similarity#scorePayload(byte[], int, int)
+ */
+public class BoostingTermQuery extends SpanTermQuery{
+
+
+ public BoostingTermQuery(Term term) {
+ super(term);
+ }
+
+
+ protected Weight createWeight(Searcher searcher) throws IOException {
+ return new BoostingTermWeight(this, searcher);
+ }
+
+ private class BoostingTermWeight extends SpanWeight implements Weight {
+
+
+ public BoostingTermWeight(BoostingTermQuery query, Searcher searcher) throws IOException {
+ super(query, searcher);
+ }
+
+
+
+
+ public Scorer scorer(IndexReader reader) throws IOException {
+ return new BoostingSpanScorer((TermSpans)query.getSpans(reader), this, similarity,
+ reader.norms(query.getField()));
+ }
+
+ class BoostingSpanScorer extends SpanScorer {
+
+ //TODO: is this the best way to allocate this?
+ byte[] payload = new byte[256];
+ private TermPositions positions;
+
+
+ public BoostingSpanScorer(TermSpans spans, Weight weight,
+ Similarity similarity, byte[] norms) throws IOException {
+ super(spans, weight, similarity, norms);
+ positions = spans.getPositions();
+
+ }
+
+ public boolean next() throws IOException {
+
+ boolean result = super.next();
+ //set the payload. super.next() properly increments the term positions
+ if (result) {
+ loadPayload();
+ }
+
+ return result;
+ }
+
+ public boolean skipTo(int target) throws IOException {
+ boolean result = super.skipTo(target);
+
+ if (result) {
+ loadPayload();
+ }
+
+ return result;
+ }
+
+ private void loadPayload() throws IOException {
+ if (positions.isPayloadAvailable()) {
+ payload = positions.getPayload(payload, 0);
+
+ } else {
+ //zero out the payload?
+ }
+
+ }
+
+ public float score() throws IOException {
+
+ int payLength = positions.getPayloadLength();
+ return super.score() * (payLength > 0 ? getSimilarity().scorePayload(payload, 0, payLength) : 1);
+ }
+
+
+ public Explanation explain(final int doc) throws IOException {
+ Explanation result = new Explanation();
+ Explanation nonPayloadExpl = super.explain(doc);
+ result.addDetail(nonPayloadExpl);
+ //QUESTION: Is there a wau to avoid this skipTo call? We need to know whether to load the payload or not
+
+ Explanation payloadBoost = new Explanation();
+ result.addDetail(payloadBoost);
+/*
+ if (skipTo(doc) == true) {
+ loadPayload();
+ }
+*/
+ float payloadScore = getSimilarity().scorePayload(payload, 0, positions.getPayloadLength());
+ payloadBoost.setValue(payloadScore);
+ //GSI: I suppose we could toString the payload, but I don't think that would be a good idea
+ payloadBoost.setDescription("scorePayload(...)");
+ result.setValue(nonPayloadExpl.getValue() * payloadScore);
+ result.setDescription("btq");
+ return result;
+ }
+ }
+
+ }
+
+
+ public boolean equals(Object o) {
+ if (!(o instanceof BoostingTermQuery))
+ return false;
+ BoostingTermQuery other = (BoostingTermQuery) o;
+ return (this.getBoost() == other.getBoost())
+ && this.term.equals(other.term);
+ }
+}
Propchange: lucene/java/trunk/src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html?view=auto&rev=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html Wed Mar 28 05:58:15 2007
@@ -0,0 +1,36 @@
+<HTML>
+ <!--
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ --><HEAD>
+ <TITLE>org.apache.lucene.search.payloads</TITLE>
+</HEAD>
+<BODY>
+<DIV>The payloads package provides Query mechanisms for finding and using payloads.
+
+ The following Query implementations are provided:
+</DIV>
+<div>
+ <ol>
+ <li><a href="./BoostingTermQuery.html">BoostingTermQuery</a> -- Boost a term's score based on the value of the payload located at that term</li>
+ </ol>
+</div>
+<DIV> </DIV>
+<DIV align="center">
+</DIV>
+</BODY>
+</HTML>
\ No newline at end of file
Propchange: lucene/java/trunk/src/java/org/apache/lucene/search/payloads/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanScorer.java Wed Mar 28 05:58:15 2007
@@ -17,27 +17,29 @@
* limitations under the License.
*/
-import java.io.IOException;
-
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.Weight;
+import java.io.IOException;
-class SpanScorer extends Scorer {
- private Spans spans;
- private Weight weight;
- private byte[] norms;
- private float value;
+/**
+ * Public for extension only.
+ */
+public class SpanScorer extends Scorer {
+ protected Spans spans;
+ protected Weight weight;
+ protected byte[] norms;
+ protected float value;
- private boolean firstTime = true;
- private boolean more = true;
+ protected boolean firstTime = true;
+ protected boolean more = true;
- private int doc;
- private float freq;
+ protected int doc;
+ protected float freq;
- SpanScorer(Spans spans, Weight weight, Similarity similarity, byte[] norms)
+ protected SpanScorer(Spans spans, Weight weight, Similarity similarity, byte[] norms)
throws IOException {
super(similarity);
this.spans = spans;
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Wed Mar 28 05:58:15 2007
@@ -17,20 +17,18 @@
* limitations under the License.
*/
-import java.io.IOException;
-
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.Set;
-
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
import org.apache.lucene.util.ToStringUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Set;
+
/** Matches spans containing a term. */
public class SpanTermQuery extends SpanQuery {
- private Term term;
+ protected Term term;
/** Construct a SpanTermQuery matching the named term's spans. */
public SpanTermQuery(Term term) { this.term = term; }
@@ -78,60 +76,7 @@
}
public Spans getSpans(final IndexReader reader) throws IOException {
- return new Spans() {
- private TermPositions positions = reader.termPositions(term);
-
- private int doc = -1;
- private int freq;
- private int count;
- private int position;
-
- public boolean next() throws IOException {
- if (count == freq) {
- if (!positions.next()) {
- doc = Integer.MAX_VALUE;
- return false;
- }
- doc = positions.doc();
- freq = positions.freq();
- count = 0;
- }
- position = positions.nextPosition();
- count++;
- return true;
- }
-
- public boolean skipTo(int target) throws IOException {
- // are we already at the correct position?
- if (doc >= target) {
- return true;
- }
-
- if (!positions.skipTo(target)) {
- doc = Integer.MAX_VALUE;
- return false;
- }
-
- doc = positions.doc();
- freq = positions.freq();
- count = 0;
-
- position = positions.nextPosition();
- count++;
-
- return true;
- }
-
- public int doc() { return doc; }
- public int start() { return position; }
- public int end() { return position + 1; }
-
- public String toString() {
- return "spans(" + SpanTermQuery.this.toString() + ")@"+
- (doc==-1?"START":(doc==Integer.MAX_VALUE)?"END":doc+"-"+position);
- }
-
- };
+ return new TermSpans(reader.termPositions(term), term);
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanWeight.java?view=diff&rev=523302&r1=523301&r2=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/spans/SpanWeight.java Wed Mar 28 05:58:15 2007
@@ -17,32 +17,27 @@
* limitations under the License.
*/
-import java.io.IOException;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.*;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.Searcher;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.ComplexExplanation;
-import org.apache.lucene.search.Similarity;
-
-class SpanWeight implements Weight {
- private Similarity similarity;
- private float value;
- private float idf;
- private float queryNorm;
- private float queryWeight;
+/**
+ * Expert-only. Public for use by other weight implementations
+ */
+public class SpanWeight implements Weight {
+ protected Similarity similarity;
+ protected float value;
+ protected float idf;
+ protected float queryNorm;
+ protected float queryWeight;
- private Set terms;
- private SpanQuery query;
+ protected Set terms;
+ protected SpanQuery query;
public SpanWeight(SpanQuery query, Searcher searcher)
throws IOException {
Added: lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java?view=auto&rev=523302
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java Wed Mar 28 05:58:15 2007
@@ -0,0 +1,101 @@
+package org.apache.lucene.search.spans;
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
+
+import java.io.IOException;
+
+/**
+ * Expert:
+ * Public for extension only
+ */
+public class TermSpans implements Spans {
+ protected TermPositions positions;
+ protected Term term;
+ protected int doc;
+ protected int freq;
+ protected int count;
+ protected int position;
+
+
+ public TermSpans(TermPositions positions, Term term) throws IOException {
+
+ this.positions = positions;
+ this.term = term;
+ doc = -1;
+ }
+
+ public boolean next() throws IOException {
+ if (count == freq) {
+ if (!positions.next()) {
+ doc = Integer.MAX_VALUE;
+ return false;
+ }
+ doc = positions.doc();
+ freq = positions.freq();
+ count = 0;
+ }
+ position = positions.nextPosition();
+ count++;
+ return true;
+ }
+
+ public boolean skipTo(int target) throws IOException {
+ // are we already at the correct position?
+ if (doc >= target) {
+ return true;
+ }
+
+ if (!positions.skipTo(target)) {
+ doc = Integer.MAX_VALUE;
+ return false;
+ }
+
+ doc = positions.doc();
+ freq = positions.freq();
+ count = 0;
+
+ position = positions.nextPosition();
+ count++;
+
+ return true;
+ }
+
+ public int doc() {
+ return doc;
+ }
+
+ public int start() {
+ return position;
+ }
+
+ public int end() {
+ return position + 1;
+ }
+
+ public String toString() {
+ return "spans(" + term.toString() + ")@" +
+ (doc == -1 ? "START" : (doc == Integer.MAX_VALUE) ? "END" : doc + "-" + position);
+ }
+
+
+ public TermPositions getPositions() {
+ return positions;
+ }
+}
Propchange: lucene/java/trunk/src/java/org/apache/lucene/search/spans/TermSpans.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java?view=auto&rev=523302
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java Wed Mar 28 05:58:15 2007
@@ -0,0 +1,188 @@
+package org.apache.lucene.search.payloads;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.search.spans.TermSpans;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.English;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public class TestBoostingTermQuery extends TestCase {
+ private IndexSearcher searcher;
+ private BoostingSimilarity similarity = new BoostingSimilarity();
+
+ public TestBoostingTermQuery(String s) {
+ super(s);
+ }
+
+ private class PayloadAnalyzer extends Analyzer {
+
+
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream result = new LowerCaseTokenizer(reader);
+ result = new PayloadFilter(result);
+ return result;
+ }
+ }
+
+ private class PayloadFilter extends TokenFilter {
+
+
+ public PayloadFilter(TokenStream input) {
+ super(input);
+ }
+
+ public Token next() throws IOException {
+ Token result = input.next();
+ if (result != null) {
+ result.setPayload(new Payload(encodePayload(result.termText()), 0, 4));
+ }
+ return result;
+ }
+ }
+
+ protected void setUp() throws IOException {
+ RAMDirectory directory = new RAMDirectory();
+ PayloadAnalyzer analyzer = new PayloadAnalyzer();
+ IndexWriter writer
+ = new IndexWriter(directory, analyzer, true);
+ writer.setSimilarity(similarity);
+ //writer.infoStream = System.out;
+ for (int i = 0; i < 1000; i++) {
+ Document doc = new Document();
+ doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.TOKENIZED));
+ writer.addDocument(doc);
+ }
+ //writer.optimize();
+ writer.close();
+
+ searcher = new IndexSearcher(directory);
+ searcher.setSimilarity(similarity);
+ }
+
+ private byte[] encodePayload(String englishInt)
+ {
+ int i = englishInt.hashCode();
+ byte[] bytes = new byte[4];
+ bytes[0] = (byte) (i >>> 24);
+ bytes[1] = (byte) (i >>> 16);
+ bytes[2] = (byte) (i >>> 8);
+ bytes[3] = (byte) i;
+ return bytes;
+ }
+
+ private int decodePayload(byte[] payload, int size)
+ {
+ //This should be equal to the hash code of the String representing the English int from English.intToEnglish
+ int result = (payload[0] << 24) | (payload[1] << 16) | (payload[2] << 8) | (payload[3]);
+
+ /*assertEquals((byte) (size >>> 24), payload[0]);
+ assertEquals((byte) (size >>> 16), payload[1]);
+ assertEquals((byte) (size >>> 8), payload[2]);
+ assertEquals((byte) size, payload[3]);*/
+
+ return result;
+ }
+
+ protected void tearDown() {
+
+ }
+
+ public void test() throws IOException {
+ BoostingTermQuery query = new BoostingTermQuery(new Term("field", "seventy"));
+ TopDocs hits = searcher.search(query, null, 100);
+ assertTrue("hits is null and it shouldn't be", hits != null);
+ assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100);
+
+ //they should all have the exact same score, because they all contain seventy once, and we set
+ //all the other similarity factors to be 1
+ //This score should be 1, since we normalize scores
+ int seventyHash = "seventy".hashCode();
+ assertTrue("score " + hits.getMaxScore() + " does not equal 'seventy' hashcode: " + seventyHash, hits.getMaxScore() == seventyHash);
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ ScoreDoc doc = hits.scoreDocs[i];
+ assertTrue("score " + doc.score + " does not equal 'seventy' hashcode: " + seventyHash, doc.score == seventyHash);
+ }
+ CheckHits.checkExplanations(query, "field", searcher);
+ Spans spans = query.getSpans(searcher.getIndexReader());
+ assertTrue("spans is null and it shouldn't be", spans != null);
+ assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans);
+ /*float score = hits.score(0);
+ for (int i =1; i < hits.length(); i++)
+ {
+ assertTrue("scores are not equal and they should be", score == hits.score(i));
+ }*/
+
+ }
+
+ public void testNoMatch() throws Exception {
+ BoostingTermQuery query = new BoostingTermQuery(new Term("field", "junk"));
+ TopDocs hits = searcher.search(query, null, 100);
+ assertTrue("hits is null and it shouldn't be", hits != null);
+ assertTrue("hits Size: " + hits.totalHits + " is not: " + 0, hits.totalHits == 0);
+
+ }
+
+
+ class BoostingSimilarity extends DefaultSimilarity
+ {
+
+ // TODO: Remove warning after API has been finalized
+ public float scorePayload(byte[] payload, int offset, int length) {
+ //we know it is size 4 here, so ignore the offset/length
+ return decodePayload(payload,4);
+ }
+
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ //Make everything else 1 so we see the effect of the payload
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ public float lengthNorm(String fieldName, int numTerms) {
+ return 1;
+ }
+
+ public float queryNorm(float sumOfSquaredWeights) {
+ return 1;
+ }
+
+ public float sloppyFreq(int distance) {
+ return 1;
+ }
+
+ public float coord(int overlap, int maxOverlap) {
+ return 1;
+ }
+
+ public float idf(int docFreq, int numDocs) {
+ return 1;
+ }
+
+ public float tf(float freq) {
+ return 1;
+ }
+ }
+}
\ No newline at end of file
Propchange: lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java
------------------------------------------------------------------------------
svn:eol-style = native