You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [18/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-too...
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Sun Aug 11 12:19:13 2013
@@ -17,6 +17,8 @@ package org.apache.lucene.search.vectorh
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -39,6 +41,7 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;
+import org.apache.lucene.util.InPlaceMergeSorter;
/**
* FieldQuery breaks down query object into terms/phrases and keeps
@@ -330,7 +333,8 @@ public class FieldQuery {
return root.searchPhrase( phraseCandidate );
}
- private QueryPhraseMap getRootMap( String fieldName ){
+ /** Get the root map for the given field name. */
+ public QueryPhraseMap getRootMap( String fieldName ){
return rootMaps.get( fieldMatch ? fieldName : null );
}
@@ -347,6 +351,7 @@ public class FieldQuery {
boolean terminal;
int slop; // valid if terminal == true and phraseHighlight == true
float boost; // valid if terminal == true
+ int[] positions; // valid if terminal == true
int termOrPhraseNumber; // valid if terminal == true
FieldQuery fieldQuery;
Map<String, QueryPhraseMap> subMap = new HashMap<String, QueryPhraseMap>();
@@ -369,38 +374,107 @@ public class FieldQuery {
return map;
}
- void add( Query query, IndexReader reader ) {
+ void add( Query query, IndexReader reader ) {
if( query instanceof TermQuery ){
addTerm( ((TermQuery)query).getTerm(), query.getBoost() );
}
else if( query instanceof PhraseQuery ){
PhraseQuery pq = (PhraseQuery)query;
- Term[] terms = pq.getTerms();
- Map<String, QueryPhraseMap> map = subMap;
- QueryPhraseMap qpm = null;
- for( Term term : terms ){
- qpm = getOrNewMap( map, term.text() );
- map = qpm.subMap;
- }
- qpm.markTerminal( pq.getSlop(), pq.getBoost() );
+ final Term[] terms = pq.getTerms();
+ final int[] positions = pq.getPositions();
+ new InPlaceMergeSorter() {
+
+ @Override
+ protected void swap(int i, int j) {
+ Term tmpTerm = terms[i];
+ terms[i] = terms[j];
+ terms[j] = tmpTerm;
+
+ int tmpPos = positions[i];
+ positions[i] = positions[j];
+ positions[j] = tmpPos;
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return positions[i] - positions[j];
+ }
+ }.sort(0, terms.length);
+
+ addToMap(pq, terms, positions, 0, subMap, pq.getSlop());
}
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
-
+
+ private int numTermsAtSamePosition(int[] positions, int i) {
+ int numTermsAtSamePosition = 1;
+ for (int j = i + 1; j < positions.length; ++j) {
+ if (positions[j] == positions[i]) {
+ ++numTermsAtSamePosition;
+ }
+ }
+ return numTermsAtSamePosition;
+ }
+
+ private void addToMap(PhraseQuery pq, Term[] terms, int[] positions, int i, Map<String, QueryPhraseMap> map, int slop) {
+ int numTermsAtSamePosition = numTermsAtSamePosition(positions, i);
+ for (int j = 0; j < numTermsAtSamePosition; ++j) {
+ QueryPhraseMap qpm = getOrNewMap(map, terms[i + j].text());
+ if (i + numTermsAtSamePosition == terms.length) {
+ qpm.markTerminal(pq.getSlop(), pq.getBoost(), uniquePositions(positions));
+ } else {
+ addToMap(pq, terms, positions, i + numTermsAtSamePosition, qpm.subMap, slop);
+ }
+ }
+ if (slop > 2 && i + numTermsAtSamePosition < terms.length) {
+ Term[] otherTerms = Arrays.copyOf(terms, terms.length);
+ int[] otherPositions = Arrays.copyOf(positions, positions.length);
+ final int nextTermAtSamePosition = numTermsAtSamePosition(positions, i + numTermsAtSamePosition);
+ System.arraycopy(terms, i + numTermsAtSamePosition, otherTerms, i, nextTermAtSamePosition);
+ System.arraycopy(positions, i + numTermsAtSamePosition, otherPositions, i, nextTermAtSamePosition);
+ System.arraycopy(terms, i, otherTerms, i + nextTermAtSamePosition, numTermsAtSamePosition);
+ System.arraycopy(positions, i, otherPositions, i + nextTermAtSamePosition, numTermsAtSamePosition);
+ addToMap(pq, otherTerms, otherPositions, i, map, slop - 2);
+ }
+ }
+
+ private int[] uniquePositions(int[] positions) {
+ int uniqueCount = 1;
+ for (int i = 1; i < positions.length; ++i) {
+ if (positions[i] != positions[i - 1]) {
+ ++uniqueCount;
+ }
+ }
+ if (uniqueCount == positions.length) {
+ return positions;
+ }
+ int[] result = new int[uniqueCount];
+ result[0] = positions[0];
+ for (int i = 1, j = 1; i < positions.length; ++i) {
+ if (positions[i] != positions[i - 1]) {
+ result[j++] = positions[i];
+ }
+ }
+ return result;
+ }
+
public QueryPhraseMap getTermMap( String term ){
return subMap.get( term );
}
private void markTerminal( float boost ){
- markTerminal( 0, boost );
+ markTerminal( 0, boost, null );
}
- private void markTerminal( int slop, float boost ){
- this.terminal = true;
- this.slop = slop;
- this.boost = boost;
- this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber();
+ private void markTerminal( int slop, float boost, int[] positions ){
+ if (slop > this.slop || (slop == this.slop && boost > this.boost)) {
+ this.terminal = true;
+ this.slop = slop;
+ this.boost = boost;
+ this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber();
+ this.positions = positions;
+ }
}
public boolean isTerminal(){
@@ -435,15 +509,20 @@ public class FieldQuery {
// if the candidate is a term, it is valid
if( phraseCandidate.size() == 1 ) return true;
+
+ assert phraseCandidate.size() == positions.length;
// else check whether the candidate is valid phrase
// compare position-gaps between terms to slop
int pos = phraseCandidate.get( 0 ).getPosition();
+ int totalDistance = 0;
for( int i = 1; i < phraseCandidate.size(); i++ ){
int nextPos = phraseCandidate.get( i ).getPosition();
- if( Math.abs( nextPos - pos - 1 ) > slop ) return false;
+ final int expectedDelta = positions[i] - positions[i - 1];
+ final int actualDelta = nextPos - pos;
+ totalDistance += Math.abs(expectedDelta - actualDelta);
pos = nextPos;
}
- return true;
+ return totalDistance <= slop;
}
}
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Sun Aug 11 12:19:13 2013
@@ -145,6 +145,13 @@ public class FieldTermStack {
}
/**
+ * Return the top TermInfo object of the stack without removing it.
+ */
+ public TermInfo peek() {
+ return termList.peek();
+ }
+
+ /**
* @param termInfo the TermInfo object to be put on the top of the stack
*/
public void push( TermInfo termInfo ){
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Sun Aug 11 12:19:13 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.search.highlig
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -248,7 +247,7 @@ public class HighlighterTest extends Bas
private String highlightField(Query query, String fieldName, String text)
throws IOException, InvalidTokenOffsetsException {
TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)
- .tokenStream(fieldName, new StringReader(text));
+ .tokenStream(fieldName, text);
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
@@ -269,8 +268,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
- new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -348,7 +346,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -377,7 +375,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -406,7 +404,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -431,7 +429,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -455,7 +453,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -479,7 +477,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -505,7 +503,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).getField(NUMERIC_FIELD_NAME).numericValue().toString();
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -533,7 +531,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -555,7 +553,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -585,7 +583,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
@@ -608,7 +606,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
@@ -639,7 +637,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -710,7 +708,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -907,7 +905,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -931,7 +929,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, null);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -955,7 +953,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -1126,7 +1124,7 @@ public class HighlighterTest extends Bas
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
@@ -1187,7 +1185,7 @@ public class HighlighterTest extends Bas
Highlighter highlighter = getHighlighter(wTerms, HighlighterTest.this);// new
// Highlighter(new
// QueryTermScorer(wTerms));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
highlighter.setTextFragmenter(new SimpleFragmenter(2));
String result = highlighter.getBestFragment(tokenStream, texts[0]).trim();
@@ -1196,7 +1194,7 @@ public class HighlighterTest extends Bas
// readjust weights
wTerms[1].setWeight(50f);
- tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
highlighter = getHighlighter(wTerms, HighlighterTest.this);
highlighter.setTextFragmenter(new SimpleFragmenter(2));
@@ -1232,7 +1230,7 @@ public class HighlighterTest extends Bas
Highlighter highlighter = getHighlighter(query, null, HighlighterTest.this);
// Get 3 best fragments and separate with a "..."
- TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s));
+ TokenStream tokenStream = analyzer.tokenStream(null, s);
String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
@@ -1257,7 +1255,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
String result = highlighter.getBestFragment(tokenStream, text);
@@ -1280,7 +1278,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);// new Highlighter(this, new
@@ -1288,7 +1286,7 @@ public class HighlighterTest extends Bas
highlighter.setTextFragmenter(new SimpleFragmenter(20));
String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);
- tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ tokenStream = analyzer.tokenStream(FIELD_NAME, text);
TextFragment fragmentResults[] = highlighter.getBestTextFragments(tokenStream, text,
true, 10);
@@ -1318,7 +1316,7 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
numHighlights = 0;
doSearching(new TermQuery(new Term(FIELD_NAME, "meat")));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);// new Highlighter(this, new
// QueryTermScorer(query));
@@ -1432,7 +1430,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME, HighlighterTest.this, false);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -1461,7 +1459,7 @@ public class HighlighterTest extends Bas
doSearching(new TermQuery(new Term(FIELD_NAME, "aninvalidquerywhichshouldyieldnoresults")));
for (String text : texts) {
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
String result = highlighter.getBestFragment(tokenStream, text);
@@ -1503,7 +1501,7 @@ public class HighlighterTest extends Bas
}
});
highlighter.setTextFragmenter(new SimpleFragmenter(2000));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(rawDocContent));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, rawDocContent);
String encodedSnippet = highlighter.getBestFragments(tokenStream, rawDocContent, 1, "");
// An ugly bit of XML creation:
@@ -1828,7 +1826,7 @@ public class HighlighterTest extends Bas
*
* for (int i = 0; i < hits.totalHits; i++) { String text =
* searcher.doc2(hits.scoreDocs[i].doc).get(FIELD_NAME); TokenStream
- * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,new StringReader(text));
+ * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,text);
* String highlightedText = highlighter.getBestFragment(tokenStream,text);
* System.out.println(highlightedText); } }
*/
@@ -1855,7 +1853,7 @@ public class HighlighterTest extends Bas
final int expectedHighlights) throws Exception {
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -2104,7 +2102,7 @@ final class SynonymTokenizer extends Tok
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
Scorer scorer = null;
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
if (mode == QUERY) {
scorer = new QueryScorer(query);
} else if (mode == QUERY_TERM) {
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java Sun Aug 11 12:19:13 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.search.highlig
* limitations under the License.
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -89,8 +88,7 @@ public class HighlightCustomQueryTest ex
private String highlightField(Query query, String fieldName,
String text) throws IOException, InvalidTokenOffsetsException {
TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE,
- true, MockTokenFilter.ENGLISH_STOPSET).tokenStream(fieldName,
- new StringReader(text));
+ true, MockTokenFilter.ENGLISH_STOPSET).tokenStream(fieldName, text);
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
MyQueryScorer scorer = new MyQueryScorer(query, fieldName, FIELD_NAME);
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java Sun Aug 11 12:19:13 2013
@@ -123,6 +123,43 @@ public class TestPostingsHighlighter ext
dir.close();
}
+ // simple test with multiple values that make a result longer than maxLength.
+ public void testMaxLengthWithMultivalue() throws Exception {
+ Directory dir = newDirectory();
+ // use simpleanalyzer for more natural tokenization (else "test." is a token)
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+ offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ Document doc = new Document();
+
+ for(int i = 0; i < 3 ; i++) {
+ Field body = new Field("body", "", offsetsType);
+ body.setStringValue("This is a multivalued field");
+ doc.add(body);
+ }
+
+ iw.addDocument(doc);
+
+ IndexReader ir = iw.getReader();
+ iw.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ PostingsHighlighter highlighter = new PostingsHighlighter(40);
+ Query query = new TermQuery(new Term("body", "field"));
+ TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
+ assertEquals(1, topDocs.totalHits);
+ String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
+ assertEquals(1, snippets.length);
+ assertTrue("Snippet should have maximum 40 characters plus the pre and post tags",
+ snippets[0].length() == (40 + "<b></b>".length()));
+
+ ir.close();
+ dir.close();
+ }
+
public void testMultipleFields() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
@@ -885,4 +922,48 @@ public class TestPostingsHighlighter ext
ir.close();
dir.close();
}
+
+ /** customizing the gap separator to force a sentence break */
+ public void testGapSeparator() throws Exception {
+ Directory dir = newDirectory();
+ // use simpleanalyzer for more natural tokenization (else "test." is a token)
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+ offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ Document doc = new Document();
+
+ Field body1 = new Field("body", "", offsetsType);
+ body1.setStringValue("This is a multivalued field");
+ doc.add(body1);
+
+ Field body2 = new Field("body", "", offsetsType);
+ body2.setStringValue("This is something different");
+ doc.add(body2);
+
+ iw.addDocument(doc);
+
+ IndexReader ir = iw.getReader();
+ iw.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ PostingsHighlighter highlighter = new PostingsHighlighter() {
+ @Override
+ protected char getMultiValuedSeparator(String field) {
+ assert field.equals("body");
+ return '\u2029';
+ }
+ };
+ Query query = new TermQuery(new Term("body", "field"));
+ TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
+ assertEquals(1, topDocs.totalHits);
+ String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
+ assertEquals(1, snippets.length);
+ assertEquals("This is a multivalued <b>field</b>\u2029", snippets[0]);
+
+ ir.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Sun Aug 11 12:19:13 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.search.vectorh
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -171,7 +170,7 @@ public abstract class AbstractTestCase e
protected List<BytesRef> analyze(String text, String field, Analyzer analyzer) throws IOException {
List<BytesRef> bytesRefs = new ArrayList<BytesRef>();
- TokenStream tokenStream = analyzer.tokenStream(field, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(field, text);
TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);
BytesRef bytesRef = termAttribute.getBytesRef();
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java Sun Aug 11 12:19:13 2013
@@ -16,10 +16,18 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
import java.io.IOException;
+import java.io.Reader;
+import java.util.Arrays;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -27,12 +35,14 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.CommonTermsQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
@@ -40,7 +50,8 @@ import org.apache.lucene.util.LuceneTest
public class FastVectorHighlighterTest extends LuceneTestCase {
-
+
+ private static final String FIELD = "text";
public void testSimpleHighlightTest() throws IOException {
Directory dir = newDirectory();
@@ -287,4 +298,128 @@ public class FastVectorHighlighterTest e
writer.close();
dir.close();
}
+
+ public void testOverlappingPhrases() throws IOException {
+ final Analyzer analyzer = new Analyzer() {
+
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ final Tokenizer source = new MockTokenizer(reader);
+ TokenStream sink = source;
+ sink = new SynonymFilter(sink);
+ return new TokenStreamComponents(source, sink);
+ }
+
+ };
+ final Directory directory = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, analyzer);
+ Document doc = new Document();
+ FieldType withVectors = new FieldType(TextField.TYPE_STORED);
+ withVectors.setStoreTermVectors(true);
+ withVectors.setStoreTermVectorPositions(true);
+ withVectors.setStoreTermVectorOffsets(true);
+ doc.add(new Field(FIELD, "a b c", withVectors));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+
+ // Disjunction of two overlapping phrase queries
+ final PhraseQuery pq1 = new PhraseQuery();
+ pq1.add(new Term(FIELD, "a"), 0);
+ pq1.add(new Term(FIELD, "b"), 1);
+ pq1.add(new Term(FIELD, "c"), 2);
+
+ final PhraseQuery pq2 = new PhraseQuery();
+ pq2.add(new Term(FIELD, "a"), 0);
+ pq2.add(new Term(FIELD, "B"), 1);
+ pq2.add(new Term(FIELD, "c"), 2);
+
+ final BooleanQuery bq = new BooleanQuery();
+ bq.add(pq1, Occur.SHOULD);
+ bq.add(pq2, Occur.SHOULD);
+
+ // Single phrase query with two terms at the same position
+ final PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term(FIELD, "a"), 0);
+ pq.add(new Term(FIELD, "b"), 1);
+ pq.add(new Term(FIELD, "B"), 1);
+ pq.add(new Term(FIELD, "c"), 2);
+
+ for (Query query : Arrays.asList(pq1, pq2, bq, pq)) {
+ assertEquals(1, new IndexSearcher(ir).search(bq, 1).totalHits);
+
+ FastVectorHighlighter highlighter = new FastVectorHighlighter();
+ FieldQuery fieldQuery = highlighter.getFieldQuery(query, ir);
+ String[] bestFragments = highlighter.getBestFragments(fieldQuery, ir, 0, FIELD, 1000, 1);
+ assertEquals("<b>a b c</b>", bestFragments[0]);
+ }
+
+ ir.close();
+ iw.close();
+ directory.close();
+ }
+
+ public void testPhraseWithGap() throws IOException {
+ final Directory directory = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
+ Document doc = new Document();
+ FieldType withVectors = new FieldType(TextField.TYPE_STORED);
+ withVectors.setStoreTermVectors(true);
+ withVectors.setStoreTermVectorPositions(true);
+ withVectors.setStoreTermVectorOffsets(true);
+ doc.add(new Field(FIELD, "a b c", withVectors));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+
+ final PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term(FIELD, "c"), 2);
+ pq.add(new Term(FIELD, "a"), 0);
+
+ assertEquals(1, new IndexSearcher(ir).search(pq, 1).totalHits);
+
+ FastVectorHighlighter highlighter = new FastVectorHighlighter();
+ FieldQuery fieldQuery = highlighter.getFieldQuery(pq, ir);
+ String[] bestFragments = highlighter.getBestFragments(fieldQuery, ir, 0, FIELD, 1000, 1);
+ assertEquals("<b>a</b> b <b>c</b>", bestFragments[0]);
+
+ ir.close();
+ iw.close();
+ directory.close();
+ }
+
+ // Simple token filter that adds 'B' as a synonym of 'b'
+ private static class SynonymFilter extends TokenFilter {
+
+ final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+
+ State pending;
+
+ protected SynonymFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (pending != null) {
+ restoreState(pending);
+ termAtt.setEmpty().append('B');
+ posIncAtt.setPositionIncrement(0);
+ pending = null;
+ return true;
+ }
+ if (!input.incrementToken()) {
+ return false;
+ }
+ if (termAtt.toString().equals("b")) {
+ pending = captureState();
+ }
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ pending = null;
+ }
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java Sun Aug 11 12:19:13 2013
@@ -120,7 +120,31 @@ public class FieldPhraseListTest extends
assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 9, fpl.phraseList.get( 0 ).getEndOffset() );
}
-
+
+ public void testProximityPhraseReverse() throws Exception {
+ make1d1fIndex( "z a a b c" );
+
+ FieldQuery fq = new FieldQuery( pqF( 2F, 3, "c", "a" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "ac(2.0)((4,5)(8,9))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 9, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testProximityPhraseWithRepeatedTerms() throws Exception {
+ make1d1fIndex( "z a a b b z d" );
+
+ FieldQuery fq = new FieldQuery( pqF( 2F, 2, "a", "b", "d" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "abd(2.0)((4,7)(12,13))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 13, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
public void test2PhrasesOverlap() throws Exception {
make1d1fIndex( "d a b c d" );
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java Sun Aug 11 12:19:13 2013
@@ -863,8 +863,8 @@ public class FieldQueryTest extends Abst
phraseCandidate.add( new TermInfo( "c", 4, 5, 4, 1 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
- // "a b c"~1
- query = pqF( 1F, 1, "a", "b", "c" );
+ // "a b c"~2
+ query = pqF( 1F, 2, "a", "b", "c" );
// phraseHighlight = true, fieldMatch = true
fq = new FieldQuery( query, true, true );
Modified: lucene/dev/branches/lucene4956/lucene/join/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/join/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/join/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/join/build.xml Sun Aug 11 12:19:13 2013
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project name="join" default="default">
<description>
Index-time and Query-time joins for normalized content
Modified: lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java Sun Aug 11 12:19:13 2013
@@ -216,7 +216,7 @@ class TermsIncludingScoreQuery extends Q
return scores[ords[scoreUpto]];
}
- public Explanation explain() throws IOException {
+ Explanation explain() throws IOException {
return new ComplexExplanation(true, score(), "Score based on join value " + termsEnum.term().utf8ToString());
}
@@ -226,27 +226,29 @@ class TermsIncludingScoreQuery extends Q
}
int nextDocOutOfOrder() throws IOException {
- if (docsEnum != null) {
- int docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- return doc = docId;
+ while (true) {
+ if (docsEnum != null) {
+ int docId = docsEnumNextDoc();
+ if (docId == DocIdSetIterator.NO_MORE_DOCS) {
+ docsEnum = null;
+ } else {
+ return doc = docId;
+ }
}
- }
- do {
if (upto == terms.size()) {
return doc = DocIdSetIterator.NO_MORE_DOCS;
}
scoreUpto = upto;
- if (termsEnum.seekExact(terms.get(ords[upto++], spare), true)) {
+ if (termsEnum.seekExact(terms.get(ords[upto++], spare))) {
docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, DocsEnum.FLAG_NONE);
}
- } while (docsEnum == null);
+ }
+ }
- return doc = docsEnum.nextDoc();
+ protected int docsEnumNextDoc() throws IOException {
+ return docsEnum.nextDoc();
}
@Override
@@ -301,47 +303,14 @@ class TermsIncludingScoreQuery extends Q
}
@Override
- int nextDocOutOfOrder() throws IOException {
- if (docsEnum != null) {
- int docId;
- do {
- docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- } while (alreadyEmittedDocs.get(docId));
+ protected int docsEnumNextDoc() throws IOException {
+ while (true) {
+ int docId = docsEnum.nextDoc();
if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- alreadyEmittedDocs.set(docId);
return docId;
}
- }
-
- for (;;) {
- do {
- if (upto == terms.size()) {
- return DocIdSetIterator.NO_MORE_DOCS;
- }
-
- scoreUpto = upto;
- if (termsEnum.seekExact(terms.get(ords[upto++], spare), true)) {
- docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, DocsEnum.FLAG_NONE);
- }
- } while (docsEnum == null);
-
- int docId;
- do {
- docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- } while (alreadyEmittedDocs.get(docId));
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- alreadyEmittedDocs.set(docId);
- return docId;
+ if (!alreadyEmittedDocs.getAndSet(docId)) {
+ return docId;//if it wasn't previously set, return it
}
}
}
@@ -368,7 +337,7 @@ class TermsIncludingScoreQuery extends Q
BytesRef spare = new BytesRef();
DocsEnum docsEnum = null;
for (int i = 0; i < terms.size(); i++) {
- if (termsEnum.seekExact(terms.get(ords[i], spare), true)) {
+ if (termsEnum.seekExact(terms.get(ords[i], spare))) {
docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
float score = TermsIncludingScoreQuery.this.scores[ords[i]];
for (int doc = docsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docsEnum.nextDoc()) {
@@ -424,7 +393,7 @@ class TermsIncludingScoreQuery extends Q
BytesRef spare = new BytesRef();
DocsEnum docsEnum = null;
for (int i = 0; i < terms.size(); i++) {
- if (termsEnum.seekExact(terms.get(ords[i], spare), true)) {
+ if (termsEnum.seekExact(terms.get(ords[i], spare))) {
docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
float score = TermsIncludingScoreQuery.this.scores[ords[i]];
for (int doc = docsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docsEnum.nextDoc()) {
Modified: lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java Sun Aug 11 12:19:13 2013
@@ -38,6 +38,7 @@ import java.util.Comparator;
class TermsQuery extends MultiTermQuery {
private final BytesRefHash terms;
+ private final int[] ords;
private final Query fromQuery; // Used for equals() only
/**
@@ -48,6 +49,7 @@ class TermsQuery extends MultiTermQuery
super(field);
this.fromQuery = fromQuery;
this.terms = terms;
+ ords = terms.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
@@ -56,7 +58,7 @@ class TermsQuery extends MultiTermQuery
return TermsEnum.EMPTY;
}
- return new SeekingTermSetTermsEnum(terms.iterator(null), this.terms);
+ return new SeekingTermSetTermsEnum(terms.iterator(null), this.terms, ords);
}
@Override
@@ -104,12 +106,12 @@ class TermsQuery extends MultiTermQuery
private BytesRef seekTerm;
private int upto = 0;
- SeekingTermSetTermsEnum(TermsEnum tenum, BytesRefHash terms) {
+ SeekingTermSetTermsEnum(TermsEnum tenum, BytesRefHash terms, int[] ords) {
super(tenum);
this.terms = terms;
-
+ this.ords = ords;
+ comparator = BytesRef.getUTF8SortedAsUnicodeComparator();
lastElement = terms.size() - 1;
- ords = terms.sort(comparator = tenum.getComparator());
lastTerm = terms.get(ords[lastElement], new BytesRef());
seekTerm = terms.get(ords[upto], spare);
}
Modified: lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Sun Aug 11 12:19:13 2013
@@ -459,7 +459,7 @@ public class TestBlockJoin extends Lucen
for(int parentDocID=0;parentDocID<numParentDocs;parentDocID++) {
Document parentDoc = new Document();
Document parentJoinDoc = new Document();
- Field id = newStringField("parentID", ""+parentDocID, Field.Store.YES);
+ Field id = new IntField("parentID", parentDocID, Field.Store.YES);
parentDoc.add(id);
parentJoinDoc.add(id);
parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
@@ -472,8 +472,8 @@ public class TestBlockJoin extends Lucen
}
if (doDeletes) {
- parentDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
- parentJoinDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
+ parentDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
+ parentJoinDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
final List<Document> joinDocs = new ArrayList<>();
@@ -497,7 +497,7 @@ public class TestBlockJoin extends Lucen
Document joinChildDoc = new Document();
joinDocs.add(joinChildDoc);
- Field childID = newStringField("childID", ""+childDocID, Field.Store.YES);
+ Field childID = new IntField("childID", childDocID, Field.Store.YES);
childDoc.add(childID);
joinChildDoc.add(childID);
@@ -522,7 +522,7 @@ public class TestBlockJoin extends Lucen
}
if (doDeletes) {
- joinChildDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
+ joinChildDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
w.addDocument(childDoc);
@@ -541,8 +541,10 @@ public class TestBlockJoin extends Lucen
if (VERBOSE) {
System.out.println("DELETE parentID=" + deleteID);
}
- w.deleteDocuments(new Term("blockID", ""+deleteID));
- joinW.deleteDocuments(new Term("blockID", ""+deleteID));
+ BytesRef term = new BytesRef();
+ NumericUtils.intToPrefixCodedBytes(deleteID, 0, term);
+ w.deleteDocuments(new Term("blockID", term));
+ joinW.deleteDocuments(new Term("blockID", term));
}
final IndexReader r = w.getReader();
Modified: lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/lucene4956/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Sun Aug 11 12:19:13 2013
@@ -557,7 +557,7 @@ public class TestJoinUtil extends Lucene
joinValues.addAll(joinValueToJoinScores.keySet());
for (BytesRef joinValue : joinValues) {
termsEnum = terms.iterator(termsEnum);
- if (termsEnum.seekExact(joinValue, true)) {
+ if (termsEnum.seekExact(joinValue)) {
docsEnum = termsEnum.docs(slowCompositeReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE);
JoinScore joinScore = joinValueToJoinScores.get(joinValue);
Modified: lucene/dev/branches/lucene4956/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene4956/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt Sun Aug 11 12:19:13 2013
@@ -1,6 +1,6 @@
Copyright (c) 2006 Dawid Weiss
-Copyright (c) 2007-2012 Dawid Weiss, Marcin MiÅkowski
+Copyright (c) 2007-2013 Dawid Weiss, Marcin MiÅkowski
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
@@ -26,4 +26,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIA
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
Modified: lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-LICENSE-BSD.txt?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-LICENSE-BSD.txt Sun Aug 11 12:19:13 2013
@@ -1,62 +1,26 @@
BSD-licensed dictionary of Polish (Morfologik)
-Copyright (c) 2012, Marcin MiÅkowski
+Morfologik Polish dictionary.
+Version: 2.0 PoliMorf
+Copyright (c) 2013, Marcin MiÅkowski
All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS âAS ISâ AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---
-
-BSD-licensed dictionary of Polish (SGJP)
-http://sgjp.pl/morfeusz/
-
-Copyright © 2011 Zygmunt Saloni, WÅodzimierz GruszczyÅski,
- Marcin WoliÅski, Robert WoÅosz
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS âAS ISâ AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-NOTICE.txt?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-NOTICE.txt (original)
+++ lucene/dev/branches/lucene4956/lucene/licenses/morfologik-polish-NOTICE.txt Sun Aug 11 12:19:13 2013
@@ -1,6 +1,3 @@
-This product includes data from BSD-licensed dictionary of Polish (Morfologik)
-(http://morfologik.blogspot.com/)
-
-This product includes data from BSD-licensed dictionary of Polish (SGJP)
-(http://sgjp.pl/morfeusz/)
+This product includes data from BSD-licensed dictionary of Polish (Morfologik, PoliMorf)
+(http://morfologik.blogspot.com/)
\ No newline at end of file
Modified: lucene/dev/branches/lucene4956/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene4956/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt Sun Aug 11 12:19:13 2013
@@ -1,6 +1,6 @@
Copyright (c) 2006 Dawid Weiss
-Copyright (c) 2007-2012 Dawid Weiss, Marcin MiÅkowski
+Copyright (c) 2007-2013 Dawid Weiss, Marcin MiÅkowski
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
@@ -26,4 +26,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIA
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
Modified: lucene/dev/branches/lucene4956/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene4956/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.index.memory;
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
@@ -291,7 +290,7 @@ public class MemoryIndex {
TokenStream stream;
try {
- stream = analyzer.tokenStream(fieldName, new StringReader(text));
+ stream = analyzer.tokenStream(fieldName, text);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -886,13 +885,13 @@ public class MemoryIndex {
@Override
- public boolean seekExact(BytesRef text, boolean useCache) {
+ public boolean seekExact(BytesRef text) {
termUpto = binarySearch(text, br, 0, info.terms.size()-1, info.terms, info.sortedTerms, BytesRef.getUTF8SortedAsUnicodeComparator());
return termUpto >= 0;
}
@Override
- public SeekStatus seekCeil(BytesRef text, boolean useCache) {
+ public SeekStatus seekCeil(BytesRef text) {
termUpto = binarySearch(text, br, 0, info.terms.size()-1, info.terms, info.sortedTerms, BytesRef.getUTF8SortedAsUnicodeComparator());
if (termUpto < 0) { // not found; choose successor
termUpto = -termUpto-1;
Modified: lucene/dev/branches/lucene4956/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sun Aug 11 12:19:13 2013
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
-import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
@@ -328,7 +327,7 @@ public class MemoryIndexTest extends Bas
// now reuse and check again
TermsEnum te = reader.terms("foo").iterator(null);
- assertTrue(te.seekExact(new BytesRef("bar"), true));
+ assertTrue(te.seekExact(new BytesRef("bar")));
disi = te.docs(null, disi, DocsEnum.FLAG_NONE);
docid = disi.docID();
assertEquals(-1, docid);
@@ -362,7 +361,7 @@ public class MemoryIndexTest extends Bas
// now reuse and check again
TermsEnum te = reader.terms("foo").iterator(null);
- assertTrue(te.seekExact(new BytesRef("bar"), true));
+ assertTrue(te.seekExact(new BytesRef("bar")));
disi = te.docsAndPositions(null, disi);
docid = disi.docID();
assertEquals(-1, docid);
@@ -378,7 +377,7 @@ public class MemoryIndexTest extends Bas
SpanQuery wrappedquery = new SpanMultiTermQueryWrapper<RegexpQuery>(regex);
MemoryIndex mindex = new MemoryIndex(random().nextBoolean(), random().nextInt(50) * 1024 * 1024);
- mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", new StringReader("hello there")));
+ mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));
// This throws an NPE
assertEquals(0, mindex.search(wrappedquery), 0.00001f);
@@ -390,7 +389,7 @@ public class MemoryIndexTest extends Bas
SpanQuery wrappedquery = new SpanOrQuery(new SpanMultiTermQueryWrapper<RegexpQuery>(regex));
MemoryIndex mindex = new MemoryIndex(random().nextBoolean(), random().nextInt(50) * 1024 * 1024);
- mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", new StringReader("hello there")));
+ mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));
// This passes though
assertEquals(0, mindex.search(wrappedquery), 0.00001f);
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java Sun Aug 11 12:19:13 2013
@@ -191,6 +191,7 @@ public abstract class Sorter {
for (int i = 0; i < maxDoc; ++i) {
newToOld.add(docs[i]);
}
+ newToOld.freeze();
for (int i = 0; i < maxDoc; ++i) {
docs[(int) newToOld.get(i)] = i;
@@ -200,6 +201,7 @@ public abstract class Sorter {
for (int i = 0; i < maxDoc; ++i) {
oldToNew.add(docs[i]);
}
+ oldToNew.freeze();
return new Sorter.DocMap() {
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java Sun Aug 11 12:19:13 2013
@@ -107,6 +107,7 @@ public final class SortingMergePolicy ex
}
}
}
+ deletes.freeze();
return deletes;
}
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Sun Aug 11 12:19:13 2013
@@ -158,7 +158,7 @@ public class SweetSpotSimilarity extends
* @see #baselineTf
*/
@Override
- public float tf(int freq) {
+ public float tf(float freq) {
return baselineTf(freq);
}
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java Sun Aug 11 12:19:13 2013
@@ -38,6 +38,15 @@ import org.apache.lucene.util.IntsRef; /
* more than one output, as this requires pushing all
* multi-output values to a final state.
*
+ * <p>NOTE: the only way to create multiple outputs is to
+ * add the same input to the FST multiple times in a row. This is
+ * how the FST maps a single input to multiple outputs (e.g. you
+ * cannot pass a List<Object> to {@link Builder#add}). If
+ * your outputs are longs, and you need at most 2, then use
+ * {@link UpToTwoPositiveIntOutputs} instead since it stores
+ * the outputs more compactly (by stealing a bit from each
+ * long value).
+ *
* <p>NOTE: this cannot wrap itself (ie you cannot make an
* FST with List<List<Object>> outputs using this).
*
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java Sun Aug 11 12:19:13 2013
@@ -17,21 +17,6 @@ package org.apache.lucene.util.fst;
* limitations under the License.
*/
-/**
- * An FST {@link Outputs} implementation where each output
- * is one or two non-negative long values. If it's a
- * single output, Long is returned; else, TwoLongs. Order
- * is preserved in the TwoLongs case, ie .first is the first
- * input/output added to Builder, and .second is the
- * second. You cannot store 0 output with this (that's
- * reserved to mean "no output")!
- *
- * NOTE: the resulting FST is not guaranteed to be minimal!
- * See {@link Builder}.
- *
- * @lucene.experimental
- */
-
import java.io.IOException;
import org.apache.lucene.store.DataInput;
@@ -46,11 +31,21 @@ import org.apache.lucene.store.DataOutpu
* second. You cannot store 0 output with this (that's
* reserved to mean "no output")!
*
- * NOTE: the resulting FST is not guaranteed to be minimal!
+ * <p>NOTE: the only way to create a TwoLongs output is to
+ * add the same input to the FST twice in a row. This is
+ * how the FST maps a single input to two outputs (e.g. you
+ * cannot pass a TwoLongs to {@link Builder#add}. If you
+ * need more than two then use {@link ListOfOutputs}, but if
+ * you only have at most 2 then this implementation will
+ * require fewer bytes as it steals one bit from each long
+ * value.
+ *
+ * <p>NOTE: the resulting FST is not guaranteed to be minimal!
* See {@link Builder}.
*
* @lucene.experimental
*/
+
public final class UpToTwoPositiveIntOutputs extends Outputs<Object> {
/** Holds two long outputs. */
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Sun Aug 11 12:19:13 2013
@@ -41,7 +41,7 @@ public class TestIndexSplitter extends L
((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
}
- LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
+ MergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setNoCFSRatio(1.0);
mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
IndexWriter iw = new IndexWriter(
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Sun Aug 11 12:19:13 2013
@@ -98,13 +98,8 @@ public abstract class SorterTestBase ext
}
@Override
- public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- return in.exactSimScorer(weight, context);
- }
-
- @Override
- public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- return in.sloppySimScorer(weight, context);
+ public SimScorer simScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+ return in.simScorer(weight, context);
}
}
Modified: lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Sun Aug 11 12:19:13 2013
@@ -246,7 +246,7 @@ public class SweetSpotSimilarityTest ext
SweetSpotSimilarity ss = new SweetSpotSimilarity() {
@Override
- public float tf(int freq) {
+ public float tf(float freq) {
return hyperbolicTf(freq);
}
};
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java Sun Aug 11 12:19:13 2013
@@ -74,7 +74,8 @@ public class CommonTermsQuery extends Qu
protected final Occur highFreqOccur;
protected float lowFreqBoost = 1.0f;
protected float highFreqBoost = 1.0f;
- protected float minNrShouldMatch = 0;
+ protected float lowFreqMinNrShouldMatch = 0;
+ protected float highFreqMinNrShouldMatch = 0;
/**
* Creates a new {@link CommonTermsQuery}
@@ -161,10 +162,18 @@ public class CommonTermsQuery extends Qu
}
protected int calcLowFreqMinimumNumberShouldMatch(int numOptional) {
- if (minNrShouldMatch >= 1.0f || minNrShouldMatch == 0.0f) {
- return (int) minNrShouldMatch;
- }
- return (int) (Math.round(minNrShouldMatch * numOptional));
+ return minNrShouldMatch(lowFreqMinNrShouldMatch, numOptional);
+ }
+
+ protected int calcHighFreqMinimumNumberShouldMatch(int numOptional) {
+ return minNrShouldMatch(highFreqMinNrShouldMatch, numOptional);
+ }
+
+ private final int minNrShouldMatch(float minNrShouldMatch, int numOptional) {
+ if (minNrShouldMatch >= 1.0f || minNrShouldMatch == 0.0f) {
+ return (int) minNrShouldMatch;
+ }
+ return (int) (Math.round(minNrShouldMatch * numOptional));
}
protected Query buildQuery(final int maxDoc,
@@ -190,11 +199,16 @@ public class CommonTermsQuery extends Qu
}
}
- final int numLowFreqClauses = lowFreq.clauses().size();
+ final int numLowFreqClauses = lowFreq.clauses().size();
+ final int numHighFreqClauses = highFreq.clauses().size();
if (lowFreqOccur == Occur.SHOULD && numLowFreqClauses > 0) {
int minMustMatch = calcLowFreqMinimumNumberShouldMatch(numLowFreqClauses);
lowFreq.setMinimumNumberShouldMatch(minMustMatch);
}
+ if (highFreqOccur == Occur.SHOULD && numHighFreqClauses > 0) {
+ int minMustMatch = calcHighFreqMinimumNumberShouldMatch(numHighFreqClauses);
+ highFreq.setMinimumNumberShouldMatch(minMustMatch);
+ }
if (lowFreq.clauses().isEmpty()) {
/*
* if lowFreq is empty we rewrite the high freq terms in a conjunction to
@@ -246,7 +260,7 @@ public class CommonTermsQuery extends Qu
assert termsEnum != null;
if (termsEnum == TermsEnum.EMPTY) continue;
- if (termsEnum.seekExact(term.bytes(), false)) {
+ if (termsEnum.seekExact(term.bytes())) {
if (termContext == null) {
contextArray[i] = new TermContext(reader.getContext(),
termsEnum.termState(), context.ord, termsEnum.docFreq(),
@@ -272,7 +286,7 @@ public class CommonTermsQuery extends Qu
}
/**
- * Specifies a minimum number of the optional BooleanClauses which must be
+ * Specifies a minimum number of the low frequent optional BooleanClauses which must be
* satisfied in order to produce a match on the low frequency terms query
* part. This method accepts a float value in the range [0..1) as a fraction
* of the actual query terms in the low frequent clause or a number
@@ -287,16 +301,44 @@ public class CommonTermsQuery extends Qu
* @param min
* the number of optional clauses that must match
*/
- public void setMinimumNumberShouldMatch(float min) {
- this.minNrShouldMatch = min;
+ public void setLowFreqMinimumNumberShouldMatch(float min) {
+ this.lowFreqMinNrShouldMatch = min;
}
/**
- * Gets the minimum number of the optional BooleanClauses which must be
+ * Gets the minimum number of the optional low frequent BooleanClauses which must be
* satisfied.
*/
- public float getMinimumNumberShouldMatch() {
- return minNrShouldMatch;
+ public float getLowFreqMinimumNumberShouldMatch() {
+ return lowFreqMinNrShouldMatch;
+ }
+
+ /**
+ * Specifies a minimum number of the high frequent optional BooleanClauses which must be
+ * satisfied in order to produce a match on the low frequency terms query
+ * part. This method accepts a float value in the range [0..1) as a fraction
+ * of the actual query terms in the low frequent clause or a number
+ * <tt>>=1</tt> as an absolut number of clauses that need to match.
+ *
+ * <p>
+ * By default no optional clauses are necessary for a match (unless there are
+ * no required clauses). If this method is used, then the specified number of
+ * clauses is required.
+ * </p>
+ *
+ * @param min
+ * the number of optional clauses that must match
+ */
+ public void setHighFreqMinimumNumberShouldMatch(float min) {
+ this.highFreqMinNrShouldMatch = min;
+ }
+
+ /**
+ * Gets the minimum number of the optional high frequent BooleanClauses which must be
+ * satisfied.
+ */
+ public float getHighFreqMinimumNumberShouldMatch() {
+ return highFreqMinNrShouldMatch;
}
@Override
@@ -308,7 +350,7 @@ public class CommonTermsQuery extends Qu
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
boolean needParens = (getBoost() != 1.0)
- || (getMinimumNumberShouldMatch() > 0);
+ || (getLowFreqMinimumNumberShouldMatch() > 0);
if (needParens) {
buffer.append("(");
}
@@ -321,9 +363,12 @@ public class CommonTermsQuery extends Qu
if (needParens) {
buffer.append(")");
}
- if (getMinimumNumberShouldMatch() > 0) {
+ if (getLowFreqMinimumNumberShouldMatch() > 0 || getHighFreqMinimumNumberShouldMatch() > 0) {
buffer.append('~');
- buffer.append(getMinimumNumberShouldMatch());
+ buffer.append("(");
+ buffer.append(getLowFreqMinimumNumberShouldMatch());
+ buffer.append(getHighFreqMinimumNumberShouldMatch());
+ buffer.append(")");
}
if (getBoost() != 1.0f) {
buffer.append(ToStringUtils.boost(getBoost()));
@@ -343,7 +388,8 @@ public class CommonTermsQuery extends Qu
result = prime * result
+ ((lowFreqOccur == null) ? 0 : lowFreqOccur.hashCode());
result = prime * result + Float.floatToIntBits(maxTermFrequency);
- result = prime * result + Float.floatToIntBits(minNrShouldMatch);
+ result = prime * result + Float.floatToIntBits(lowFreqMinNrShouldMatch);
+ result = prime * result + Float.floatToIntBits(highFreqMinNrShouldMatch);
result = prime * result + ((terms == null) ? 0 : terms.hashCode());
return result;
}
@@ -363,7 +409,8 @@ public class CommonTermsQuery extends Qu
if (lowFreqOccur != other.lowFreqOccur) return false;
if (Float.floatToIntBits(maxTermFrequency) != Float
.floatToIntBits(other.maxTermFrequency)) return false;
- if (minNrShouldMatch != other.minNrShouldMatch) return false;
+ if (lowFreqMinNrShouldMatch != other.lowFreqMinNrShouldMatch) return false;
+ if (highFreqMinNrShouldMatch != other.highFreqMinNrShouldMatch) return false;
if (terms == null) {
if (other.terms != null) return false;
} else if (!terms.equals(other.terms)) return false;
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java Sun Aug 11 12:19:13 2013
@@ -193,7 +193,7 @@ public final class TermsFilter extends F
for (int i = termsAndField.start; i < termsAndField.end; i++) {
spare.offset = offsets[i];
spare.length = offsets[i+1] - offsets[i];
- if (termsEnum.seekExact(spare, false)) { // don't use cache since we could pollute the cache here easily
+ if (termsEnum.seekExact(spare)) {
docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE); // no freq since we don't need them
if (result == null) {
if (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java Sun Aug 11 12:19:13 2013
@@ -69,7 +69,7 @@ public class JoinDocFreqValueSource exte
{
try {
terms.get(doc, ref);
- if (termsEnum.seekExact(ref, true)) {
+ if (termsEnum.seekExact(ref)) {
return termsEnum.docFreq();
} else {
return 0;
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java Sun Aug 11 12:19:13 2013
@@ -27,23 +27,18 @@ public class MaxFloatFunction extends Mu
public MaxFloatFunction(ValueSource[] sources) {
super(sources);
}
-
- @Override
+
+ @Override
protected String name() {
return "max";
}
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- boolean first = true;
- float val = 0.0f;
+ if (valsArr.length == 0) return 0.0f;
+ float val = Float.NEGATIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- if (first) {
- first = false;
- val = vals.floatVal(doc);
- } else {
- val = Math.max(vals.floatVal(doc),val);
- }
+ val = Math.max(vals.floatVal(doc), val);
}
return val;
}
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java Sun Aug 11 12:19:13 2013
@@ -35,15 +35,10 @@ public class MinFloatFunction extends Mu
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- boolean first = true;
- float val = 0.0f;
+ if (valsArr.length == 0) return 0.0f;
+ float val = Float.POSITIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- if (first) {
- first = false;
- val = vals.floatVal(doc);
- } else {
- val = Math.min(vals.floatVal(doc),val);
- }
+ val = Math.min(vals.floatVal(doc), val);
}
return val;
}