You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/02/28 23:44:42 UTC
svn commit: r1294876 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/
lucene/test-framework/src/java/org/apache/lucene/util/
Author: mikemccand
Date: Tue Feb 28 22:44:41 2012
New Revision: 1294876
URL: http://svn.apache.org/viewvc?rev=1294876&view=rev
Log:
LUCENE-3824: don't do pointless by-value cmp in TermOrdVal/DocValuesComparator.setBottom
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Tue Feb 28 22:44:41 2012
@@ -886,31 +886,17 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
+ final int docOrd = this.order[doc];
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
- return bottomOrd - this.order[doc];
+ return bottomOrd - docOrd;
+ } else if (bottomOrd >= docOrd) {
+ // the equals case always means bottom is > doc
+ // (because we set bottomOrd to the lower bound in
+ // setBottom):
+ return 1;
} else {
- // ord is only approx comparable: if they are not
- // equal, we can use that; if they are equal, we
- // must fallback to compare by value
- final int order = this.order[doc];
- final int cmp = bottomOrd - order;
- if (cmp != 0) {
- return cmp;
- }
-
- final String val2 = lookup[order];
- if (bottomValue == null) {
- if (val2 == null) {
- return 0;
- }
- // bottom wins
- return -1;
- } else if (val2 == null) {
- // doc wins
- return 1;
- }
- return bottomValue.compareTo(val2);
+ return -1;
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java Tue Feb 28 22:44:41 2012
@@ -68,7 +68,7 @@ public abstract class TopFieldCollector
if (queueFull) {
if ((reverseMul * comparator.compareBottom(doc)) <= 0) {
// since docs are visited in doc Id order, if compare is 0, it means
- // this document is largest than anything else in the queue, and
+ // this document is larger than anything else in the queue, and
// therefore not competitive.
return;
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java Tue Feb 28 22:44:41 2012
@@ -22,9 +22,14 @@ import java.io.Serializable;
import java.text.Collator;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Locale;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -41,9 +46,10 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.FieldValueHitQueue.Entry;
-import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.DocIdBitSet;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.BeforeClass;
@@ -1293,4 +1299,141 @@ public class TestSort extends LuceneTest
return data[doc][2]==null ? missingVal : Integer.parseInt(data[doc][2]);
}
+ private static class RandomFilter extends Filter {
+ private final Random random;
+ private float density;
+ private final List<String> docValues;
+ public final List<String> matchValues = Collections.synchronizedList(new ArrayList<String>());
+
+ // density should be 0.0 ... 1.0
+ public RandomFilter(Random random, float density, List<String> docValues) {
+ this.random = random;
+ this.density = density;
+ this.docValues = docValues;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+ final int maxDoc = reader.maxDoc();
+ final int[] docIDToID = FieldCache.DEFAULT.getInts(reader, "id");
+ final FixedBitSet bits = new FixedBitSet(maxDoc);
+ for(int docID=0;docID<maxDoc;docID++) {
+ if (random.nextFloat() <= density) {
+ bits.set(docID);
+ //System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
+ matchValues.add(docValues.get(docIDToID[docID]));
+ }
+ }
+
+ return bits;
+ }
+ }
+
+ public void testRandomStringSort() throws Exception {
+ final int NUM_DOCS = atLeast(100);
+ final Directory dir = newDirectory();
+ final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ final boolean allowDups = random.nextBoolean();
+ final Set<String> seen = new HashSet<String>();
+ final int maxLength = _TestUtil.nextInt(random, 5, 100);
+ if (VERBOSE) {
+ System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups);
+ }
+ int numDocs = 0;
+ final List<String> docValues = new ArrayList<String>();
+ // TODO: deletions
+ while (numDocs < NUM_DOCS) {
+ final String s;
+ if (random.nextBoolean()) {
+ s = _TestUtil.randomSimpleString(random, maxLength);
+ } else {
+ s = _TestUtil.randomUnicodeString(random, maxLength);
+ }
+
+ if (!allowDups) {
+ if (seen.contains(s)) {
+ continue;
+ }
+ seen.add(s);
+ }
+
+ if (VERBOSE) {
+ System.out.println(" " + numDocs + ": s=" + s);
+ }
+
+ final Document doc = new Document();
+ doc.add(newField("string", s, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(newField("id", ""+numDocs, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
+ docValues.add(s);
+ writer.addDocument(doc);
+ numDocs++;
+
+ if (random.nextInt(40) == 17) {
+ // force flush
+ writer.getReader().close();
+ }
+ }
+
+ final IndexReader r = writer.getReader();
+ writer.close();
+ if (VERBOSE) {
+ System.out.println(" reader=" + r);
+ }
+
+ final IndexSearcher s = newSearcher(r, false);
+ final int ITERS = atLeast(100);
+ for(int iter=0;iter<ITERS;iter++) {
+ final boolean reverse = random.nextBoolean();
+ final TopFieldDocs hits;
+ final SortField sf = new SortField("string", SortField.STRING, reverse);
+ final Sort sort = new Sort(sf);
+ final int hitCount = _TestUtil.nextInt(random, 1, r.maxDoc() + 20);
+ final RandomFilter f = new RandomFilter(random, random.nextFloat(), docValues);
+ if (random.nextBoolean()) {
+ hits = s.search(new ConstantScoreQuery(f),
+ hitCount,
+ sort);
+ } else {
+ hits = s.search(new MatchAllDocsQuery(),
+ f,
+ hitCount,
+ sort);
+ }
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse);
+ }
+
+ // Compute expected results:
+ Collections.sort(f.matchValues);
+ if (reverse) {
+ Collections.reverse(f.matchValues);
+ }
+ final List<String> expected = f.matchValues;
+ if (VERBOSE) {
+ System.out.println(" expected:");
+ for(int idx=0;idx<expected.size();idx++) {
+ System.out.println(" " + idx + ": " + expected.get(idx));
+ if (idx == hitCount-1) {
+ break;
+ }
+ }
+ }
+
+ if (VERBOSE) {
+ System.out.println(" actual:");
+ for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
+ final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
+ System.out.println(" " + hitIDX + ": " + fd.fields[0]);
+ }
+ }
+ for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
+ final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
+ assertEquals(expected.get(hitIDX), (String) fd.fields[0]);
+ }
+ }
+
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Tue Feb 28 22:44:41 2012
@@ -207,8 +207,8 @@ public class _TestUtil {
return buf.toString();
}
- public static String randomSimpleString(Random r) {
- final int end = r.nextInt(10);
+ public static String randomSimpleString(Random r, int maxLength) {
+ final int end = r.nextInt(maxLength);
if (end == 0) {
// allow 0 length
return "";
@@ -220,6 +220,10 @@ public class _TestUtil {
return new String(buffer, 0, end);
}
+ public static String randomSimpleString(Random r) {
+ return randomSimpleString(r, 10);
+ }
+
/** Returns random string, including full unicode range. */
public static String randomUnicodeString(Random r) {
return randomUnicodeString(r, 20);