You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/06/03 20:38:07 UTC
svn commit: r951104 [2/2] - in /lucene/dev/trunk: lucene/
lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/
lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/
lucene/src/java/org/apache/lucene/index/ lucene/src/java/...
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFieldCache.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFieldCache.java Thu Jun 3 18:38:05 2010
@@ -23,14 +23,20 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
+import java.util.Random;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
public class TestFieldCache extends LuceneTestCase {
protected IndexReader reader;
- private static final int NUM_DOCS = 1000;
+ private int NUM_DOCS;
+ private String[] unicodeStrings;
public TestFieldCache(String s) {
super(s);
@@ -39,14 +45,17 @@ public class TestFieldCache extends Luce
@Override
protected void setUp() throws Exception {
super.setUp();
+ Random r = newRandom();
+ NUM_DOCS = 1000 * _TestUtil.getRandomMultiplier();
RAMDirectory directory = new RAMDirectory();
- IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(500));
long theLong = Long.MAX_VALUE;
double theDouble = Double.MAX_VALUE;
byte theByte = Byte.MAX_VALUE;
short theShort = Short.MAX_VALUE;
int theInt = Integer.MAX_VALUE;
float theFloat = Float.MAX_VALUE;
+ unicodeStrings = new String[NUM_DOCS];
for (int i = 0; i < NUM_DOCS; i++){
Document doc = new Document();
doc.add(new Field("theLong", String.valueOf(theLong--), Field.Store.NO, Field.Index.NOT_ANALYZED));
@@ -55,10 +64,28 @@ public class TestFieldCache extends Luce
doc.add(new Field("theShort", String.valueOf(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theInt", String.valueOf(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theFloat", String.valueOf(theFloat--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+
+ // sometimes skip the field:
+ if (r.nextInt(40) != 17) {
+ String s = null;
+ if (i > 0 && r.nextInt(3) == 1) {
+ // reuse past string -- try to find one that's not null
+ for(int iter=0;iter<10 && s==null;iter++) {
+ s = unicodeStrings[r.nextInt(i)];
+ }
+ if (s == null) {
+ s = _TestUtil.randomUnicodeString(r, 250);
+ }
+ } else {
+ s = _TestUtil.randomUnicodeString(r, 250);
+ }
+ unicodeStrings[i] = s;
+ doc.add(new Field("theRandomUnicodeString", unicodeStrings[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+ }
writer.addDocument(doc);
}
+ reader = writer.getReader();
writer.close();
- reader = IndexReader.open(directory, true);
}
public void testInfoStream() throws Exception {
@@ -129,5 +156,43 @@ public class TestFieldCache extends Luce
assertTrue(floats[i] + " does not equal: " + (Float.MAX_VALUE - i), floats[i] == (Float.MAX_VALUE - i));
}
+
+ // getTermsIndex
+ FieldCache.DocTermsIndex termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
+ assertSame("Second request to cache return same array", termsIndex, cache.getTermsIndex(reader, "theRandomUnicodeString"));
+ assertTrue("doubles Size: " + termsIndex.size() + " is not: " + NUM_DOCS, termsIndex.size() == NUM_DOCS);
+ final BytesRef br = new BytesRef();
+ for (int i = 0; i < NUM_DOCS; i++) {
+ final BytesRef term = termsIndex.getTerm(i, br);
+ final String s = term == null ? null : term.utf8ToString();
+ assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
+ }
+ // test bad field
+ termsIndex = cache.getTermsIndex(reader, "bogusfield");
+
+ // getTerms
+ FieldCache.DocTerms terms = cache.getTerms(reader, "theRandomUnicodeString");
+ assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString"));
+ assertTrue("doubles Size: " + terms.size() + " is not: " + NUM_DOCS, terms.size() == NUM_DOCS);
+ for (int i = 0; i < NUM_DOCS; i++) {
+ final BytesRef term = terms.getTerm(i, br);
+ final String s = term == null ? null : term.utf8ToString();
+ assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
+ }
+
+ // test bad field
+ terms = cache.getTerms(reader, "bogusfield");
+
+ FieldCache.DEFAULT.purge(reader);
+ }
+
+ public void testEmptyIndex() throws Exception {
+ Directory dir = new MockRAMDirectory();
+ IndexWriter writer= new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(500));
+ IndexReader r = writer.getReader();
+ FieldCache.DocTerms terms = FieldCache.DEFAULT.getTerms(r, "foobar");
+ FieldCache.DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(r, "foobar");
+ r.close();
+ dir.close();
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java Thu Jun 3 18:38:05 2010
@@ -43,6 +43,8 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.search.FieldValueHitQueue.Entry;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -1012,4 +1014,29 @@ public class TestSort extends LuceneTest
}
}
+ public void testEmptyStringVsNullStringSort() throws Exception {
+ Directory dir = new MockRAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document doc = new Document();
+ doc.add(new Field("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ doc.add(new Field("t", "1", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ w.addDocument(doc);
+ w.commit();
+ doc = new Document();
+ doc.add(new Field("t", "1", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ w.addDocument(doc);
+
+ IndexReader r = w.getReader();
+ w.close();
+ IndexSearcher s = new IndexSearcher(r);
+ TopDocs hits = s.search(new TermQuery(new Term("t", "1")), null, 10, new Sort(new SortField("f", SortField.STRING)));
+ assertEquals(2, hits.totalHits);
+ // null sorts first
+ assertEquals(1, hits.scoreDocs[0].doc);
+ assertEquals(0, hits.scoreDocs[1].doc);
+ r.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java Thu Jun 3 18:38:05 2010
@@ -207,7 +207,7 @@ public class TestOrdValues extends Funct
log("compare (should differ): " + innerArray + " to "
+ q.valSrc.getValues(reader).getInnerArray());
assertNotSame(
- "different values shuold be loaded for a different field!",
+ "different values should be loaded for a different field!",
innerArray, q.valSrc.getValues(reader).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Thu Jun 3 18:38:05 2010
@@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
@@ -112,7 +111,7 @@ public class TestFieldCacheSanityChecker
cache.purgeAllCaches();
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER);
- cache.getStrings(readerX, "theInt");
+ cache.getTerms(readerX, "theInt");
cache.getBytes(readerX, "theByte");
// // //
@@ -135,9 +134,9 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getStrings(readerA, "theString");
- cache.getStrings(readerB, "theString");
- cache.getStrings(readerX, "theString");
+ cache.getTerms(readerA, "theString");
+ cache.getTerms(readerB, "theString");
+ cache.getTerms(readerX, "theString");
cache.getBytes(readerX, "theByte");
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java Thu Jun 3 18:38:05 2010
@@ -118,7 +118,11 @@ public class _TestUtil {
/** Returns random string, including full unicode range. */
public static String randomUnicodeString(Random r) {
- final int end = r.nextInt(20);
+ return randomUnicodeString(r, 20);
+ }
+
+ public static String randomUnicodeString(Random r, int maxLength) {
+ final int end = r.nextInt(maxLength);
if (end == 0) {
// allow 0 length
return "";
@@ -126,6 +130,7 @@ public class _TestUtil {
final char[] buffer = new char[end];
for (int i = 0; i < end; i++) {
int t = r.nextInt(5);
+ //buffer[i] = (char) (97 + r.nextInt(26));
if (0 == t && i < end - 1) {
// Make a surrogate pair
// High surrogate
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/FieldFacetStats.java Thu Jun 3 18:38:05 2010
@@ -17,11 +17,11 @@ package org.apache.solr.handler.componen
*/
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.schema.FieldType;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -37,12 +37,9 @@ import java.util.Map;
public class FieldFacetStats {
public final String name;
- final FieldCache.StringIndex si;
+ final FieldCache.DocTermsIndex si;
final FieldType ft;
- final String[] terms;
- final int[] termNum;
-
final int startTermIndex;
final int endTermIndex;
final int nTerms;
@@ -53,16 +50,16 @@ public class FieldFacetStats {
final List<HashMap<String, Integer>> facetStatsTerms;
- public FieldFacetStats(String name, FieldCache.StringIndex si, FieldType ft, int numStatsTerms) {
+ private final BytesRef tempBR = new BytesRef();
+
+ public FieldFacetStats(String name, FieldCache.DocTermsIndex si, FieldType ft, int numStatsTerms) {
this.name = name;
this.si = si;
this.ft = ft;
this.numStatsTerms = numStatsTerms;
- terms = si.lookup;
- termNum = si.order;
startTermIndex = 1;
- endTermIndex = terms.length;
+ endTermIndex = si.numOrd();
nTerms = endTermIndex - startTermIndex;
facetStatsValues = new HashMap<String, StatsValues>();
@@ -76,21 +73,27 @@ public class FieldFacetStats {
}
}
- String getTermText(int docID) {
- return terms[termNum[docID]];
+ BytesRef getTermText(int docID, BytesRef ret) {
+ final int ord = si.getOrd(docID);
+ if (ord == 0) {
+ return null;
+ } else {
+ return si.lookup(ord, ret);
+ }
}
-
public boolean facet(int docID, Double v) {
- int term = termNum[docID];
+ int term = si.getOrd(docID);
int arrIdx = term - startTermIndex;
if (arrIdx >= 0 && arrIdx < nTerms) {
- String key = ft.indexedToReadable(terms[term]);
+ final BytesRef br = si.lookup(term, tempBR);
+ String key = ft.indexedToReadable(br == null ? null : br.utf8ToString());
StatsValues stats = facetStatsValues.get(key);
if (stats == null) {
stats = new StatsValues();
facetStatsValues.put(key, stats);
}
+
if (v != null) {
stats.accumulate(v);
} else {
@@ -107,10 +110,11 @@ public class FieldFacetStats {
// Currently only used by UnInvertedField stats
public boolean facetTermNum(int docID, int statsTermNum) {
- int term = termNum[docID];
+ int term = si.getOrd(docID);
int arrIdx = term - startTermIndex;
if (arrIdx >= 0 && arrIdx < nTerms) {
- String key = ft.indexedToReadable(terms[term]);
+ final BytesRef br = si.lookup(term, tempBR);
+ String key = br == null ? null : br.utf8ToString();
HashMap<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
Integer statsTermCount = statsTermCounts.get(key);
if (statsTermCount == null) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java Thu Jun 3 18:38:05 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.search.FieldCom
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -33,7 +34,6 @@ import org.apache.solr.common.params.Sha
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
@@ -259,6 +259,14 @@ public class QueryComponent extends Sear
val = ft.toObject(field);
}
+ // Must do the same conversion when sorting by a
+ // String field in Lucene, which returns the terms
+ // data as BytesRef:
+ if (val instanceof BytesRef) {
+ field.setValue(((BytesRef)val).utf8ToString());
+ val = ft.toObject(field);
+ }
+
vals.add(val);
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Thu Jun 3 18:38:05 2010
@@ -44,6 +44,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.DOMUtil;
@@ -99,7 +100,7 @@ public class QueryElevationComponent ext
final String analyzed;
final BooleanClause[] exclude;
final BooleanQuery include;
- final Map<String,Integer> priority;
+ final Map<BytesRef,Integer> priority;
// use singletons so hashCode/equals on Sort will just work
final FieldComparatorSource comparatorSource;
@@ -111,12 +112,12 @@ public class QueryElevationComponent ext
this.include = new BooleanQuery();
this.include.setBoost( 0 );
- this.priority = new HashMap<String, Integer>();
+ this.priority = new HashMap<BytesRef, Integer>();
int max = elevate.size()+5;
for( String id : elevate ) {
TermQuery tq = new TermQuery( new Term( idField, id ) );
include.add( tq, BooleanClause.Occur.SHOULD );
- this.priority.put( id, max-- );
+ this.priority.put( new BytesRef(id), max-- );
}
if( exclude == null || exclude.isEmpty() ) {
@@ -445,18 +446,19 @@ public class QueryElevationComponent ext
}
class ElevationComparatorSource extends FieldComparatorSource {
- private final Map<String,Integer> priority;
+ private final Map<BytesRef,Integer> priority;
- public ElevationComparatorSource( final Map<String,Integer> boosts) {
+ public ElevationComparatorSource( final Map<BytesRef,Integer> boosts) {
this.priority = boosts;
}
public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator() {
- FieldCache.StringIndex idIndex;
+ FieldCache.DocTermsIndex idIndex;
private final int[] values = new int[numHits];
int bottomVal;
+ private final BytesRef tempBR = new BytesRef();
public int compare(int slot1, int slot2) {
return values[slot2] - values[slot1]; // values will be small enough that there is no overflow concern
@@ -467,7 +469,7 @@ class ElevationComparatorSource extends
}
private int docVal(int doc) throws IOException {
- String id = idIndex.lookup[idIndex.order[doc]];
+ BytesRef id = idIndex.getTerm(doc, tempBR);
Integer prio = priority.get(id);
return prio == null ? 0 : prio.intValue();
}
@@ -481,7 +483,7 @@ class ElevationComparatorSource extends
}
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- idIndex = FieldCache.DEFAULT.getStringIndex(reader, fieldname);
+ idIndex = FieldCache.DEFAULT.getTermsIndex(reader, fieldname);
}
public Comparable value(int slot) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/StatsComponent.java Thu Jun 3 18:38:05 2010
@@ -22,15 +22,13 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.search.FieldCache;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.lucene.util.BytesRef;
+import org.apache.noggit.CharArr;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.handler.component.StatsValues;
-import org.apache.solr.handler.component.FieldFacetStats;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
@@ -39,7 +37,6 @@ import org.apache.solr.search.DocIterato
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.request.UnInvertedField;
-import org.apache.solr.core.SolrCore;
/**
* Stats component calculates simple statistics on numeric field values
@@ -249,9 +246,9 @@ class SimpleStats {
public NamedList getFieldCacheStats(String fieldName, String[] facet ) {
FieldType ft = searcher.getSchema().getFieldType(fieldName);
- FieldCache.StringIndex si = null;
+ FieldCache.DocTermsIndex si = null;
try {
- si = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), fieldName);
+ si = FieldCache.DEFAULT.getTermsIndex(searcher.getReader(), fieldName);
}
catch (IOException e) {
throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
@@ -266,23 +263,27 @@ class SimpleStats {
for( String f : facet ) {
ft = searcher.getSchema().getFieldType(f);
try {
- si = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), f);
+ si = FieldCache.DEFAULT.getTermsIndex(searcher.getReader(), f);
}
catch (IOException e) {
throw new RuntimeException( "failed to open field cache for: "+f, e );
}
finfo[i++] = new FieldFacetStats( f, si, ft, 0 );
}
-
-
+
+ final BytesRef tempBR = new BytesRef();
+ final CharArr spare = new CharArr();
+
DocIterator iter = docs.iterator();
while (iter.hasNext()) {
int docID = iter.nextDoc();
- String raw = all.getTermText(docID);
+ BytesRef raw = all.getTermText(docID, tempBR);
Double v = null;
if( raw != null ) {
- v = Double.parseDouble( all.ft.indexedToReadable(raw) );
- allstats.accumulate( v );
+ spare.reset();
+ all.ft.indexedToReadable(raw, spare);
+ v = Double.parseDouble(spare.toString());
+ allstats.accumulate(v);
}
else {
allstats.missing++;
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java Thu Jun 3 18:38:05 2010
@@ -5,12 +5,11 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
-import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexReader;
import org.apache.solr.search.SolrIndexSearcher;
@@ -100,7 +99,7 @@ class PerSegmentSingleValuedFaceting {
}
@Override
protected boolean lessThan(SegFacet a, SegFacet b) {
- return a.terms[a.pos].compareTo(b.terms[b.pos]) < 0;
+ return a.si.lookup(a.pos, a.tempBR).compareTo(b.si.lookup(b.pos, b.tempBR)) < 0;
}
};
@@ -150,9 +149,11 @@ class PerSegmentSingleValuedFaceting {
collector = new IndexSortedFacetCollector(offset, limit, mincount);
}
+ final BytesRef tempBR = new BytesRef();
+
while (queue.size() > 0) {
SegFacet seg = queue.top();
- String val = seg.terms[seg.pos];
+ BytesRef val = seg.si.lookup(seg.pos, tempBR);
int count = 0;
do {
@@ -167,7 +168,7 @@ class PerSegmentSingleValuedFaceting {
} else {
seg = queue.updateTop();
}
- } while (seg != null && val.compareTo(seg.terms[seg.pos]) == 0);
+ } while (seg != null && val.compareTo(seg.si.lookup(seg.pos, seg.tempBR)) == 0);
boolean stop = collector.collect(val, count);
if (stop) break;
@@ -192,20 +193,6 @@ class PerSegmentSingleValuedFaceting {
return res;
}
-
-
-
-
- // first element of the fieldcache is null, so we need this comparator.
- private static final Comparator nullStrComparator = new Comparator() {
- public int compare(Object o1, Object o2) {
- if (o1==null) return (o2==null) ? 0 : -1;
- else if (o2==null) return 1;
- return ((String)o1).compareTo((String)o2);
- }
- };
-
-
class SegFacet {
SolrIndexReader reader;
int readerOffset;
@@ -215,32 +202,31 @@ class PerSegmentSingleValuedFaceting {
this.readerOffset = readerOffset;
}
- int[] ords;
- String[] terms;
-
+ FieldCache.DocTermsIndex si;
int startTermIndex;
int endTermIndex;
int[] counts;
int pos; // only used during merge with other segments
+ final BytesRef tempBR = new BytesRef();
+
void countTerms() throws IOException {
- FieldCache.StringIndex si = FieldCache.DEFAULT.getStringIndex(reader, fieldName);
- final String[] terms = this.terms = si.lookup;
- final int[] termNum = this.ords = si.order;
+ si = FieldCache.DEFAULT.getTermsIndex(reader, fieldName);
// SolrCore.log.info("reader= " + reader + " FC=" + System.identityHashCode(si));
if (prefix!=null) {
- startTermIndex = Arrays.binarySearch(terms,prefix,nullStrComparator);
+ startTermIndex = si.binarySearchLookup(new BytesRef(prefix), tempBR);
if (startTermIndex<0) startTermIndex=-startTermIndex-1;
// find the end term. \uffff isn't a legal unicode char, but only compareTo
// is used, so it should be fine, and is guaranteed to be bigger than legal chars.
// TODO: switch to binarySearch version that takes start/end in Java6
- endTermIndex = Arrays.binarySearch(terms,prefix+"\uffff\uffff\uffff\uffff",nullStrComparator);
+ endTermIndex = si.binarySearchLookup(new BytesRef(prefix+"\uffff\uffff\uffff\uffff"), tempBR);
+ assert endTermIndex < 0;
endTermIndex = -endTermIndex-1;
} else {
startTermIndex=0;
- endTermIndex=terms.length;
+ endTermIndex=si.numOrd();
}
final int nTerms=endTermIndex-startTermIndex;
@@ -251,17 +237,17 @@ class PerSegmentSingleValuedFaceting {
DocIdSet idSet = baseSet.getDocIdSet(reader);
DocIdSetIterator iter = idSet.iterator();
- if (startTermIndex==0 && endTermIndex==terms.length) {
+ if (startTermIndex==0 && endTermIndex==si.numOrd()) {
// specialized version when collecting counts for all terms
int doc;
while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
- counts[termNum[doc]]++;
+ counts[si.getOrd(doc)]++;
}
} else {
// version that adjusts term numbers because we aren't collecting the full range
int doc;
while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
- int term = termNum[doc];
+ int term = si.getOrd(doc);
int arrIdx = term-startTermIndex;
if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
}
@@ -276,7 +262,7 @@ class PerSegmentSingleValuedFaceting {
abstract class FacetCollector {
/*** return true to stop collection */
- public abstract boolean collect(String term, int count);
+ public abstract boolean collect(BytesRef term, int count);
public abstract NamedList getFacetCounts();
}
@@ -299,12 +285,12 @@ class CountSortedFacetCollector extends
}
@Override
- public boolean collect(String term, int count) {
+ public boolean collect(BytesRef term, int count) {
if (count > min) {
// NOTE: we use c>min rather than c>=min as an optimization because we are going in
// index order, so we already know that the keys are ordered. This can be very
// important if a lot of the counts are repeated (like zero counts would be).
- queue.add(new SimpleFacets.CountPair<String,Integer>(term, count));
+ queue.add(new SimpleFacets.CountPair<String,Integer>(term.utf8ToString(), count));
if (queue.size()>=maxsize) min=queue.last().val;
}
return false;
@@ -340,7 +326,7 @@ class IndexSortedFacetCollector extends
}
@Override
- public boolean collect(String term, int count) {
+ public boolean collect(BytesRef term, int count) {
if (count < mincount) {
return false;
}
@@ -351,7 +337,7 @@ class IndexSortedFacetCollector extends
}
if (limit > 0) {
- res.add(term, count);
+ res.add(term.utf8ToString(), count);
limit--;
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/request/SimpleFacets.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/request/SimpleFacets.java Thu Jun 3 18:38:05 2010
@@ -21,7 +21,6 @@ import org.apache.lucene.index.*;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.noggit.CharArr;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
@@ -356,15 +355,6 @@ public class SimpleFacets {
}
- // first element of the fieldcache is null, so we need this comparator.
- private static final Comparator nullStrComparator = new Comparator() {
- public int compare(Object o1, Object o2) {
- if (o1==null) return (o2==null) ? 0 : -1;
- else if (o2==null) return 1;
- return ((String)o1).compareTo((String)o2);
- }
- };
-
/**
* Use the Lucene FieldCache to get counts for each unique field value in <code>docs</code>.
* The field must have at most one indexed token per document.
@@ -386,27 +376,38 @@ public class SimpleFacets {
FieldType ft = searcher.getSchema().getFieldType(fieldName);
NamedList res = new NamedList();
- FieldCache.StringIndex si = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), fieldName);
- final String[] terms = si.lookup;
- final int[] termNum = si.order;
+ FieldCache.DocTermsIndex si = FieldCache.DEFAULT.getTermsIndex(searcher.getReader(), fieldName);
- if (prefix!=null && prefix.length()==0) prefix=null;
+ final BytesRef prefixRef;
+ if (prefix == null) {
+ prefixRef = null;
+ } else if (prefix.length()==0) {
+ prefix = null;
+ prefixRef = null;
+ } else {
+ prefixRef = new BytesRef(prefix);
+ }
+
+ final BytesRef br = new BytesRef();
int startTermIndex, endTermIndex;
if (prefix!=null) {
- startTermIndex = Arrays.binarySearch(terms,prefix,nullStrComparator);
+ startTermIndex = si.binarySearchLookup(prefixRef, br);
if (startTermIndex<0) startTermIndex=-startTermIndex-1;
// find the end term. \uffff isn't a legal unicode char, but only compareTo
// is used, so it should be fine, and is guaranteed to be bigger than legal chars.
- endTermIndex = Arrays.binarySearch(terms,prefix+"\uffff\uffff\uffff\uffff",nullStrComparator);
+ endTermIndex = si.binarySearchLookup(new BytesRef(prefix+"\uffff\uffff\uffff\uffff"), br);
+ assert endTermIndex < 0;
endTermIndex = -endTermIndex-1;
} else {
startTermIndex=1;
- endTermIndex=terms.length;
+ endTermIndex=si.numOrd();
}
final int nTerms=endTermIndex-startTermIndex;
+ CharArr spare = new CharArr();
+
if (nTerms>0 && docs.size() >= mincount) {
// count collection array only needs to be as big as the number of terms we are
@@ -415,7 +416,7 @@ public class SimpleFacets {
DocIterator iter = docs.iterator();
while (iter.hasNext()) {
- int term = termNum[iter.nextDoc()];
+ int term = si.getOrd(iter.nextDoc());
int arrIdx = term-startTermIndex;
if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
}
@@ -429,7 +430,7 @@ public class SimpleFacets {
if (sort.equals(FacetParams.FACET_SORT_COUNT) || sort.equals(FacetParams.FACET_SORT_COUNT_LEGACY)) {
int maxsize = limit>0 ? offset+limit : Integer.MAX_VALUE-1;
maxsize = Math.min(maxsize, nTerms);
- final BoundedTreeSet<CountPair<String,Integer>> queue = new BoundedTreeSet<CountPair<String,Integer>>(maxsize);
+ final BoundedTreeSet<CountPair<BytesRef,Integer>> queue = new BoundedTreeSet<CountPair<BytesRef,Integer>>(maxsize);
int min=mincount-1; // the smallest value in the top 'N' values
for (int i=0; i<nTerms; i++) {
int c = counts[i];
@@ -437,15 +438,17 @@ public class SimpleFacets {
// NOTE: we use c>min rather than c>=min as an optimization because we are going in
// index order, so we already know that the keys are ordered. This can be very
// important if a lot of the counts are repeated (like zero counts would be).
- queue.add(new CountPair<String,Integer>(terms[startTermIndex+i], c));
+ queue.add(new CountPair<BytesRef,Integer>(si.lookup(startTermIndex+i, new BytesRef()), c));
if (queue.size()>=maxsize) min=queue.last().val;
}
}
// now select the right page from the results
- for (CountPair<String,Integer> p : queue) {
+ for (CountPair<BytesRef,Integer> p : queue) {
if (--off>=0) continue;
if (--lim<0) break;
- res.add(ft.indexedToReadable(p.key), p.val);
+ spare.reset();
+ ft.indexedToReadable(p.key, spare);
+ res.add(spare.toString(), p.val);
}
} else {
// add results in index order
@@ -461,7 +464,9 @@ public class SimpleFacets {
int c = counts[i];
if (c<mincount || --off>=0) continue;
if (--lim<0) break;
- res.add(ft.indexedToReadable(terms[startTermIndex+i]), c);
+ spare.reset();
+ ft.indexedToReadable(si.lookup(startTermIndex+i, br), spare);
+ res.add(spare.toString(), c);
}
}
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/request/UnInvertedField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/request/UnInvertedField.java Thu Jun 3 18:38:05 2010
@@ -653,11 +653,11 @@ public class UnInvertedField {
int i = 0;
final FieldFacetStats[] finfo = new FieldFacetStats[facet.length];
//Initialize facetstats, if facets have been passed in
- FieldCache.StringIndex si;
+ FieldCache.DocTermsIndex si;
for (String f : facet) {
FieldType facet_ft = searcher.getSchema().getFieldType(f);
try {
- si = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), f);
+ si = FieldCache.DEFAULT.getTermsIndex(searcher.getReader(), f);
}
catch (IOException e) {
throw new RuntimeException("failed to open field cache for: " + f, e);
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/DateField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/DateField.java Thu Jun 3 18:38:05 2010
@@ -443,7 +443,7 @@ class DateFieldSource extends FieldCache
}
public int intVal(int doc) {
- int ord=order[doc];
+ int ord=termsIndex.getOrd(doc);
return ord;
}
@@ -456,8 +456,15 @@ class DateFieldSource extends FieldCache
}
public String strVal(int doc) {
- int ord=order[doc];
- return ft.indexedToReadable(lookup[ord]);
+ int ord=termsIndex.getOrd(doc);
+ if (ord == 0) {
+ return null;
+ } else {
+ BytesRef br = termsIndex.lookup(ord, new BytesRef());
+ CharArr spare = new CharArr();
+ ft.indexedToReadable(br, spare);
+ return spare.toString();
+ }
}
public String toString(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableDoubleField.java Thu Jun 3 18:38:05 2010
@@ -122,8 +122,8 @@ class SortableDoubleFieldSource extends
}
public double doubleVal(int doc) {
- int ord=order[doc];
- return ord==0 ? def : NumberUtils.SortableStr2double(lookup[ord]);
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? def : NumberUtils.SortableStr2double(termsIndex.lookup(ord, new BytesRef()));
}
public String strVal(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableFloatField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableFloatField.java Thu Jun 3 18:38:05 2010
@@ -110,8 +110,8 @@ class SortableFloatFieldSource extends F
}
public float floatVal(int doc) {
- int ord=order[doc];
- return ord==0 ? def : NumberUtils.SortableStr2float(lookup[ord]);
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? def : NumberUtils.SortableStr2float(termsIndex.lookup(ord, new BytesRef()));
}
public int intVal(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableIntField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableIntField.java Thu Jun 3 18:38:05 2010
@@ -118,8 +118,8 @@ class SortableIntFieldSource extends Fie
}
public int intVal(int doc) {
- int ord=order[doc];
- return ord==0 ? def : NumberUtils.SortableStr2int(lookup[ord],0,3);
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? def : NumberUtils.SortableStr2int(termsIndex.lookup(ord, new BytesRef()),0,3);
}
public long longVal(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableLongField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableLongField.java Thu Jun 3 18:38:05 2010
@@ -119,8 +119,8 @@ class SortableLongFieldSource extends Fi
}
public long longVal(int doc) {
- int ord=order[doc];
- return ord==0 ? def : NumberUtils.SortableStr2long(lookup[ord],0,5);
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? def : NumberUtils.SortableStr2long(termsIndex.lookup(ord, new BytesRef()),0,5);
}
public double doubleVal(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/StrField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/StrField.java Thu Jun 3 18:38:05 2010
@@ -20,6 +20,7 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.response.XMLWriter;
import org.apache.solr.search.function.ValueSource;
@@ -77,7 +78,7 @@ class StrFieldSource extends FieldCacheS
}
public int intVal(int doc) {
- int ord=order[doc];
+ int ord=termsIndex.getOrd(doc);
return ord;
}
@@ -90,8 +91,12 @@ class StrFieldSource extends FieldCacheS
}
public String strVal(int doc) {
- int ord=order[doc];
- return lookup[ord];
+ int ord=termsIndex.getOrd(doc);
+ if (ord == 0) {
+ return null;
+ } else {
+ return termsIndex.lookup(ord, new BytesRef()).utf8ToString();
+ }
}
public String toString(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java Thu Jun 3 18:38:05 2010
@@ -19,6 +19,7 @@ package org.apache.solr.search;
import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -53,33 +54,34 @@ public class MissingStringLastComparator
// be extended or have it's values accessed.
class MissingLastOrdComparator extends FieldComparator {
private static final int NULL_ORD = Integer.MAX_VALUE;
- private final String nullVal;
+ private final BytesRef nullVal;
private final int[] ords;
- private final String[] values;
+ private final BytesRef[] values;
private final int[] readerGen;
private int currentReaderGen = -1;
- private String[] lookup;
- private int[] order;
+ private FieldCache.DocTermsIndex termsIndex;
private final String field;
private int bottomSlot = -1;
private int bottomOrd;
- private String bottomValue;
+ private BytesRef bottomValue;
private final boolean reversed;
private final int sortPos;
+ private final BytesRef tempBR = new BytesRef();
public MissingLastOrdComparator(int numHits, String field, int sortPos, boolean reversed, String nullVal) {
ords = new int[numHits];
- values = new String[numHits];
+ values = new BytesRef[numHits];
readerGen = new int[numHits];
this.sortPos = sortPos;
this.reversed = reversed;
this.field = field;
- this.nullVal = nullVal;
+ this.nullVal = nullVal == null ? null : new BytesRef(nullVal);
}
+ @Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
int cmp = ords[slot1] - ords[slot2];
@@ -88,8 +90,8 @@ public class MissingStringLastComparator
}
}
- final String val1 = values[slot1];
- final String val2 = values[slot2];
+ final BytesRef val1 = values[slot1];
+ final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
@@ -104,43 +106,40 @@ public class MissingStringLastComparator
public int compareBottom(int doc) {
assert bottomSlot != -1;
- int order = this.order[doc];
+ int order = termsIndex.getOrd(doc);
int ord = (order == 0) ? NULL_ORD : order;
final int cmp = bottomOrd - ord;
if (cmp != 0) {
return cmp;
}
- final String val2 = lookup[order];
-
// take care of the case where both vals are null
- if (bottomValue == val2) return 0;
-
- return bottomValue.compareTo(val2);
+ if (bottomOrd == NULL_ORD) return 0;
+ return bottomValue.compareTo(termsIndex.lookup(order, tempBR));
}
private void convert(int slot) {
readerGen[slot] = currentReaderGen;
int index = 0;
- String value = values[slot];
+ BytesRef value = values[slot];
if (value == null) {
// should already be done
- // ords[slot] = NULL_ORD;
+ assert ords[slot] == NULL_ORD;
return;
}
if (sortPos == 0 && bottomSlot != -1 && bottomSlot != slot) {
// Since we are the primary sort, the entries in the
// queue are bounded by bottomOrd:
- assert bottomOrd < lookup.length;
+ assert bottomOrd < termsIndex.numOrd();
if (reversed) {
- index = binarySearch(lookup, value, bottomOrd, lookup.length-1);
+ index = binarySearch(tempBR, termsIndex, value, bottomOrd, termsIndex.numOrd()-1);
} else {
- index = binarySearch(lookup, value, 0, bottomOrd);
+ index = binarySearch(tempBR, termsIndex, value, 0, bottomOrd);
}
} else {
// Full binary search
- index = binarySearch(lookup, value);
+ index = binarySearch(tempBR, termsIndex, value);
}
if (index < 0) {
@@ -149,26 +148,35 @@ public class MissingStringLastComparator
ords[slot] = index;
}
+ @Override
public void copy(int slot, int doc) {
- final int ord = order[doc];
- ords[slot] = ord == 0 ? NULL_ORD : ord;
+ final int ord = termsIndex.getOrd(doc);
assert ord >= 0;
- values[slot] = lookup[ord];
+ if (ord == 0) {
+ ords[slot] = NULL_ORD;
+ values[slot] = null;
+ } else {
+ ords[slot] = ord;
+ if (values[slot] == null) {
+ values[slot] = new BytesRef();
+ }
+ termsIndex.lookup(ord, values[slot]);
+ }
readerGen[slot] = currentReaderGen;
}
+ @Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- FieldCache.StringIndex currentReaderValues = FieldCache.DEFAULT.getStringIndex(reader, field);
+ termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
currentReaderGen++;
- order = currentReaderValues.order;
- lookup = currentReaderValues.lookup;
- assert lookup.length > 0;
+ assert termsIndex.numOrd() > 0;
if (bottomSlot != -1) {
convert(bottomSlot);
bottomOrd = ords[bottomSlot];
}
}
+ @Override
public void setBottom(final int bottom) {
bottomSlot = bottom;
if (readerGen[bottom] != currentReaderGen) {
@@ -180,12 +188,13 @@ public class MissingStringLastComparator
bottomValue = values[bottom];
}
+ @Override
public Comparable value(int slot) {
Comparable v = values[slot];
return v==null ? nullVal : v;
}
- public String[] getValues() {
+ public BytesRef[] getValues() {
return values;
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocValues.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocValues.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocValues.java Thu Jun 3 18:38:05 2010
@@ -19,9 +19,6 @@ package org.apache.solr.search.function;
import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexReader;
-import org.apache.solr.util.NumberUtils;
-
-import java.io.IOException;
/**
* Represents field values as different types.
@@ -45,6 +42,7 @@ public abstract class DocValues {
public int intVal(int doc) { throw new UnsupportedOperationException(); }
public long longVal(int doc) { throw new UnsupportedOperationException(); }
public double doubleVal(int doc) { throw new UnsupportedOperationException(); }
+ // TODO: should we make a termVal, returns BytesRef?
public String strVal(int doc) { throw new UnsupportedOperationException(); }
public abstract String toString(int doc);
@@ -56,6 +54,8 @@ public abstract class DocValues {
public void intVal(int doc, int [] vals) { throw new UnsupportedOperationException(); }
public void longVal(int doc, long [] vals) { throw new UnsupportedOperationException(); }
public void doubleVal(int doc, double [] vals) { throw new UnsupportedOperationException(); }
+
+ // TODO: should we make a termVal, fills BytesRef[]?
public void strVal(int doc, String [] vals) { throw new UnsupportedOperationException(); }
public Explanation explain(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java Thu Jun 3 18:38:05 2010
@@ -18,8 +18,6 @@
package org.apache.solr.search.function;
import org.apache.lucene.index.IndexReader;
-import org.apache.solr.search.function.DocValues;
-import org.apache.solr.search.function.ValueSource;
import java.io.IOException;
import java.util.Map;
@@ -61,24 +59,24 @@ public class OrdFieldSource extends Valu
}
public float floatVal(int doc) {
- return (float)order[doc];
+ return (float)termsIndex.getOrd(doc);
}
public int intVal(int doc) {
- return order[doc];
+ return termsIndex.getOrd(doc);
}
public long longVal(int doc) {
- return (long)order[doc];
+ return (long)termsIndex.getOrd(doc);
}
public double doubleVal(int doc) {
- return (double)order[doc];
+ return (double)termsIndex.getOrd(doc);
}
public String strVal(int doc) {
// the string value of the ordinal, not the string itself
- return Integer.toString(order[doc]);
+ return Integer.toString(termsIndex.getOrd(doc));
}
public String toString(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java Thu Jun 3 18:38:05 2010
@@ -18,15 +18,13 @@
package org.apache.solr.search.function;
import org.apache.lucene.index.IndexReader;
-import org.apache.solr.search.function.DocValues;
-import org.apache.solr.search.function.ValueSource;
import org.apache.lucene.search.FieldCache;
import java.io.IOException;
import java.util.Map;
/**
- * Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getStringIndex()
+ * Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getTermsIndex()
* and reverses the order.
* <br>
* The native lucene index order is used to assign an ordinal value for each field value.
@@ -58,31 +56,30 @@ public class ReverseOrdFieldSource exten
}
public DocValues getValues(Map context, IndexReader reader) throws IOException {
- final FieldCache.StringIndex sindex = FieldCache.DEFAULT.getStringIndex(reader, field);
+ final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(reader, field);
- final int arr[] = sindex.order;
- final int end = sindex.lookup.length;
+ final int end = sindex.numOrd();
return new DocValues() {
public float floatVal(int doc) {
- return (float)(end - arr[doc]);
+ return (float)(end - sindex.getOrd(doc));
}
public int intVal(int doc) {
- return (int)(end - arr[doc]);
+ return (int)(end - sindex.getOrd(doc));
}
public long longVal(int doc) {
- return (long)(end - arr[doc]);
+ return (long)(end - sindex.getOrd(doc));
}
public double doubleVal(int doc) {
- return (double)(end - arr[doc]);
+ return (double)(end - sindex.getOrd(doc));
}
public String strVal(int doc) {
// the string value of the ordinal, not the string itself
- return Integer.toString((end - arr[doc]));
+ return Integer.toString((end - sindex.getOrd(doc)));
}
public String toString(int doc) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java Thu Jun 3 18:38:05 2010
@@ -19,6 +19,7 @@ package org.apache.solr.search.function;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -26,49 +27,47 @@ import java.io.IOException;
* Serves as base class for DocValues based on StringIndex
**/
public abstract class StringIndexDocValues extends DocValues {
- protected final FieldCache.StringIndex index;
- protected final int[] order;
- protected final String[] lookup;
+ protected final FieldCache.DocTermsIndex termsIndex;
protected final ValueSource vs;
public StringIndexDocValues(ValueSource vs, IndexReader reader, String field) throws IOException {
try {
- index = FieldCache.DEFAULT.getStringIndex(reader, field);
+ termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
} catch (RuntimeException e) {
throw new StringIndexException(field, e);
}
- order = index.order;
- lookup = index.lookup;
this.vs = vs;
}
protected abstract String toTerm(String readableValue);
- @Override
+ @Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
// TODO: are lowerVal and upperVal in indexed form or not?
lowerVal = lowerVal == null ? null : toTerm(lowerVal);
upperVal = upperVal == null ? null : toTerm(upperVal);
- int lower = Integer.MIN_VALUE;
- if (lowerVal != null) {
- lower = index.binarySearchLookup(lowerVal);
- if (lower < 0) {
- lower = -lower-1;
- } else if (!includeLower) {
- lower++;
- }
- }
-
- int upper = Integer.MAX_VALUE;
- if (upperVal != null) {
- upper = index.binarySearchLookup(upperVal);
- if (upper < 0) {
- upper = -upper-2;
- } else if (!includeUpper) {
- upper--;
- }
- }
+ final BytesRef spare = new BytesRef();
+
+ int lower = Integer.MIN_VALUE;
+ if (lowerVal != null) {
+ lower = termsIndex.binarySearchLookup(new BytesRef(lowerVal), spare);
+ if (lower < 0) {
+ lower = -lower-1;
+ } else if (!includeLower) {
+ lower++;
+ }
+ }
+
+ int upper = Integer.MAX_VALUE;
+ if (upperVal != null) {
+ upper = termsIndex.binarySearchLookup(new BytesRef(upperVal), spare);
+ if (upper < 0) {
+ upper = -upper-2;
+ } else if (!includeUpper) {
+ upper--;
+ }
+ }
final int ll = lower;
final int uu = upper;
@@ -76,7 +75,7 @@ public abstract class StringIndexDocValu
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
- int ord = order[doc];
+ int ord = termsIndex.getOrd(doc);
return ord >= ll && ord <= uu;
}
};
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/util/ByteUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/util/ByteUtils.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/util/ByteUtils.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/util/ByteUtils.java Thu Jun 3 18:38:05 2010
@@ -17,7 +17,6 @@
package org.apache.solr.util;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.noggit.CharArr;
@@ -31,8 +30,8 @@ public class ByteUtils {
*/
public static int UTF8toUTF16(byte[] utf8, int offset, int len, char[] out, int out_offset) {
int out_start = out_offset;
-
- while (offset < len) {
+ final int limit = offset + len;
+ while (offset < limit) {
int b = utf8[offset++]&0xff;
if (b < 0xc0) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/util/NumberUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/util/NumberUtils.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/util/NumberUtils.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/util/NumberUtils.java Thu Jun 3 18:38:05 2010
@@ -17,6 +17,8 @@
package org.apache.solr.util;
+import org.apache.lucene.util.BytesRef;
+
/**
* @version $Id$
*/
@@ -41,6 +43,11 @@ public class NumberUtils {
return Integer.toString(ival);
}
+ public static String SortableStr2int(BytesRef val) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2int(val.utf8ToString());
+ }
+
public static String long2sortableStr(long val) {
char[] arr = new char[5];
@@ -57,6 +64,11 @@ public class NumberUtils {
return Long.toString(ival);
}
+ public static String SortableStr2long(BytesRef val) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2long(val.utf8ToString());
+ }
+
//
// IEEE floating point format is defined so that it sorts correctly
// when interpreted as a signed integer (or signed long in the case
@@ -85,6 +97,11 @@ public class NumberUtils {
return Float.intBitsToFloat(f);
}
+ public static float SortableStr2float(BytesRef val) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2float(val.utf8ToString());
+ }
+
public static String SortableStr2floatStr(String val) {
return Float.toString(SortableStr2float(val));
}
@@ -106,6 +123,11 @@ public class NumberUtils {
return Double.longBitsToDouble(f);
}
+ public static double SortableStr2double(BytesRef val) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2double(val.utf8ToString());
+ }
+
public static String SortableStr2doubleStr(String val) {
return Double.toString(SortableStr2double(val));
}
@@ -131,6 +153,11 @@ public class NumberUtils {
return val;
}
+ public static int SortableStr2int(BytesRef sval, int offset, int len) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2int(sval.utf8ToString(), offset, len);
+ }
+
// uses binary representation of an int to build a string of
// chars that will sort correctly. Only char ranges
// less than 0xd800 will be used to avoid UCS-16 surrogates.
@@ -155,5 +182,8 @@ public class NumberUtils {
return val;
}
-
+ public static long SortableStr2long(BytesRef sval, int offset, int len) {
+ // TODO: operate directly on BytesRef
+ return SortableStr2long(sval.utf8ToString(), offset, len);
+ }
}
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java Thu Jun 3 18:38:05 2010
@@ -18,7 +18,6 @@
package org.apache.solr.handler.component;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
@@ -26,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
@@ -34,7 +34,6 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.QueryElevationComponent.ElevationObj;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -247,7 +246,7 @@ public class QueryElevationComponentTest
IndexReader reader = core.getSearcher().get().getReader();
Map<String, ElevationObj> map = comp.getElevationMap(reader, core);
- assertTrue( map.get( "aaa" ).priority.containsKey( "A" ) );
+ assertTrue( map.get( "aaa" ).priority.containsKey( new BytesRef("A") ) );
assertNull( map.get( "bbb" ) );
// now change the file
@@ -258,6 +257,6 @@ public class QueryElevationComponentTest
reader = core.getSearcher().get().getReader();
map = comp.getElevationMap(reader, core);
assertNull( map.get( "aaa" ) );
- assertTrue( map.get( "bbb" ).priority.containsKey( "B" ) );
+ assertTrue( map.get( "bbb" ).priority.containsKey( new BytesRef("B") ) );
}
}
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestIndexSearcher.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestIndexSearcher.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestIndexSearcher.java Thu Jun 3 18:38:05 2010
@@ -21,7 +21,6 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.search.FieldCache;
import java.util.Map;
import java.io.IOException;
@@ -32,18 +31,6 @@ public class TestIndexSearcher extends A
public String getSolrConfigFile() { return "solrconfig.xml"; }
public String getCoreName() { return "basic"; }
-
- public void setUp() throws Exception {
- // if you override setUp or tearDown, you better call
- // the super classes version
- super.setUp();
- }
- public void tearDown() throws Exception {
- // if you override setUp or tearDown, you better call
- // the super classes version
- super.tearDown();
- }
-
private String getStringVal(SolrQueryRequest sqr, String field, int doc) throws IOException {
SchemaField sf = sqr.getSchema().getField(field);
ValueSource vs = sf.getType().getValueSource(sf, null);
@@ -80,9 +67,6 @@ public class TestIndexSearcher extends A
// Didn't work w/ older versions of lucene2.9 going from segment -> multi
assertEquals(r1.getLeafReaders()[0], r2.getLeafReaders()[0]);
- // make sure the String returned is the exact same instance (i.e. same FieldCache instance)
- assertTrue(sval1 == getStringVal(sr2,"v_s",0));
-
assertU(adoc("id","5", "v_f","3.14159"));
assertU(adoc("id","6", "v_f","8983", "v_s","string6"));
assertU(commit());
@@ -121,7 +105,6 @@ public class TestIndexSearcher extends A
SolrQueryRequest sr5 = req("q","foo");
SolrIndexReader r5 = sr5.getSearcher().getReader();
- String beforeDelete = getStringVal(sr5, "v_s",1);
assertU(delI("1"));
assertU(commit());
@@ -129,8 +112,6 @@ public class TestIndexSearcher extends A
SolrIndexReader r6 = sr4.getSearcher().getReader();
assertEquals(1, r6.getLeafReaders()[0].numDocs()); // only a single doc left in the first segment
assertTrue( !r5.getLeafReaders()[0].equals(r6.getLeafReaders()[0]) ); // readers now different
- String afterDelete = getStringVal(sr6, "v_s",1);
- assertTrue( beforeDelete == afterDelete ); // same field cache is used even though deletions are different
sr5.close();
sr6.close();
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestSort.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestSort.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestSort.java Thu Jun 3 18:38:05 2010
@@ -115,7 +115,6 @@ public class TestSort extends AbstractSo
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.SCORE));
// hit both use-cases of sort-missing-last
sfields.add( Sorting.getStringSortField("f", reverse, sortMissingLast, !sortMissingLast) );
- int sortIdx = sfields.size() - 1;
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.SCORE));
Sort sort = new Sort(sfields.toArray(new SortField[sfields.size()]));
@@ -174,7 +173,6 @@ public class TestSort extends AbstractSo
ScoreDoc[] sdocs = topDocs.scoreDocs;
for (int j=0; j<sdocs.length; j++) {
int id = sdocs[j].doc;
- String s = (String)((FieldDoc)sdocs[j]).fields[sortIdx];
if (id != collectedDocs.get(j).doc) {
log.error("Error at pos " + j);
}