You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/06/03 20:38:07 UTC
svn commit: r951104 [1/2] - in /lucene/dev/trunk: lucene/
lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/
lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/
lucene/src/java/org/apache/lucene/index/ lucene/src/java/...
Author: mikemccand
Date: Thu Jun 3 18:38:05 2010
New Revision: 951104
URL: http://svn.apache.org/viewvc?rev=951104&view=rev
Log:
LUCENE-2380: cutover to shared byte[] (BytesRef) instead of String, in FieldCache
Added:
lucene/dev/trunk/lucene/MIGRATE.txt (with props)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
- copied, changed from r944313, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java (with props)
Removed:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
lucene/dev/trunk/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldDocSortedHitQueue.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRef.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct16.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct32.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct64.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct8.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/FieldFacetStats.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/trunk/solr/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
lucene/dev/trunk/solr/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/trunk/solr/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/DateField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableDoubleField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableFloatField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableIntField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/SortableLongField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/StrField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocValues.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java
lucene/dev/trunk/solr/src/java/org/apache/solr/util/ByteUtils.java
lucene/dev/trunk/solr/src/java/org/apache/solr/util/NumberUtils.java
lucene/dev/trunk/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestIndexSearcher.java
lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestSort.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Jun 3 18:38:05 2010
@@ -73,6 +73,13 @@ Changes in backwards compatibility polic
not unicode code units. For example, a Wildcard "?" represents any unicode
character. Furthermore, the rest of the automaton package and RegexpQuery use
true Unicode codepoint representation. (Robert Muir, Mike McCandless)
+
+* LUCENE-2380: The String-based FieldCache methods (getStrings,
+ getStringIndex) have been replaced with BytesRef-based equivalents
+ (getTerms, getTermsIndex). Also, the sort values (returned in
+ FieldDoc.fields) when sorting by SortField.STRING or
+ SortField.STRING_VAL are now BytesRef instances. See MIGRATE.txt
+ for more details. (Mike McCandless)
* LUCENE-2480: Though not a change in backwards compatibility policy, pre-3.0
indexes are no longer supported. You should upgrade to 3.x first, then run
@@ -611,6 +618,11 @@ Optimizations
* LUCENE-2360: Small speedup to recycling of reused per-doc RAM in
IndexWriter (Robert Muir, Mike McCandless)
+* LUCENE-2380: The terms field cache methods (getTerms,
+ getTermsIndex), which replace the older String equivalents
+ (getStrings, getStringIndex), consume quite a bit less RAM in most
+ cases. (Mike McCandless)
+
Build
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
Added: lucene/dev/trunk/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/MIGRATE.txt?rev=951104&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/MIGRATE.txt (added)
+++ lucene/dev/trunk/lucene/MIGRATE.txt Thu Jun 3 18:38:05 2010
@@ -0,0 +1,45 @@
+
+LUCENE-2380
+
+ * The field values returned when sorting by SortField.STRING are now
+ BytesRef. You can call value.utf8ToString() to convert back to
+ string, if necessary.
+
+ * In FieldCache, getStrings (returning String[]) has been replaced
+ with getTerms (returning a FieldCache.DocTerms instance).
+ DocTerms provides a getTerm method, taking a docID and a BytesRef
+ to fill (which must not be null), and it fills it in with the
+ reference to the bytes for that term.
+
+ If you had code like this before:
+
+ String[] values = FieldCache.DEFAULT.getStrings(reader, field);
+ ...
+ String aValue = values[docID];
+
+ you can do this instead:
+
+ DocTerms values = FieldCache.DEFAULT.getTerms(reader, field);
+ ...
+ BytesRef term = new BytesRef();
+ String aValue = values.get(docID, term).utf8ToString();
+
+ Note however that it can be costly to convert to String, so it's
+ better to work directly with the BytesRef.
+
+ * StringComparatorLocale is now more CPU costly than it was before
+ (it was already very CPU costly since it does not compare using
+ indexed collation keys; use CollationKeyFilter for better
+ performance), since it converts BytesRef -> String on the fly.
+ Also, the field values returned when sorting by SortField.STRING
+ are now BytesRef.
+
+ * FieldComparator.StringOrdValComparator has been renamed to
+ TermOrdValComparator, and now uses BytesRef for its values.
+ Likewise for StringValComparator, renamed to TermValComparator.
+ This means when sorting by SortField.STRING or
+ SortField.STRING_VAL (or directly invoking these comparators) the
+ values returned in the FieldDoc.fields array will be BytesRef not
+ String. You can call the .utf8ToString() method on the BytesRef
+ instances, if necessary.
+
Propchange: lucene/dev/trunk/lucene/MIGRATE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Thu Jun 3 18:38:05 2010
@@ -48,9 +48,10 @@ import org.apache.lucene.index.LogMergeP
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.search.FieldCache.StringIndex;
+import org.apache.lucene.search.FieldCache.DocTermsIndex;
import org.apache.lucene.search.FieldCache;
/**
@@ -326,11 +327,12 @@ public class TestPerfTasksLogic extends
Benchmark benchmark = execBenchmark(algLines);
IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory(), true);
- StringIndex idx = FieldCache.DEFAULT.getStringIndex(r, "country");
+ DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(r, "country");
final int maxDoc = r.maxDoc();
assertEquals(1000, maxDoc);
+ BytesRef br = new BytesRef();
for(int i=0;i<1000;i++) {
- assertNotNull("doc " + i + " has null country", idx.lookup[idx.order[i]]);
+ assertNotNull("doc " + i + " has null country", idx.getTerm(i, br));
}
r.close();
}
Modified: lucene/dev/trunk/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java Thu Jun 3 18:38:05 2010
@@ -21,9 +21,11 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.FieldCache.DocTerms;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.FilteredDocIdSet;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.spatial.tier.DistanceFilter;
import org.apache.lucene.spatial.tier.DistanceUtils;
@@ -62,7 +64,8 @@ public class GeoHashDistanceFilter exten
@Override
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
- final String[] geoHashValues = FieldCache.DEFAULT.getStrings(reader, geoHashField);
+ final DocTerms geoHashValues = FieldCache.DEFAULT.getTerms(reader, geoHashField);
+ final BytesRef br = new BytesRef();
final int docBase = nextDocBase;
nextDocBase += reader.maxDoc();
@@ -70,8 +73,10 @@ public class GeoHashDistanceFilter exten
return new FilteredDocIdSet(startingFilter.getDocIdSet(reader)) {
@Override
public boolean match(int doc) {
-
- String geoHash = geoHashValues[doc];
+
+ // TODO: cutover to BytesRef so we don't have to
+ // make String here
+ String geoHash = geoHashValues.getTerm(doc, br).utf8ToString();
double[] coords = GeoHashUtils.decode(geoHash);
double x = coords[0];
double y = coords[1];
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Jun 3 18:38:05 2010
@@ -820,7 +820,7 @@ final class DocumentsWriter {
boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm)
throws CorruptIndexException, IOException {
-
+
// This call is synchronized but fast
final DocumentsWriterThreadState state = getThreadState(doc, delTerm);
@@ -1301,6 +1301,8 @@ final class DocumentsWriter {
final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK;
+ /* if you increase this, you must fix field cache impl for
+ * getTerms/getTermsIndex requires <= 32768 */
final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2;
private class ByteBlockAllocator extends ByteBlockPool.Allocator {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java Thu Jun 3 18:38:05 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.PackedInts;
import java.util.HashMap;
@@ -82,6 +83,7 @@ public class SimpleStandardTermsIndexRea
// all fields share this single logical byte[]
private final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
+ private PagedBytes.Reader termBytesReader;
final HashMap<FieldInfo,FieldIndexReader> fields = new HashMap<FieldInfo,FieldIndexReader>();
@@ -135,7 +137,7 @@ public class SimpleStandardTermsIndexRea
if (success) {
indexLoaded = true;
}
- termBytes.finish();
+ termBytesReader = termBytes.freeze();
} else {
this.in = in;
}
@@ -347,7 +349,7 @@ public class SimpleStandardTermsIndexRea
private void fillResult(int idx, TermsIndexResult result) {
final long offset = termOffsets.get(idx);
final int length = (int) (termOffsets.get(1+idx) - offset);
- termBytes.fill(result.term, termBytesStart + offset, length);
+ termBytesReader.fill(result.term, termBytesStart + offset, length);
result.position = idx * totalIndexInterval;
result.offset = termsStart + termsDictOffsets.get(idx);
}
@@ -361,7 +363,7 @@ public class SimpleStandardTermsIndexRea
final long offset = termOffsets.get(mid);
final int length = (int) (termOffsets.get(1+mid) - offset);
- termBytes.fill(result.term, termBytesStart + offset, length);
+ termBytesReader.fill(result.term, termBytesStart + offset, length);
int delta = termComp.compare(term, result.term);
if (delta < 0) {
@@ -382,7 +384,7 @@ public class SimpleStandardTermsIndexRea
final long offset = termOffsets.get(hi);
final int length = (int) (termOffsets.get(1+hi) - offset);
- termBytes.fill(result.term, termBytesStart + offset, length);
+ termBytesReader.fill(result.term, termBytesStart + offset, length);
result.position = hi*totalIndexInterval;
result.offset = termsStart + termsDictOffsets.get(hi);
@@ -411,7 +413,7 @@ public class SimpleStandardTermsIndexRea
indexLoaded = true;
in.close();
- termBytes.finish();
+ termBytesReader = termBytes.freeze();
}
}
@@ -438,5 +440,8 @@ public class SimpleStandardTermsIndexRea
if (in != null && !indexLoaded) {
in.close();
}
+ if (termBytesReader != null) {
+ termBytesReader.close();
+ }
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCache.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCache.java Thu Jun 3 18:38:05 2010
@@ -44,50 +44,6 @@ public interface FieldCache {
Object value;
}
- /** Indicator for StringIndex values in the cache. */
- // NOTE: the value assigned to this constant must not be
- // the same as any of those in SortField!!
- public static final int STRING_INDEX = -1;
-
-
- /** Expert: Stores term text values and document ordering data. */
- public static class StringIndex {
-
- public int binarySearchLookup(String key) {
- // this special case is the reason that Arrays.binarySearch() isn't useful.
- if (key == null)
- return 0;
-
- int low = 1;
- int high = lookup.length-1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- int cmp = lookup[mid].compareTo(key);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid; // key found
- }
- return -(low + 1); // key not found.
- }
-
- /** All the term values, in natural order. */
- public final String[] lookup;
-
- /** For each document, an index into the lookup array. */
- public final int[] order;
-
- /** Creates one of these objects */
- public StringIndex (int[] values, String[] lookup) {
- this.order = values;
- this.lookup = lookup;
- }
- }
-
/**
* Marker interface as super-interface to all parsers. It
* is used to specify a custom parser to {@link
@@ -490,28 +446,112 @@ public interface FieldCache {
public double[] getDoubles(IndexReader reader, String field, DoubleParser parser)
throws IOException;
+ /** Returned by {@link #getTerms} */
+ public abstract static class DocTerms {
+ /** The BytesRef argument must not be null; the method
+ * returns the same BytesRef, or an empty (length=0)
+ * BytesRef if the doc did not have this field or was
+ * deleted. */
+ public abstract BytesRef getTerm(int docID, BytesRef ret);
+
+ /** Returns true if this doc has this field and is not
+ * deleted. */
+ public abstract boolean exists(int docID);
+
+ /** Number of documents */
+ public abstract int size();
+ }
+
/** Checks the internal cache for an appropriate entry, and if none
- * is found, reads the term values in <code>field</code> and returns an array
- * of size <code>reader.maxDoc()</code> containing the value each document
- * has in the given field.
+ * is found, reads the term values in <code>field</code>
+ * and returns a {@link DocTerms} instance, providing a
+ * method to retrieve the term (as a BytesRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public String[] getStrings (IndexReader reader, String field)
+ public DocTerms getTerms (IndexReader reader, String field)
throws IOException;
+ /** Expert: just like {@link #getTerms(IndexReader,String)},
+ * but you can specify whether more RAM should be consumed in exchange for
+ * faster lookups (default is "true"). Note that the
+ * first call for a given reader and field "wins",
+ * subsequent calls will share the same cache entry. */
+ public DocTerms getTerms (IndexReader reader, String field, boolean fasterButMoreRAM)
+ throws IOException;
+
+ /** Returned by {@link #getTermsIndex} */
+ public abstract static class DocTermsIndex {
+
+ public int binarySearchLookup(BytesRef key, BytesRef spare) {
+ // this special case is the reason that Arrays.binarySearch() isn't useful.
+ if (key == null)
+ return 0;
+
+ int low = 1;
+ int high = numOrd()-1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int cmp = lookup(mid, spare).compareTo(key);
+
+ if (cmp < 0)
+ low = mid + 1;
+ else if (cmp > 0)
+ high = mid - 1;
+ else
+ return mid; // key found
+ }
+ return -(low + 1); // key not found.
+ }
+
+ /** The BytesRef argument must not be null; the method
+ * returns the same BytesRef, or an empty (length=0)
+ * BytesRef if this ord is the null ord (0). */
+ public abstract BytesRef lookup(int ord, BytesRef reuse);
+
+ /** Convenience method, to lookup the Term for a doc.
+ * If this doc is deleted or did not have this field,
+ * this will return an empty (length=0) BytesRef. */
+ public BytesRef getTerm(int docID, BytesRef reuse) {
+ return lookup(getOrd(docID), reuse);
+ }
+
+ /** Returns sort ord for this document. Ord 0 is
+ * reserved for docs that are deleted or did not have
+ * this field. */
+ public abstract int getOrd(int docID);
+
+ /** Returns total unique ord count; this includes +1 for
+ * the null ord (always 0). */
+ public abstract int numOrd();
+
+ /** Number of documents */
+ public abstract int size();
+ }
+
/** Checks the internal cache for an appropriate entry, and if none
- * is found reads the term values in <code>field</code> and returns
- * an array of them in natural order, along with an array telling
- * which element in the term array each document uses.
+ * is found, reads the term values in <code>field</code>
+ * and returns a {@link DocTerms} instance, providing a
+ * method to retrieve the term (as a BytesRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
- * @return Array of terms and index into the array for each document.
+ * @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public StringIndex getStringIndex (IndexReader reader, String field)
+ public DocTermsIndex getTermsIndex (IndexReader reader, String field)
+ throws IOException;
+
+
+ /** Expert: just like {@link
+ * #getTermsIndex(IndexReader,String)}, but you can specify
+ * whether more RAM should be consumed in exchange for
+ * faster lookups (default is "true"). Note that the
+ * first call for a given reader and field "wins",
+ * subsequent calls will share the same cache entry. */
+ public DocTermsIndex getTermsIndex (IndexReader reader, String field, boolean fasterButMoreRAM)
throws IOException;
/**
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Thu Jun 3 18:38:05 2010
@@ -30,7 +30,11 @@ import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.FieldCacheSanityChecker;
@@ -57,8 +61,8 @@ class FieldCacheImpl implements FieldCac
caches.put(Float.TYPE, new FloatCache(this));
caches.put(Long.TYPE, new LongCache(this));
caches.put(Double.TYPE, new DoubleCache(this));
- caches.put(String.class, new StringCache(this));
- caches.put(StringIndex.class, new StringIndexCache(this));
+ caches.put(DocTermsIndex.class, new DocTermsIndexCache(this));
+ caches.put(DocTerms.class, new DocTermsCache(this));
}
public synchronized void purgeAllCaches() {
@@ -638,76 +642,222 @@ class FieldCacheImpl implements FieldCac
}
}
- // inherit javadocs
- public String[] getStrings(IndexReader reader, String field)
- throws IOException {
- return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser)null));
+ private static class DocTermsIndexImpl extends DocTermsIndex {
+ private final PagedBytes.Reader bytes;
+ private final PackedInts.Reader termOrdToBytesOffset;
+ private final PackedInts.Reader docToTermOrd;
+ private final int numOrd;
+
+ public DocTermsIndexImpl(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, PackedInts.Reader docToTermOrd, int numOrd) {
+ this.bytes = bytes;
+ this.docToTermOrd = docToTermOrd;
+ this.termOrdToBytesOffset = termOrdToBytesOffset;
+ this.numOrd = numOrd;
+ }
+
+ @Override
+ public int numOrd() {
+ return numOrd;
+ }
+
+ @Override
+ public int getOrd(int docID) {
+ return (int) docToTermOrd.get(docID);
+ }
+
+ @Override
+ public int size() {
+ return docToTermOrd.size();
+ }
+
+ @Override
+ public BytesRef lookup(int ord, BytesRef ret) {
+ return bytes.fillUsingLengthPrefix(ret, termOrdToBytesOffset.get(ord));
+ }
+ }
+
+ private static boolean DEFAULT_FASTER_BUT_MORE_RAM = true;
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field) throws IOException {
+ return getTermsIndex(reader, field, DEFAULT_FASTER_BUT_MORE_RAM);
+ }
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)));
}
- static final class StringCache extends Cache {
- StringCache(FieldCache wrapper) {
+ static class DocTermsIndexCache extends Cache {
+ DocTermsIndexCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
- String field = StringHelper.intern(entryKey.field);
- final String[] retArray = new String[reader.maxDoc()];
+ String field = StringHelper.intern(entryKey.field);
Terms terms = MultiFields.getTerms(reader, field);
+
+ final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();
+
+ final PagedBytes bytes = new PagedBytes(15);
+
+ int startBytesBPV;
+ int startTermsBPV;
+ int startNumUniqueTerms;
+
+ if (terms != null) {
+ // Try for coarse estimate for number of bits; this
+ // should be an underestimate most of the time, which
+ // is fine -- GrowableWriter will reallocate as needed
+ long numUniqueTerms = 0;
+ try {
+ numUniqueTerms = terms.getUniqueTermCount();
+ } catch (UnsupportedOperationException uoe) {
+ numUniqueTerms = -1;
+ }
+ if (numUniqueTerms != -1) {
+ startBytesBPV = PackedInts.bitsRequired(numUniqueTerms*4);
+ startTermsBPV = PackedInts.bitsRequired(numUniqueTerms);
+ if (numUniqueTerms > Integer.MAX_VALUE-1) {
+ throw new IllegalStateException("this field has too many (" + numUniqueTerms + ") unique terms");
+ }
+ startNumUniqueTerms = (int) numUniqueTerms;
+ } else {
+ startBytesBPV = 1;
+ startTermsBPV = 1;
+ startNumUniqueTerms = 1;
+ }
+ } else {
+ startBytesBPV = 1;
+ startTermsBPV = 1;
+ startNumUniqueTerms = 1;
+ }
+
+ GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1+startNumUniqueTerms, fasterButMoreRAM);
+ final GrowableWriter docToTermOrd = new GrowableWriter(startTermsBPV, reader.maxDoc(), false);
+
+ // 0 is reserved for "unset"
+ bytes.copyUsingLengthPrefix(new BytesRef());
+ int termOrd = 1;
+
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
DocsEnum docs = null;
+
while(true) {
final BytesRef term = termsEnum.next();
if (term == null) {
break;
}
+ if (termOrd == termOrdToBytesOffset.size()) {
+ // NOTE: this code only runs if the incoming
+ // reader impl doesn't implement
+ // getUniqueTermCount (which should be uncommon)
+ termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
+ }
+ termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
+ bytes.copyUsingLengthPrefix(term);
docs = termsEnum.docs(delDocs, docs);
- final String termval = term.utf8ToString();
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
break;
}
- retArray[docID] = termval;
+ docToTermOrd.set(docID, termOrd);
}
+ termOrd++;
+ }
+
+ if (termOrdToBytesOffset.size() > termOrd) {
+ termOrdToBytesOffset = termOrdToBytesOffset.resize(termOrd);
}
}
- return retArray;
+
+ // maybe an int-only impl?
+ return new DocTermsIndexImpl(bytes.freeze(), termOrdToBytesOffset.getMutable(), docToTermOrd.getMutable(), termOrd);
}
}
- // inherit javadocs
- public StringIndex getStringIndex(IndexReader reader, String field)
- throws IOException {
- return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser)null));
+ private static class DocTermsImpl extends DocTerms {
+ private final PagedBytes.Reader bytes;
+ private final PackedInts.Reader docToOffset;
+
+ public DocTermsImpl(PagedBytes.Reader bytes, PackedInts.Reader docToOffset) {
+ this.bytes = bytes;
+ this.docToOffset = docToOffset;
+ }
+
+ @Override
+ public int size() {
+ return docToOffset.size();
+ }
+
+ @Override
+ public boolean exists(int docID) {
+ return docToOffset.get(docID) == 0;
+ }
+
+ @Override
+ public BytesRef getTerm(int docID, BytesRef ret) {
+ final int pointer = (int) docToOffset.get(docID);
+ return bytes.fillUsingLengthPrefix(ret, pointer);
+ }
+ }
+
+ // TODO: this if DocTermsIndex was already created, we
+ // should share it...
+ public DocTerms getTerms(IndexReader reader, String field) throws IOException {
+ return getTerms(reader, field, DEFAULT_FASTER_BUT_MORE_RAM);
+ }
+
+ public DocTerms getTerms(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return (DocTerms) caches.get(DocTerms.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)));
}
- static final class StringIndexCache extends Cache {
- StringIndexCache(FieldCache wrapper) {
+ static final class DocTermsCache extends Cache {
+ DocTermsCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
- String field = StringHelper.intern(entryKey.field);
- final int[] retArray = new int[reader.maxDoc()];
- String[] mterms = new String[reader.maxDoc()+1];
- //System.out.println("FC: getStringIndex field=" + field);
+ String field = StringHelper.intern(entryKey.field);
Terms terms = MultiFields.getTerms(reader, field);
- int t = 0; // current term number
+ final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();
+
+ // Holds the actual term data, expanded.
+ final PagedBytes bytes = new PagedBytes(15);
- // an entry for documents that have no terms in this field
- // should a document with no terms be at top or bottom?
- // this puts them at the top - if it is changed, FieldDocSortedHitQueue
- // needs to change as well.
- mterms[t++] = null;
+ int startBPV;
+
+ if (terms != null) {
+ // Try for coarse estimate for number of bits; this
+ // should be an underestimate most of the time, which
+ // is fine -- GrowableWriter will reallocate as needed
+ long numUniqueTerms = 0;
+ try {
+ numUniqueTerms = terms.getUniqueTermCount();
+ } catch (UnsupportedOperationException uoe) {
+ numUniqueTerms = -1;
+ }
+ if (numUniqueTerms != -1) {
+ startBPV = PackedInts.bitsRequired(numUniqueTerms*4);
+ } else {
+ startBPV = 1;
+ }
+ } else {
+ startBPV = 1;
+ }
+
+ final GrowableWriter docToOffset = new GrowableWriter(startBPV, reader.maxDoc(), fasterButMoreRAM);
+
+ // pointer==0 means not set
+ bytes.copyUsingLengthPrefix(new BytesRef());
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
@@ -718,42 +868,22 @@ class FieldCacheImpl implements FieldCac
if (term == null) {
break;
}
-
- // store term text
- mterms[t] = term.utf8ToString();
- //System.out.println("FC: ord=" + t + " term=" + term.toBytesString());
-
+ final long pointer = bytes.copyUsingLengthPrefix(term);
docs = termsEnum.docs(delDocs, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
break;
}
- //System.out.println("FC: docID=" + docID);
- retArray[docID] = t;
+ docToOffset.set(docID, pointer);
}
- t++;
}
}
- if (t == 0) {
- // if there are no terms, make the term array
- // have a single null entry
- mterms = new String[1];
- } else if (t < mterms.length) {
- // if there are less terms than documents,
- // trim off the dead array space
- String[] newTerms = new String[t];
- System.arraycopy (mterms, 0, newTerms, 0, t);
- mterms = newTerms;
- }
-
- StringIndex value = new StringIndex (retArray, mterms);
- //System.out.println("FC: done\n");
- return value;
+ // maybe an int-only impl?
+ return new DocTermsImpl(bytes.freeze(), docToOffset.getMutable());
}
}
-
private volatile PrintStream infoStream;
public void setInfoStream(PrintStream stream) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Thu Jun 3 18:38:05 2010
@@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.document.NumericField; // for javadocs
/**
@@ -83,10 +84,11 @@ public abstract class FieldCacheRangeFil
return new FieldCacheRangeFilter<String>(field, null, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
- final FieldCache.StringIndex fcsi = FieldCache.DEFAULT.getStringIndex(reader, field);
- final int lowerPoint = fcsi.binarySearchLookup(lowerVal);
- final int upperPoint = fcsi.binarySearchLookup(upperVal);
-
+ final FieldCache.DocTermsIndex fcsi = FieldCache.DEFAULT.getTermsIndex(reader, field);
+ final BytesRef spare = new BytesRef();
+ final int lowerPoint = fcsi.binarySearchLookup(lowerVal == null ? null : new BytesRef(lowerVal), spare);
+ final int upperPoint = fcsi.binarySearchLookup(upperVal == null ? null : new BytesRef(upperVal), spare);
+
final int inclusiveLowerPoint, inclusiveUpperPoint;
// Hints:
@@ -125,7 +127,8 @@ public abstract class FieldCacheRangeFil
return new FieldCacheDocIdSet(reader, true) {
@Override
final boolean matchDoc(int doc) {
- return fcsi.order[doc] >= inclusiveLowerPoint && fcsi.order[doc] <= inclusiveUpperPoint;
+ final int docOrd = fcsi.getOrd(doc);
+ return docOrd >= inclusiveLowerPoint && docOrd <= inclusiveUpperPoint;
}
};
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java Thu Jun 3 18:38:05 2010
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.TermDocs; // for javadocs
/**
@@ -108,19 +109,20 @@ public class FieldCacheTermsFilter exten
@Override
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
- return new FieldCacheTermsFilterDocIdSet(getFieldCache().getStringIndex(reader, field));
+ return new FieldCacheTermsFilterDocIdSet(getFieldCache().getTermsIndex(reader, field));
}
protected class FieldCacheTermsFilterDocIdSet extends DocIdSet {
- private FieldCache.StringIndex fcsi;
+ private FieldCache.DocTermsIndex fcsi;
private OpenBitSet openBitSet;
- public FieldCacheTermsFilterDocIdSet(FieldCache.StringIndex fcsi) {
+ public FieldCacheTermsFilterDocIdSet(FieldCache.DocTermsIndex fcsi) {
this.fcsi = fcsi;
- openBitSet = new OpenBitSet(this.fcsi.lookup.length);
+ openBitSet = new OpenBitSet(this.fcsi.size());
+ final BytesRef spare = new BytesRef();
for (int i=0;i<terms.length;i++) {
- int termNumber = this.fcsi.binarySearchLookup(terms[i]);
+ int termNumber = this.fcsi.binarySearchLookup(new BytesRef(terms[i]), spare);
if (termNumber > 0) {
openBitSet.fastSet(termNumber);
}
@@ -149,7 +151,7 @@ public class FieldCacheTermsFilter exten
@Override
public int nextDoc() {
try {
- while (!openBitSet.fastGet(fcsi.order[++doc])) {}
+ while (!openBitSet.fastGet(fcsi.getOrd(++doc))) {}
} catch (ArrayIndexOutOfBoundsException e) {
doc = NO_MORE_DOCS;
}
@@ -160,7 +162,7 @@ public class FieldCacheTermsFilter exten
public int advance(int target) {
try {
doc = target;
- while (!openBitSet.fastGet(fcsi.order[doc])) {
+ while (!openBitSet.fastGet(fcsi.getOrd(doc))) {
doc++;
}
} catch (ArrayIndexOutOfBoundsException e) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java Thu Jun 3 18:38:05 2010
@@ -28,7 +28,9 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.FieldCache.FloatParser;
import org.apache.lucene.search.FieldCache.IntParser;
import org.apache.lucene.search.FieldCache.ShortParser;
-import org.apache.lucene.search.FieldCache.StringIndex;
+import org.apache.lucene.search.FieldCache.DocTermsIndex;
+import org.apache.lucene.search.FieldCache.DocTerms;
+import org.apache.lucene.util.BytesRef;
/**
* Expert: a FieldComparator compares hits so as to determine their
@@ -616,14 +618,19 @@ public abstract class FieldComparator {
}
/** Sorts by a field's value using the Collator for a
- * given Locale.*/
+ * given Locale.
+ *
+ * <p><b>WARNING</b>: this is likely very slow; you'll
+ * get much better performance using the
+ * CollationKeyAnalyzer or ICUCollationKeyAnalyzer. */
public static final class StringComparatorLocale extends FieldComparator {
private final String[] values;
- private String[] currentReaderValues;
+ private DocTerms currentDocTerms;
private final String field;
final Collator collator;
private String bottom;
+ private final BytesRef tempBR = new BytesRef();
StringComparatorLocale(int numHits, String field, Locale locale) {
values = new String[numHits];
@@ -648,7 +655,7 @@ public abstract class FieldComparator {
@Override
public int compareBottom(int doc) {
- final String val2 = currentReaderValues[doc];
+ final String val2 = currentDocTerms.getTerm(doc, tempBR).utf8ToString();
if (bottom == null) {
if (val2 == null) {
return 0;
@@ -662,12 +669,17 @@ public abstract class FieldComparator {
@Override
public void copy(int slot, int doc) {
- values[slot] = currentReaderValues[doc];
+ final BytesRef br = currentDocTerms.getTerm(doc, tempBR);
+ if (br == null) {
+ values[slot] = null;
+ } else {
+ values[slot] = br.utf8ToString();
+ }
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getStrings(reader, field);
+ currentDocTerms = FieldCache.DEFAULT.getTerms(reader, field);
}
@Override
@@ -677,39 +689,40 @@ public abstract class FieldComparator {
@Override
public Comparable<?> value(int slot) {
- return values[slot];
+ final String s = values[slot];
+ return s == null ? null : new BytesRef(values[slot]);
}
}
- /** Sorts by field's natural String sort order, using
+ /** Sorts by field's natural Term sort order, using
* ordinals. This is functionally equivalent to {@link
- * StringValComparator}, but it first resolves the string
+ * TermValComparator}, but it first resolves the string
* to their relative ordinal positions (using the index
* returned by {@link FieldCache#getStringIndex}), and
* does most comparisons using the ordinals. For medium
* to large results, this comparator will be much faster
- * than {@link StringValComparator}. For very small
+ * than {@link TermValComparator}. For very small
* result sets it may be slower. */
- public static final class StringOrdValComparator extends FieldComparator {
+ public static final class TermOrdValComparator extends FieldComparator {
private final int[] ords;
- private final String[] values;
+ private final BytesRef[] values;
private final int[] readerGen;
private int currentReaderGen = -1;
- private String[] lookup;
- private int[] order;
+ private DocTermsIndex termsIndex;
private final String field;
private int bottomSlot = -1;
private int bottomOrd;
- private String bottomValue;
+ private BytesRef bottomValue;
private final boolean reversed;
private final int sortPos;
+ private final BytesRef tempBR = new BytesRef();
- public StringOrdValComparator(int numHits, String field, int sortPos, boolean reversed) {
+ public TermOrdValComparator(int numHits, String field, int sortPos, boolean reversed) {
ords = new int[numHits];
- values = new String[numHits];
+ values = new BytesRef[numHits];
readerGen = new int[numHits];
this.sortPos = sortPos;
this.reversed = reversed;
@@ -725,8 +738,8 @@ public abstract class FieldComparator {
}
}
- final String val1 = values[slot1];
- final String val2 = values[slot2];
+ final BytesRef val1 = values[slot1];
+ final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
@@ -741,47 +754,48 @@ public abstract class FieldComparator {
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
- int order = this.order[doc];
+ int order = termsIndex.getOrd(doc);
final int cmp = bottomOrd - order;
if (cmp != 0) {
return cmp;
}
- final String val2 = lookup[order];
if (bottomValue == null) {
- if (val2 == null) {
+ if (order == 0) {
+ // unset
return 0;
}
// bottom wins
return -1;
- } else if (val2 == null) {
+ } else if (order == 0) {
// doc wins
return 1;
}
- return bottomValue.compareTo(val2);
+ termsIndex.lookup(order, tempBR);
+ return bottomValue.compareTo(tempBR);
}
private void convert(int slot) {
readerGen[slot] = currentReaderGen;
int index = 0;
- String value = values[slot];
+ BytesRef value = values[slot];
if (value == null) {
- ords[slot] = 0;
+ // 0 ord is null for all segments
+ assert ords[slot] == 0;
return;
}
if (sortPos == 0 && bottomSlot != -1 && bottomSlot != slot) {
// Since we are the primary sort, the entries in the
// queue are bounded by bottomOrd:
- assert bottomOrd < lookup.length;
if (reversed) {
- index = binarySearch(lookup, value, bottomOrd, lookup.length-1);
+ index = binarySearch(tempBR, termsIndex, value, bottomOrd, termsIndex.numOrd()-1);
} else {
- index = binarySearch(lookup, value, 0, bottomOrd);
+ index = binarySearch(tempBR, termsIndex, value, 0, bottomOrd);
}
} else {
// Full binary search
- index = binarySearch(lookup, value);
+ index = binarySearch(tempBR, termsIndex, value);
}
if (index < 0) {
@@ -792,20 +806,24 @@ public abstract class FieldComparator {
@Override
public void copy(int slot, int doc) {
- final int ord = order[doc];
- ords[slot] = ord;
- assert ord >= 0;
- values[slot] = lookup[ord];
+ final int ord = termsIndex.getOrd(doc);
+ if (ord == 0) {
+ values[slot] = null;
+ } else {
+ ords[slot] = ord;
+ assert ord >= 0;
+ if (values[slot] == null) {
+ values[slot] = new BytesRef();
+ }
+ termsIndex.lookup(ord, values[slot]);
+ }
readerGen[slot] = currentReaderGen;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- StringIndex currentReaderValues = FieldCache.DEFAULT.getStringIndex(reader, field);
+ termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
currentReaderGen++;
- order = currentReaderValues.order;
- lookup = currentReaderValues.lookup;
- assert lookup.length > 0;
if (bottomSlot != -1) {
convert(bottomSlot);
bottomOrd = ords[bottomSlot];
@@ -819,8 +837,6 @@ public abstract class FieldComparator {
convert(bottomSlot);
}
bottomOrd = ords[bottom];
- assert bottomOrd >= 0;
- assert bottomOrd < lookup.length;
bottomValue = values[bottom];
}
@@ -829,10 +845,6 @@ public abstract class FieldComparator {
return values[slot];
}
- public String[] getValues() {
- return values;
- }
-
public int getBottomSlot() {
return bottomSlot;
}
@@ -842,26 +854,27 @@ public abstract class FieldComparator {
}
}
- /** Sorts by field's natural String sort order. All
- * comparisons are done using String.compareTo, which is
+ /** Sorts by field's natural Term sort order. All
+ * comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets. */
- public static final class StringValComparator extends FieldComparator {
+ public static final class TermValComparator extends FieldComparator {
- private String[] values;
- private String[] currentReaderValues;
+ private BytesRef[] values;
+ private DocTerms docTerms;
private final String field;
- private String bottom;
+ private BytesRef bottom;
+ private final BytesRef tempBR = new BytesRef();
- StringValComparator(int numHits, String field) {
- values = new String[numHits];
+ TermValComparator(int numHits, String field) {
+ values = new BytesRef[numHits];
this.field = field;
}
@Override
public int compare(int slot1, int slot2) {
- final String val1 = values[slot1];
- final String val2 = values[slot2];
+ final BytesRef val1 = values[slot1];
+ final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
@@ -876,7 +889,7 @@ public abstract class FieldComparator {
@Override
public int compareBottom(int doc) {
- final String val2 = currentReaderValues[doc];
+ BytesRef val2 = docTerms.getTerm(doc, tempBR);
if (bottom == null) {
if (val2 == null) {
return 0;
@@ -890,12 +903,15 @@ public abstract class FieldComparator {
@Override
public void copy(int slot, int doc) {
- values[slot] = currentReaderValues[doc];
+ if (values[slot] == null) {
+ values[slot] = new BytesRef();
+ }
+ docTerms.getTerm(doc, values[slot]);
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getStrings(reader, field);
+ docTerms = FieldCache.DEFAULT.getTerms(reader, field);
}
@Override
@@ -909,15 +925,15 @@ public abstract class FieldComparator {
}
}
- final protected static int binarySearch(String[] a, String key) {
- return binarySearch(a, key, 0, a.length-1);
+ final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key) {
+ return binarySearch(br, a, key, 1, a.numOrd()-1);
}
- final protected static int binarySearch(String[] a, String key, int low, int high) {
+ final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key, int low, int high) {
while (low <= high) {
int mid = (low + high) >>> 1;
- String midVal = a[mid];
+ BytesRef midVal = a.lookup(mid, br);
int cmp;
if (midVal != null) {
cmp = midVal.compareTo(key);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldDocSortedHitQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldDocSortedHitQueue.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldDocSortedHitQueue.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldDocSortedHitQueue.java Thu Jun 3 18:38:05 2010
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.BytesRef;
import java.text.Collator;
import java.util.Locale;
@@ -99,8 +100,8 @@ class FieldDocSortedHitQueue extends Pri
for (int i=0; i<n && c==0; ++i) {
final int type = fields[i].getType();
if (type == SortField.STRING) {
- final String s1 = (String) docA.fields[i];
- final String s2 = (String) docB.fields[i];
+ final BytesRef s1 = (BytesRef) docA.fields[i];
+ final BytesRef s2 = (BytesRef) docB.fields[i];
// null values need to be sorted first, because of how FieldCache.getStringIndex()
// works - in that routine, any documents without a value in the given field are
// put first. If both are null, the next SortField is used
@@ -111,7 +112,7 @@ class FieldDocSortedHitQueue extends Pri
} else if (fields[i].getLocale() == null) {
c = s1.compareTo(s2);
} else {
- c = collators[i].compare(s1, s2);
+ c = collators[i].compare(s1.utf8ToString(), s2.utf8ToString());
}
} else {
c = docA.fields[i].compareTo(docB.fields[i]);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java Thu Jun 3 18:38:05 2010
@@ -83,10 +83,6 @@ implements Serializable {
* uses ordinals to do the sorting. */
public static final int STRING_VAL = 11;
- // IMPLEMENTATION NOTE: the FieldCache.STRING_INDEX is in the same "namespace"
- // as the above static int values. Any new values must not have the same value
- // as FieldCache.STRING_INDEX.
-
/** Represents sorting by document score (relevancy). */
public static final SortField FIELD_SCORE = new SortField (null, SCORE);
@@ -413,10 +409,10 @@ implements Serializable {
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
case SortField.STRING:
- return new FieldComparator.StringOrdValComparator(numHits, field, sortPos, reverse);
+ return new FieldComparator.TermOrdValComparator(numHits, field, sortPos, reverse);
case SortField.STRING_VAL:
- return new FieldComparator.StringValComparator(numHits, field);
+ return new FieldComparator.TermValComparator(numHits, field);
default:
throw new IllegalStateException("Illegal sort type: " + type);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java Thu Jun 3 18:38:05 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.search.functio
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.FieldCache.DocTermsIndex;
import java.io.IOException;
@@ -69,18 +70,18 @@ public class OrdFieldSource extends Valu
/*(non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
@Override
public DocValues getValues(IndexReader reader) throws IOException {
- final int[] arr = FieldCache.DEFAULT.getStringIndex(reader, field).order;
+ final DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
return new DocValues() {
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#floatVal(int) */
@Override
public float floatVal(int doc) {
- return arr[doc];
+ return termsIndex.getOrd(doc);
}
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#strVal(int) */
@Override
public String strVal(int doc) {
// the string value of the ordinal, not the string itself
- return Integer.toString(arr[doc]);
+ return Integer.toString(termsIndex.getOrd(doc));
}
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#toString(int) */
@Override
@@ -90,7 +91,7 @@ public class OrdFieldSource extends Valu
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#getInnerArray() */
@Override
Object getInnerArray() {
- return arr;
+ return termsIndex;
}
};
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java Thu Jun 3 18:38:05 2010
@@ -70,21 +70,20 @@ public class ReverseOrdFieldSource exten
/*(non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
@Override
public DocValues getValues(IndexReader reader) throws IOException {
- final FieldCache.StringIndex sindex = FieldCache.DEFAULT.getStringIndex(reader, field);
+ final FieldCache.DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
- final int arr[] = sindex.order;
- final int end = sindex.lookup.length;
+ final int end = termsIndex.numOrd();
return new DocValues() {
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#floatVal(int) */
@Override
public float floatVal(int doc) {
- return (end - arr[doc]);
+ return (end - termsIndex.getOrd(doc));
}
/* (non-Javadoc) @see org.apache.lucene.search.function.DocValues#intVal(int) */
@Override
public int intVal(int doc) {
- return end - arr[doc];
+ return end - termsIndex.getOrd(doc);
}
/* (non-Javadoc) @see org.apache.lucene.search.function.DocValues#strVal(int) */
@Override
@@ -100,7 +99,7 @@ public class ReverseOrdFieldSource exten
/*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#getInnerArray() */
@Override
Object getInnerArray() {
- return arr;
+ return termsIndex;
}
};
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRef.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRef.java Thu Jun 3 18:38:05 2010
@@ -19,12 +19,17 @@ package org.apache.lucene.util;
import java.util.Comparator;
import java.io.UnsupportedEncodingException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.Externalizable;
+import java.io.IOException;
/** Represents byte[], as a slice (offset + length) into an
* existing byte[].
*
* @lucene.experimental */
-public final class BytesRef implements Comparable<BytesRef> {
+public final class BytesRef implements Comparable<BytesRef>, Externalizable {
+
public static final byte[] EMPTY_BYTES = new byte[0];
/** The contents of the BytesRef. Should never be {@code null}. */
@@ -78,6 +83,19 @@ public final class BytesRef implements C
copy(other);
}
+ /* // maybe?
+ public BytesRef(BytesRef other, boolean shallow) {
+ this();
+ if (shallow) {
+ offset = other.offset;
+ length = other.length;
+ bytes = other.bytes;
+ } else {
+ copy(other);
+ }
+ }
+ */
+
/**
* Copies the UTF8 bytes for this string.
*
@@ -316,4 +334,25 @@ public final class BytesRef implements C
return this == other;
}
}
+
+ public void writeExternal(ObjectOutput out)
+ throws IOException
+ {
+ out.writeInt(length);
+ if (length > 0) {
+ out.write(bytes, offset, length);
+ }
+ }
+
+ public void readExternal( ObjectInput in ) throws
+ IOException, ClassNotFoundException {
+ length = in.readInt();
+ offset = 0;
+ if (length > 0) {
+ bytes = new byte[length];
+ in.read(bytes, 0, length);
+ } else {
+ bytes = null;
+ }
+ }
}
Copied: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java (from r944313, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java?p2=lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java&r1=944313&r2=951104&rev=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java Thu Jun 3 18:38:05 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.index.codecs.standard;
+package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,9 +17,6 @@ package org.apache.lucene.index.codecs.s
* limitations under the License.
*/
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.store.IndexInput;
import java.util.List;
@@ -28,19 +25,88 @@ import java.io.Closeable;
import java.io.IOException;
/** Represents a logical byte[] as a series of pages. You
- * can write-once into the logical byte[], using copy, and
- * then retrieve slices (BytesRef) into it using fill. */
-class PagedBytes implements Closeable {
+ * can write-once into the logical byte[] (append only),
+ * using copy, and then retrieve slices (BytesRef) into it
+ * using fill.
+ *
+ * <p>@lucene.internal</p>*/
+public final class PagedBytes {
private final List<byte[]> blocks = new ArrayList<byte[]>();
private final int blockSize;
private final int blockBits;
private final int blockMask;
private int upto;
private byte[] currentBlock;
- private final CloseableThreadLocal<byte[]> threadBuffers = new CloseableThreadLocal<byte[]>();
private static final byte[] EMPTY_BYTES = new byte[0];
+ public final static class Reader implements Closeable {
+ private final byte[][] blocks;
+ private final int blockBits;
+ private final int blockMask;
+ private final int blockSize;
+ private final CloseableThreadLocal<byte[]> threadBuffers = new CloseableThreadLocal<byte[]>();
+
+ public Reader(PagedBytes pagedBytes) {
+ blocks = new byte[pagedBytes.blocks.size()][];
+ for(int i=0;i<blocks.length;i++) {
+ blocks[i] = pagedBytes.blocks.get(i);
+ }
+ blockBits = pagedBytes.blockBits;
+ blockMask = pagedBytes.blockMask;
+ blockSize = pagedBytes.blockSize;
+ }
+
+ /** Get a slice out of the byte array. */
+ public BytesRef fill(BytesRef b, long start, int length) {
+ assert length >= 0: "length=" + length;
+ final int index = (int) (start >> blockBits);
+ final int offset = (int) (start & blockMask);
+ b.length = length;
+ if (blockSize - offset >= length) {
+ // Within block
+ b.bytes = blocks[index];
+ b.offset = offset;
+ } else {
+ // Split
+ byte[] buffer = threadBuffers.get();
+ if (buffer == null) {
+ buffer = new byte[length];
+ threadBuffers.set(buffer);
+ } else if (buffer.length < length) {
+ buffer = ArrayUtil.grow(buffer, length);
+ threadBuffers.set(buffer);
+ }
+ b.bytes = buffer;
+ b.offset = 0;
+ System.arraycopy(blocks[index], offset, buffer, 0, blockSize-offset);
+ System.arraycopy(blocks[1+index], 0, buffer, blockSize-offset, length-(blockSize-offset));
+ }
+ return b;
+ }
+
+ /** Reads length as 1 or 2 byte vInt prefix, starting @ start */
+ public BytesRef fillUsingLengthPrefix(BytesRef b, long start) {
+ final int index = (int) (start >> blockBits);
+ final int offset = (int) (start & blockMask);
+ final byte[] block = b.bytes = blocks[index];
+
+ if ((block[offset] & 128) == 0) {
+ b.length = block[offset];
+ b.offset = offset+1;
+ } else {
+ b.length = (((int) (block[offset] & 0x7f)) << 8) | (block[1+offset] & 0xff);
+ b.offset = offset+2;
+ assert b.length > 0;
+ }
+ return b;
+ }
+
+ public void close() {
+ threadBuffers.close();
+ }
+ }
+
/** 1<<blockBits must be bigger than biggest single
* BytesRef slice that will be pulled */
public PagedBytes(int blockBits) {
@@ -69,13 +135,40 @@ class PagedBytes implements Closeable {
} else {
in.readBytes(currentBlock, upto, (int) byteCount, false);
upto += byteCount;
- byteCount = 0;
+ break;
+ }
+ }
+ }
+
+ /** Copy BytesRef in */
+ public void copy(BytesRef bytes) throws IOException {
+ int byteCount = bytes.length;
+ int bytesUpto = bytes.offset;
+ while (byteCount > 0) {
+ int left = blockSize - upto;
+ if (left == 0) {
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ left = blockSize;
+ }
+ if (left < byteCount) {
+ System.arraycopy(bytes.bytes, bytesUpto, currentBlock, upto, left);
+ upto = blockSize;
+ byteCount -= left;
+ bytesUpto += left;
+ } else {
+ System.arraycopy(bytes.bytes, bytesUpto, currentBlock, upto, (int) byteCount);
+ upto += byteCount;
+ break;
}
}
}
/** Commits final byte[], trimming it if necessary. */
- public void finish() {
+ public Reader freeze() {
if (upto < blockSize) {
final byte[] newBlock = new byte[upto];
System.arraycopy(currentBlock, 0, newBlock, 0, upto);
@@ -86,6 +179,7 @@ class PagedBytes implements Closeable {
}
blocks.add(currentBlock);
currentBlock = null;
+ return new Reader(this);
}
public long getPointer() {
@@ -96,34 +190,32 @@ class PagedBytes implements Closeable {
}
}
- /** Get a slice out of the byte array. */
- public void fill(BytesRef b, long start, int length) {
- assert length >= 0: "length=" + length;
- final int index = (int) (start >> blockBits);
- final int offset = (int) (start & blockMask);
- b.length = length;
- if (blockSize - offset >= length) {
- // Within block
- b.bytes = blocks.get(index);
- b.offset = offset;
+ /** Copy bytes in, writing the length as a 1 or 2 byte
+ * vInt prefix. */
+ public long copyUsingLengthPrefix(BytesRef bytes) throws IOException {
+
+ if (upto + bytes.length + 2 > blockSize) {
+ if (bytes.length + 2 > blockSize) {
+ throw new IllegalArgumentException("block size " + blockSize + " is too small to store length " + bytes.length + " bytes");
+ }
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ }
+
+ final long pointer = getPointer();
+
+ if (bytes.length < 128) {
+ currentBlock[upto++] = (byte) bytes.length;
} else {
- // Split
- byte[] buffer = threadBuffers.get();
- if (buffer == null) {
- buffer = new byte[length];
- threadBuffers.set(buffer);
- } else if (buffer.length < length) {
- buffer = ArrayUtil.grow(buffer, length);
- threadBuffers.set(buffer);
- }
- b.bytes = buffer;
- b.offset = 0;
- System.arraycopy(blocks.get(index), offset, buffer, 0, blockSize-offset);
- System.arraycopy(blocks.get(1+index), 0, buffer, blockSize-offset, length-(blockSize-offset));
+ currentBlock[upto++] = (byte) (0x80 | (bytes.length >> 8));
+ currentBlock[upto++] = (byte) (bytes.length & 0xff);
}
- }
+ System.arraycopy(bytes.bytes, bytes.offset, currentBlock, upto, bytes.length);
+ upto += bytes.length;
- public void close() {
- threadBuffers.close();
+ return pointer;
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct16.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct16.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct16.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct16.java Thu Jun 3 18:38:05 2010
@@ -29,19 +29,19 @@ import java.util.Arrays;
class Direct16 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
- private short[] blocks;
+ private short[] values;
private static final int BITS_PER_VALUE = 16;
public Direct16(int valueCount) {
super(valueCount, BITS_PER_VALUE);
- blocks = new short[valueCount];
+ values = new short[valueCount];
}
public Direct16(IndexInput in, int valueCount) throws IOException {
super(valueCount, BITS_PER_VALUE);
- short[] blocks = new short[valueCount];
+ short[] values = new short[valueCount];
for(int i=0;i<valueCount;i++) {
- blocks[i] = in.readShort();
+ values[i] = in.readShort();
}
final int mod = valueCount % 4;
if (mod != 0) {
@@ -52,35 +52,35 @@ class Direct16 extends PackedInts.Reader
}
}
- this.blocks = blocks;
+ this.values = values;
}
/**
- * Creates an array backed by the given blocks.
+ * Creates an array backed by the given values.
* </p><p>
- * Note: The blocks are used directly, so changes to the given block will
+ * Note: The values are used directly, so changes to the values will
* affect the structure.
- * @param blocks used as the internal backing array.
+ * @param values used as the internal backing array.
*/
- public Direct16(short[] blocks) {
- super(blocks.length, BITS_PER_VALUE);
- this.blocks = blocks;
+ public Direct16(short[] values) {
+ super(values.length, BITS_PER_VALUE);
+ this.values = values;
}
public long get(final int index) {
- return 0xFFFFL & blocks[index];
+ return 0xFFFFL & values[index];
}
public void set(final int index, final long value) {
- blocks[index] = (short)(value & 0xFFFF);
+ values[index] = (short)(value & 0xFFFF);
}
public long ramBytesUsed() {
return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- blocks.length * RamUsageEstimator.NUM_BYTES_SHORT;
+ values.length * RamUsageEstimator.NUM_BYTES_SHORT;
}
public void clear() {
- Arrays.fill(blocks, (short)0);
+ Arrays.fill(values, (short)0);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct32.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct32.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct32.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct32.java Thu Jun 3 18:38:05 2010
@@ -29,54 +29,54 @@ import java.util.Arrays;
class Direct32 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
- private int[] blocks;
+ private int[] values;
private static final int BITS_PER_VALUE = 32;
public Direct32(int valueCount) {
super(valueCount, BITS_PER_VALUE);
- blocks = new int[valueCount];
+ values = new int[valueCount];
}
public Direct32(IndexInput in, int valueCount) throws IOException {
super(valueCount, BITS_PER_VALUE);
- int[] blocks = new int[valueCount];
+ int[] values = new int[valueCount];
for(int i=0;i<valueCount;i++) {
- blocks[i] = in.readInt();
+ values[i] = in.readInt();
}
final int mod = valueCount % 2;
if (mod != 0) {
in.readInt();
}
- this.blocks = blocks;
+ this.values = values;
}
/**
- * Creates an array backed by the given blocks.
+ * Creates an array backed by the given values.
* </p><p>
- * Note: The blocks are used directly, so changes to the given block will
+ * Note: The values are used directly, so changes to the given values will
* affect the structure.
- * @param blocks used as the internal backing array.
+ * @param values used as the internal backing array.
*/
- public Direct32(int[] blocks) {
- super(blocks.length, BITS_PER_VALUE);
- this.blocks = blocks;
+ public Direct32(int[] values) {
+ super(values.length, BITS_PER_VALUE);
+ this.values = values;
}
public long get(final int index) {
- return 0xFFFFFFFFL & blocks[index];
+ return 0xFFFFFFFFL & values[index];
}
public void set(final int index, final long value) {
- blocks[index] = (int)(value & 0xFFFFFFFF);
+ values[index] = (int)(value & 0xFFFFFFFF);
}
public long ramBytesUsed() {
return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ values.length * RamUsageEstimator.NUM_BYTES_INT;
}
public void clear() {
- Arrays.fill(blocks, 0);
+ Arrays.fill(values, 0);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct64.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct64.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct64.java Thu Jun 3 18:38:05 2010
@@ -29,51 +29,51 @@ import java.util.Arrays;
class Direct64 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
- private long[] blocks;
+ private long[] values;
private static final int BITS_PER_VALUE = 64;
public Direct64(int valueCount) {
super(valueCount, BITS_PER_VALUE);
- blocks = new long[valueCount];
+ values = new long[valueCount];
}
public Direct64(IndexInput in, int valueCount) throws IOException {
super(valueCount, BITS_PER_VALUE);
- long[] blocks = new long[valueCount];
+ long[] values = new long[valueCount];
for(int i=0;i<valueCount;i++) {
- blocks[i] = in.readLong();
+ values[i] = in.readLong();
}
- this.blocks = blocks;
+ this.values = values;
}
/**
- * Creates an array backed by the given blocks.
+ * Creates an array backed by the given values.
* </p><p>
- * Note: The blocks are used directly, so changes to the given block will
+ * Note: The values are used directly, so changes to the given values will
* affect the structure.
- * @param blocks used as the internal backing array.
+ * @param values used as the internal backing array.
*/
- public Direct64(long[] blocks) {
- super(blocks.length, BITS_PER_VALUE);
- this.blocks = blocks;
+ public Direct64(long[] values) {
+ super(values.length, BITS_PER_VALUE);
+ this.values = values;
}
public long get(final int index) {
- return blocks[index];
+ return values[index];
}
public void set(final int index, final long value) {
- blocks[index] = value;
+ values[index] = value;
}
public long ramBytesUsed() {
return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- blocks.length * RamUsageEstimator.NUM_BYTES_LONG;
+ values.length * RamUsageEstimator.NUM_BYTES_LONG;
}
public void clear() {
- Arrays.fill(blocks, 0L);
+ Arrays.fill(values, 0L);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct8.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct8.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct8.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Direct8.java Thu Jun 3 18:38:05 2010
@@ -29,20 +29,20 @@ import java.util.Arrays;
class Direct8 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
- private byte[] blocks;
+ private byte[] values;
private static final int BITS_PER_VALUE = 8;
public Direct8(int valueCount) {
super(valueCount, BITS_PER_VALUE);
- blocks = new byte[valueCount];
+ values = new byte[valueCount];
}
public Direct8(IndexInput in, int valueCount)
throws IOException {
super(valueCount, BITS_PER_VALUE);
- byte[] blocks = new byte[valueCount];
+ byte[] values = new byte[valueCount];
for(int i=0;i<valueCount;i++) {
- blocks[i] = in.readByte();
+ values[i] = in.readByte();
}
final int mod = valueCount % 8;
if (mod != 0) {
@@ -53,34 +53,34 @@ class Direct8 extends PackedInts.ReaderI
}
}
- this.blocks = blocks;
+ this.values = values;
}
/**
- * Creates an array backed by the given blocks.
+ * Creates an array backed by the given values.
* </p><p>
- * Note: The blocks are used directly, so changes to the given block will
+ * Note: The values are used directly, so changes to the given values will
* affect the structure.
- * @param blocks used as the internal backing array.
+ * @param values used as the internal backing array.
*/
- public Direct8(byte[] blocks) {
- super(blocks.length, BITS_PER_VALUE);
- this.blocks = blocks;
+ public Direct8(byte[] values) {
+ super(values.length, BITS_PER_VALUE);
+ this.values = values;
}
public long get(final int index) {
- return 0xFFL & blocks[index];
+ return 0xFFL & values[index];
}
public void set(final int index, final long value) {
- blocks[index] = (byte)(value & 0xFF);
+ values[index] = (byte)(value & 0xFF);
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + blocks.length;
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.length;
}
public void clear() {
- Arrays.fill(blocks, (byte)0);
+ Arrays.fill(values, (byte)0);
}
}
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=951104&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java Thu Jun 3 18:38:05 2010
@@ -0,0 +1,93 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implements {@link PackedInts.Mutable}, but grows the
+ * bit count of the underlying packed ints on-demand.
+ *
+ * <p>@lucene.internal</p>
+ */
+
+public class GrowableWriter implements PackedInts.Mutable {
+
+ private long currentMaxValue;
+ private PackedInts.Mutable current;
+ private final boolean roundFixedSize;
+
+ public GrowableWriter(int startBitsPerValue, int valueCount, boolean roundFixedSize) {
+ this.roundFixedSize = roundFixedSize;
+ current = PackedInts.getMutable(valueCount, getSize(startBitsPerValue));
+ currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+ }
+
+ private final int getSize(int bpv) {
+ if (roundFixedSize) {
+ return PackedInts.getNextFixedSize(bpv);
+ } else {
+ return bpv;
+ }
+ }
+
+ public long get(int index) {
+ return current.get(index);
+ }
+
+ public int size() {
+ return current.size();
+ }
+
+ public int getBitsPerValue() {
+ return current.getBitsPerValue();
+ }
+
+ public PackedInts.Mutable getMutable() {
+ return current;
+ }
+
+ public void set(int index, long value) {
+ if (value >= currentMaxValue) {
+ int bpv = getBitsPerValue();
+ while(currentMaxValue <= value && currentMaxValue != Long.MAX_VALUE) {
+ bpv++;
+ currentMaxValue *= 2;
+ }
+ final int valueCount = size();
+ PackedInts.Mutable next = PackedInts.getMutable(valueCount, getSize(bpv));
+ for(int i=0;i<valueCount;i++) {
+ next.set(i, current.get(i));
+ }
+ current = next;
+ currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+ }
+ current.set(index, value);
+ }
+
+ public void clear() {
+ current.clear();
+ }
+
+ public GrowableWriter resize(int newSize) {
+ GrowableWriter next = new GrowableWriter(getBitsPerValue(), newSize, roundFixedSize);
+ final int limit = Math.min(size(), newSize);
+ for(int i=0;i<limit;i++) {
+ next.set(i, get(i));
+ }
+ return next;
+ }
+}
Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java Thu Jun 3 18:38:05 2010
@@ -209,7 +209,7 @@ public class PackedInts {
* @lucene.internal
*/
public static Mutable getMutable(
- int valueCount, int bitsPerValue) throws IOException {
+ int valueCount, int bitsPerValue) {
switch (bitsPerValue) {
case 8:
return new Direct8(valueCount);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java?rev=951104&r1=951103&r2=951104&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java Thu Jun 3 18:38:05 2010
@@ -24,13 +24,14 @@ import org.apache.lucene.index.*;
import org.apache.lucene.search.FieldValueHitQueue.Entry;
import org.apache.lucene.store.*;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class TestElevationComparator extends LuceneTestCase {
- private final Map<String,Integer> priority = new HashMap<String,Integer>();
+ private final Map<BytesRef,Integer> priority = new HashMap<BytesRef,Integer>();
//@Test
public void testSorting() throws Throwable {
@@ -109,7 +110,7 @@ public class TestElevationComparator ext
int max = (vals.length / 2) + 5;
for (int i = 0; i < vals.length - 1; i += 2) {
q.add(new TermQuery(new Term(vals[i], vals[i + 1])), BooleanClause.Occur.SHOULD);
- priority.put(vals[i + 1], Integer.valueOf(max--));
+ priority.put(new BytesRef(vals[i + 1]), Integer.valueOf(max--));
// System.out.println(" pri doc=" + vals[i+1] + " pri=" + (1+max));
}
return q;
@@ -125,9 +126,9 @@ public class TestElevationComparator ext
}
class ElevationComparatorSource extends FieldComparatorSource {
- private final Map<String,Integer> priority;
+ private final Map<BytesRef,Integer> priority;
- public ElevationComparatorSource(final Map<String,Integer> boosts) {
+ public ElevationComparatorSource(final Map<BytesRef,Integer> boosts) {
this.priority = boosts;
}
@@ -135,8 +136,9 @@ class ElevationComparatorSource extends
public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator() {
- FieldCache.StringIndex idIndex;
+ FieldCache.DocTermsIndex idIndex;
private final int[] values = new int[numHits];
+ private final BytesRef tempBR = new BytesRef();
int bottomVal;
@Override
@@ -150,9 +152,14 @@ class ElevationComparatorSource extends
}
private int docVal(int doc) throws IOException {
- String id = idIndex.lookup[idIndex.order[doc]];
- Integer prio = priority.get(id);
- return prio == null ? 0 : prio.intValue();
+ int ord = idIndex.getOrd(doc);
+ if (ord == 0) {
+ return 0;
+ } else {
+ BytesRef id = idIndex.lookup(ord, tempBR);
+ Integer prio = priority.get(id);
+ return prio == null ? 0 : prio.intValue();
+ }
}
@Override
@@ -167,7 +174,7 @@ class ElevationComparatorSource extends
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- idIndex = FieldCache.DEFAULT.getStringIndex(reader, fieldname);
+ idIndex = FieldCache.DEFAULT.getTermsIndex(reader, fieldname);
}
@Override