You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/11/11 02:06:49 UTC
svn commit: r1200675 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/test/ lucene/contrib/analyzers/common/
lucene/src/java/org/apache/lucene/analysis/
lucene/src/java/org/apache/lucene/search/
lucene/src/test/org/apache/lucene/ana...
Author: mikemccand
Date: Fri Nov 11 01:06:49 2011
New Revision: 1200675
URL: http://svn.apache.org/viewvc?rev=1200675&view=rev
Log:
LUCENE-3443: allow setting docsWithField bits when pulling FC entries
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharArraySet.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestWordlistLoader.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1200675&r1=1200674&r2=1200675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Fri Nov 11 01:06:49 2011
@@ -128,6 +128,11 @@ Optimizations
* LUCENE-2205: Very substantial (3-5X) RAM reduction required to hold
the terms index on opening an IndexReader (Aaron McCurry via Mike McCandless)
+* LUCENE-3443: FieldCache can now set docsWithField, and create an
+ array, in a single pass. This results in faster init time for apps
+ that need both (such as sorting by a field with a missing value).
+ (Mike McCandless)
+
Test Cases
* LUCENE-3420: Disable the finalness checks in TokenStream and Analyzer
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java?rev=1200675&r1=1200674&r2=1200675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java Fri Nov 11 01:06:49 2011
@@ -344,6 +344,21 @@ public interface FieldCache {
public byte[] getBytes (IndexReader reader, String field, ByteParser parser)
throws IOException;
+ /** Checks the internal cache for an appropriate entry, and if none is found,
+ * reads the terms in <code>field</code> as bytes and returns an array of
+ * size <code>reader.maxDoc()</code> of the value each document has in the
+ * given field.
+ * @param reader Used to get field values.
+ * @param field Which field contains the bytes.
+ * @param parser Computes byte for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public byte[] getBytes (IndexReader reader, String field, ByteParser parser, boolean setDocsWithField)
+ throws IOException;
+
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as shorts and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
@@ -369,6 +384,21 @@ public interface FieldCache {
public short[] getShorts (IndexReader reader, String field, ShortParser parser)
throws IOException;
+ /** Checks the internal cache for an appropriate entry, and if none is found,
+ * reads the terms in <code>field</code> as shorts and returns an array of
+ * size <code>reader.maxDoc()</code> of the value each document has in the
+ * given field.
+ * @param reader Used to get field values.
+ * @param field Which field contains the shorts.
+ * @param parser Computes short for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public short[] getShorts (IndexReader reader, String field, ShortParser parser, boolean setDocsWithField)
+ throws IOException;
+
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as integers and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
@@ -394,6 +424,21 @@ public interface FieldCache {
public int[] getInts (IndexReader reader, String field, IntParser parser)
throws IOException;
+ /** Checks the internal cache for an appropriate entry, and if none is found,
+ * reads the terms in <code>field</code> as integers and returns an array of
+ * size <code>reader.maxDoc()</code> of the value each document has in the
+ * given field.
+ * @param reader Used to get field values.
+ * @param field Which field contains the integers.
+ * @param parser Computes integer for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public int[] getInts (IndexReader reader, String field, IntParser parser, boolean setDocsWithField)
+ throws IOException;
+
/** Checks the internal cache for an appropriate entry, and if
* none is found, reads the terms in <code>field</code> as floats and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
@@ -419,6 +464,21 @@ public interface FieldCache {
public float[] getFloats (IndexReader reader, String field,
FloatParser parser) throws IOException;
+ /** Checks the internal cache for an appropriate entry, and if
+ * none is found, reads the terms in <code>field</code> as floats and returns an array
+ * of size <code>reader.maxDoc()</code> of the value each document
+ * has in the given field.
+ * @param reader Used to get field values.
+ * @param field Which field contains the floats.
+ * @param parser Computes float for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public float[] getFloats (IndexReader reader, String field,
+ FloatParser parser, boolean setDocsWithField) throws IOException;
+
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as longs and returns an array
@@ -447,7 +507,22 @@ public interface FieldCache {
*/
public long[] getLongs(IndexReader reader, String field, LongParser parser)
throws IOException;
-
+ /**
+ * Checks the internal cache for an appropriate entry, and if none is found,
+ * reads the terms in <code>field</code> as longs and returns an array of
+ * size <code>reader.maxDoc()</code> of the value each document has in the
+ * given field.
+ *
+ * @param reader Used to get field values.
+ * @param field Which field contains the longs.
+ * @param parser Computes integer for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public long[] getLongs(IndexReader reader, String field, LongParser parser, boolean setDocsWithField)
+ throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
@@ -478,6 +553,23 @@ public interface FieldCache {
public double[] getDoubles(IndexReader reader, String field, DoubleParser parser)
throws IOException;
+ /**
+ * Checks the internal cache for an appropriate entry, and if none is found,
+ * reads the terms in <code>field</code> as doubles and returns an array of
+ * size <code>reader.maxDoc()</code> of the value each document has in the
+ * given field.
+ *
+ * @param reader Used to get field values.
+ * @param field Which field contains the doubles.
+ * @param parser Computes integer for string values.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
+ * @return The values in the given field for each document.
+ * @throws IOException If any error occurs.
+ */
+ public double[] getDoubles(IndexReader reader, String field, DoubleParser parser, boolean setDocsWithField)
+ throws IOException;
+
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code> and returns an array
* of size <code>reader.maxDoc()</code> containing the value each document
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1200675&r1=1200674&r2=1200675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Fri Nov 11 01:06:49 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -149,15 +148,15 @@ class FieldCacheImpl implements FieldCac
this.wrapper = null;
}
- Cache(FieldCache wrapper) {
+ Cache(FieldCacheImpl wrapper) {
this.wrapper = wrapper;
}
- final FieldCache wrapper;
+ final FieldCacheImpl wrapper;
final Map<Object,Map<Entry,Object>> readerCache = new WeakHashMap<Object,Map<Entry,Object>>();
- protected abstract Object createValue(IndexReader reader, Entry key)
+ protected abstract Object createValue(IndexReader reader, Entry key, boolean setDocsWithField)
throws IOException;
/** Remove this reader from the cache, if present. */
@@ -168,7 +167,28 @@ class FieldCacheImpl implements FieldCac
}
}
- public Object get(IndexReader reader, Entry key) throws IOException {
+ /** Sets the key to the value for the provided reader;
+ * if the key is already set then this doesn't change it. */
+ public void put(IndexReader reader, Entry key, Object value) {
+ final Object readerKey = reader.getCoreCacheKey();
+ synchronized (readerCache) {
+ Map<Entry,Object> innerCache = readerCache.get(readerKey);
+ if (innerCache == null) {
+ // First time this reader is using FieldCache
+ innerCache = new HashMap<Entry,Object>();
+ readerCache.put(readerKey, innerCache);
+ reader.addReaderFinishedListener(purgeReader);
+ }
+ if (innerCache.get(key) == null) {
+ innerCache.put(key, value);
+ } else {
+ // Another thread beat us to it; leave the current
+ // value
+ }
+ }
+ }
+
+ public Object get(IndexReader reader, Entry key, boolean setDocsWithField) throws IOException {
Map<Entry,Object> innerCache;
Object value;
final Object readerKey = reader.getCoreCacheKey();
@@ -192,7 +212,7 @@ class FieldCacheImpl implements FieldCac
synchronized (value) {
CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
- progress.value = createValue(reader, key);
+ progress.value = createValue(reader, key, setDocsWithField);
synchronized (readerCache) {
innerCache.put(key, progress.value);
}
@@ -267,31 +287,39 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public byte[] getBytes (IndexReader reader, String field) throws IOException {
- return getBytes(reader, field, null);
+ return getBytes(reader, field, null, false);
}
// inherit javadocs
public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
throws IOException {
- return (byte[]) caches.get(Byte.TYPE).get(reader, new Entry(field, parser));
+ return getBytes(reader, field, parser, false);
+ }
+
+ public byte[] getBytes(IndexReader reader, String field, ByteParser parser, boolean setDocsWithField)
+ throws IOException {
+ return (byte[]) caches.get(Byte.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class ByteCache extends Cache {
- ByteCache(FieldCache wrapper) {
+ ByteCache(FieldCacheImpl wrapper) {
super(wrapper);
}
+
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
ByteParser parser = (ByteParser) entry.custom;
if (parser == null) {
- return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
+ return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER, setDocsWithField);
}
- final byte[] retArray = new byte[reader.maxDoc()];
+ final int maxDoc = reader.maxDoc();
+ final byte[] retArray = new byte[maxDoc];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
@@ -299,7 +327,15 @@ class FieldCacheImpl implements FieldCac
byte termval = parser.parseByte(term.text());
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -307,38 +343,49 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
return retArray;
}
}
// inherit javadocs
public short[] getShorts (IndexReader reader, String field) throws IOException {
- return getShorts(reader, field, null);
+ return getShorts(reader, field, null, false);
}
// inherit javadocs
public short[] getShorts(IndexReader reader, String field, ShortParser parser)
throws IOException {
- return (short[]) caches.get(Short.TYPE).get(reader, new Entry(field, parser));
+ return getShorts(reader, field, parser, false);
+ }
+
+ // inherit javadocs
+ public short[] getShorts(IndexReader reader, String field, ShortParser parser, boolean setDocsWithField)
+ throws IOException {
+ return (short[]) caches.get(Short.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class ShortCache extends Cache {
- ShortCache(FieldCache wrapper) {
+ ShortCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
ShortParser parser = (ShortParser) entry.custom;
if (parser == null) {
- return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
+ return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER, setDocsWithField);
}
- final short[] retArray = new short[reader.maxDoc()];
+ final int maxDoc = reader.maxDoc();
+ final short[] retArray = new short[maxDoc];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
@@ -346,7 +393,15 @@ class FieldCacheImpl implements FieldCac
short termval = parser.parseShort(term.text());
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -354,10 +409,34 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
return retArray;
}
}
+ // null Bits means no docs matched
+ void setDocsWithField(IndexReader reader, String field, Bits docsWithField) {
+ final int maxDoc = reader.maxDoc();
+ final Bits bits;
+ if (docsWithField == null) {
+ bits = new Bits.MatchNoBits(maxDoc);
+ } else if (docsWithField instanceof FixedBitSet) {
+ final int numSet = ((FixedBitSet) docsWithField).cardinality();
+ if (numSet >= maxDoc) {
+ // The cardinality of the BitSet is maxDoc if all documents have a value.
+ assert numSet == maxDoc;
+ bits = new Bits.MatchAllBits(maxDoc);
+ } else {
+ bits = docsWithField;
+ }
+ } else {
+ bits = docsWithField;
+ }
+ caches.get(DocsWithFieldCache.class).put(reader, new Entry(field, null), bits);
+ }
+
// inherit javadocs
public int[] getInts (IndexReader reader, String field) throws IOException {
return getInts(reader, field, null);
@@ -366,40 +445,57 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public int[] getInts(IndexReader reader, String field, IntParser parser)
throws IOException {
- return (int[]) caches.get(Integer.TYPE).get(reader, new Entry(field, parser));
+ return getInts(reader, field, parser, false);
+ }
+
+ // inherit javadocs
+ public int[] getInts(IndexReader reader, String field, IntParser parser, boolean setDocsWithField)
+ throws IOException {
+ return (int[]) caches.get(Integer.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class IntCache extends Cache {
- IntCache(FieldCache wrapper) {
+ IntCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
IntParser parser = (IntParser) entry.custom;
if (parser == null) {
try {
- return wrapper.getInts(reader, field, DEFAULT_INT_PARSER);
+ return wrapper.getInts(reader, field, DEFAULT_INT_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
- return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
+ return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER, setDocsWithField);
}
}
+ final int maxDoc = reader.maxDoc();
int[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
if (term==null || term.field() != field) break;
int termval = parser.parseInt(term.text());
- if (retArray == null) // late init
- retArray = new int[reader.maxDoc()];
+ if (retArray == null) { // late init
+ retArray = new int[maxDoc];
+ }
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -407,24 +503,28 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
- if (retArray == null) // no values
- retArray = new int[reader.maxDoc()];
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
+ if (retArray == null) { // no values
+ retArray = new int[maxDoc];
+ }
return retArray;
}
}
public Bits getDocsWithField(IndexReader reader, String field)
throws IOException {
- return (Bits) caches.get(DocsWithFieldCache.class).get(reader, new Entry(field, null));
+ return (Bits) caches.get(DocsWithFieldCache.class).get(reader, new Entry(field, null), false);
}
static final class DocsWithFieldCache extends Cache {
- DocsWithFieldCache(FieldCache wrapper) {
+ DocsWithFieldCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
final Entry entry = entryKey;
final String field = entry.field;
@@ -462,47 +562,63 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public float[] getFloats (IndexReader reader, String field)
throws IOException {
- return getFloats(reader, field, null);
+ return getFloats(reader, field, null, false);
}
// inherit javadocs
public float[] getFloats(IndexReader reader, String field, FloatParser parser)
throws IOException {
+ return getFloats(reader, field, parser, false);
+ }
- return (float[]) caches.get(Float.TYPE).get(reader, new Entry(field, parser));
+ public float[] getFloats(IndexReader reader, String field, FloatParser parser, boolean setDocsWithField)
+ throws IOException {
+
+ return (float[]) caches.get(Float.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class FloatCache extends Cache {
- FloatCache(FieldCache wrapper) {
+ FloatCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
FloatParser parser = (FloatParser) entry.custom;
if (parser == null) {
try {
- return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER);
+ return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
- return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
+ return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER, setDocsWithField);
}
- }
+ }
+ final int maxDoc = reader.maxDoc();
float[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
if (term==null || term.field() != field) break;
float termval = parser.parseFloat(term.text());
- if (retArray == null) // late init
- retArray = new float[reader.maxDoc()];
+ if (retArray == null) { // late init
+ retArray = new float[maxDoc];
+ }
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -510,53 +626,73 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
- if (retArray == null) // no values
- retArray = new float[reader.maxDoc()];
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
+ if (retArray == null) { // no values
+ retArray = new float[maxDoc];
+ }
return retArray;
}
}
-
public long[] getLongs(IndexReader reader, String field) throws IOException {
- return getLongs(reader, field, null);
+ return getLongs(reader, field, null, false);
}
// inherit javadocs
public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser)
throws IOException {
- return (long[]) caches.get(Long.TYPE).get(reader, new Entry(field, parser));
+ return getLongs(reader, field, parser, false);
+ }
+
+ // inherit javadocs
+ public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser, boolean setDocsWithField)
+ throws IOException {
+ return (long[]) caches.get(Long.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class LongCache extends Cache {
- LongCache(FieldCache wrapper) {
+ LongCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entry)
+ protected Object createValue(IndexReader reader, Entry entry, boolean setDocsWithField)
throws IOException {
String field = entry.field;
FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom;
if (parser == null) {
try {
- return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER);
+ return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
- return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
+ return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER, setDocsWithField);
}
}
+ final int maxDoc = reader.maxDoc();
long[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term(field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
if (term==null || term.field() != field) break;
long termval = parser.parseLong(term.text());
- if (retArray == null) // late init
- retArray = new long[reader.maxDoc()];
+ if (retArray == null) { // late init
+ retArray = new long[maxDoc];
+ }
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -564,8 +700,12 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
- if (retArray == null) // no values
- retArray = new long[reader.maxDoc()];
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
+ if (retArray == null) { // no values
+ retArray = new long[maxDoc];
+ }
return retArray;
}
}
@@ -573,46 +713,63 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public double[] getDoubles(IndexReader reader, String field)
throws IOException {
- return getDoubles(reader, field, null);
+ return getDoubles(reader, field, null, false);
}
// inherit javadocs
public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser)
throws IOException {
- return (double[]) caches.get(Double.TYPE).get(reader, new Entry(field, parser));
+ return getDoubles(reader, field, parser, false);
+ }
+
+ // inherit javadocs
+ public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser, boolean setDocsWithField)
+ throws IOException {
+ return (double[]) caches.get(Double.TYPE).get(reader, new Entry(field, parser), setDocsWithField);
}
static final class DoubleCache extends Cache {
- DoubleCache(FieldCache wrapper) {
+ DoubleCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom;
if (parser == null) {
try {
- return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
+ return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
- return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
+ return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER, setDocsWithField);
}
}
+ final int maxDoc = reader.maxDoc();
double[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
+ FixedBitSet docsWithField = null;
try {
do {
Term term = termEnum.term();
if (term==null || term.field() != field) break;
double termval = parser.parseDouble(term.text());
- if (retArray == null) // late init
- retArray = new double[reader.maxDoc()];
+ if (retArray == null) { // late init
+ retArray = new double[maxDoc];
+ }
termDocs.seek (termEnum);
while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
+ final int docID = termDocs.doc();
+ retArray[docID] = termval;
+ if (setDocsWithField) {
+ if (docsWithField == null) {
+ // Lazy init
+ docsWithField = new FixedBitSet(maxDoc);
+ }
+ docsWithField.set(docID);
+ }
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
@@ -620,8 +777,12 @@ class FieldCacheImpl implements FieldCac
termDocs.close();
termEnum.close();
}
- if (retArray == null) // no values
- retArray = new double[reader.maxDoc()];
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, field, docsWithField);
+ }
+ if (retArray == null) { // no values
+ retArray = new double[maxDoc];
+ }
return retArray;
}
}
@@ -629,16 +790,16 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public String[] getStrings(IndexReader reader, String field)
throws IOException {
- return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser)null));
+ return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser)null), false);
}
static final class StringCache extends Cache {
- StringCache(FieldCache wrapper) {
+ StringCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
String field = StringHelper.intern(entryKey.field);
final String[] retArray = new String[reader.maxDoc()];
@@ -674,16 +835,16 @@ class FieldCacheImpl implements FieldCac
// inherit javadocs
public StringIndex getStringIndex(IndexReader reader, String field)
throws IOException {
- return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser)null));
+ return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser)null), false);
}
static final class StringIndexCache extends Cache {
- StringIndexCache(FieldCache wrapper) {
+ StringIndexCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
- protected Object createValue(IndexReader reader, Entry entryKey)
+ protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
String field = StringHelper.intern(entryKey.field);
final int[] retArray = new int[reader.maxDoc()];
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1200675&r1=1200674&r2=1200675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java Fri Nov 11 01:06:49 2011
@@ -186,8 +186,7 @@ public abstract class FieldComparator<T>
public static abstract class NumericComparator<T extends Number> extends FieldComparator<T> {
protected final T missingValue;
protected final String field;
-
- protected Bits docsWithField = null;
+ protected Bits docsWithField;
public NumericComparator(String field, T missingValue) {
this.field = field;
@@ -199,27 +198,23 @@ public abstract class FieldComparator<T>
if (missingValue != null) {
docsWithField = FieldCache.DEFAULT.getDocsWithField(reader, field);
// optimization to remove unneeded checks on the bit interface:
- if (docsWithField instanceof Bits.MatchAllBits)
+ if (docsWithField instanceof Bits.MatchAllBits) {
docsWithField = null;
+ }
} else {
docsWithField = null;
}
}
-
}
/** Parses field's values as byte (using {@link
* FieldCache#getBytes} and sorts by ascending value */
public static final class ByteComparator extends NumericComparator<Byte> {
private final byte[] values;
+ private final ByteParser parser;
private byte[] currentReaderValues;
- private ByteParser parser;
private byte bottom;
- ByteComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue) {
super(field, missingValue);
values = new byte[numHits];
@@ -234,22 +229,28 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
byte v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
return bottom - v2;
}
@Override
public void copy(int slot, int doc) {
byte v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getBytes(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getBytes(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
@@ -314,14 +315,10 @@ public abstract class FieldComparator<T>
* FieldCache#getDoubles} and sorts by ascending value */
public static final class DoubleComparator extends NumericComparator<Double> {
private final double[] values;
+ private final DoubleParser parser;
private double[] currentReaderValues;
- private DoubleParser parser;
private double bottom;
- DoubleComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
super(field, missingValue);
values = new double[numHits];
@@ -344,8 +341,11 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
double v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -358,14 +358,17 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
double v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getDoubles(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getDoubles(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
@@ -384,14 +387,10 @@ public abstract class FieldComparator<T>
* FieldCache#getFloats} and sorts by ascending value */
public static final class FloatComparator extends NumericComparator<Float> {
private final float[] values;
+ private final FloatParser parser;
private float[] currentReaderValues;
- private FloatParser parser;
private float bottom;
- FloatComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
super(field, missingValue);
values = new float[numHits];
@@ -418,8 +417,11 @@ public abstract class FieldComparator<T>
// TODO: are there sneaky non-branch ways to compute
// sign of float?
float v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -432,14 +434,17 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
float v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getFloats(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getFloats(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
@@ -458,14 +463,10 @@ public abstract class FieldComparator<T>
* FieldCache#getInts} and sorts by ascending value */
public static final class IntComparator extends NumericComparator<Integer> {
private final int[] values;
+ private final IntParser parser;
private int[] currentReaderValues;
- private IntParser parser;
private int bottom; // Value of bottom of queue
- IntComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
super(field, missingValue);
values = new int[numHits];
@@ -496,8 +497,11 @@ public abstract class FieldComparator<T>
// Cannot return bottom - values[slot2] because that
// may overflow
int v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -510,14 +514,17 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
int v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
@@ -536,14 +543,10 @@ public abstract class FieldComparator<T>
* FieldCache#getLongs} and sorts by ascending value */
public static final class LongComparator extends NumericComparator<Long> {
private final long[] values;
+ private final LongParser parser;
private long[] currentReaderValues;
- private LongParser parser;
private long bottom;
- LongComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
super(field, missingValue);
values = new long[numHits];
@@ -570,8 +573,11 @@ public abstract class FieldComparator<T>
// TODO: there are sneaky non-branch ways to compute
// -1/+1/0 sign
long v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -584,14 +590,17 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
long v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getLongs(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getLongs(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
@@ -678,14 +687,10 @@ public abstract class FieldComparator<T>
* FieldCache#getShorts} and sorts by ascending value */
public static final class ShortComparator extends NumericComparator<Short> {
private final short[] values;
+ private final ShortParser parser;
private short[] currentReaderValues;
- private ShortParser parser;
private short bottom;
- ShortComparator(int numHits, String field, FieldCache.Parser parser) {
- this(numHits, field, parser, null);
- }
-
ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue) {
super(field, missingValue);
values = new short[numHits];
@@ -700,22 +705,28 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
short v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
return bottom - v2;
}
@Override
public void copy(int slot, int doc) {
short v2 = currentReaderValues[doc];
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc))
+ // Test for v2 == 0 to save Bits.get method call for
+ // the common case (doc has value and value is non-zero):
+ if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
+ }
values[slot] = v2;
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = FieldCache.DEFAULT.getShorts(reader, field, parser);
+ currentReaderValues = FieldCache.DEFAULT.getShorts(reader, field, parser, missingValue != null);
super.setNextReader(reader, docBase);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1200675&r1=1200674&r2=1200675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java Fri Nov 11 01:06:49 2011
@@ -16,17 +16,22 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
-import java.io.IOException;
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
public class TestFieldCache extends LuceneTestCase {
protected IndexReader reader;
@@ -56,6 +61,9 @@ public class TestFieldCache extends Luce
if (i%2 == 0) {
doc.add(newField("sparse", String.valueOf(i), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
+ if (i%2 == 0) {
+ doc.add(new NumericField("numInt").setIntValue(i));
+ }
writer.addDocument(doc);
}
writer.close();
@@ -84,7 +92,7 @@ public class TestFieldCache extends Luce
public void test() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
- double [] doubles = cache.getDoubles(reader, "theDouble");
+ double [] doubles = cache.getDoubles(reader, "theDouble", null, random.nextBoolean());
assertSame("Second request to cache return same array", doubles, cache.getDoubles(reader, "theDouble"));
assertSame("Second request with explicit parser return same array", doubles, cache.getDoubles(reader, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER));
assertTrue("doubles Size: " + doubles.length + " is not: " + NUM_DOCS, doubles.length == NUM_DOCS);
@@ -93,7 +101,7 @@ public class TestFieldCache extends Luce
}
- long [] longs = cache.getLongs(reader, "theLong");
+ long [] longs = cache.getLongs(reader, "theLong", null, random.nextBoolean());
assertSame("Second request to cache return same array", longs, cache.getLongs(reader, "theLong"));
assertSame("Second request with explicit parser return same array", longs, cache.getLongs(reader, "theLong", FieldCache.DEFAULT_LONG_PARSER));
assertTrue("longs Size: " + longs.length + " is not: " + NUM_DOCS, longs.length == NUM_DOCS);
@@ -102,7 +110,7 @@ public class TestFieldCache extends Luce
}
- byte [] bytes = cache.getBytes(reader, "theByte");
+ byte [] bytes = cache.getBytes(reader, "theByte", null, random.nextBoolean());
assertSame("Second request to cache return same array", bytes, cache.getBytes(reader, "theByte"));
assertSame("Second request with explicit parser return same array", bytes, cache.getBytes(reader, "theByte", FieldCache.DEFAULT_BYTE_PARSER));
assertTrue("bytes Size: " + bytes.length + " is not: " + NUM_DOCS, bytes.length == NUM_DOCS);
@@ -111,7 +119,7 @@ public class TestFieldCache extends Luce
}
- short [] shorts = cache.getShorts(reader, "theShort");
+ short [] shorts = cache.getShorts(reader, "theShort", null, random.nextBoolean());
assertSame("Second request to cache return same array", shorts, cache.getShorts(reader, "theShort"));
assertSame("Second request with explicit parser return same array", shorts, cache.getShorts(reader, "theShort", FieldCache.DEFAULT_SHORT_PARSER));
assertTrue("shorts Size: " + shorts.length + " is not: " + NUM_DOCS, shorts.length == NUM_DOCS);
@@ -120,7 +128,7 @@ public class TestFieldCache extends Luce
}
- int [] ints = cache.getInts(reader, "theInt");
+ int [] ints = cache.getInts(reader, "theInt", null, random.nextBoolean());
assertSame("Second request to cache return same array", ints, cache.getInts(reader, "theInt"));
assertSame("Second request with explicit parser return same array", ints, cache.getInts(reader, "theInt", FieldCache.DEFAULT_INT_PARSER));
assertTrue("ints Size: " + ints.length + " is not: " + NUM_DOCS, ints.length == NUM_DOCS);
@@ -129,7 +137,7 @@ public class TestFieldCache extends Luce
}
- float [] floats = cache.getFloats(reader, "theFloat");
+ float [] floats = cache.getFloats(reader, "theFloat", null, random.nextBoolean());
assertSame("Second request to cache return same array", floats, cache.getFloats(reader, "theFloat"));
assertSame("Second request with explicit parser return same array", floats, cache.getFloats(reader, "theFloat", FieldCache.DEFAULT_FLOAT_PARSER));
assertTrue("floats Size: " + floats.length + " is not: " + NUM_DOCS, floats.length == NUM_DOCS);
@@ -154,4 +162,110 @@ public class TestFieldCache extends Luce
assertEquals(i%2 == 0, docsWithField.get(i));
}
}
+ public void testDocsWithField() throws Exception {
+ FieldCache cache = FieldCache.DEFAULT;
+ cache.purgeAllCaches();
+ assertEquals(0, cache.getCacheEntries().length);
+ double[] doubles = cache.getDoubles(reader, "theDouble", null, true);
+
+ // The double[] takes two slots (one w/ null parser, one
+ // w/ real parser), and docsWithField should also
+ // have been populated:
+ assertEquals(3, cache.getCacheEntries().length);
+ Bits bits = cache.getDocsWithField(reader, "theDouble");
+
+ // No new entries should appear:
+ assertEquals(3, cache.getCacheEntries().length);
+ assertTrue(bits instanceof Bits.MatchAllBits);
+
+ int[] ints = cache.getInts(reader, "sparse", null, true);
+ assertEquals(6, cache.getCacheEntries().length);
+ Bits docsWithField = cache.getDocsWithField(reader, "sparse");
+ assertEquals(6, cache.getCacheEntries().length);
+ for (int i = 0; i < docsWithField.length(); i++) {
+ if (i%2 == 0) {
+ assertTrue(docsWithField.get(i));
+ assertEquals(i, ints[i]);
+ } else {
+ assertFalse(docsWithField.get(i));
+ }
+ }
+
+ int[] numInts = cache.getInts(reader, "numInt", null, random.nextBoolean());
+ docsWithField = cache.getDocsWithField(reader, "numInt");
+ for (int i = 0; i < docsWithField.length(); i++) {
+ if (i%2 == 0) {
+ assertTrue(docsWithField.get(i));
+ assertEquals(i, numInts[i]);
+ } else {
+ assertFalse(docsWithField.get(i));
+ }
+ }
+ }
+
+ public void testGetDocsWithFieldThreadSafety() throws Exception {
+ final FieldCache cache = FieldCache.DEFAULT;
+ cache.purgeAllCaches();
+
+ int NUM_THREADS = 3;
+ Thread[] threads = new Thread[NUM_THREADS];
+ final AtomicBoolean failed = new AtomicBoolean();
+ final AtomicInteger iters = new AtomicInteger();
+ final int NUM_ITER = 200 * RANDOM_MULTIPLIER;
+ final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS,
+ new Runnable() {
+ // @Override not until java 1.6
+ public void run() {
+ cache.purgeAllCaches();
+ iters.incrementAndGet();
+ }
+ });
+ for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
+ threads[threadIDX] = new Thread() {
+ @Override
+ public void run() {
+
+ try {
+ while(!failed.get()) {
+ final int op = random.nextInt(3);
+ if (op == 0) {
+ // Purge all caches & resume, once all
+ // threads get here:
+ restart.await();
+ if (iters.get() >= NUM_ITER) {
+ break;
+ }
+ } else if (op == 1) {
+ Bits docsWithField = cache.getDocsWithField(reader, "sparse");
+ for (int i = 0; i < docsWithField.length(); i++) {
+ assertEquals(i%2 == 0, docsWithField.get(i));
+ }
+ } else {
+ int[] ints = cache.getInts(reader, "sparse", null, true);
+ Bits docsWithField = cache.getDocsWithField(reader, "sparse");
+ for (int i = 0; i < docsWithField.length(); i++) {
+ if (i%2 == 0) {
+ assertTrue(docsWithField.get(i));
+ assertEquals(i, ints[i]);
+ } else {
+ assertFalse(docsWithField.get(i));
+ }
+ }
+ }
+ }
+ } catch (Throwable t) {
+ failed.set(true);
+ restart.reset();
+ throw new RuntimeException(t);
+ }
+ }
+ };
+ threads[threadIDX].start();
+ }
+
+ for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
+ threads[threadIDX].join();
+ }
+ assertFalse(failed.get());
+ }
}