You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/08/30 17:57:58 UTC
svn commit: r1519003 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/index/
lucene/core/src/test/org/apache...
Author: rmuir
Date: Fri Aug 30 15:57:58 2013
New Revision: 1519003
URL: http://svn.apache.org/r1519003
Log:
LUCENE-5183: remove BinaryDocValues.MISSING
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
lucene/dev/branches/branch_4x/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Aug 30 15:57:58 2013
@@ -172,6 +172,9 @@ API Changes
a value for the field. Previously it only did this if the SortedDocValues
was produced by uninversion on the FieldCache. (Robert Muir)
+* LUCENE-5183: remove BinaryDocValues.MISSING. In order to determine a document
+ is missing a field, use getDocsWithField instead. (Robert Muir)
+
Changes in Runtime Behavior
* LUCENE-5178: DocValues codec consumer APIs (iterables) return null values
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java Fri Aug 30 15:57:58 2013
@@ -30,17 +30,12 @@ public abstract class BinaryDocValues {
/** Lookup the value for document. */
public abstract void get(int docID, BytesRef result);
-
- /**
- * Indicates the value was missing for the document.
- */
- public static final byte[] MISSING = new byte[0];
- /** An empty BinaryDocValues which returns {@link #MISSING} for every document */
+ /** An empty BinaryDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document */
public static final BinaryDocValues EMPTY = new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
- result.bytes = MISSING;
+ result.bytes = BytesRef.EMPTY_BYTES;
result.offset = 0;
result.length = 0;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java Fri Aug 30 15:57:58 2013
@@ -60,7 +60,7 @@ public abstract class SortedDocValues ex
public void get(int docID, BytesRef result) {
int ord = getOrd(docID);
if (ord == -1) {
- result.bytes = MISSING;
+ result.bytes = BytesRef.EMPTY_BYTES;
result.length = 0;
result.offset = 0;
} else {
@@ -68,7 +68,7 @@ public abstract class SortedDocValues ex
}
}
- /** An empty SortedDocValues which returns {@link #MISSING} for every document */
+ /** An empty SortedDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document */
public static final SortedDocValues EMPTY = new SortedDocValues() {
@Override
public int getOrd(int docID) {
@@ -77,7 +77,7 @@ public abstract class SortedDocValues ex
@Override
public void lookupOrd(int ord, BytesRef result) {
- result.bytes = MISSING;
+ result.bytes = BytesRef.EMPTY_BYTES;
result.offset = 0;
result.length = 0;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Fri Aug 30 15:57:58 2013
@@ -621,18 +621,19 @@ public interface FieldCache {
* method to retrieve the term (as a BytesRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
+ * @param setDocsWithField If true then {@link #getDocsWithField} will
+ * also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public BinaryDocValues getTerms (AtomicReader reader, String field)
- throws IOException;
+ public BinaryDocValues getTerms (AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
- /** Expert: just like {@link #getTerms(AtomicReader,String)},
+ /** Expert: just like {@link #getTerms(AtomicReader,String,boolean)},
* but you can specify whether more RAM should be consumed in exchange for
* faster lookups (default is "true"). Note that the
* first call for a given reader and field "wins",
* subsequent calls will share the same cache entry. */
- public BinaryDocValues getTerms (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
+ public BinaryDocValues getTerms (AtomicReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Fri Aug 30 15:57:58 2013
@@ -1268,7 +1268,7 @@ class FieldCacheImpl implements FieldCac
public void get(int docID, BytesRef ret) {
final int pointer = (int) docToOffset.get(docID);
if (pointer == 0) {
- ret.bytes = MISSING;
+ ret.bytes = BytesRef.EMPTY_BYTES;
ret.offset = 0;
ret.length = 0;
} else {
@@ -1279,11 +1279,11 @@ class FieldCacheImpl implements FieldCac
// TODO: this if DocTermsIndex was already created, we
// should share it...
- public BinaryDocValues getTerms(AtomicReader reader, String field) throws IOException {
- return getTerms(reader, field, PackedInts.FAST);
+ public BinaryDocValues getTerms(AtomicReader reader, String field, boolean setDocsWithField) throws IOException {
+ return getTerms(reader, field, setDocsWithField, PackedInts.FAST);
}
- public BinaryDocValues getTerms(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException {
+ public BinaryDocValues getTerms(AtomicReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException {
BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
if (valuesIn == null) {
valuesIn = reader.getSortedDocValues(field);
@@ -1304,7 +1304,7 @@ class FieldCacheImpl implements FieldCac
return BinaryDocValues.EMPTY;
}
- return (BinaryDocValues) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
+ return (BinaryDocValues) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), setDocsWithField);
}
static final class BinaryDocValuesCache extends Cache {
@@ -1313,7 +1313,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
// TODO: would be nice to first check if DocTermsIndex
@@ -1382,8 +1382,22 @@ class FieldCacheImpl implements FieldCac
}
}
+ final PackedInts.Reader offsetReader = docToOffset.getMutable();
+ if (setDocsWithField) {
+ wrapper.setDocsWithField(reader, key.field, new Bits() {
+ @Override
+ public boolean get(int index) {
+ return offsetReader.get(index) != 0;
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ });
+ }
// maybe an int-only impl?
- return new BinaryDocValuesImpl(bytes.freeze(true), docToOffset.getMutable());
+ return new BinaryDocValuesImpl(bytes.freeze(true), offsetReader);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Fri Aug 30 15:57:58 2013
@@ -1099,6 +1099,9 @@ public abstract class FieldComparator<T>
return values[slot];
}
}
+
+ // just used internally in this comparator
+ private static final byte[] MISSING_BYTES = new byte[0];
/** Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
@@ -1108,6 +1111,7 @@ public abstract class FieldComparator<T>
private BytesRef[] values;
private BinaryDocValues docTerms;
+ private Bits docsWithField;
private final String field;
private BytesRef bottom;
private final BytesRef tempBR = new BytesRef();
@@ -1136,12 +1140,15 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
docTerms.get(doc, tempBR);
- if (bottom.bytes == BinaryDocValues.MISSING) {
- if (tempBR.bytes == BinaryDocValues.MISSING) {
+ if (tempBR.length == 0 && docsWithField.get(doc) == false) {
+ tempBR.bytes = MISSING_BYTES;
+ }
+ if (bottom.bytes == MISSING_BYTES) {
+ if (tempBR.bytes == MISSING_BYTES) {
return 0;
}
return -1;
- } else if (tempBR.bytes == BinaryDocValues.MISSING) {
+ } else if (tempBR.bytes == MISSING_BYTES) {
return 1;
}
return bottom.compareTo(tempBR);
@@ -1153,11 +1160,15 @@ public abstract class FieldComparator<T>
values[slot] = new BytesRef();
}
docTerms.get(doc, values[slot]);
+ if (values[slot].length == 0 && docsWithField.get(doc) == false) {
+ values[slot].bytes = MISSING_BYTES;
+ }
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- docTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
+ docTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, true);
+ docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
return this;
}
@@ -1187,6 +1198,9 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, BytesRef value) {
docTerms.get(doc, tempBR);
+ if (tempBR.length == 0 && docsWithField.get(doc) == false) {
+ tempBR.bytes = MISSING_BYTES;
+ }
return tempBR.compareTo(value);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Fri Aug 30 15:57:58 2013
@@ -184,7 +184,7 @@ public class TestDocValuesIndexing exten
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field");
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field", false);
BytesRef bytes1 = new BytesRef();
s.get(0, bytes1);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java Fri Aug 30 15:57:58 2013
@@ -81,7 +81,7 @@ public class TestDocValuesWithThreads ex
//NumericDocValues ndv = ar.getNumericDocValues("number");
FieldCache.Longs ndv = FieldCache.DEFAULT.getLongs(ar, "number", false);
//BinaryDocValues bdv = ar.getBinaryDocValues("bytes");
- BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes");
+ BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes", false);
SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
startingGun.await();
int iters = atLeast(1000);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java Fri Aug 30 15:57:58 2013
@@ -257,12 +257,13 @@ public class TestFieldCache extends Luce
termsIndex = cache.getTermsIndex(reader, "bogusfield");
// getTerms
- BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString");
- assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString"));
+ BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString", true);
+ assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString", true));
+ Bits bits = cache.getDocsWithField(reader, "theRandomUnicodeString");
for (int i = 0; i < NUM_DOCS; i++) {
terms.get(i, br);
final BytesRef term;
- if (br.bytes == BinaryDocValues.MISSING) {
+ if (!bits.get(i)) {
term = null;
} else {
term = br;
@@ -272,7 +273,7 @@ public class TestFieldCache extends Luce
}
// test bad field
- terms = cache.getTerms(reader, "bogusfield");
+ terms = cache.getTerms(reader, "bogusfield", false);
// getDocTermOrds
SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField");
@@ -312,7 +313,7 @@ public class TestFieldCache extends Luce
writer.close();
IndexReader r = DirectoryReader.open(dir);
AtomicReader reader = SlowCompositeReaderWrapper.wrap(r);
- FieldCache.DEFAULT.getTerms(reader, "foobar");
+ FieldCache.DEFAULT.getTerms(reader, "foobar", true);
FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
FieldCache.DEFAULT.purge(reader);
r.close();
@@ -468,7 +469,7 @@ public class TestFieldCache extends Luce
fail();
} catch (IllegalStateException expected) {}
- BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary");
+ BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
binary.get(0, scratch);
assertEquals("binary value", scratch.utf8ToString());
@@ -501,7 +502,7 @@ public class TestFieldCache extends Luce
fail();
} catch (IllegalStateException expected) {}
- binary = FieldCache.DEFAULT.getTerms(ar, "sorted");
+ binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
binary.get(0, scratch);
assertEquals("sorted value", scratch.utf8ToString());
@@ -525,7 +526,7 @@ public class TestFieldCache extends Luce
assertEquals(42, numeric.get(0));
try {
- FieldCache.DEFAULT.getTerms(ar, "numeric");
+ FieldCache.DEFAULT.getTerms(ar, "numeric", true);
fail();
} catch (IllegalStateException expected) {}
@@ -555,7 +556,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- FieldCache.DEFAULT.getTerms(ar, "sortedset");
+ FieldCache.DEFAULT.getTerms(ar, "sortedset", true);
fail();
} catch (IllegalStateException expected) {}
@@ -617,14 +618,14 @@ public class TestFieldCache extends Luce
assertEquals(0, doubles.get(0), 0.0D);
BytesRef scratch = new BytesRef();
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
sortedSet.setDocument(0);
@@ -682,14 +683,14 @@ public class TestFieldCache extends Luce
assertEquals(0, doubles.get(0), 0.0D);
BytesRef scratch = new BytesRef();
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
sortedSet.setDocument(0);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Fri Aug 30 15:57:58 2013
@@ -120,7 +120,7 @@ public class TestFieldCacheSanityChecker
cache.purgeAllCaches();
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
- cache.getTerms(readerX, "theInt");
+ cache.getTerms(readerX, "theInt", false);
cache.getBytes(readerX, "theByte", false);
// // //
@@ -143,9 +143,9 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getTerms(readerA, "theInt");
- cache.getTerms(readerB, "theInt");
- cache.getTerms(readerX, "theInt");
+ cache.getTerms(readerA, "theInt", false);
+ cache.getTerms(readerB, "theInt", false);
+ cache.getTerms(readerX, "theInt", false);
cache.getBytes(readerX, "theByte", false);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java Fri Aug 30 15:57:58 2013
@@ -56,7 +56,7 @@ public class TestFailOnFieldCacheInsanit
public void testDummy() throws Exception {
makeIndex();
assertNotNull(FieldCache.DEFAULT.getTermsIndex(subR, "ints"));
- assertNotNull(FieldCache.DEFAULT.getTerms(subR, "ints"));
+ assertNotNull(FieldCache.DEFAULT.getTerms(subR, "ints", false));
// NOTE: do not close reader/directory, else it
// purges FC entries
}
Modified: lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java Fri Aug 30 15:57:58 2013
@@ -109,7 +109,7 @@ abstract class TermsCollector extends Co
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
- fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
+ fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, false);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java Fri Aug 30 15:57:58 2013
@@ -129,7 +129,7 @@ abstract class TermsWithScoreCollector e
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
- fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
+ fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, false);
}
static class Avg extends SV {
Modified: lucene/dev/branches/branch_4x/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/branch_4x/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Fri Aug 30 15:57:58 2013
@@ -52,6 +52,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase.Slow;
@@ -504,13 +505,14 @@ public class TestJoinUtil extends Lucene
private Scorer scorer;
private BinaryDocValues terms;
+ private Bits docsWithField;
private final BytesRef spare = new BytesRef();
@Override
public void collect(int doc) throws IOException {
terms.get(doc, spare);
BytesRef joinValue = spare;
- if (joinValue.bytes == BinaryDocValues.MISSING) {
+ if (joinValue.length == 0 && !docsWithField.get(doc)) {
return;
}
@@ -523,7 +525,8 @@ public class TestJoinUtil extends Lucene
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
- terms = FieldCache.DEFAULT.getTerms(context.reader(), fromField);
+ terms = FieldCache.DEFAULT.getTerms(context.reader(), fromField, true);
+ docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), fromField);
}
@Override
@@ -621,7 +624,7 @@ public class TestJoinUtil extends Lucene
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
- terms = FieldCache.DEFAULT.getTerms(context.reader(), toField);
+ terms = FieldCache.DEFAULT.getTerms(context.reader(), toField, false);
docBase = context.docBase;
}
Modified: lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java Fri Aug 30 15:57:58 2013
@@ -148,8 +148,16 @@ public abstract class DocTermsIndexDocVa
@Override
public void fillValue(int doc) {
- termsIndex.get(doc, mval.value);
- mval.exists = mval.value.bytes != SortedDocValues.MISSING;
+ int ord = termsIndex.getOrd(doc);
+ if (ord == -1) {
+ mval.value.bytes = BytesRef.EMPTY_BYTES;
+ mval.value.offset = 0;
+ mval.value.length = 0;
+ mval.exists = false;
+ } else {
+ termsIndex.lookupOrd(ord, mval.value);
+ mval.exists = true;
+ }
}
};
}
Modified: lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Fri Aug 30 15:57:58 2013
@@ -45,8 +45,8 @@ public class BytesRefFieldSource extends
// To be sorted or not to be sorted, that is the question
// TODO: do it cleaner?
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field, true);
final Bits docsWithField = FieldCache.DEFAULT.getDocsWithField(readerContext.reader(), field);
- final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
return new FunctionValues() {
@Override
Modified: lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java Fri Aug 30 15:57:58 2013
@@ -56,7 +56,7 @@ public class JoinDocFreqValueSource exte
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException
{
- final BinaryDocValues terms = cache.getTerms(readerContext.reader(), field, PackedInts.FAST);
+ final BinaryDocValues terms = cache.getTerms(readerContext.reader(), field, false, PackedInts.FAST);
final IndexReader top = ReaderUtil.getTopLevelContext(readerContext).reader();
Terms t = MultiFields.getTerms(top, qfield);
final TermsEnum termsEnum = t == null ? TermsEnum.EMPTY : t.iterator(null);
Modified: lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java Fri Aug 30 15:57:58 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
/** Sorts by a field's value using the given Collator
@@ -39,6 +40,7 @@ public final class SlowCollatedStringCom
private final String[] values;
private BinaryDocValues currentDocTerms;
+ private Bits docsWithField;
private final String field;
final Collator collator;
private String bottom;
@@ -68,7 +70,7 @@ public final class SlowCollatedStringCom
@Override
public int compareBottom(int doc) {
currentDocTerms.get(doc, tempBR);
- final String val2 = tempBR.bytes == BinaryDocValues.MISSING ? null : tempBR.utf8ToString();
+ final String val2 = tempBR.length == 0 && docsWithField.get(doc) == false ? null : tempBR.utf8ToString();
if (bottom == null) {
if (val2 == null) {
return 0;
@@ -83,7 +85,7 @@ public final class SlowCollatedStringCom
@Override
public void copy(int slot, int doc) {
currentDocTerms.get(doc, tempBR);
- if (tempBR.bytes == BinaryDocValues.MISSING) {
+ if (tempBR.length == 0 && docsWithField.get(doc) == false) {
values[slot] = null;
} else {
values[slot] = tempBR.utf8ToString();
@@ -92,7 +94,8 @@ public final class SlowCollatedStringCom
@Override
public FieldComparator<String> setNextReader(AtomicReaderContext context) throws IOException {
- currentDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
+ currentDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, true);
+ docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
return this;
}
@@ -124,7 +127,7 @@ public final class SlowCollatedStringCom
public int compareDocToValue(int doc, String value) {
currentDocTerms.get(doc, tempBR);
final String docValue;
- if (tempBR.bytes == BinaryDocValues.MISSING) {
+ if (tempBR.length == 0 && docsWithField.get(doc) == false) {
docValue = null;
} else {
docValue = tempBR.utf8ToString();
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1519003&r1=1519002&r2=1519003&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Fri Aug 30 15:57:58 2013
@@ -2635,7 +2635,7 @@ public abstract class BaseDocValuesForma
AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field");
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
for(int docID=0;docID<docBytes.size();docID++) {
Document doc = ar.document(docID);
BytesRef bytes = new BytesRef();
@@ -2707,7 +2707,7 @@ public abstract class BaseDocValuesForma
AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field");
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
for(int docID=0;docID<docBytes.size();docID++) {
Document doc = ar.document(docID);
BytesRef bytes = new BytesRef();