You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/20 20:55:09 UTC
svn commit: r1448371 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/index/
lucene/core/src/test/org/apache...
Author: rmuir
Date: Wed Feb 20 19:55:08 2013
New Revision: 1448371
URL: http://svn.apache.org/r1448371
Log:
LUCENE-4790: FieldCache.getDocTermOrds back to the future bug
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Feb 20 19:55:08 2013
@@ -223,6 +223,10 @@ Bug Fixes
is enhanced to roll up beyond the bottom cell level. (David Smiley,
Florian Schilling)
+* LUCENE-4790: Fix FieldCacheImpl.getDocTermOrds to not bake deletes into the
+ cached datastructure. Otherwise this can cause inconsistencies with readers
+ at different points in time. (Robert Muir)
+
Documentation
* LUCENE-4718: Fixed documentation of oal.queryparser.classic.
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Wed Feb 20 19:55:08 2013
@@ -173,30 +173,29 @@ public class DocTermOrds {
}
/** Inverts all terms */
- public DocTermOrds(AtomicReader reader, String field) throws IOException {
- this(reader, field, null, Integer.MAX_VALUE);
+ public DocTermOrds(AtomicReader reader, Bits liveDocs, String field) throws IOException {
+ this(reader, liveDocs, field, null, Integer.MAX_VALUE);
}
/** Inverts only terms starting w/ prefix */
- public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix) throws IOException {
- this(reader, field, termPrefix, Integer.MAX_VALUE);
+ public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix) throws IOException {
+ this(reader, liveDocs, field, termPrefix, Integer.MAX_VALUE);
}
/** Inverts only terms starting w/ prefix, and only terms
* whose docFreq (not taking deletions into account) is
* <= maxTermDocFreq */
- public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix, int maxTermDocFreq) throws IOException {
- this(reader, field, termPrefix, maxTermDocFreq, DEFAULT_INDEX_INTERVAL_BITS);
- uninvert(reader, termPrefix);
+ public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix, int maxTermDocFreq) throws IOException {
+ this(reader, liveDocs, field, termPrefix, maxTermDocFreq, DEFAULT_INDEX_INTERVAL_BITS);
}
/** Inverts only terms starting w/ prefix, and only terms
* whose docFreq (not taking deletions into account) is
* <= maxTermDocFreq, with a custom indexing interval
* (default is every 128nd term). */
- public DocTermOrds(AtomicReader reader, String field, BytesRef termPrefix, int maxTermDocFreq, int indexIntervalBits) throws IOException {
+ public DocTermOrds(AtomicReader reader, Bits liveDocs, String field, BytesRef termPrefix, int maxTermDocFreq, int indexIntervalBits) throws IOException {
this(field, maxTermDocFreq, indexIntervalBits);
- uninvert(reader, termPrefix);
+ uninvert(reader, liveDocs, termPrefix);
}
/** Subclass inits w/ this, but be sure you then call
@@ -257,14 +256,14 @@ public class DocTermOrds {
protected void visitTerm(TermsEnum te, int termNum) throws IOException {
}
- /** Invoked during {@link #uninvert(AtomicReader,BytesRef)}
+ /** Invoked during {@link #uninvert(AtomicReader,Bits,BytesRef)}
* to record the document frequency for each uninverted
* term. */
protected void setActualDocFreq(int termNum, int df) throws IOException {
}
/** Call this only once (if you subclass!) */
- protected void uninvert(final AtomicReader reader, final BytesRef termPrefix) throws IOException {
+ protected void uninvert(final AtomicReader reader, Bits liveDocs, final BytesRef termPrefix) throws IOException {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info != null && info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
@@ -304,8 +303,6 @@ public class DocTermOrds {
boolean testedOrd = false;
- final Bits liveDocs = reader.getLiveDocs();
-
// we need a minimum of 9 bytes, but round up to 12 since the space would
// be wasted with most allocators anyway.
byte[] tempArr = new byte[12];
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Wed Feb 20 19:55:08 2013
@@ -1399,7 +1399,7 @@ class FieldCacheImpl implements FieldCac
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
- return new DocTermOrds(reader, key.field);
+ return new DocTermOrds(reader, null, key.field);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java Wed Feb 20 19:55:08 2013
@@ -63,7 +63,7 @@ public class TestDocTermOrds extends Luc
w.close();
final AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
- final DocTermOrds dto = new DocTermOrds(ar, "field");
+ final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field");
SortedSetDocValues iter = dto.iterator(ar.terms("field").iterator(null));
iter.setDocument(0);
@@ -295,7 +295,7 @@ public class TestDocTermOrds extends Luc
private void verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef) throws Exception {
- final DocTermOrds dto = new DocTermOrds(r,
+ final DocTermOrds dto = new DocTermOrds(r, r.getLiveDocs(),
"field",
prefixRef,
Integer.MAX_VALUE,
@@ -372,4 +372,34 @@ public class TestDocTermOrds extends Luc
assertEquals(answers.length, upto);
}
}
+
+ public void testBackToTheFuture() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+
+ Document doc = new Document();
+ doc.add(newStringField("foo", "bar", Field.Store.NO));
+ iw.addDocument(doc);
+
+ doc = new Document();
+ doc.add(newStringField("foo", "baz", Field.Store.NO));
+ iw.addDocument(doc);
+
+ DirectoryReader r1 = DirectoryReader.open(iw, true);
+
+ iw.deleteDocuments(new Term("foo", "baz"));
+ DirectoryReader r2 = DirectoryReader.open(iw, true);
+
+ FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r2), "foo");
+
+ SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r1), "foo");
+ assertEquals(2, v.getValueCount());
+ v.setDocument(1);
+ assertEquals(1, v.nextOrd());
+
+ iw.close();
+ r1.close();
+ r2.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java Wed Feb 20 19:55:08 2013
@@ -473,7 +473,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- new DocTermOrds(ar, "binary");
+ new DocTermOrds(ar, null, "binary");
fail();
} catch (IllegalStateException expected) {}
@@ -487,7 +487,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- new DocTermOrds(ar, "sorted");
+ new DocTermOrds(ar, null, "sorted");
fail();
} catch (IllegalStateException expected) {}
@@ -530,7 +530,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- new DocTermOrds(ar, "numeric");
+ new DocTermOrds(ar, null, "numeric");
fail();
} catch (IllegalStateException expected) {}
@@ -555,7 +555,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- new DocTermOrds(ar, "sortedset");
+ new DocTermOrds(ar, null, "sortedset");
fail();
} catch (IllegalStateException expected) {}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1448371&r1=1448370&r2=1448371&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Wed Feb 20 19:55:08 2013
@@ -23,6 +23,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
@@ -174,7 +175,8 @@ public class UnInvertedField extends Doc
final String prefix = TrieField.getMainValuePrefix(searcher.getSchema().getFieldType(field));
this.searcher = searcher;
try {
- uninvert(searcher.getAtomicReader(), prefix == null ? null : new BytesRef(prefix));
+ AtomicReader r = searcher.getAtomicReader();
+ uninvert(r, r.getLiveDocs(), prefix == null ? null : new BytesRef(prefix));
} catch (IllegalStateException ise) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ise.getMessage());
}