You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/12/11 17:31:37 UTC
svn commit: r889688 - in /lucene/java/branches/lucene_3_0: ./ contrib/
contrib/highlighter/src/test/
contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
src/java/org/ap...
Author: mikemccand
Date: Fri Dec 11 16:31:36 2009
New Revision: 889688
URL: http://svn.apache.org/viewvc?rev=889688&view=rev
Log:
LUCENE-2144: fix instantiated to handle termDocs(null) correctly
Added:
lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedAllTermDocs.java
- copied unchanged from r889432, lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedAllTermDocs.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AbstractAllTermDocs.java
- copied, changed from r889432, lucene/java/trunk/src/java/org/apache/lucene/index/AbstractAllTermDocs.java
Modified:
lucene/java/branches/lucene_3_0/ (props changed)
lucene/java/branches/lucene_3_0/CHANGES.txt (props changed)
lucene/java/branches/lucene_3_0/build.xml (props changed)
lucene/java/branches/lucene_3_0/contrib/ (props changed)
lucene/java/branches/lucene_3_0/contrib/CHANGES.txt (contents, props changed)
lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/ (props changed)
lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java
lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (contents, props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AllTermDocs.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4:748824
/lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
-/lucene/java/trunk:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336
-/lucene/java/trunk/CHANGES.txt:881213,881315,881466,882374,882464,882672,882807,882888,882977,883074-883075,883554,883654,883661,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/CHANGES.txt:881213,881315,881466,882374,882464,882672,882807,882888,882977,883074-883075,883554,883654,883661,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/build.xml:748824
/lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336
-/lucene/java/trunk/build.xml:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887617,888247
+/lucene/java/trunk/build.xml:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887617,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/contrib:748824
/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
-/lucene/java/trunk/contrib:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/contrib:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Modified: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/CHANGES.txt?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/contrib/CHANGES.txt Fri Dec 11 16:31:36 2009
@@ -2,6 +2,12 @@
======================= 3.0 branch (not yet released) =======================
+Bug fixes
+
+ * LUCENE-2144: Fix InstantiatedIndex to handle termDocs(null)
+ correctly (enumerate all non-deleted docs). (Karl Wettin via Mike
+ McCandless)
+
API Changes
* LUCENE-2108: Add SpellChecker.close, to close the underlying
Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
-/lucene/java/trunk/contrib/CHANGES.txt:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/contrib/CHANGES.txt:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
-/lucene/java/trunk/contrib/highlighter/src/test:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/contrib/highlighter/src/test:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Modified: lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Fri Dec 11 16:31:36 2009
@@ -35,6 +35,7 @@
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.util.BitVector;
/**
* Represented as a coupled graph of class instances, this
@@ -61,8 +62,7 @@
private InstantiatedDocument[] documentsByNumber;
- /** todo: should this be a BitSet? */
- private Set<Integer> deletedDocuments;
+ private BitVector deletedDocuments;
private Map<String, Map<String, InstantiatedTerm>> termsByFieldAndText;
private InstantiatedTerm[] orderedTerms;
@@ -85,7 +85,6 @@
orderedTerms = new InstantiatedTerm[0];
documentsByNumber = new InstantiatedDocument[0];
normsByFieldNameAndDocumentNumber = new HashMap<String, byte[]>();
- deletedDocuments = new HashSet<Integer>();
}
@@ -174,11 +173,14 @@
documentsByNumber = new InstantiatedDocument[sourceIndexReader.maxDoc()];
+ if (sourceIndexReader.hasDeletions()) {
+ deletedDocuments = new BitVector(sourceIndexReader.maxDoc());
+ }
// create documents
for (int i = 0; i < sourceIndexReader.maxDoc(); i++) {
- if (sourceIndexReader.isDeleted(i)) {
- deletedDocuments.add(i);
+ if (sourceIndexReader.hasDeletions() && sourceIndexReader.isDeleted(i)) {
+ deletedDocuments.set(i);
} else {
InstantiatedDocument document = new InstantiatedDocument();
// copy stored fields from source reader
@@ -329,10 +331,13 @@
this.normsByFieldNameAndDocumentNumber = normsByFieldNameAndDocumentNumber;
}
- public Set<Integer> getDeletedDocuments() {
+ public BitVector getDeletedDocuments() {
return deletedDocuments;
}
+ void setDeletedDocuments(BitVector deletedDocuments) {
+ this.deletedDocuments = deletedDocuments;
+ }
void setOrderedTerms(InstantiatedTerm[] orderedTerms) {
this.orderedTerms = orderedTerms;
Modified: lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Fri Dec 11 16:31:36 2009
@@ -29,16 +29,9 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.index.TermPositions;
-import org.apache.lucene.index.TermVectorMapper;
+import org.apache.lucene.index.*;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitVector;
/**
* An InstantiatedIndexReader is not a snapshot in time, it is completely in
@@ -105,9 +98,9 @@
return index;
}
- private Set<InstantiatedDocument> deletedDocuments = new HashSet<InstantiatedDocument>();
- private Set<Integer> deletedDocumentNumbers = new HashSet<Integer>();
- private Map<String,List<NormUpdate>> updatedNormsByFieldNameAndDocumentNumber = null;
+ private BitVector uncommittedDeletedDocuments;
+
+ private Map<String,List<NormUpdate>> uncommittedNormsByFieldNameAndDocumentNumber = null;
private class NormUpdate {
private int doc;
@@ -121,7 +114,15 @@
@Override
public int numDocs() {
- return getIndex().getDocumentsByNumber().length - index.getDeletedDocuments().size() - deletedDocuments.size();
+ // todo i suppose this value could be cached, but array#length and bitvector#count is fast.
+ int numDocs = getIndex().getDocumentsByNumber().length;
+ if (uncommittedDeletedDocuments != null) {
+ numDocs -= uncommittedDeletedDocuments.count();
+ }
+ if (index.getDeletedDocuments() != null) {
+ numDocs -= index.getDeletedDocuments().count();
+ }
+ return numDocs;
}
@Override
@@ -130,28 +131,39 @@
}
@Override
- public boolean isDeleted(int n) {
- return getIndex().getDeletedDocuments().contains(n) || deletedDocumentNumbers.contains(n);
+ public boolean hasDeletions() {
+ return index.getDeletedDocuments() != null || uncommittedDeletedDocuments != null;
}
+
@Override
- public boolean hasDeletions() {
- return getIndex().getDeletedDocuments().size() > 0 || deletedDocumentNumbers.size() > 0;
+ public boolean isDeleted(int n) {
+ return (index.getDeletedDocuments() != null && index.getDeletedDocuments().get(n))
+ || (uncommittedDeletedDocuments != null && uncommittedDeletedDocuments.get(n));
}
+
@Override
protected void doDelete(int docNum) throws IOException {
- if (!getIndex().getDeletedDocuments().contains(docNum)) {
- if (deletedDocumentNumbers.add(docNum)) {
- deletedDocuments.add(getIndex().getDocumentsByNumber()[docNum]);
- }
+
+ // dont delete if already deleted
+ if ((index.getDeletedDocuments() != null && index.getDeletedDocuments().get(docNum))
+ || (uncommittedDeletedDocuments != null && uncommittedDeletedDocuments.get(docNum))) {
+ return;
}
+
+ if (uncommittedDeletedDocuments == null) {
+ uncommittedDeletedDocuments = new BitVector(maxDoc());
+ }
+
+ uncommittedDeletedDocuments.set(docNum);
}
@Override
protected void doUndeleteAll() throws IOException {
- deletedDocumentNumbers.clear();
- deletedDocuments.clear();
+ // todo: read/write lock
+ uncommittedDeletedDocuments = null;
+ // todo: read/write unlock
}
@Override
@@ -161,25 +173,30 @@
boolean updated = false;
// 1. update norms
- if (updatedNormsByFieldNameAndDocumentNumber != null) {
- for (Map.Entry<String,List<NormUpdate>> e : updatedNormsByFieldNameAndDocumentNumber.entrySet()) {
+ if (uncommittedNormsByFieldNameAndDocumentNumber != null) {
+ for (Map.Entry<String,List<NormUpdate>> e : uncommittedNormsByFieldNameAndDocumentNumber.entrySet()) {
byte[] norms = getIndex().getNormsByFieldNameAndDocumentNumber().get(e.getKey());
for (NormUpdate normUpdate : e.getValue()) {
norms[normUpdate.doc] = normUpdate.value;
}
}
- updatedNormsByFieldNameAndDocumentNumber = null;
+ uncommittedNormsByFieldNameAndDocumentNumber = null;
updated = true;
}
// 2. remove deleted documents
- if (deletedDocumentNumbers.size() > 0) {
- for (Integer doc : deletedDocumentNumbers) {
- getIndex().getDeletedDocuments().add(doc);
+ if (uncommittedDeletedDocuments != null) {
+ if (index.getDeletedDocuments() == null) {
+ index.setDeletedDocuments(uncommittedDeletedDocuments);
+ } else {
+ for (int d = 0; d< uncommittedDeletedDocuments.size(); d++) {
+ if (uncommittedDeletedDocuments.get(d)) {
+ index.getDeletedDocuments().set(d);
+ }
+ }
}
- deletedDocumentNumbers.clear();
- deletedDocuments.clear();
+ uncommittedDeletedDocuments = null;
updated = true;
@@ -299,9 +316,9 @@
if (norms == null) {
return new byte[0]; // todo a static final zero length attribute?
}
- if (updatedNormsByFieldNameAndDocumentNumber != null) {
+ if (uncommittedNormsByFieldNameAndDocumentNumber != null) {
norms = norms.clone();
- List<NormUpdate> updated = updatedNormsByFieldNameAndDocumentNumber.get(field);
+ List<NormUpdate> updated = uncommittedNormsByFieldNameAndDocumentNumber.get(field);
if (updated != null) {
for (NormUpdate normUpdate : updated) {
norms[normUpdate.doc] = normUpdate.value;
@@ -322,13 +339,13 @@
@Override
protected void doSetNorm(int doc, String field, byte value) throws IOException {
- if (updatedNormsByFieldNameAndDocumentNumber == null) {
- updatedNormsByFieldNameAndDocumentNumber = new HashMap<String,List<NormUpdate>>(getIndex().getNormsByFieldNameAndDocumentNumber().size());
+ if (uncommittedNormsByFieldNameAndDocumentNumber == null) {
+ uncommittedNormsByFieldNameAndDocumentNumber = new HashMap<String,List<NormUpdate>>(getIndex().getNormsByFieldNameAndDocumentNumber().size());
}
- List<NormUpdate> list = updatedNormsByFieldNameAndDocumentNumber.get(field);
+ List<NormUpdate> list = uncommittedNormsByFieldNameAndDocumentNumber.get(field);
if (list == null) {
list = new LinkedList<NormUpdate>();
- updatedNormsByFieldNameAndDocumentNumber.put(field, list);
+ uncommittedNormsByFieldNameAndDocumentNumber.put(field, list);
}
list.add(new NormUpdate(doc, value));
}
@@ -367,6 +384,18 @@
return new InstantiatedTermDocs(this);
}
+
+ @Override
+ public TermDocs termDocs(Term term) throws IOException {
+ if (term == null) {
+ return new InstantiatedAllTermDocs(this);
+ } else {
+ InstantiatedTermDocs termDocs = new InstantiatedTermDocs(this);
+ termDocs.seek(term);
+ return termDocs;
+ }
+ }
+
@Override
public TermPositions termPositions() throws IOException {
return new InstantiatedTermPositions(this);
@@ -411,7 +440,7 @@
@Override
public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
InstantiatedDocument doc = getIndex().getDocumentsByNumber()[docNumber];
- for (Map.Entry<String,List<InstantiatedTermDocumentInformation>> e : doc.getVectorSpace().entrySet()) {
+ for (Map.Entry<String, List<InstantiatedTermDocumentInformation>> e : doc.getVectorSpace().entrySet()) {
mapper.setExpectations(e.getKey(), e.getValue().size(), true, true);
for (InstantiatedTermDocumentInformation tdi : e.getValue()) {
mapper.map(tdi.getTerm().text(), tdi.getTermPositions().length, tdi.getTermOffsets(), tdi.getTermPositions());
Modified: lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Fri Dec 11 16:31:36 2009
@@ -47,6 +47,7 @@
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BitVector;
/**
* This class, similar to {@link org.apache.lucene.index.IndexWriter}, has no locking mechanism.
@@ -407,6 +408,18 @@
termDocumentInformationFactoryByDocument.clear();
fieldNameBuffer.clear();
+
+ // update deleted documents bitset
+ if (index.getDeletedDocuments() != null) {
+ BitVector deletedDocuments = new BitVector(index.getDocumentsByNumber().length);
+ for (int i = 0; i < index.getDeletedDocuments().size(); i++) {
+ if (index.getDeletedDocuments().get(i)) {
+ deletedDocuments.set(i);
+ }
+ }
+ index.setDeletedDocuments(deletedDocuments);
+ }
+
index.setVersion(System.currentTimeMillis());
// todo unlock
Modified: lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java Fri Dec 11 16:31:36 2009
@@ -60,11 +60,14 @@
currentDocumentIndex++;
if (currentDocumentIndex < currentTerm.getAssociatedDocuments().length) {
currentDocumentInformation = currentTerm.getAssociatedDocuments()[currentDocumentIndex];
- if (reader.hasDeletions() && reader.isDeleted(currentDocumentInformation.getDocument().getDocumentNumber())) {
+ if (reader.isDeleted(currentDocumentInformation.getDocument().getDocumentNumber())) {
return next();
} else {
return true;
}
+ } else {
+ // mimic SegmentTermDocs
+ currentDocumentIndex = currentTerm.getAssociatedDocuments().length -1;
}
}
return false;
@@ -111,6 +114,8 @@
int pos = currentTerm.seekCeilingDocumentInformationIndex(target, startOffset);
if (pos == -1) {
+ // mimic SegmentTermDocs that positions at the last index
+ currentDocumentIndex = currentTerm.getAssociatedDocuments().length -1;
return false;
}
Modified: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Fri Dec 11 16:31:36 2009
@@ -41,8 +41,6 @@
import org.apache.lucene.index.TermPositions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.AttributeImpl;
/**
@@ -104,53 +102,155 @@
}
instantiatedIndexWriter.close();
+
testEqualBehaviour(dir, ii);
- testTermDocs(dir, ii);
}
- private void testTermDocs(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception {
+ private void testTermDocsSomeMore(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception {
IndexReader aprioriReader = IndexReader.open(aprioriIndex, false);
IndexReader testReader = testIndex.indexReaderFactory();
- TermEnum aprioriTermEnum = aprioriReader.terms(new Term("c", "danny"));
+ // test seek
- TermDocs aprioriTermDocs = aprioriReader.termDocs(aprioriTermEnum.term());
- TermDocs testTermDocs = testReader.termDocs(aprioriTermEnum.term());
+ Term t = new Term("c", "danny");
+ TermEnum aprioriTermEnum = aprioriReader.terms(t);
+ TermEnum testTermEnum = testReader.terms(t);
- assertEquals(aprioriTermDocs.next(), testTermDocs.next());
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ assertEquals(aprioriTermEnum.term(), testTermEnum.term());
- assertEquals(aprioriTermDocs.skipTo(100), testTermDocs.skipTo(100));
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ t = aprioriTermEnum.term();
- assertEquals(aprioriTermDocs.next(), testTermDocs.next());
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ aprioriTermEnum.close();
+ testTermEnum.close();
+
+ TermDocs aprioriTermDocs = aprioriReader.termDocs(t);
+ TermDocs testTermDocs = testReader.termDocs(t);
assertEquals(aprioriTermDocs.next(), testTermDocs.next());
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
- assertEquals(aprioriTermDocs.skipTo(110), testTermDocs.skipTo(110));
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ if (aprioriTermDocs.skipTo(4)) {
+ assertTrue(testTermDocs.skipTo(4));
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.skipTo(4));
+ }
- assertEquals(aprioriTermDocs.skipTo(10), testTermDocs.skipTo(10));
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ if (aprioriTermDocs.next()) {
+ assertTrue(testTermDocs.next());
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.next());
+ }
- assertEquals(aprioriTermDocs.skipTo(210), testTermDocs.skipTo(210));
- assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+ // beyond this point all next and skipto will return false
+
+ if (aprioriTermDocs.skipTo(100)) {
+ assertTrue(testTermDocs.skipTo(100));
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.skipTo(100));
+ }
+
+
+ if (aprioriTermDocs.next()) {
+ assertTrue(testTermDocs.next());
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.next());
+ }
+
+ if (aprioriTermDocs.skipTo(110)) {
+ assertTrue(testTermDocs.skipTo(110));
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.skipTo(110));
+ }
+
+ if (aprioriTermDocs.skipTo(10)) {
+ assertTrue(testTermDocs.skipTo(10));
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.skipTo(10));
+ }
+
+
+ if (aprioriTermDocs.skipTo(210)) {
+ assertTrue(testTermDocs.skipTo(210));
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ } else {
+ assertFalse(testTermDocs.skipTo(210));
+ }
aprioriTermDocs.close();
- aprioriReader.close();
+ testTermDocs.close();
+
+
+
+ // test seek null (AllTermDocs)
+ aprioriTermDocs = aprioriReader.termDocs(null);
+ testTermDocs = testReader.termDocs(null);
+
+ while (aprioriTermDocs.next()) {
+ assertTrue(testTermDocs.next());
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ }
+ assertFalse(testTermDocs.next());
+
+
+ aprioriTermDocs.close();
+ testTermDocs.close();
+
+
+ // test seek default
+ aprioriTermDocs = aprioriReader.termDocs();
+ testTermDocs = testReader.termDocs();
+
+ // this is invalid use of the API,
+ // but if the response differs then it's an indication that something might have changed.
+ // in 2.9 and 3.0 the two TermDocs-implementations returned different values at this point.
+// assertEquals("Descripency during invalid use of the TermDocs API, see comments in test code for details.",
+// aprioriTermDocs.next(), testTermDocs.next());
+
+ // start using the API the way one is supposed to use it
+
+ t = new Term("", "");
+ aprioriTermDocs.seek(t);
+ testTermDocs.seek(t);
+
+ while (aprioriTermDocs.next()) {
+ assertTrue(testTermDocs.next());
+ assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+ assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+ }
+ assertFalse(testTermDocs.next());
+ aprioriTermDocs.close();
testTermDocs.close();
+
+
+ // clean up
+ aprioriReader.close();
testReader.close();
}
+
private void assembleDocument(Document document, int i) {
document.add(new Field("a", i + " Do you really want to go and live in that house all winter?", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
if (i > 0) {
@@ -213,18 +313,57 @@
*/
protected void testEqualBehaviour(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception {
+
testEquals(aprioriIndex, testIndex);
- // delete a few documents
- IndexReader ir = IndexReader.open(aprioriIndex, false);
- ir.deleteDocument(3);
- ir.deleteDocument(8);
- ir.close();
+ // delete a few documents
+ IndexReader air = IndexReader.open(aprioriIndex, false);
+ InstantiatedIndexReader tir = testIndex.indexReaderFactory();
+
+ assertEquals(air.isCurrent(), tir.isCurrent());
+ assertEquals(air.hasDeletions(), tir.hasDeletions());
+ assertEquals(air.maxDoc(), tir.maxDoc());
+ assertEquals(air.numDocs(), tir.numDocs());
+ assertEquals(air.numDeletedDocs(), tir.numDeletedDocs());
+
+ air.deleteDocument(3);
+ tir.deleteDocument(3);
+
+ assertEquals(air.isCurrent(), tir.isCurrent());
+ assertEquals(air.hasDeletions(), tir.hasDeletions());
+ assertEquals(air.maxDoc(), tir.maxDoc());
+ assertEquals(air.numDocs(), tir.numDocs());
+ assertEquals(air.numDeletedDocs(), tir.numDeletedDocs());
+
+ air.deleteDocument(8);
+ tir.deleteDocument(8);
+
+ assertEquals(air.isCurrent(), tir.isCurrent());
+ assertEquals(air.hasDeletions(), tir.hasDeletions());
+ assertEquals(air.maxDoc(), tir.maxDoc());
+ assertEquals(air.numDocs(), tir.numDocs());
+ assertEquals(air.numDeletedDocs(), tir.numDeletedDocs());
+
+ // this (in 3.0) commits the deletions
+ air.close();
+ tir.close();
+
+ air = IndexReader.open(aprioriIndex, false);
+ tir = testIndex.indexReaderFactory();
+
+ assertEquals(air.isCurrent(), tir.isCurrent());
+ assertEquals(air.hasDeletions(), tir.hasDeletions());
+ assertEquals(air.maxDoc(), tir.maxDoc());
+ assertEquals(air.numDocs(), tir.numDocs());
+ assertEquals(air.numDeletedDocs(), tir.numDeletedDocs());
+
+ for (int d =0; d<air.maxDoc(); d++) {
+ assertEquals(air.isDeleted(d), tir.isDeleted(d));
+ }
+
+ air.close();
+ tir.close();
- ir = testIndex.indexReaderFactory();
- ir.deleteDocument(3);
- ir.deleteDocument(8);
- ir.close();
// make sure they still equal
testEquals(aprioriIndex, testIndex);
@@ -232,6 +371,8 @@
protected void testEquals(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception {
+ testTermDocsSomeMore(aprioriIndex, testIndex);
+
IndexReader aprioriReader = IndexReader.open(aprioriIndex, false);
IndexReader testReader = testIndex.indexReaderFactory();
Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -0,0 +1,5 @@
+/lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036
+/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336
+/lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:889463
+/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Copied: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AbstractAllTermDocs.java (from r889432, lucene/java/trunk/src/java/org/apache/lucene/index/AbstractAllTermDocs.java)
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AbstractAllTermDocs.java?p2=lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AbstractAllTermDocs.java&p1=lucene/java/trunk/src/java/org/apache/lucene/index/AbstractAllTermDocs.java&r1=889432&r2=889688&rev=889688&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/AbstractAllTermDocs.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AbstractAllTermDocs.java Fri Dec 11 16:31:36 2009
@@ -19,6 +19,13 @@
import java.io.IOException;
+/** Base class for enumerating all but deleted docs.
+ *
+ * <p>NOTE: this class is meant only to be used internally
+ * by Lucene; it's only public so it can be shared across
+ * packages. This means the API is freely subject to
+ * change, and, the class could be removed entirely, in any
+ * Lucene release. Use directly at your own risk! */
public abstract class AbstractAllTermDocs implements TermDocs {
protected int maxDoc;
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AllTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AllTermDocs.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AllTermDocs.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/AllTermDocs.java Fri Dec 11 16:31:36 2009
@@ -18,69 +18,19 @@
package org.apache.lucene.index;
import org.apache.lucene.util.BitVector;
-import java.io.IOException;
-class AllTermDocs implements TermDocs {
+class AllTermDocs extends AbstractAllTermDocs {
+
protected BitVector deletedDocs;
- protected int maxDoc;
- protected int doc = -1;
protected AllTermDocs(SegmentReader parent) {
+ super(parent.maxDoc());
synchronized (parent) {
this.deletedDocs = parent.deletedDocs;
}
- this.maxDoc = parent.maxDoc();
- }
-
- public void seek(Term term) throws IOException {
- if (term==null) {
- doc = -1;
- } else {
- throw new UnsupportedOperationException();
- }
- }
-
- public void seek(TermEnum termEnum) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- public int doc() {
- return doc;
- }
-
- public int freq() {
- return 1;
- }
-
- public boolean next() throws IOException {
- return skipTo(doc+1);
- }
-
- public int read(int[] docs, int[] freqs) throws IOException {
- final int length = docs.length;
- int i = 0;
- while (i < length && doc < maxDoc) {
- if (deletedDocs == null || !deletedDocs.get(doc)) {
- docs[i] = doc;
- freqs[i] = 1;
- ++i;
- }
- doc++;
- }
- return i;
- }
-
- public boolean skipTo(int target) throws IOException {
- doc = target;
- while (doc < maxDoc) {
- if (deletedDocs == null || !deletedDocs.get(doc)) {
- return true;
- }
- doc++;
- }
- return false;
}
- public void close() throws IOException {
+ public boolean isDeleted(int doc) {
+ return deletedDocs != null && deletedDocs.get(doc);
}
}
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java?rev=889688&r1=889687&r2=889688&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java Fri Dec 11 16:31:36 2009
@@ -931,7 +931,7 @@
}
/**
- * @see IndexReader#getFieldNames(IndexReader.FieldOption fldOption)
+ * @see IndexReader#getFieldNames(org.apache.lucene.index.IndexReader.FieldOption)
*/
@Override
public Collection<String> getFieldNames(IndexReader.FieldOption fieldOption) {
Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336
-/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881984,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881984,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,3 +1,3 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036
-/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,3 +1,3 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,3 +1,3 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,3 +1,3 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036
-/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887532,887602,888247,889431-889432
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 16:31:36 2009
@@ -1,4 +1,4 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
-/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883079,883554,884870,886257,886911,887532,887602,888247
+/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883079,883554,884870,886257,886911,887532,887602,888247,889431-889432