You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/01/25 17:27:01 UTC
svn commit: r1063338 - in /lucene/dev/branches/bulkpostings: ./ lucene/
lucene/src/java/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/codecs/preflex/
lucene/src/java/org/apache/lucene/index/codecs/pulsing/
lucene/src/java/org/apache/...
Author: simonw
Date: Tue Jan 25 16:27:00 2011
New Revision: 1063338
URL: http://svn.apache.org/viewvc?rev=1063338&view=rev
Log:
merged revision 1063272 to 1063333
Added:
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
- copied unchanged from r1063336, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
Modified:
lucene/dev/branches/bulkpostings/ (props changed)
lucene/dev/branches/bulkpostings/lucene/ (props changed)
lucene/dev/branches/bulkpostings/lucene/CHANGES.txt
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java
lucene/dev/branches/bulkpostings/solr/ (props changed)
lucene/dev/branches/bulkpostings/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/bulkpostings/solr/KEYS (props changed)
lucene/dev/branches/bulkpostings/solr/LICENSE.txt (props changed)
lucene/dev/branches/bulkpostings/solr/NOTICE.txt (props changed)
lucene/dev/branches/bulkpostings/solr/README.txt (props changed)
lucene/dev/branches/bulkpostings/solr/build.xml (props changed)
lucene/dev/branches/bulkpostings/solr/client/ (props changed)
lucene/dev/branches/bulkpostings/solr/common-build.xml (props changed)
lucene/dev/branches/bulkpostings/solr/contrib/ (props changed)
lucene/dev/branches/bulkpostings/solr/example/ (props changed)
lucene/dev/branches/bulkpostings/solr/lib/ (props changed)
lucene/dev/branches/bulkpostings/solr/site/ (props changed)
lucene/dev/branches/bulkpostings/solr/src/ (props changed)
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/branches/bulkpostings/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/bulkpostings/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/CHANGES.txt?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/lucene/CHANGES.txt Tue Jan 25 16:27:00 2011
@@ -486,6 +486,9 @@ Changes in runtime behavior
* LUCENE-2829: Improve the performance of "primary key" lookup use
case (running a TermQuery that matches one document) on a
multi-segment index. (Robert Muir, Mike McCandless)
+
+* LUCENE-2010: Segments with 100% deleted documents are now removed on
+ IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
API Changes
@@ -905,6 +908,9 @@ Optimizations
* LUCENE-2824: Optimize BufferedIndexInput to do less bounds checks.
(Robert Muir)
+* LUCENE-2010: Segments with 100% deleted documents are now removed on
+ IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
+
Build
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Tue Jan 25 16:27:00 2011
@@ -270,9 +270,9 @@ class BufferedDeletes {
}
private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool,
- SegmentInfo info,
- SegmentDeletes coalescedDeletes,
- SegmentDeletes segmentDeletes) throws IOException {
+ SegmentInfo info,
+ SegmentDeletes coalescedDeletes,
+ SegmentDeletes segmentDeletes) throws IOException {
assert readerPool.infoIsLive(info);
assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Jan 25 16:27:00 2011
@@ -710,6 +710,9 @@ class DirectoryReader extends IndexReade
for (int i = 0; i < subReaders.length; i++)
subReaders[i].commit();
+ // Remove segments that contain only 100% deleted docs:
+ segmentInfos.pruneDeletedSegments();
+
// Sync all files we just wrote
directory.sync(segmentInfos.files(directory, false));
segmentInfos.commit(directory);
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Jan 25 16:27:00 2011
@@ -1195,7 +1195,14 @@ public abstract class IndexReader implem
return n;
}
- /** Undeletes all documents currently marked as deleted in this index.
+ /** Undeletes all documents currently marked as deleted in
+ * this index.
+ *
+ * <p>NOTE: this is only a best-effort process. For
+ * example, if all documents in a given segment were
+ * deleted, Lucene now drops that segment from the index,
+ * which means its documents will not be recovered by this
+ * method.
*
* @throws StaleReaderException if the index has changed
* since this reader was opened
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jan 25 16:27:00 2011
@@ -3276,6 +3276,15 @@ public class IndexWriter implements Clos
}
}
+ private boolean keepFullyDeletedSegments;
+
+ /** Only for testing.
+ *
+ * @lucene.internal */
+ void keepFullyDeletedSegments() {
+ keepFullyDeletedSegments = true;
+ }
+
// called only from assert
private boolean filesExist(SegmentInfos toSync) throws IOException {
Collection<String> files = toSync.files(directory, false);
@@ -3334,6 +3343,10 @@ public class IndexWriter implements Clos
readerPool.commit();
toSync = (SegmentInfos) segmentInfos.clone();
+ if (!keepFullyDeletedSegments) {
+ toSync.pruneDeletedSegments();
+ }
+
assert filesExist(toSync);
if (commitUserData != null)
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Tue Jan 25 16:27:00 2011
@@ -308,6 +308,19 @@ public final class SegmentInfos extends
}
}
+ /** Prunes any segment whose docs are all deleted. */
+ public void pruneDeletedSegments() {
+ int segIdx = 0;
+ while(segIdx < size()) {
+ final SegmentInfo info = info(segIdx);
+ if (info.getDelCount() == info.docCount) {
+ remove(segIdx);
+ } else {
+ segIdx++;
+ }
+ }
+ }
+
/**
* Returns a copy of this instance, also copying each
* SegmentInfo.
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Jan 25 16:27:00 2011
@@ -1002,7 +1002,7 @@ public class PreFlexFields extends Field
private final class PreDocsEnum extends DocsEnum {
final private SegmentTermDocs docs;
-
+ private int docID = -1;
PreDocsEnum() throws IOException {
docs = new SegmentTermDocs(freqStream, getTermsDict(), fieldInfos);
}
@@ -1020,18 +1020,18 @@ public class PreFlexFields extends Field
@Override
public int nextDoc() throws IOException {
if (docs.next()) {
- return docs.doc();
+ return docID = docs.doc();
} else {
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
}
@Override
public int advance(int target) throws IOException {
if (docs.skipTo(target)) {
- return docs.doc();
+ return docID = docs.doc();
} else {
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
}
@@ -1042,13 +1042,13 @@ public class PreFlexFields extends Field
@Override
public int docID() {
- return docs.doc();
+ return docID;
}
}
private final class PreDocsAndPositionsEnum extends DocsAndPositionsEnum {
final private SegmentTermPositions pos;
-
+ private int docID = -1;
PreDocsAndPositionsEnum() throws IOException {
pos = new SegmentTermPositions(freqStream, proxStream, getTermsDict(), fieldInfos);
}
@@ -1066,18 +1066,18 @@ public class PreFlexFields extends Field
@Override
public int nextDoc() throws IOException {
if (pos.next()) {
- return pos.doc();
+ return docID = pos.doc();
} else {
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
}
@Override
public int advance(int target) throws IOException {
if (pos.skipTo(target)) {
- return pos.doc();
+ return docID = pos.doc();
} else {
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
}
@@ -1088,16 +1088,18 @@ public class PreFlexFields extends Field
@Override
public int docID() {
- return pos.doc();
+ return docID;
}
@Override
public int nextPosition() throws IOException {
+ assert docID != NO_MORE_DOCS;
return pos.nextPosition();
}
@Override
public boolean hasPayload() {
+ assert docID != NO_MORE_DOCS;
return pos.isPayloadAvailable();
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java Tue Jan 25 16:27:00 2011
@@ -287,7 +287,7 @@ public class PulsingPostingsReaderImpl e
while(true) {
if (postings.eof()) {
//System.out.println("PR END");
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
final int code = postings.readVInt();
@@ -345,7 +345,7 @@ public class PulsingPostingsReaderImpl e
if (doc >= target)
return doc;
}
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
}
@@ -614,7 +614,7 @@ public class PulsingPostingsReaderImpl e
if (postings.eof()) {
//System.out.println("PR END");
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
final int code = postings.readVInt();
@@ -652,7 +652,7 @@ public class PulsingPostingsReaderImpl e
return doc;
}
}
- return NO_MORE_DOCS;
+ return docID = NO_MORE_DOCS;
}
@Override
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java Tue Jan 25 16:27:00 2011
@@ -99,7 +99,9 @@ public class OrdFieldSource extends Valu
/*(non-Javadoc) @see java.lang.Object#equals(java.lang.Object) */
@Override
public boolean equals(Object o) {
- if (o.getClass() != OrdFieldSource.class) return false;
+ if (o == this) return true;
+ if (o == null) return false;
+ if (o.getClass() != OrdFieldSource.class) return false;
OrdFieldSource other = (OrdFieldSource)o;
return this.field.equals(other.field);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java Tue Jan 25 16:27:00 2011
@@ -107,7 +107,9 @@ public class ReverseOrdFieldSource exten
/*(non-Javadoc) @see java.lang.Object#equals(java.lang.Object) */
@Override
public boolean equals(Object o) {
- if (o.getClass() != ReverseOrdFieldSource.class) return false;
+ if (o == this) return true;
+ if (o == null) return false;
+ if (o.getClass() != ReverseOrdFieldSource.class) return false;
ReverseOrdFieldSource other = (ReverseOrdFieldSource)o;
return this.field.equals(other.field);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Jan 25 16:27:00 2011
@@ -428,7 +428,7 @@ public class TestAddIndexes extends Luce
);
writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux)));
- assertEquals(1060, writer.maxDoc());
+ assertEquals(1020, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
dir.close();
@@ -480,7 +480,7 @@ public class TestAddIndexes extends Luce
);
writer.addIndexes(aux, aux2);
- assertEquals(1060, writer.maxDoc());
+ assertEquals(1040, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
dir.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 16:27:00 2011
@@ -360,7 +360,7 @@ public class TestIndexReader extends Luc
// CREATE A NEW READER and re-test
reader = IndexReader.open(dir, false);
- assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
+ assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
reader.close();
reader2.close();
@@ -697,7 +697,6 @@ public class TestIndexReader extends Luc
// CREATE A NEW READER and re-test
reader = IndexReader.open(dir, false);
- assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
@@ -838,7 +837,6 @@ public class TestIndexReader extends Luc
writer.close();
IndexReader reader = IndexReader.open(dir, false);
reader.deleteDocument(0);
- reader.deleteDocument(1);
reader.close();
reader = IndexReader.open(dir, false);
reader.undeleteAll();
@@ -855,7 +853,6 @@ public class TestIndexReader extends Luc
writer.close();
IndexReader reader = IndexReader.open(dir, false);
reader.deleteDocument(0);
- reader.deleteDocument(1);
reader.close();
reader = IndexReader.open(dir, false);
reader.undeleteAll();
@@ -1290,9 +1287,6 @@ public class TestIndexReader extends Luc
// Open another reader to confirm that everything is deleted
reader2 = IndexReader.open(dir, false);
- assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
- assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
- assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Jan 25 16:27:00 2011
@@ -1211,7 +1211,6 @@ public class TestIndexReaderReopen exten
IndexReader r = IndexReader.open(dir, false);
assertEquals(0, r.numDocs());
- assertEquals(4, r.maxDoc());
Collection<IndexCommit> commits = IndexReader.listCommits(dir);
for (final IndexCommit commit : commits) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jan 25 16:27:00 2011
@@ -102,19 +102,12 @@ public class TestIndexWriter extends Luc
}
reader.close();
- // test doc count before segments are merged/index is optimized
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
- assertEquals(100, writer.maxDoc());
- writer.close();
-
reader = IndexReader.open(dir, true);
- assertEquals(100, reader.maxDoc());
assertEquals(60, reader.numDocs());
reader.close();
// optimize the index and check that the new doc count is correct
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
- assertEquals(100, writer.maxDoc());
assertEquals(60, writer.numDocs());
writer.optimize();
assertEquals(60, writer.maxDoc());
@@ -1432,7 +1425,6 @@ public class TestIndexWriter extends Luc
w.close();
IndexReader ir = IndexReader.open(dir, true);
- assertEquals(1, ir.maxDoc());
assertEquals(0, ir.numDocs());
ir.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Jan 25 16:27:00 2011
@@ -567,24 +567,25 @@ public class TestIndexWriterExceptions e
System.out.println("TEST: open reader");
}
IndexReader reader = IndexReader.open(dir, true);
- int expected = 3+(1-i)*2;
- assertEquals(expected, reader.docFreq(new Term("contents", "here")));
- assertEquals(expected, reader.maxDoc());
- int numDel = 0;
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
- assertNotNull(delDocs);
- for(int j=0;j<reader.maxDoc();j++) {
- if (delDocs.get(j))
- numDel++;
- else {
- reader.document(j);
- reader.getTermFreqVectors(j);
+ if (i == 0) {
+ int expected = 5;
+ assertEquals(expected, reader.docFreq(new Term("contents", "here")));
+ assertEquals(expected, reader.maxDoc());
+ int numDel = 0;
+ final Bits delDocs = MultiFields.getDeletedDocs(reader);
+ assertNotNull(delDocs);
+ for(int j=0;j<reader.maxDoc();j++) {
+ if (delDocs.get(j))
+ numDel++;
+ else {
+ reader.document(j);
+ reader.getTermFreqVectors(j);
+ }
}
+ assertEquals(1, numDel);
}
reader.close();
- assertEquals(1, numDel);
-
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
analyzer).setMaxBufferedDocs(10));
doc = new Document();
@@ -596,10 +597,10 @@ public class TestIndexWriterExceptions e
writer.close();
reader = IndexReader.open(dir, true);
- expected = 19+(1-i)*2;
+ int expected = 19+(1-i)*2;
assertEquals(expected, reader.docFreq(new Term("contents", "here")));
assertEquals(expected, reader.maxDoc());
- numDel = 0;
+ int numDel = 0;
assertNull(MultiFields.getDeletedDocs(reader));
for(int j=0;j<reader.maxDoc();j++) {
reader.document(j);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java Tue Jan 25 16:27:00 2011
@@ -6,6 +6,7 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Random;
+import java.lang.reflect.Method;
import junit.framework.Assert;
@@ -172,6 +173,16 @@ public class QueryUtils {
}
w.commit();
w.deleteDocuments( new MatchAllDocsQuery() );
+ try {
+ // Carefully invoke what is a package-private (test
+ // only, internal) method on IndexWriter:
+ Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
+ m.setAccessible(true);
+ m.invoke(w);
+ } catch (Exception e) {
+ // Should not happen?
+ throw new RuntimeException(e);
+ }
w.commit();
if (0 < numDeletedDocs)
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java Tue Jan 25 16:27:00 2011
@@ -253,5 +253,14 @@ public class TestOrdValues extends Funct
private String testName() {
return getClass().getName() + "." + getName();
}
+
+ // LUCENE-1250
+ public void testEqualsNull() throws Exception {
+ OrdFieldSource ofs = new OrdFieldSource("f");
+ assertFalse(ofs.equals(null));
+
+ ReverseOrdFieldSource rofs = new ReverseOrdFieldSource("f");
+ assertFalse(rofs.equals(null));
+ }
}
Modified: lucene/dev/branches/bulkpostings/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/CHANGES.txt?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/CHANGES.txt Tue Jan 25 16:27:00 2011
@@ -161,6 +161,8 @@ Bug Fixes
* SOLR-2127: Fixed serialization of default core and indentation of solr.xml when serializing.
(Ephraim Ofir, Mark Miller)
+* SOLR-482: Provide more exception handling in CSVLoader (gsingers)
+
Other Changes
----------------------
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1063338&r1=1063337&r2=1063338&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Jan 25 16:27:00 2011
@@ -305,12 +305,27 @@ abstract class CSVLoader extends Content
private void input_err(String msg, String[] line, int lineno) {
StringBuilder sb = new StringBuilder();
- sb.append(errHeader+", line="+lineno + ","+msg+"\n\tvalues={");
- for (String val: line) { sb.append("'"+val+"',"); }
+ sb.append(errHeader).append(", line=").append(lineno).append(",").append(msg).append("\n\tvalues={");
+ for (String val: line) {
+ sb.append("'").append(val).append("',"); }
sb.append('}');
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString());
}
+ private void input_err(String msg, String[] lines, int lineNo, Throwable e) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(errHeader).append(", line=").append(lineNo).append(",").append(msg).append("\n\tvalues={");
+ if (lines != null) {
+ for (String val : lines) {
+ sb.append("'").append(val).append("',");
+ }
+ } else {
+ sb.append("NO LINES AVAILABLE");
+ }
+ sb.append('}');
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString(), e);
+ }
+
/** load the CSV input */
public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws IOException {
errHeader = "CSVLoader: input=" + stream.getSourceInfo();
@@ -341,7 +356,13 @@ abstract class CSVLoader extends Content
// read the rest of the CSV file
for(;;) {
int line = parser.getLineNumber(); // for error reporting in MT mode
- String[] vals = parser.getLine();
+ String[] vals = null;
+ try {
+ vals = parser.getLine();
+ } catch (IOException e) {
+ //Catch the exception and rethrow it with more line information
+ input_err("can't read line: " + line, null, line, e);
+ }
if (vals==null) break;
if (vals.length != fields.length) {