You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/12/08 14:26:12 UTC
svn commit: r1043406 - in /lucene/dev/branches/branch_3x/lucene: ./
backwards/src/test/org/apache/lucene/index/
contrib/misc/src/java/org/apache/lucene/index/
src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Author: simonw
Date: Wed Dec 8 13:26:11 2010
New Revision: 1043406
URL: http://svn.apache.org/viewvc?rev=1043406&view=rev
Log:
LUCENE-2805: SegmentInfos shouldn't blindly increment version on commit
Modified:
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Wed Dec 8 13:26:11 2010
@@ -116,6 +116,11 @@ Changes in runtime behavior
* LUCENE-2790: LogMergePolicy.useCompoundFile's logic now factors in noCFSRatio
to determine whether the passed in segment should be compound.
(Shai Erera, Earwin Burrfoot)
+
+* LUCENE-2805: IndexWriter now increments the index version on every change to
+ the index instead of for every commit. Committing or closing the IndexWriter
+ without any changes to the index will not cause any index version increment.
+ (Simon Willnauer, Mike Mccandless)
API Changes
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java Wed Dec 8 13:26:11 2010
@@ -165,7 +165,7 @@ public class TestIndexWriterReader exten
assertTrue(r1.isCurrent());
writer.commit();
- assertFalse(r1.isCurrent());
+ assertTrue(r1.isCurrent());
assertEquals(200, r1.maxDoc());
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Wed Dec 8 13:26:11 2010
@@ -125,6 +125,7 @@ public class IndexSplitter {
int idx = getIdx(n);
infos.remove(idx);
}
+ infos.changed();
infos.commit(fsDir);
}
@@ -143,6 +144,7 @@ public class IndexSplitter {
copyFile(srcFile, destFile);
}
}
+ destInfos.changed();
destInfos.commit(destFSDir);
// System.out.println("destDir:"+destDir.getAbsolutePath());
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java Wed Dec 8 13:26:11 2010
@@ -766,6 +766,7 @@ public class CheckIndex {
public void fixIndex(Status result) throws IOException {
if (result.partial)
throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
+ result.newSegments.changed();
result.newSegments.commit(result.dir);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Wed Dec 8 13:26:11 2010
@@ -765,6 +765,7 @@ class DirectoryReader extends IndexReade
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
segmentInfos, null, null);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
+ segmentInfos.changed();
// Checkpoint the state we are about to change, in
// case we have to roll back:
@@ -777,7 +778,6 @@ class DirectoryReader extends IndexReade
// Sync all files we just wrote
directory.sync(segmentInfos.files(directory, false));
-
segmentInfos.commit(directory);
success = true;
} finally {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Dec 8 13:26:11 2010
@@ -1099,6 +1099,7 @@ public class IndexWriter implements Clos
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
+ segmentInfos.changed();
} else {
segmentInfos.read(directory);
@@ -1115,6 +1116,7 @@ public class IndexWriter implements Clos
oldInfos.read(directory, commit.getSegmentsFileName());
segmentInfos.replace(oldInfos);
changeCount++;
+ segmentInfos.changed();
if (infoStream != null)
message("init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
}
@@ -1132,12 +1134,14 @@ public class IndexWriter implements Clos
conf.getIndexDeletionPolicy(),
segmentInfos, infoStream, docWriter);
- if (deleter.startingCommitDeleted)
+ if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
+ segmentInfos.changed();
+ }
docWriter.setMaxBufferedDeleteTerms(conf.getMaxBufferedDeleteTerms());
docWriter.setRAMBufferSizeMB(conf.getRAMBufferSizeMB());
@@ -2311,6 +2315,7 @@ public class IndexWriter implements Clos
// name that was previously returned which can cause
// problems at least with ConcurrentMergeScheduler.
changeCount++;
+ segmentInfos.changed();
return "_" + Integer.toString(segmentInfos.counter++, Character.MAX_RADIX);
}
}
@@ -2812,6 +2817,7 @@ public class IndexWriter implements Clos
// Mark that the index has changed
++changeCount;
+ segmentInfos.changed();
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteAll");
} finally {
@@ -2893,6 +2899,7 @@ public class IndexWriter implements Clos
*/
private synchronized void checkpoint() throws IOException {
changeCount++;
+ segmentInfos.changed();
deleter.checkpoint(segmentInfos, false);
}
@@ -4461,6 +4468,7 @@ public class IndexWriter implements Clos
}
toSync.remove(toSync.size()-1);
changeCount++;
+ segmentInfos.changed();
}
}
assert filesExist(toSync);
@@ -4675,17 +4683,7 @@ public class IndexWriter implements Clos
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
- if (!infos.equals(segmentInfos)) {
- // if any structural changes (new segments), we are
- // stale
- return false;
- } else if (infos.getGeneration() != segmentInfos.getGeneration()) {
- // if any commit took place since we were opened, we
- // are stale
- return false;
- } else {
- return !docWriter.anyChanges();
- }
+ return infos.version == segmentInfos.version && !docWriter.anyChanges();
}
synchronized boolean isClosed() {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Wed Dec 8 13:26:11 2010
@@ -97,7 +97,7 @@ public final class SegmentInfos extends
* counts how often the index has been changed by adding or deleting docs.
* starting with the current time in milliseconds forces to create unique version numbers.
*/
- private long version = System.currentTimeMillis();
+ long version = System.currentTimeMillis();
private long generation = 0; // generation of the "segments_N" for the next commit
private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
@@ -335,8 +335,7 @@ public final class SegmentInfos extends
try {
segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
- segnOutput.writeLong(++version); // every write changes
- // the index
+ segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
for (int i = 0; i < size(); i++) {
@@ -758,7 +757,6 @@ public final class SegmentInfos extends
void updateGeneration(SegmentInfos other) {
lastGeneration = other.lastGeneration;
generation = other.generation;
- version = other.version;
}
final void rollbackCommit(Directory dir) throws IOException {
@@ -789,7 +787,12 @@ public final class SegmentInfos extends
* segments file, but writes an invalid checksum at the
* end, so that it is not visible to readers. Once this
* is called you must call {@link #finishCommit} to complete
- * the commit or {@link #rollbackCommit} to abort it. */
+ * the commit or {@link #rollbackCommit} to abort it.
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
final void prepareCommit(Directory dir) throws IOException {
if (pendingSegnOutput != null)
throw new IllegalStateException("prepareCommit was already called");
@@ -875,7 +878,12 @@ public final class SegmentInfos extends
}
/** Writes & syncs to the Directory dir, taking care to
- * remove the segments file on exception */
+ * remove the segments file on exception
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
final void commit(Directory dir) throws IOException {
prepareCommit(dir);
finishCommit(dir);
@@ -926,4 +934,10 @@ public final class SegmentInfos extends
}
return count;
}
+
+ /** Call this before committing if changes have been made to the
+ * segments. */
+ public void changed() {
+ version++;
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1043406&r1=1043405&r2=1043406&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Wed Dec 8 13:26:11 2010
@@ -56,7 +56,54 @@ public class TestIndexWriterReader exten
td.close();
return count;
}
-
+
+ public void testAddCloseOpen() throws IOException {
+ Directory dir1 = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+ for (int i = 0; i < 97 ; i++) {
+ IndexReader reader = writer.getReader();
+ if (i == 0) {
+ writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ } else {
+ int previous = random.nextInt(i);
+ // a check if the reader is current here could fail since there might be
+ // merges going on.
+ switch (random.nextInt(5)) {
+ case 0:
+ case 1:
+ case 2:
+ writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ break;
+ case 3:
+ writer.updateDocument(new Term("id", "" + previous), createDocument(
+ previous, "x", 1 + random.nextInt(5)));
+ break;
+ case 4:
+ writer.deleteDocuments(new Term("id", "" + previous));
+ }
+ }
+ assertFalse(reader.isCurrent());
+ reader.close();
+ }
+ writer.optimize(); // make sure all merging is done etc.
+ IndexReader reader = writer.getReader();
+ writer.commit(); // no changes that are not visible to the reader
+ assertTrue(reader.isCurrent());
+ writer.close();
+ assertTrue(reader.isCurrent()); // all changes are visible to the reader
+ iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+ writer = new IndexWriter(dir1, iwc);
+ assertTrue(reader.isCurrent());
+ writer.addDocument(createDocument(1, "x", 1+random.nextInt(5)));
+ assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
+ writer.close();
+ assertFalse(reader.isCurrent()); // segments written
+ reader.close();
+ dir1.close();
+ }
+
public void testUpdateDocument() throws Exception {
boolean optimize = true;
@@ -166,7 +213,7 @@ public class TestIndexWriterReader exten
assertTrue(r1.isCurrent());
writer.commit();
- assertFalse(r1.isCurrent());
+ assertTrue(r1.isCurrent()); // we have seen all changes - no change after opening the NRT reader
assertEquals(200, r1.maxDoc());