You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/02/24 23:33:47 UTC
svn commit: r1662131 - in /lucene/dev/branches/lucene_solr_4_10/lucene: ./
core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/
Author: mikemccand
Date: Tue Feb 24 22:33:47 2015
New Revision: 1662131
URL: http://svn.apache.org/r1662131
Log:
LUCENE_6287: fix merge + commit concurrency issue when 4.x first kisses a 3.x index that can cause missing _N.si corruption
Added:
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/manysegments.362.zip (with props)
Modified:
lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt?rev=1662131&r1=1662130&r2=1662131&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt Tue Feb 24 22:33:47 2015
@@ -30,6 +30,11 @@ Bug fixes
index directory cause index corruption on upgrade (Robert Muir, Mike
McCandless)
+* LUCENE-6287: Fix concurrency bug in IndexWriter that could cause
+ index corruption (missing _N.si files) the first time 4.x kisses a
+ 3.x index if merges are also running. (Simon Willnauer, Mike
+ McCandless)
+
API Changes
* LUCENE-6212: Deprecate IndexWriter APIs that accept per-document Analyzer.
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1662131&r1=1662130&r2=1662131&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Feb 24 22:33:47 2015
@@ -4529,6 +4529,15 @@ public class IndexWriter implements Clos
toSync.prepareCommit(directory);
//System.out.println("DONE prepareCommit");
+ // prepareCommit may have created new _N.si and _N_upgraded.si
+ // (marker) files if the segments file was 3.x, so we must now
+ // (while still under IW's monitor lock) incRef these files in
+ // IFD to protect them until the commit is done:
+ final Collection<String> newFiles = toSync.files(directory, false);
+ deleter.incRef(newFiles);
+ deleter.decRef(filesToCommit);
+ filesToCommit = newFiles;
+
pendingCommitSet = true;
pendingCommit = toSync;
}
@@ -4536,10 +4545,8 @@ public class IndexWriter implements Clos
// This call can take a long time -- 10s of seconds
// or more. We do it without syncing on this:
boolean success = false;
- final Collection<String> filesToSync;
try {
- filesToSync = toSync.files(directory, false);
- directory.sync(filesToSync);
+ directory.sync(filesToCommit);
success = true;
} finally {
if (!success) {
@@ -4550,7 +4557,7 @@ public class IndexWriter implements Clos
}
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "done all syncs: " + filesToSync);
+ infoStream.message("IW", "done all syncs: " + filesToCommit);
}
assert testPoint("midStartCommitSuccess");
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java?rev=1662131&r1=1662130&r2=1662131&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java Tue Feb 24 22:33:47 2015
@@ -1040,4 +1040,50 @@ public class TestBackwardsCompatibility3
// Causes FNFE on _0.si during check index before the fix:
dir.close();
}
+
+ // LUCENE-6287: copy this back to a 3.6.x checkout, run it, then cd to
+ // the index dir and run "zip manysegments.362.zip *" and copy to this dir:
+ /*
+ public void testCreateManySegmentsIndex() throws Exception {
+ // we use a real directory name that is not cleaned up, because this method is only used to create backwards indexes:
+ File indexDir = new File(LuceneTestCase.TEMP_DIR, "manysegments");
+ System.out.println("TEST: indexDir " + indexDir);
+ _TestUtil.rmDir(indexDir);
+ Directory dir = newFSDirectory(indexDir);
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
+ .setMaxBufferedDocs(2);
+ TieredMergePolicy tmp = (TieredMergePolicy) conf.getMergePolicy();
+ tmp.setSegmentsPerTier(1000);
+ tmp.setFloorSegmentMB(.0000001);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ //writer.setInfoStream(System.out);
+
+ for(int i=0;i<200;i++) {
+ Document doc = new Document();
+ doc.add(new Field("content", "doc " + i, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+ }
+ assertEquals("wrong doc count", 200, writer.maxDoc());
+ writer.close();
+ dir.close();
+ }
+ */
+
+ static final String manySegments3xIndex = "manysegments.362.zip";
+
+ // LUCENE-6287
+ public void testMergeDuringUpgrade() throws Exception {
+ // NOTE: this fails only rarely
+ for(int i=0;i<10;i++) {
+ File indexPath = createTempDir("manysegments.362");
+ TestUtil.unzip(getDataFile(manySegments3xIndex), indexPath);
+ Directory dir = newFSDirectory(indexPath);
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.addDocument(new Document());
+ writer.commit();
+ writer.rollback();
+ dir.close();
+ }
+ }
}
Added: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/manysegments.362.zip
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/manysegments.362.zip?rev=1662131&view=auto
==============================================================================
Binary file - no diff available.