You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/08/26 23:52:30 UTC
svn commit: r1620746 - in /lucene/dev/branches/lucene_solr_4_10: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/
Author: mikemccand
Date: Tue Aug 26 21:52:30 2014
New Revision: 1620746
URL: http://svn.apache.org/r1620746
Log:
LUCENE-5907: fix index corruption when opening a pre-4.x index and opening and closing an NRT reader
Modified:
lucene/dev/branches/lucene_solr_4_10/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/core/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt Tue Aug 26 21:52:30 2014
@@ -178,6 +178,12 @@ Bug Fixes
buffer size was reduced, and scanner buffer growth was disabled, resulting
in much, much faster tokenization for these text sequences.
(Chris Geeringh, Robert Muir, Steve Rowe)
+
+* LUCENE-5907: Fix corruption case when opening a pre-4.x index with
+ IndexWriter, then opening an NRT reader from that writer, then
+ calling commit from the writer, then closing the NRT reader. This
+ case would remove the wrong files from the index leading to a
+ corrupt index. (Mike McCandless)
Test Framework
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java Tue Aug 26 21:52:30 2014
@@ -324,7 +324,18 @@ public class SegmentCommitInfo {
@Override
public SegmentCommitInfo clone() {
- SegmentCommitInfo other = new SegmentCommitInfo(info, delCount, delGen, fieldInfosGen, docValuesGen);
+ return clone(false);
+ }
+
+ /** Clones {@code this}, optionally also cloning the {@link SegmentInfo}. */
+ SegmentCommitInfo clone(boolean cloneSegmentInfo) {
+ SegmentInfo otherInfo;
+ if (cloneSegmentInfo) {
+ otherInfo = info.clone();
+ } else {
+ otherInfo = info;
+ }
+ SegmentCommitInfo other = new SegmentCommitInfo(otherInfo, delCount, delGen, fieldInfosGen, docValuesGen);
// Not clear that we need to carry over nextWriteDelGen
// (i.e. do we ever clone after a failed write and
// before the next successful write?), but just do it to
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue Aug 26 21:52:30 2014
@@ -21,6 +21,7 @@ package org.apache.lucene.index;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -344,4 +345,21 @@ public final class SegmentInfo {
public Map<String,String> attributes() {
return attributes;
}
+
+ private static Map<String,String> cloneMap(Map<String,String> map) {
+ if (map != null) {
+ return new HashMap<String,String>(map);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public SegmentInfo clone() {
+ SegmentInfo other = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, cloneMap(diagnostics), cloneMap(attributes));
+ if (setFiles != null) {
+ other.setFiles(new HashSet<>(setFiles));
+ }
+ return other;
+ }
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Tue Aug 26 21:52:30 2014
@@ -643,6 +643,11 @@ public final class SegmentInfos implemen
@Override
public SegmentInfos clone() {
+ return clone(false);
+ }
+
+ /** Deep clone of this {@code SegmentInfos}, optionally also fully cloning the {@link SegmentInfo}. */
+ SegmentInfos clone(boolean cloneSegmentInfo) {
try {
final SegmentInfos sis = (SegmentInfos) super.clone();
// deep clone, first recreate all collections:
@@ -650,7 +655,7 @@ public final class SegmentInfos implemen
for(final SegmentCommitInfo info : this) {
assert info.info.getCodec() != null;
// dont directly access segments, use add method!!!
- sis.add(info.clone());
+ sis.add(info.clone(cloneSegmentInfo));
}
sis.userData = new HashMap<>(userData);
return sis;
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Tue Aug 26 21:52:30 2014
@@ -74,6 +74,9 @@ final class StandardDirectoryReader exte
/** Used by near real-time search */
static DirectoryReader open(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException {
+
+ assert Thread.holdsLock(writer);
+
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
@@ -82,7 +85,10 @@ final class StandardDirectoryReader exte
List<SegmentReader> readers = new ArrayList<>();
final Directory dir = writer.getDirectory();
- final SegmentInfos segmentInfos = infos.clone();
+ // LUCENE-5907: must make a deep clone, even of the SegmentInfo, so that if its files changed e.g. due to upgrade from 3.x index, we
+ // don't later corrupt the index when this reader is closed:
+ final SegmentInfos segmentInfos = infos.clone(true);
+
int infosUpto = 0;
boolean success = false;
try {
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java?rev=1620746&r1=1620745&r2=1620746&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility3x.java Tue Aug 26 21:52:30 2014
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -57,8 +56,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -948,4 +945,39 @@ public class TestBackwardsCompatibility3
TestUtil.checkIndex(dir);
dir.close();
}
+
+ // LUCENE-5907
+ public void testUpgradeWithNRTReader() throws Exception {
+ for (String name : oldNames) {
+ Directory dir = newDirectory(oldIndexDirs.get(name));
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND));
+ writer.addDocument(new Document());
+ DirectoryReader r = DirectoryReader.open(writer, true);
+ writer.commit();
+ r.close();
+ writer.forceMerge(1);
+ writer.commit();
+ writer.rollback();
+ new SegmentInfos().read(dir);
+ dir.close();
+ }
+ }
+
+ // LUCENE-5907
+ public void testUpgradeThenMultipleCommits() throws Exception {
+ for (String name : oldNames) {
+ Directory dir = newDirectory(oldIndexDirs.get(name));
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND));
+ writer.addDocument(new Document());
+ writer.commit();
+ writer.addDocument(new Document());
+ writer.commit();
+ writer.close();
+ dir.close();
+ }
+ }
}