You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/28 16:21:03 UTC
svn commit: r1207291 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/
java/org/apache/lucene/index/codecs/simpletext/
test-framework/java/org/apache/lucene/store/ test/org/apache/lucene/index/
Author: rmuir
Date: Mon Nov 28 15:21:02 2011
New Revision: 1207291
URL: http://svn.apache.org/viewvc?rev=1207291&view=rev
Log:
LUCENE-3601: fix addIndexes(3.x index) where we get a broken CFS, termvectors unreferenced files on init failures, and crash all indexes before checkindex in tests
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Nov 28 15:21:02 2011
@@ -2344,10 +2344,7 @@ public class IndexWriter implements Clos
* <p>
* <b>NOTE:</b> this method only copies the segments of the incoming indexes
* and does not merge them. Therefore deleted documents are not removed and
- * the new segments are not merged with the existing ones. Also, if the merge
- * policy allows compound files, then any segment that is not compound is
- * converted to such. However, if the segment is compound, it is copied as-is
- * even if the merge policy does not allow compound files.
+ * the new segments are not merged with the existing ones.
*
* <p>This requires this index not be among those to be added.
*
@@ -2369,9 +2366,7 @@ public class IndexWriter implements Clos
infoStream.message("IW", "flush at addIndexes(Directory...)");
flush(false, true);
- int docCount = 0;
List<SegmentInfo> infos = new ArrayList<SegmentInfo>();
- Comparator<String> versionComparator = StringHelper.getVersionComparator();
for (Directory dir : dirs) {
if (infoStream != null) {
infoStream.message("IW", "addIndexes: process directory " + dir);
@@ -2383,7 +2378,6 @@ public class IndexWriter implements Clos
for (SegmentInfo info : sis) {
assert !infos.contains(info): "dup info dir=" + info.dir + " name=" + info.name;
- docCount += info.docCount;
String newSegName = newSegmentName();
String dsName = info.getDocStoreSegment();
@@ -2391,23 +2385,9 @@ public class IndexWriter implements Clos
infoStream.message("IW", "addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
- // create CFS only if the source segment is not CFS, and MP agrees it
- // should be CFS.
- boolean createCFS;
- synchronized (this) { // Guard segmentInfos
- createCFS = !info.getUseCompoundFile()
- && mergePolicy.useCompoundFile(segmentInfos, info)
- // optimize case only for segments that don't share doc stores
- && versionComparator.compare(info.getVersion(), "3.1") >= 0;
- }
-
IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(), true, -1));
- if (createCFS) {
- copySegmentIntoCFS(info, newSegName, context);
- } else {
- copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context);
- }
+ copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context);
infos.add(info);
}
@@ -2515,31 +2495,6 @@ public class IndexWriter implements Clos
}
}
- /** Copies the segment into the IndexWriter's directory, as a compound segment. */
- private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException {
- String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
- Collection<String> files = info.files();
- final CompoundFileDirectory cfsdir = new CompoundFileDirectory(directory, segFileName, context, true);
- try {
- for (String file : files) {
- String newFileName = segName + IndexFileNames.stripSegmentName(file);
- if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- && !IndexFileNames.isSeparateNormsFile(file)) {
- info.dir.copy(cfsdir, file, file, context);
- } else {
- assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
- info.dir.copy(directory, file, newFileName, context);
- }
- }
- } finally {
- IOUtils.close(cfsdir);
- }
-
- info.dir = directory;
- info.name = segName;
- info.setUseCompoundFile(true);
- }
-
/** Copies the segment files as-is into the IndexWriter's directory. */
private void copySegmentAsIs(SegmentInfo info, String segName,
Map<String, String> dsNames, Set<String> dsFilesCopied, IOContext context)
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java Mon Nov 28 15:21:02 2011
@@ -64,7 +64,7 @@ public final class DefaultTermVectorsWri
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(tvx, tvd, tvf);
+ abort();
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java Mon Nov 28 15:21:02 2011
@@ -63,7 +63,15 @@ public class SimpleTextTermVectorsWriter
public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
this.directory = directory;
this.segment = segment;
- out = directory.createOutput(IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION), context);
+ boolean success = false;
+ try {
+ out = directory.createOutput(IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION), context);
+ success = true;
+ } finally {
+ if (!success) {
+ abort();
+ }
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon Nov 28 15:21:02 2011
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -196,10 +197,13 @@ public class MockDirectoryWrapper extend
while(it.hasNext()) {
String name = it.next();
- int damage = randomState.nextInt(4);
+ int damage = randomState.nextInt(5);
+ String action = null;
if (damage == 0) {
+ action = "deleted";
deleteFile(name, true);
} else if (damage == 1) {
+ action = "zeroed";
// Zero out file entirely
long length = fileLength(name);
byte[] zeroes = new byte[256];
@@ -212,17 +216,25 @@ public class MockDirectoryWrapper extend
}
out.close();
} else if (damage == 2) {
+ action = "partially truncated";
// Partially Truncate the file:
IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState));
out.setLength(fileLength(name)/2);
out.close();
+ } else if (damage == 3) {
+ // The file survived intact:
+ action = "didn't change";
} else {
+ action = "fully truncated";
// Totally truncate the file to zero bytes
deleteFile(name, true);
IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState));
out.setLength(0);
out.close();
}
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
+ }
}
}
@@ -504,10 +516,15 @@ public class MockDirectoryWrapper extend
}
open = false;
if (checkIndexOnClose) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
- }
if (IndexReader.indexExists(this)) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("\nNOTE: MockDirectoryWrapper: now crash");
+ }
+ unSyncedFiles.remove("segments.gen"); // otherwise we add minutes to the tests: LUCENE-3605
+ crash(); // corrumpt any unsynced-files
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
+ }
_TestUtil.checkIndex(this);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Mon Nov 28 15:21:02 2011
@@ -1086,65 +1086,6 @@ public class TestAddIndexes extends Luce
dir.close();
}
- // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
- // a CFS, given MP preferences
- public void testCopyIntoCFS() throws Exception {
- // create an index, no CFS (so we can assert that existing segments are not affected)
- Directory target = newDirectory();
- LogMergePolicy lmp = newLogMergePolicy(false);
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
- IndexWriter w = new IndexWriter(target, conf);
- w.addDocument(new Document());
- w.commit();
- assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
-
- // prepare second index, no-CFS too + .del file + separate norms file
- Directory src = newDirectory();
- LogMergePolicy lmp2 = newLogMergePolicy(false);
- IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMergePolicy(lmp2);
- IndexWriter w2 = new IndexWriter(src, conf2);
- Document doc = new Document();
- doc.add(new Field("c", "some text", TextField.TYPE_STORED));
- w2.addDocument(doc);
- doc = new Document();
- doc.add(new StringField("d", "delete"));
- w2.addDocument(doc);
- w2.commit();
- w2.deleteDocuments(new Term("d", "delete"));
- w2.commit();
- w2.close();
-
- // create separate norms file
- IndexReader r = IndexReader.open(src, false);
- r.setNorm(0, "c", (byte) 1);
- r.close();
- assertTrue(".del file not found", src.fileExists("_0_1.del"));
- assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
-
- // Case 1: force 'CFS' on target
- lmp.setUseCompoundFile(true);
- lmp.setNoCFSRatio(1.0);
- w.addIndexes(src);
- w.commit();
- assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
- assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
- assertTrue(".del file not found", target.fileExists("_1_1.del"));
- assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
-
- // Case 2: LMP disallows CFS
- lmp.setUseCompoundFile(false);
- w.addIndexes(src);
- w.commit();
- assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
-
- w.close();
-
- // cleanup
- src.close();
- target.close();
- }
-
private static class UnRegisteredCodec extends Codec {
public UnRegisteredCodec() {
super("NotRegistered");
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java Mon Nov 28 15:21:02 2011
@@ -39,7 +39,7 @@ public class TestDirectoryReader extends
@Override
public void setUp() throws Exception {
super.setUp();
- dir = newDirectory();
+ dir = createDirectory();
doc1 = new Document();
doc2 = new Document();
DocHelper.setupDoc(doc1);
@@ -57,6 +57,10 @@ public class TestDirectoryReader extends
dir.close();
super.tearDown();
}
+
+ protected Directory createDirectory() throws IOException {
+ return newDirectory();
+ }
protected IndexReader openReader() throws IOException {
IndexReader reader;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java?rev=1207291&r1=1207290&r2=1207291&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java Mon Nov 28 15:21:02 2011
@@ -19,10 +19,20 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
public class TestMultiReader extends TestDirectoryReader {
+ // TODO: files are never fsynced if you do what this test is doing,
+ // so the checkindex is disabled.
+ @Override
+ protected Directory createDirectory() throws IOException {
+ MockDirectoryWrapper mdw = newDirectory();
+ mdw.setCheckIndexOnClose(false);
+ return mdw;
+ }
+
@Override
protected IndexReader openReader() throws IOException {
IndexReader reader;