You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/28 17:30:58 UTC
svn commit: r1207392 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ lucene/backwards/src/test/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/ lucen...
Author: rmuir
Date: Mon Nov 28 16:30:56 2011
New Revision: 1207392
URL: http://svn.apache.org/viewvc?rev=1207392&view=rev
Log:
LUCENE-3601: backport all tests and don't copy into CFS on addindexes
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiReader.java
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/solrconfig-icucollate.xml (props changed)
lucene/dev/branches/branch_3x/solr/core/src/test-files/solr/conf/solrconfig-basic.xml (props changed)
lucene/dev/branches/branch_3x/solr/solrj/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexes.java Mon Nov 28 16:30:56 2011
@@ -986,65 +986,6 @@ public class TestAddIndexes extends Luce
}
- // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
- // a CFS, given MP preferences
- public void testCopyIntoCFS() throws Exception {
- // create an index, no CFS (so we can assert that existing segments are not affected)
- Directory target = newDirectory();
- LogMergePolicy lmp = newLogMergePolicy(false);
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
- IndexWriter w = new IndexWriter(target, conf);
- w.addDocument(new Document());
- w.commit();
- assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
-
- // prepare second index, no-CFS too + .del file + separate norms file
- Directory src = newDirectory();
- LogMergePolicy lmp2 = newLogMergePolicy(false);
- IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMergePolicy(lmp2);
- IndexWriter w2 = new IndexWriter(src, conf2);
- Document doc = new Document();
- doc.add(new Field("c", "some text", Store.YES, Index.ANALYZED));
- w2.addDocument(doc);
- doc = new Document();
- doc.add(new Field("d", "delete", Store.NO, Index.NOT_ANALYZED_NO_NORMS));
- w2.addDocument(doc);
- w2.commit();
- w2.deleteDocuments(new Term("d", "delete"));
- w2.commit();
- w2.close();
-
- // create separate norms file
- IndexReader r = IndexReader.open(src, false);
- r.setNorm(0, "c", (byte) 1);
- r.close();
- assertTrue(".del file not found", src.fileExists("_0_1.del"));
- assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
-
- // Case 1: force 'CFS' on target
- lmp.setUseCompoundFile(true);
- lmp.setNoCFSRatio(1.0);
- w.addIndexes(src);
- w.commit();
- assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
- assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
- assertTrue(".del file not found", target.fileExists("_1_1.del"));
- assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
-
- // Case 2: LMP disallows CFS
- lmp.setUseCompoundFile(false);
- w.addIndexes(src);
- w.commit();
- assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
-
- w.close();
-
- // cleanup
- src.close();
- target.close();
- }
-
// LUCENE-3575
public void testFieldNamesChanged() throws IOException {
Directory d1 = newDirectory();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Nov 28 16:30:56 2011
@@ -3095,10 +3095,7 @@ public class IndexWriter implements Clos
* <p>
* <b>NOTE:</b> this method only copies the segments of the incomning indexes
* and does not merge them. Therefore deleted documents are not removed and
- * the new segments are not merged with the existing ones. Also, if the merge
- * policy allows compound files, then any segment that is not compound is
- * converted to such. However, if the segment is compound, it is copied as-is
- * even if the merge policy does not allow compound files.
+ * the new segments are not merged with the existing ones.
*
* <p>
* <p>This requires this index not be among those to be added.
@@ -3121,7 +3118,6 @@ public class IndexWriter implements Clos
message("flush at addIndexes(Directory...)");
flush(false, true);
- int docCount = 0;
List<SegmentInfo> infos = new ArrayList<SegmentInfo>();
Comparator<String> versionComparator = StringHelper.getVersionComparator();
for (Directory dir : dirs) {
@@ -3135,7 +3131,6 @@ public class IndexWriter implements Clos
for (SegmentInfo info : sis) {
assert !infos.contains(info): "dup info dir=" + info.dir + " name=" + info.name;
- docCount += info.docCount;
String newSegName = newSegmentName();
String dsName = info.getDocStoreSegment();
@@ -3143,21 +3138,8 @@ public class IndexWriter implements Clos
message("addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
- // create CFS only if the source segment is not CFS, and MP agrees it
- // should be CFS.
- boolean createCFS;
- synchronized (this) { // Guard segmentInfos
- createCFS = !info.getUseCompoundFile()
- && mergePolicy.useCompoundFile(segmentInfos, info)
- // optimize case only for segments that don't share doc stores
- && versionComparator.compare(info.getVersion(), "3.1") >= 0;
- }
+ copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied);
- if (createCFS) {
- copySegmentIntoCFS(info, newSegName);
- } else {
- copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied);
- }
infos.add(info);
}
}
@@ -3265,31 +3247,7 @@ public class IndexWriter implements Clos
handleOOM(oom, "addIndexes(IndexReader...)");
}
}
-
- /** Copies the segment into the IndexWriter's directory, as a compound segment. */
- private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException {
- String segFileName = IndexFileNames.segmentFileName(segName, IndexFileNames.COMPOUND_FILE_EXTENSION);
- Collection<String> files = info.files();
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName);
- for (String file : files) {
- String newFileName = segName + IndexFileNames.stripSegmentName(file);
- if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- && !IndexFileNames.isSeparateNormsFile(file)) {
- cfsWriter.addFile(file, info.dir);
- } else {
- assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
- info.dir.copy(directory, file, newFileName);
- }
- }
-
- // Create the .cfs
- cfsWriter.close();
-
- info.dir = directory;
- info.name = segName;
- info.setUseCompoundFile(true);
- }
-
+
/** Copies the segment files as-is into the IndexWriter's directory. */
private void copySegmentAsIs(SegmentInfo info, String segName,
Map<String, String> dsNames, Set<String> dsFilesCopied)
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon Nov 28 16:30:56 2011
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -207,10 +208,13 @@ public class MockDirectoryWrapper extend
while(it.hasNext()) {
String name = it.next();
- int damage = randomState.nextInt(4);
+ int damage = randomState.nextInt(5);
+ String action = null;
if (damage == 0) {
+ action = "deleted";
deleteFile(name, true);
} else if (damage == 1) {
+ action = "zeroed";
// Zero out file entirely
long length = fileLength(name);
byte[] zeroes = new byte[256];
@@ -223,17 +227,25 @@ public class MockDirectoryWrapper extend
}
out.close();
} else if (damage == 2) {
+ action = "partially truncated";
// Partially Truncate the file:
IndexOutput out = delegate.createOutput(name);
out.setLength(fileLength(name)/2);
out.close();
+ } else if (damage == 3) {
+ // The file survived intact:
+ action = "didn't change";
} else {
+ action = "fully truncated";
// Totally truncate the file to zero bytes
deleteFile(name, true);
IndexOutput out = delegate.createOutput(name);
out.setLength(0);
out.close();
}
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
+ }
}
}
@@ -506,11 +518,18 @@ public class MockDirectoryWrapper extend
throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
}
open = false;
- if (checkIndexOnClose && IndexReader.indexExists(this)) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
- }
- _TestUtil.checkIndex(this);
+ if (checkIndexOnClose) {
+ if (IndexReader.indexExists(this)) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("\nNOTE: MockDirectoryWrapper: now crash");
+ }
+ unSyncedFiles.remove("segments.gen"); // otherwise we add minutes to the tests: LUCENE-3605
+ crash(); // corrumpt any unsynced-files
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
+ }
+ _TestUtil.checkIndex(this);
+ }
}
delegate.close();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Mon Nov 28 16:30:56 2011
@@ -986,65 +986,6 @@ public class TestAddIndexes extends Luce
}
- // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
- // a CFS, given MP preferences
- public void testCopyIntoCFS() throws Exception {
- // create an index, no CFS (so we can assert that existing segments are not affected)
- Directory target = newDirectory();
- LogMergePolicy lmp = newLogMergePolicy(false);
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
- IndexWriter w = new IndexWriter(target, conf);
- w.addDocument(new Document());
- w.commit();
- assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
-
- // prepare second index, no-CFS too + .del file + separate norms file
- Directory src = newDirectory();
- LogMergePolicy lmp2 = newLogMergePolicy(false);
- IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMergePolicy(lmp2);
- IndexWriter w2 = new IndexWriter(src, conf2);
- Document doc = new Document();
- doc.add(new Field("c", "some text", Store.YES, Index.ANALYZED));
- w2.addDocument(doc);
- doc = new Document();
- doc.add(new Field("d", "delete", Store.NO, Index.NOT_ANALYZED_NO_NORMS));
- w2.addDocument(doc);
- w2.commit();
- w2.deleteDocuments(new Term("d", "delete"));
- w2.commit();
- w2.close();
-
- // create separate norms file
- IndexReader r = IndexReader.open(src, false);
- r.setNorm(0, "c", (byte) 1);
- r.close();
- assertTrue(".del file not found", src.fileExists("_0_1.del"));
- assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
-
- // Case 1: force 'CFS' on target
- lmp.setUseCompoundFile(true);
- lmp.setNoCFSRatio(1.0);
- w.addIndexes(src);
- w.commit();
- assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
- assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
- assertTrue(".del file not found", target.fileExists("_1_1.del"));
- assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
-
- // Case 2: LMP disallows CFS
- lmp.setUseCompoundFile(false);
- w.addIndexes(src);
- w.commit();
- assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
-
- w.close();
-
- // cleanup
- src.close();
- target.close();
- }
-
// LUCENE-3575
public void testFieldNamesChanged() throws IOException {
Directory d1 = newDirectory();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java Mon Nov 28 16:30:56 2011
@@ -38,7 +38,7 @@ public class TestDirectoryReader extends
@Override
public void setUp() throws Exception {
super.setUp();
- dir = newDirectory();
+ dir = createDirectory();
doc1 = new Document();
doc2 = new Document();
DocHelper.setupDoc(doc1);
@@ -56,6 +56,10 @@ public class TestDirectoryReader extends
dir.close();
super.tearDown();
}
+
+ protected Directory createDirectory() throws IOException {
+ return newDirectory();
+ }
protected IndexReader openReader() throws IOException {
IndexReader reader;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Mon Nov 28 16:30:56 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.File;
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -441,6 +442,10 @@ public class TestFieldsReader extends Lu
return fsDir.createOutput(name);
}
@Override
+ public void sync(Collection<String> names) throws IOException {
+ fsDir.sync(names);
+ }
+ @Override
public void close() throws IOException {
fsDir.close();
}
@@ -491,6 +496,9 @@ public class TestFieldsReader extends Lu
Directory dir = new FaultyFSDirectory(indexDir);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
+ if (VERBOSE) {
+ writer.setInfoStream(System.out);
+ }
for(int i=0;i<2;i++)
writer.addDocument(testDoc);
writer.forceMerge(1);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiReader.java?rev=1207392&r1=1207391&r2=1207392&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiReader.java Mon Nov 28 16:30:56 2011
@@ -19,8 +19,20 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+
public class TestMultiReader extends TestDirectoryReader {
+ // TODO: files are never fsynced if you do what this test is doing,
+ // so the checkindex is disabled.
+ @Override
+ protected Directory createDirectory() throws IOException {
+ MockDirectoryWrapper mdw = newDirectory();
+ mdw.setCheckIndexOnClose(false);
+ return mdw;
+ }
+
@Override
protected IndexReader openReader() throws IOException {
IndexReader reader;