You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/10/14 21:40:14 UTC
svn commit: r1183471 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/test/ lucene/src/test-framework/org/apache/lucene/store/
lucene/src/test/org/apache/lucene/index/ solr/
Author: mikemccand
Date: Fri Oct 14 19:40:14 2011
New Revision: 1183471
URL: http://svn.apache.org/viewvc?rev=1183471&view=rev
Log:
LUCENE-3515: port over-cloning test case to 3.x
Added:
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
- copied, changed from r1183464, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1183471&r1=1183470&r2=1183471&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Fri Oct 14 19:40:14 2011
@@ -3,6 +3,11 @@ Lucene Change Log
For more information on past and future Lucene versions, please see:
http://s.apache.org/luceneversions
+* LUCENE-3515: Fix terrible merge performance versus 3.x, especially
+ when the directory isn't MMapDirectory, due to failing to reuse
+ DocsAndPositionsEnum while merging (Marc Sturlese, Erick Erickson,
+ Robert Muir, Simon Willnauer, Mike McCandless)
+
======================= Lucene 3.5.0 =======================
Changes in backwards compatibility policy
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1183471&r1=1183470&r2=1183471&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Fri Oct 14 19:40:14 2011
@@ -30,6 +30,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.LuceneTestCase;
@@ -73,6 +74,8 @@ public class MockDirectoryWrapper extend
private ThrottledIndexOutput throttledOutput;
private Throttling throttling = Throttling.SOMETIMES;
+ final AtomicInteger inputCloneCount = new AtomicInteger();
+
// use this for tracking files for crash.
// additionally: provides debugging information in case you leave one open
private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap<Closeable,Exception>());
@@ -116,6 +119,10 @@ public class MockDirectoryWrapper extend
init();
}
+ public int getInputCloneCount() {
+ return inputCloneCount.get();
+ }
+
public void setTrackDiskUsage(boolean v) {
trackDiskUsage = v;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java?rev=1183471&r1=1183470&r2=1183471&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Fri Oct 14 19:40:14 2011
@@ -57,6 +57,7 @@ public class MockIndexInputWrapper exten
@Override
public Object clone() {
+ dir.inputCloneCount.incrementAndGet();
IndexInput iiclone = (IndexInput) delegate.clone();
MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone);
clone.isClone = true;
Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (from r1183464, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java&r1=1183464&r2=1183471&rev=1183471&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java Fri Oct 14 19:40:14 2011
@@ -21,8 +21,7 @@ import java.util.*;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
@@ -36,9 +35,6 @@ public class TestForTooMuchCloning exten
// Make sure we don't clone IndexInputs too frequently
// during merging:
public void test() throws Exception {
- String codec = CodecProvider.getDefault().getFieldCodec("field");
- // TODO: once LUCENE-3517 is fixed, remove this:
- assumeFalse("PulsingCodec fails this test because of over-cloning", codec.equals("Pulsing") || codec.equals("MockRandom"));
final MockDirectoryWrapper dir = newDirectory();
final TieredMergePolicy tmp = new TieredMergePolicy();
tmp.setMaxMergeAtOnce(2);
@@ -52,7 +48,7 @@ public class TestForTooMuchCloning exten
sb.append(' ');
}
final Document doc = new Document();
- doc.add(new TextField("field", sb.toString()));
+ doc.add(newField("field", sb.toString(), Field.Store.NO, Field.Index.ANALYZED));
w.addDocument(doc);
}
final IndexReader r = w.getReader();
@@ -68,8 +64,8 @@ public class TestForTooMuchCloning exten
// cutover to filter rewrite and reuse a single DocsEnum
// across all terms;
final TopDocs hits = s.search(new TermRangeQuery("field",
- new BytesRef(),
- new BytesRef("\uFFFF"),
+ "",
+ "\uFFFF",
true,
true), 10);
assertTrue(hits.totalHits > 0);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1183471&r1=1183470&r2=1183471&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Fri Oct 14 19:40:14 2011
@@ -35,7 +35,8 @@ public class TestRollingUpdates extends
final LineFileDocs docs = new LineFileDocs(random);
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- final int SIZE = atLeast(20);
+ w.setInfoStream(VERBOSE ? System.out : null);
+ final int SIZE = atLeast(TEST_NIGHTLY ? 100 : 20);
int id = 0;
IndexReader r = null;
final int numUpdates = (int) (SIZE * (2+random.nextDouble()));