You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/26 11:57:15 UTC
svn commit: r1641795 - in /lucene/dev/trunk: ./ lucene/ lucene/core/
lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/util/
Author: mikemccand
Date: Wed Nov 26 10:57:14 2014
New Revision: 1641795
URL: http://svn.apache.org/r1641795
Log:
add fun test case for reindexing using ParallelLeafReader
Added:
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
- copied, changed from r1641789, lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/core/ (props changed)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReaderManager.java
lucene/dev/trunk/lucene/test-framework/ (props changed)
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexReader.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexReader.java Wed Nov 26 10:57:14 2014
@@ -19,7 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.Bits; // javadocs
import org.apache.lucene.util.IOUtils;
import java.io.Closeable;
@@ -30,7 +30,6 @@ import java.util.List;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-// javadocs
/**
IndexReader is an abstract class, providing an interface for accessing a
@@ -99,7 +98,7 @@ public abstract class IndexReader implem
*/
public static interface ReaderClosedListener {
/** Invoked when the {@link IndexReader} is closed. */
- public void onClose(IndexReader reader);
+ public void onClose(IndexReader reader) throws IOException;
}
private final Set<ReaderClosedListener> readerClosedListeners =
@@ -191,7 +190,7 @@ public abstract class IndexReader implem
*/
public final void incRef() {
if (!tryIncRef()) {
- ensureOpen();
+ ensureOpen();
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Wed Nov 26 10:57:14 2014
@@ -3831,6 +3831,14 @@ public class IndexWriter implements Clos
merge.readers.set(i, null);
}
}
+
+ try {
+ merge.mergeFinished();
+ } catch (Throwable t) {
+ if (th == null) {
+ th = t;
+ }
+ }
// If any error occured, throw it.
if (!suppressExceptions) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReader.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReader.java Wed Nov 26 10:57:14 2014
@@ -80,7 +80,7 @@ public abstract class LeafReader extends
public static interface CoreClosedListener {
/** Invoked when the shared core of the original {@code
* SegmentReader} has closed. */
- public void onClose(Object ownerCoreCacheKey);
+ public void onClose(Object ownerCoreCacheKey) throws IOException;
}
private static class CoreClosedListenerWrapper implements ReaderClosedListener {
@@ -92,7 +92,7 @@ public abstract class LeafReader extends
}
@Override
- public void onClose(IndexReader reader) {
+ public void onClose(IndexReader reader) throws IOException {
listener.onClose(reader.getCoreCacheKey());
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java Wed Nov 26 10:57:14 2014
@@ -66,4 +66,9 @@ public final class LeafReaderContext ext
public LeafReader reader() {
return reader;
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ return "LeafReaderContext(" + reader + " docBase=" + docBase + " ord=" + ord + ")";
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java Wed Nov 26 10:57:14 2014
@@ -129,6 +129,10 @@ public abstract class MergePolicy {
totalDocCount = count;
}
+ /** Called by {@link IndexWriter} after the merge is done and all readers have been closed. */
+ public void mergeFinished() throws IOException {
+ }
+
/** Expert: Get the list of readers to merge. Note that this list does not
* necessarily match the list of segments to merge and should only be used
* to feed SegmentMerger to initialize a merge. When a {@link OneMerge}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java Wed Nov 26 10:57:14 2014
@@ -28,7 +28,6 @@ import java.util.TreeMap;
import org.apache.lucene.util.Bits;
-
/** An {@link LeafReader} which reads multiple, parallel indexes. Each index
* added must have the same number of documents, but typically each contains
* different fields. Deletions are taken from the first reader.
@@ -322,4 +321,10 @@ public class ParallelLeafReader extends
reader.checkIntegrity();
}
}
+
+ /** Returns the {@link LeafReader}s that were passed on init. */
+ public LeafReader[] getParallelReaders() {
+ ensureOpen();
+ return parallelReaders;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReaderManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReaderManager.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReaderManager.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReaderManager.java Wed Nov 26 10:57:14 2014
@@ -67,6 +67,19 @@ public final class ReaderManager extends
current = DirectoryReader.open(dir);
}
+ /**
+ * Creates and returns a new ReaderManager from the given
+ * already-opened {@link DirectoryReader}, stealing
+ * the incoming reference.
+ *
+ * @param reader the directoryReader to use for future reopens
+ *
+ * @throws IOException If there is a low-level I/O error
+ */
+ public ReaderManager(DirectoryReader reader) throws IOException {
+ current = reader;
+ }
+
@Override
protected void decRef(DirectoryReader reference) throws IOException {
reference.decRef();
Copied: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java (from r1641789, lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java?p2=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java&p1=lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java&r1=1641789&r2=1641795&rev=1641795&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java Wed Nov 26 10:57:14 2014
@@ -667,7 +667,7 @@ public class TestDemoParallelLeafReader
// Slowly parse the stored field into a new doc values field:
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
- Document oldDoc = reader.document(i);
+ StoredDocument oldDoc = reader.document(i);
Document newDoc = new Document();
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
newDoc.add(new NumericDocValuesField("number", value));
@@ -718,7 +718,7 @@ public class TestDemoParallelLeafReader
// Must slowly parse the stored field into a new doc values field:
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
- Document oldDoc = reader.document(i);
+ StoredDocument oldDoc = reader.document(i);
Document newDoc = new Document();
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, value));
@@ -731,7 +731,7 @@ public class TestDemoParallelLeafReader
assertNotNull("oldSchemaGen=" + oldSchemaGen, oldValues);
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
- Document oldDoc = reader.document(i);
+ StoredDocument oldDoc = reader.document(i);
Document newDoc = new Document();
newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, oldValues.get(i)));
w.addDocument(newDoc);
@@ -761,7 +761,7 @@ public class TestDemoParallelLeafReader
int maxDoc = r.maxDoc();
boolean failed = false;
for(int i=0;i<maxDoc;i++) {
- Document oldDoc = r.document(i);
+ StoredDocument oldDoc = r.document(i);
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
if (value != numbers.get(i)) {
if (DEBUG) System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i) + " numbers=" + numbers);
@@ -804,7 +804,7 @@ public class TestDemoParallelLeafReader
// Must slowly parse the stored field into a new doc values field:
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
- Document oldDoc = reader.document(i);
+ StoredDocument oldDoc = reader.document(i);
Document newDoc = new Document();
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
newDoc.add(new NumericDocValuesField("number", newSchemaGen*value));
@@ -817,7 +817,7 @@ public class TestDemoParallelLeafReader
assertNotNull("oldSchemaGen=" + oldSchemaGen, oldValues);
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
- Document oldDoc = reader.document(i);
+ StoredDocument oldDoc = reader.document(i);
Document newDoc = new Document();
newDoc.add(new NumericDocValuesField("number", newSchemaGen*(oldValues.get(i)/oldSchemaGen)));
w.addDocument(newDoc);
@@ -851,7 +851,7 @@ public class TestDemoParallelLeafReader
int maxDoc = r.maxDoc();
boolean failed = false;
for(int i=0;i<maxDoc;i++) {
- Document oldDoc = r.document(i);
+ StoredDocument oldDoc = r.document(i);
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
value *= schemaGen;
if (value != numbers.get(i)) {
@@ -1111,7 +1111,7 @@ public class TestDemoParallelLeafReader
if (numbers != null) {
int maxDoc = leaf.maxDoc();
for(int i=0;i<maxDoc;i++) {
- Document doc = leaf.document(i);
+ StoredDocument doc = leaf.document(i);
long value = Long.parseLong(doc.get("text").split(" ")[1]);
long dvValue = numbers.get(i);
if (value == 0) {
@@ -1275,7 +1275,7 @@ public class TestDemoParallelLeafReader
boolean failed = false;
long t0 = System.currentTimeMillis();
for(int i=0;i<maxDoc;i++) {
- Document oldDoc = r.document(i);
+ StoredDocument oldDoc = r.document(i);
long value = multiplier * Long.parseLong(oldDoc.get("text").split(" ")[1]);
if (value != numbers.get(i)) {
System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i) + " numbers=" + numbers);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1641795&r1=1641794&r2=1641795&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Nov 26 10:57:14 2014
@@ -56,8 +56,10 @@ import java.util.concurrent.atomic.Atomi
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -76,8 +78,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.ReaderClosedListener;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
@@ -104,8 +106,8 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.AssertingIndexSearcher;
import org.apache.lucene.search.DocIdSetIterator;
@@ -116,12 +118,12 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.FlushInfo;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IOContext.Context;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.RateLimitedDirectoryWrapper;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
@@ -138,7 +140,6 @@ import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
-
import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.MixWithSuiteName;
@@ -149,16 +150,16 @@ import com.carrotsearch.randomizedtestin
import com.carrotsearch.randomizedtesting.annotations.SeedDecorators;
import com.carrotsearch.randomizedtesting.annotations.TestGroup;
import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup.Group;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule;
@@ -863,6 +864,11 @@ public abstract class LuceneTestCase ext
}
/** create a new index writer config with random defaults */
+ public static IndexWriterConfig newIndexWriterConfig() {
+ return newIndexWriterConfig(new MockAnalyzer(random()));
+ }
+
+ /** create a new index writer config with random defaults */
public static IndexWriterConfig newIndexWriterConfig(Analyzer a) {
return newIndexWriterConfig(random(), a);
}