You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/01/26 00:10:21 UTC
svn commit: r1063510 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/src/java/org/apache/lucene/index/ luce...
Author: mikemccand
Date: Tue Jan 25 23:10:20 2011
New Revision: 1063510
URL: http://svn.apache.org/viewvc?rev=1063510&view=rev
Log:
LUCENE-2474: add expert ReaderFinishedListener API, for external caches to evict entries for readers
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Jan 25 23:10:20 2011
@@ -488,6 +488,11 @@ New features
* LUCENE-2720: Segments now record the code version which created them.
(Shai Erera, Mike McCandless, Uwe Schindler)
+
+* LUCENE-2474: Added expert ReaderFinishedListener API to
+ IndexReader, to allow apps that maintain external per-segment caches
+ to evict entries when a segment is finished. (Shay Banon, Yonik
+ Seeley, Mike McCandless)
Optimizations
Modified: lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Tue Jan 25 23:10:20 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.store.instanti
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -46,6 +47,7 @@ public class InstantiatedIndexReader ext
public InstantiatedIndexReader(InstantiatedIndex index) {
super();
this.index = index;
+ readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
}
/**
Modified: lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Jan 25 23:10:20 2011
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@@ -735,6 +736,7 @@ public class MemoryIndex implements Seri
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
+ readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
}
private Info getInfo(String fieldName) {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Jan 25 23:10:20 2011
@@ -36,8 +36,6 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
-
/**
* An IndexReader which reads indexes with multiple segments.
*/
@@ -75,20 +73,28 @@ class DirectoryReader extends IndexReade
SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
if (readOnly)
- return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+ return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, null);
else
- return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+ return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, null);
}
}.run(commit);
}
/** Construct reading the named set of readers. */
- DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+ DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor,
+ Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+
+ if (readerFinishedListeners == null) {
+ this.readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
+ } else {
+ this.readerFinishedListeners = readerFinishedListeners;
+ }
+
// To reduce the chance of hitting FileNotFound
// (and having to retry), we open segments in
// reverse because IndexWriter merges & deletes
@@ -99,6 +105,7 @@ class DirectoryReader extends IndexReade
boolean success = false;
try {
readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
+ readers[i].readerFinishedListeners = this.readerFinishedListeners;
success = true;
} finally {
if (!success) {
@@ -123,6 +130,7 @@ class DirectoryReader extends IndexReade
this.readOnly = true;
segmentInfos = (SegmentInfos) infos.clone();// make sure we clone otherwise we share mutable state with IW
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ readerFinishedListeners = writer.getReaderFinishedListeners();
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
@@ -137,6 +145,7 @@ class DirectoryReader extends IndexReade
final SegmentInfo info = infos.info(i);
assert info.dir == dir;
readers[i] = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor);
+ readers[i].readerFinishedListeners = readerFinishedListeners;
success = true;
} finally {
if (!success) {
@@ -159,11 +168,14 @@ class DirectoryReader extends IndexReade
/** This constructor is only used for {@link #reopen()} */
DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
- Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+ Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor,
+ Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ assert readerFinishedListeners != null;
+ this.readerFinishedListeners = readerFinishedListeners;
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
@@ -203,8 +215,10 @@ class DirectoryReader extends IndexReade
// this is a new reader; in case we hit an exception we can close it safely
newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor);
+ newReader.readerFinishedListeners = readerFinishedListeners;
} else {
newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);
+ assert newReader.readerFinishedListeners == readerFinishedListeners;
}
if (newReader == newReaders[i]) {
// this reader will be shared between the old and the new one,
@@ -344,6 +358,7 @@ class DirectoryReader extends IndexReade
writeLock = null;
hasChanges = false;
}
+ assert newReader.readerFinishedListeners != null;
return newReader;
}
@@ -378,7 +393,9 @@ class DirectoryReader extends IndexReade
// TODO: right now we *always* make a new reader; in
// the future we could have write make some effort to
// detect that no changes have occurred
- return writer.getReader();
+ IndexReader reader = writer.getReader();
+ reader.readerFinishedListeners = readerFinishedListeners;
+ return reader;
}
private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
@@ -446,9 +463,9 @@ class DirectoryReader extends IndexReade
private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryReader reader;
if (openReadOnly) {
- reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+ reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, readerFinishedListeners);
} else {
- reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+ reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, readerFinishedListeners);
}
return reader;
}
@@ -857,11 +874,6 @@ class DirectoryReader extends IndexReade
}
}
- // NOTE: only needed in case someone had asked for
- // FieldCache for top-level reader (which is generally
- // not a good idea):
- FieldCache.DEFAULT.purge(this);
-
if (writer != null) {
// Since we just closed, writer may now be able to
// delete unused files:
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Jan 25 23:10:20 2011
@@ -20,11 +20,12 @@ package org.apache.lucene.index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
+import java.util.HashSet;
+import java.util.Collections;
/** A <code>FilterIndexReader</code> contains another IndexReader, which it
* uses as its basic source of data, possibly transforming the data along the
@@ -108,6 +109,7 @@ public class FilterIndexReader extends I
public FilterIndexReader(IndexReader in) {
super();
this.in = in;
+ readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
}
@Override
@@ -244,11 +246,6 @@ public class FilterIndexReader extends I
@Override
protected void doClose() throws IOException {
in.close();
-
- // NOTE: only needed in case someone had asked for
- // FieldCache for top-level reader (which is generally
- // not a good idea):
- FieldCache.DEFAULT.purge(this);
}
@@ -305,4 +302,17 @@ public class FilterIndexReader extends I
buffer.append(')');
return buffer.toString();
}
+
+ @Override
+ public void addReaderFinishedListener(ReaderFinishedListener listener) {
+ super.addReaderFinishedListener(listener);
+ in.addReaderFinishedListener(listener);
+ }
+
+ @Override
+ public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+ super.removeReaderFinishedListener(listener);
+ in.removeReaderFinishedListener(listener);
+ }
}
+
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Jan 25 23:10:20 2011
@@ -28,6 +28,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Closeable;
import java.util.Collection;
+import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@@ -77,6 +78,65 @@ import java.util.concurrent.atomic.Atomi
public abstract class IndexReader implements Cloneable,Closeable {
/**
+ * A custom listener that's invoked when the IndexReader
+ * is finished.
+ *
+ * <p>For a SegmentReader, this listener is called only
+ * once all SegmentReaders sharing the same core are
+ * closed. At this point it is safe for apps to evict
+ * this reader from any caches keyed on {@link
+ * #getCoreCacheKey}. This is the same interface that
+ * {@link FieldCache} uses, internally, to evict
+ * entries.</p>
+ *
+ * <p>For other readers, this listener is called when they
+ * are closed.</p>
+ *
+ * @lucene.experimental
+ */
+ public static interface ReaderFinishedListener {
+ public void finished(IndexReader reader);
+ }
+
+ // Impls must set this if they may call add/removeReaderFinishedListener:
+ protected volatile Collection<ReaderFinishedListener> readerFinishedListeners;
+
+ /** Expert: adds a {@link ReaderFinishedListener}. The
+ * provided listener is also added to any sub-readers, if
+ * this is a composite reader. Also, any reader reopened
+ * or cloned from this one will also copy the listeners at
+ * the time of reopen.
+ *
+ * @lucene.experimental */
+ public void addReaderFinishedListener(ReaderFinishedListener listener) {
+ readerFinishedListeners.add(listener);
+ }
+
+ /** Expert: remove a previously added {@link ReaderFinishedListener}.
+ *
+ * @lucene.experimental */
+ public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+ readerFinishedListeners.remove(listener);
+ }
+
+ protected void notifyReaderFinishedListeners() {
+ // Defensive (should never be null -- all impls must set
+ // this):
+ if (readerFinishedListeners != null) {
+
+ // Clone the set so that we don't have to sync on
+ // readerFinishedListeners while invoking them:
+ for(ReaderFinishedListener listener : new HashSet<ReaderFinishedListener>(readerFinishedListeners)) {
+ listener.finished(this);
+ }
+ }
+ }
+
+ protected void readerFinished() {
+ notifyReaderFinishedListeners();
+ }
+
+ /**
* Constants describing field properties, for example used for
* {@link IndexReader#getFieldNames(FieldOption)}.
*/
@@ -190,6 +250,7 @@ public abstract class IndexReader implem
refCount.incrementAndGet();
}
}
+ readerFinished();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jan 25 23:10:20 2011
@@ -30,6 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Collections;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.Analyzer;
@@ -460,6 +461,13 @@ public class IndexWriter implements Clos
return r;
}
+ // Used for all SegmentReaders we open
+ private final Collection<IndexReader.ReaderFinishedListener> readerFinishedListeners = Collections.synchronizedSet(new HashSet<IndexReader.ReaderFinishedListener>());
+
+ Collection<IndexReader.ReaderFinishedListener> getReaderFinishedListeners() throws IOException {
+ return readerFinishedListeners;
+ }
+
/** Holds shared SegmentReader instances. IndexWriter uses
* SegmentReaders for 1) applying deletes, 2) doing
* merges, 3) handing out a real-time reader. This pool
@@ -669,6 +677,7 @@ public class IndexWriter implements Clos
// synchronized
// Returns a ref, which we xfer to readerMap:
sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor);
+ sr.readerFinishedListeners = readerFinishedListeners;
if (info.dir == directory) {
// Only pool if reader is not external
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java Tue Jan 25 23:10:20 2011
@@ -22,6 +22,8 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.HashSet;
+import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
@@ -29,7 +31,6 @@ import org.apache.lucene.index.Directory
import org.apache.lucene.index.DirectoryReader.MultiTermEnum;
import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
/** An IndexReader which reads multiple indexes, appending
* their content. */
@@ -84,6 +85,7 @@ public class MultiReader extends IndexRe
hasDeletions = true;
}
starts[subReaders.length] = maxDoc;
+ readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
}
/**
@@ -416,11 +418,6 @@ public class MultiReader extends IndexRe
subReaders[i].close();
}
}
-
- // NOTE: only needed in case someone had asked for
- // FieldCache for top-level reader (which is generally
- // not a good idea):
- FieldCache.DEFAULT.purge(this);
}
@Override
@@ -456,4 +453,20 @@ public class MultiReader extends IndexRe
public IndexReader[] getSequentialSubReaders() {
return subReaders;
}
+
+ @Override
+ public void addReaderFinishedListener(ReaderFinishedListener listener) {
+ super.addReaderFinishedListener(listener);
+ for(IndexReader sub : subReaders) {
+ sub.addReaderFinishedListener(listener);
+ }
+ }
+
+ @Override
+ public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+ super.removeReaderFinishedListener(listener);
+ for(IndexReader sub : subReaders) {
+ sub.removeReaderFinishedListener(listener);
+ }
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java Tue Jan 25 23:10:20 2011
@@ -21,7 +21,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import java.io.IOException;
import java.util.*;
@@ -68,6 +67,7 @@ public class ParallelReader extends Inde
public ParallelReader(boolean closeSubReaders) throws IOException {
super();
this.incRefReaders = !closeSubReaders;
+ readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
}
/** {@inheritDoc} */
@@ -486,8 +486,6 @@ public class ParallelReader extends Inde
readers.get(i).close();
}
}
-
- FieldCache.DEFAULT.purge(this);
}
@Override
@@ -661,6 +659,21 @@ public class ParallelReader extends Inde
}
}
+ @Override
+ public void addReaderFinishedListener(ReaderFinishedListener listener) {
+ super.addReaderFinishedListener(listener);
+ for (IndexReader reader : readers) {
+ reader.addReaderFinishedListener(listener);
+ }
+ }
+
+ @Override
+ public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+ super.removeReaderFinishedListener(listener);
+ for (IndexReader reader : readers) {
+ reader.removeReaderFinishedListener(listener);
+ }
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java Tue Jan 25 23:10:20 2011
@@ -21,15 +21,17 @@ import org.apache.lucene.store.Directory
import java.io.IOException;
import java.util.Map;
+import java.util.Collection;
class ReadOnlyDirectoryReader extends DirectoryReader {
- ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor) throws IOException {
- super(directory, sis, deletionPolicy, true, termInfosIndexDivisor);
+ ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor,
+ Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
+ super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, readerFinishedListeners);
}
ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, Map<String,byte[]> oldNormsCache, boolean doClone,
- int termInfosIndexDivisor) throws IOException {
- super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor);
+ int termInfosIndexDivisor, Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
+ super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor, readerFinishedListeners);
}
ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java Tue Jan 25 23:10:20 2011
@@ -38,7 +38,6 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.CloseableThreadLocal;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
/**
* @lucene.experimental
@@ -242,13 +241,9 @@ public class SegmentReader extends Index
storeCFSReader.close();
}
- // Force FieldCache to evict our entries at this
- // point. If the exception occurred while
- // initializing the core readers, then
- // origInstance will be null, and we don't want
- // to call FieldCache.purge (it leads to NPE):
+ // Now, notify any ReaderFinished listeners:
if (origInstance != null) {
- FieldCache.DEFAULT.purge(origInstance);
+ origInstance.notifyReaderFinishedListeners();
}
}
}
@@ -715,6 +710,7 @@ public class SegmentReader extends Index
clone.si = si;
clone.readBufferSize = readBufferSize;
clone.pendingDeleteCount = pendingDeleteCount;
+ clone.readerFinishedListeners = readerFinishedListeners;
if (!openReadOnly && hasChanges) {
// My pending changes transfer to the new reader
@@ -1368,4 +1364,14 @@ public class SegmentReader extends Index
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
+
+ @Override
+ protected void readerFinished() {
+ // Do nothing here -- we have more careful control on
+ // when to notify that a SegmentReader has finished,
+ // because a given core is shared across many cloned
+ // SegmentReaders. We only notify once that core is no
+ // longer used (all SegmentReaders sharing it have been
+ // closed).
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Jan 25 23:10:20 2011
@@ -132,6 +132,13 @@ class FieldCacheImpl implements FieldCac
static final class StopFillCacheException extends RuntimeException {
}
+ final static IndexReader.ReaderFinishedListener purgeReader = new IndexReader.ReaderFinishedListener() {
+ // @Override -- not until Java 1.6
+ public void finished(IndexReader reader) {
+ FieldCache.DEFAULT.purge(reader);
+ }
+ };
+
/** Expert: Internal cache. */
abstract static class Cache {
Cache() {
@@ -164,8 +171,10 @@ class FieldCacheImpl implements FieldCac
synchronized (readerCache) {
innerCache = readerCache.get(readerKey);
if (innerCache == null) {
+ // First time this reader is using FieldCache
innerCache = new HashMap<Entry,Object>();
readerCache.put(readerKey, innerCache);
+ reader.addReaderFinishedListener(purgeReader);
value = null;
} else {
value = innerCache.get(key);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 23:10:20 2011
@@ -1790,4 +1790,42 @@ public class TestIndexReader extends Luc
assertTrue(IndexReader.indexExists(dir));
dir.close();
}
+
+ // LUCENE-2474
+ public void testReaderFinishedListener() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ writer.addDocument(new Document());
+ writer.commit();
+ writer.addDocument(new Document());
+ writer.commit();
+ final IndexReader reader = writer.getReader();
+ final int[] closeCount = new int[1];
+ final IndexReader.ReaderFinishedListener listener = new IndexReader.ReaderFinishedListener() {
+ public void finished(IndexReader reader) {
+ closeCount[0]++;
+ }
+ };
+
+ reader.addReaderFinishedListener(listener);
+
+ reader.close();
+
+ // Just the top reader
+ assertEquals(1, closeCount[0]);
+ writer.close();
+
+ // Now also the subs
+ assertEquals(3, closeCount[0]);
+
+ IndexReader reader2 = IndexReader.open(dir);
+ reader2.addReaderFinishedListener(listener);
+
+ closeCount[0] = 0;
+ reader2.close();
+ assertEquals(3, closeCount[0]);
+ dir.close();
+ }
}