You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2012/02/14 13:55:44 UTC
svn commit: r1243906 - in /lucene/dev/branches/branch_3x/lucene: ./
core/src/java/org/apache/lucene/search/
core/src/test/org/apache/lucene/search/
Author: shaie
Date: Tue Feb 14 12:55:43 2012
New Revision: 1243906
URL: http://svn.apache.org/viewvc?rev=1243906&view=rev
Log:
LUCENE-3761: Generalize SearcherManager
Added:
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java (with props)
Modified:
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Feb 14 12:55:43 2012
@@ -69,7 +69,12 @@ Changes in backwards compatibility polic
IndexCommit.getTimestamp and .getVersion and
IndexReader.lastModified and getCurrentVersion (Andrzej Bialecki,
Robert Muir, Mike McCandless)
-
+
+* LUCENE-3761: Generalize SearcherManager into an abstract ReferenceManager.
+ SearcherManager remains a concrete class, but due to the refactoring, the
+ method maybeReopen has been deprecated in favor of maybeRefresh().
+ (Shai Erera, Mike McCandless, Simon Willnauer)
+
Security fixes
* LUCENE-3588: Try harder to prevent SIGSEGV on cloned MMapIndexInputs:
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java Tue Feb 14 12:55:43 2012
@@ -344,7 +344,7 @@ public class NRTManager implements Close
}
boolean setSearchGen;
if (!mgr.isSearcherCurrent()) {
- setSearchGen = mgr.maybeReopen();
+ setSearchGen = mgr.maybeRefresh();
} else {
setSearchGen = true;
}
Added: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java?rev=1243906&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java (added)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java Tue Feb 14 12:55:43 2012
@@ -0,0 +1,164 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+
+import org.apache.lucene.store.AlreadyClosedException;
+
+/**
+ * Utility class to safely share instances of a certain type across multiple
+ * threads, while periodically refreshing them. This class ensures each
+ * reference is closed only once all threads have finished using it. It is
+ * recommended to consult the documentation of {@link ReferenceManager}
+ * implementations for their {@link #maybeRefresh()} semantics.
+ *
+ * @param <G>
+ * the concrete type that will be {@link #acquire() acquired} and
+ * {@link #release(Object) released}.
+ *
+ * @lucene.experimental
+ */
+public abstract class ReferenceManager<G> implements Closeable {
+
+ private static final String REFERENCE_MANAGER_IS_CLOSED_MSG = "this ReferenceManager is closed";
+
+ protected volatile G current;
+
+ private final Semaphore reopenLock = new Semaphore(1);
+
+ private void ensureOpen() {
+ if (current == null) {
+ throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG);
+ }
+ }
+
+ private synchronized void swapReference(G newReference) throws IOException {
+ ensureOpen();
+ final G oldReference = current;
+ current = newReference;
+ release(oldReference);
+ }
+
+ /** Decrement reference counting on the given reference. */
+ protected abstract void decRef(G reference) throws IOException;
+
+ /**
+ * Refresh the given reference if needed. Returns {@code null} if no refresh
+ * was needed, otherwise a new refreshed reference.
+ */
+ protected abstract G refreshIfNeeded(G referenceToRefresh) throws IOException;
+
+ /**
+ * Try to increment reference counting on the given reference. Return true if
+ * the operation was successful.
+ */
+ protected abstract boolean tryIncRef(G reference);
+
+ /**
+ * Obtain the current reference. You must match every call to acquire with one
+ * call to {@link #release}; it's best to do so in a finally clause, and set
+ * the reference to {@code null} to prevent accidental usage after it has been
+ * released.
+ */
+ public final G acquire() {
+ G ref;
+ do {
+ if ((ref = current) == null) {
+ throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG);
+ }
+ } while (!tryIncRef(ref));
+ return ref;
+ }
+
+ /**
+ * Close this ReferenceManager to future {@link #acquire() acquiring}. Any
+ * references that were previously {@link #acquire() acquired} won't be
+ * affected, and they should still be {@link #release released} when they are
+ * not needed anymore.
+ */
+ public final synchronized void close() throws IOException {
+ if (current != null) {
+ // make sure we can call this more than once
+ // closeable javadoc says:
+ // if this is already closed then invoking this method has no effect.
+ swapReference(null);
+ }
+ }
+
+ /**
+ * You must call this, periodically, if you want that {@link #acquire()} will
+ * return refreshed instances.
+ *
+ * <p>
+ * <b>Threads</b>: it's fine for more than one thread to call this at once.
+ * Only the first thread will attempt the refresh; subsequent threads will see
+ * that another thread is already handling refresh and will return
+ * immediately. Note that this means if another thread is already refreshing
+ * then subsequent threads will return right away without waiting for the
+ * refresh to complete.
+ *
+ * <p>
+ * This method returns true if the reference was in fact refreshed, or if the
+ * current reference has no pending changes.
+ */
+ public final boolean maybeRefresh() throws IOException {
+ ensureOpen();
+ // Ensure only 1 thread does reopen at once; other threads just return immediately:
+ if (reopenLock.tryAcquire()) {
+ try {
+ final G reference = acquire();
+ try {
+ G newReference = refreshIfNeeded(reference);
+ if (newReference != null) {
+ assert newReference != reference : "refreshIfNeeded should return null if refresh wasn't needed";
+ boolean success = false;
+ try {
+ swapReference(newReference);
+ success = true;
+ } finally {
+ if (!success) {
+ release(newReference);
+ }
+ }
+ }
+ } finally {
+ release(reference);
+ }
+ return true;
+ } finally {
+ reopenLock.release();
+ }
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Release the refernce previously obtained via {@link #acquire()}.
+ * <p>
+ * <b>NOTE:</b> it's safe to call this after {@link #close()}.
+ */
+ public final void release(G reference) throws IOException {
+ assert reference != null;
+ decRef(reference);
+ }
+
+}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java Tue Feb 14 12:55:43 2012
@@ -17,16 +17,10 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import java.io.Closeable;
import java.io.IOException;
-import java.util.concurrent.Semaphore;
-import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.search.NRTManager; // javadocs
-import org.apache.lucene.search.IndexSearcher; // javadocs
-import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
/**
@@ -50,27 +44,19 @@ import org.apache.lucene.store.Directory
* </pre>
*
* <p>
- * In addition you should periodically call {@link #maybeReopen}. While it's
+ * In addition you should periodically call {@link #maybeRefresh}. While it's
* possible to call this just before running each query, this is discouraged
* since it penalizes the unlucky queries that do the reopen. It's better to use
* a separate background thread, that periodically calls maybeReopen. Finally,
* be sure to call {@link #close} once you are done.
*
- * <p>
- * <b>NOTE</b>: if you have an {@link IndexWriter}, it's better to use
- * {@link NRTManager} since that class pulls near-real-time readers from the
- * IndexWriter.
- *
* @see SearcherFactory
*
* @lucene.experimental
*/
+public final class SearcherManager extends ReferenceManager<IndexSearcher> {
-public final class SearcherManager implements Closeable {
-
- private volatile IndexSearcher currentSearcher;
private final SearcherFactory searcherFactory;
- private final Semaphore reopenLock = new Semaphore(1);
/**
* Creates and returns a new SearcherManager from the given {@link IndexWriter}.
@@ -93,7 +79,27 @@ public final class SearcherManager imple
searcherFactory = new SearcherFactory();
}
this.searcherFactory = searcherFactory;
- currentSearcher = searcherFactory.newSearcher(IndexReader.open(writer, applyAllDeletes));
+ current = searcherFactory.newSearcher(IndexReader.open(writer, applyAllDeletes));
+ }
+
+ @Override
+ protected void decRef(IndexSearcher reference) throws IOException {
+ reference.getIndexReader().decRef();
+ }
+
+ @Override
+ protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException {
+ final IndexReader newReader = IndexReader.openIfChanged(referenceToRefresh.getIndexReader());
+ if (newReader == null) {
+ return null;
+ } else {
+ return searcherFactory.newSearcher(newReader);
+ }
+ }
+
+ @Override
+ protected boolean tryIncRef(IndexSearcher reference) {
+ return reference.getIndexReader().tryIncRef();
}
/**
@@ -110,63 +116,12 @@ public final class SearcherManager imple
searcherFactory = new SearcherFactory();
}
this.searcherFactory = searcherFactory;
- currentSearcher = searcherFactory.newSearcher(IndexReader.open(dir));
+ current = searcherFactory.newSearcher(IndexReader.open(dir));
}
- /**
- * You must call this, periodically, to perform a reopen. This calls
- * {@link IndexReader#openIfChanged(IndexReader)} with the underlying reader, and if that returns a
- * new reader, it's warmed (if you provided a {@link SearcherFactory} and then
- * swapped into production.
- *
- * <p>
- * <b>Threads</b>: it's fine for more than one thread to call this at once.
- * Only the first thread will attempt the reopen; subsequent threads will see
- * that another thread is already handling reopen and will return immediately.
- * Note that this means if another thread is already reopening then subsequent
- * threads will return right away without waiting for the reader reopen to
- * complete.
- * </p>
- *
- * <p>
- * This method returns true if a new reader was in fact opened or
- * if the current searcher has no pending changes.
- * </p>
- */
- public boolean maybeReopen() throws IOException {
- ensureOpen();
- // Ensure only 1 thread does reopen at once; other
- // threads just return immediately:
- if (reopenLock.tryAcquire()) {
- try {
- // IR.openIfChanged preserves NRT and applyDeletes
- // in the newly returned reader:
- final IndexReader newReader;
- final IndexSearcher searcherToReopen = acquire();
- try {
- newReader = IndexReader.openIfChanged(searcherToReopen.getIndexReader());
- } finally {
- release(searcherToReopen);
- }
- if (newReader != null) {
- final IndexSearcher newSearcher = searcherFactory.newSearcher(newReader);
- boolean success = false;
- try {
- swapSearcher(newSearcher);
- success = true;
- } finally {
- if (!success) {
- release(newSearcher);
- }
- }
- }
- return true;
- } finally {
- reopenLock.release();
- }
- } else {
- return false;
- }
+ /** @deprecated see {@link #maybeRefresh()}. */
+ public final boolean maybeReopen() throws IOException {
+ return maybeRefresh();
}
/**
@@ -174,8 +129,7 @@ public final class SearcherManager imple
* ie. reader was opened, otherwise <code>false</code>.
* @see IndexReader#isCurrent()
*/
- public boolean isSearcherCurrent() throws CorruptIndexException,
- IOException {
+ public boolean isSearcherCurrent() throws IOException {
final IndexSearcher searcher = acquire();
try {
return searcher.getIndexReader().isCurrent();
@@ -183,56 +137,5 @@ public final class SearcherManager imple
release(searcher);
}
}
-
- /**
- * Release the searcher previously obtained with {@link #acquire}.
- *
- * <p>
- * <b>NOTE</b>: it's safe to call this after {@link #close}.
- */
- public void release(IndexSearcher searcher) throws IOException {
- assert searcher != null;
- searcher.getIndexReader().decRef();
- }
-
- /**
- * Close this SearcherManager to future searching. Any searches still in
- * process in other threads won't be affected, and they should still call
- * {@link #release} after they are done.
- */
- public synchronized void close() throws IOException {
- if (currentSearcher != null) {
- // make sure we can call this more than once
- // closeable javadoc says:
- // if this is already closed then invoking this method has no effect.
- swapSearcher(null);
- }
- }
-
- /**
- * Obtain the current IndexSearcher. You must match every call to acquire with
- * one call to {@link #release}; it's best to do so in a finally clause.
- */
- public IndexSearcher acquire() {
- IndexSearcher searcher;
- do {
- if ((searcher = currentSearcher) == null) {
- throw new AlreadyClosedException("this SearcherManager is closed");
- }
- } while (!searcher.getIndexReader().tryIncRef());
- return searcher;
- }
-
- private void ensureOpen() {
- if (currentSearcher == null) {
- throw new AlreadyClosedException("this SearcherManager is closed");
- }
- }
-
- private synchronized void swapSearcher(IndexSearcher newSearcher) throws IOException {
- ensureOpen();
- final IndexSearcher oldSearcher = currentSearcher;
- currentSearcher = newSearcher;
- release(oldSearcher);
- }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java Tue Feb 14 12:55:43 2012
@@ -31,9 +31,9 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
-import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.NamedThreadFactory;
@@ -55,7 +55,7 @@ public class TestSearcherManager extends
if (!isNRT) {
writer.commit();
}
- assertTrue(mgr.maybeReopen() || mgr.isSearcherCurrent());
+ assertTrue(mgr.maybeRefresh() || mgr.isSearcherCurrent());
return mgr.acquire();
}
@@ -101,7 +101,7 @@ public class TestSearcherManager extends
Thread.sleep(_TestUtil.nextInt(random, 1, 100));
writer.commit();
Thread.sleep(_TestUtil.nextInt(random, 1, 5));
- if (mgr.maybeReopen()) {
+ if (mgr.maybeRefresh()) {
lifetimeMGR.prune(pruner);
}
}
@@ -130,7 +130,7 @@ public class TestSearcherManager extends
// synchronous to your search threads, but still we
// test as apps will presumably do this for
// simplicity:
- if (mgr.maybeReopen()) {
+ if (mgr.maybeRefresh()) {
lifetimeMGR.prune(pruner);
}
}
@@ -196,6 +196,7 @@ public class TestSearcherManager extends
final AtomicBoolean triedReopen = new AtomicBoolean(false);
final ExecutorService es = random.nextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
final SearcherFactory factory = new SearcherFactory() {
+ @Override
public IndexSearcher newSearcher(IndexReader r) throws IOException {
try {
if (triedReopen.get()) {
@@ -232,7 +233,7 @@ public class TestSearcherManager extends
if (VERBOSE) {
System.out.println("NOW call maybeReopen");
}
- searcherManager.maybeReopen();
+ searcherManager.maybeRefresh();
success.set(true);
} catch (AlreadyClosedException e) {
// expected
@@ -275,4 +276,41 @@ public class TestSearcherManager extends
es.awaitTermination(1, TimeUnit.SECONDS);
}
}
+
+ public void testCloseTwice() throws Exception {
+ // test that we can close SM twice (per Closeable's contract).
+ Directory dir = newDirectory();
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+ SearcherManager sm = new SearcherManager(dir, null);
+ sm.close();
+ sm.close();
+ dir.close();
+ }
+
+ public void testEnsureOpen() throws Exception {
+ Directory dir = newDirectory();
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+ SearcherManager sm = new SearcherManager(dir, null);
+ IndexSearcher s = sm.acquire();
+ sm.close();
+
+ // this should succeed;
+ sm.release(s);
+
+ try {
+ // this should fail
+ sm.acquire();
+ } catch (AlreadyClosedException e) {
+ // ok
+ }
+
+ try {
+ // this should fail
+ sm.maybeRefresh();
+ } catch (AlreadyClosedException e) {
+ // ok
+ }
+ dir.close();
+ }
+
}