You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2012/02/14 13:55:44 UTC

svn commit: r1243906 - in /lucene/dev/branches/branch_3x/lucene: ./ core/src/java/org/apache/lucene/search/ core/src/test/org/apache/lucene/search/

Author: shaie
Date: Tue Feb 14 12:55:43 2012
New Revision: 1243906

URL: http://svn.apache.org/viewvc?rev=1243906&view=rev
Log:
LUCENE-3761: Generalize SearcherManager

Added:
    lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java   (with props)
Modified:
    lucene/dev/branches/branch_3x/lucene/CHANGES.txt
    lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
    lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
    lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java

Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Feb 14 12:55:43 2012
@@ -69,7 +69,12 @@ Changes in backwards compatibility polic
   IndexCommit.getTimestamp and .getVersion and
   IndexReader.lastModified and getCurrentVersion (Andrzej Bialecki,
   Robert Muir, Mike McCandless)
-   
+
+* LUCENE-3761: Generalize SearcherManager into an abstract ReferenceManager.
+  SearcherManager remains a concrete class, but due to the refactoring, the 
+  method maybeReopen has been deprecated in favor of maybeRefresh().
+  (Shai Erera, Mike McCandless, Simon Willnauer)
+  
 Security fixes
 
 * LUCENE-3588: Try harder to prevent SIGSEGV on cloned MMapIndexInputs:

Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/NRTManager.java Tue Feb 14 12:55:43 2012
@@ -344,7 +344,7 @@ public class NRTManager implements Close
         }
         boolean setSearchGen;
         if (!mgr.isSearcherCurrent()) {
-          setSearchGen = mgr.maybeReopen();
+          setSearchGen = mgr.maybeRefresh();
         } else {
           setSearchGen = true;
         }

Added: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java?rev=1243906&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java (added)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java Tue Feb 14 12:55:43 2012
@@ -0,0 +1,164 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+
+import org.apache.lucene.store.AlreadyClosedException;
+
+/**
+ * Utility class to safely share instances of a certain type across multiple
+ * threads, while periodically refreshing them. This class ensures each
+ * reference is closed only once all threads have finished using it. It is
+ * recommended to consult the documentation of {@link ReferenceManager}
+ * implementations for their {@link #maybeRefresh()} semantics.
+ * 
+ * @param <G>
+ *          the concrete type that will be {@link #acquire() acquired} and
+ *          {@link #release(Object) released}.
+ * 
+ * @lucene.experimental
+ */
+public abstract class ReferenceManager<G> implements Closeable {
+
+  private static final String REFERENCE_MANAGER_IS_CLOSED_MSG = "this ReferenceManager is closed";
+  
+  protected volatile G current;
+  
+  private final Semaphore reopenLock = new Semaphore(1);
+  
+  private void ensureOpen() {
+    if (current == null) {
+      throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG);
+    }
+  }
+  
+  private synchronized void swapReference(G newReference) throws IOException {
+    ensureOpen();
+    final G oldReference = current;
+    current = newReference;
+    release(oldReference);
+  }
+  
+  /** Decrement reference counting on the given reference. */
+  protected abstract void decRef(G reference) throws IOException;
+  
+  /**
+   * Refresh the given reference if needed. Returns {@code null} if no refresh
+   * was needed, otherwise a new refreshed reference.
+   */
+  protected abstract G refreshIfNeeded(G referenceToRefresh) throws IOException;
+
+  /**
+   * Try to increment reference counting on the given reference. Return true if
+   * the operation was successful.
+   */
+  protected abstract boolean tryIncRef(G reference);
+
+  /**
+   * Obtain the current reference. You must match every call to acquire with one
+   * call to {@link #release}; it's best to do so in a finally clause, and set
+   * the reference to {@code null} to prevent accidental usage after it has been
+   * released.
+   */
+  public final G acquire() {
+    G ref;
+    do {
+      if ((ref = current) == null) {
+        throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG);
+      }
+    } while (!tryIncRef(ref));
+    return ref;
+  }
+
+  /**
+   * Close this ReferenceManager to future {@link #acquire() acquiring}. Any
+   * references that were previously {@link #acquire() acquired} won't be
+   * affected, and they should still be {@link #release released} when they are
+   * not needed anymore.
+   */
+  public final synchronized void close() throws IOException {
+    if (current != null) {
+      // make sure we can call this more than once
+      // closeable javadoc says:
+      // if this is already closed then invoking this method has no effect.
+      swapReference(null);
+    }
+  }
+
+  /**
+   * You must call this, periodically, if you want that {@link #acquire()} will
+   * return refreshed instances.
+   * 
+   * <p>
+   * <b>Threads</b>: it's fine for more than one thread to call this at once.
+   * Only the first thread will attempt the refresh; subsequent threads will see
+   * that another thread is already handling refresh and will return
+   * immediately. Note that this means if another thread is already refreshing
+   * then subsequent threads will return right away without waiting for the
+   * refresh to complete.
+   * 
+   * <p>
+   * This method returns true if the reference was in fact refreshed, or if the
+   * current reference has no pending changes.
+   */
+  public final boolean maybeRefresh() throws IOException {
+    ensureOpen();
+    // Ensure only 1 thread does reopen at once; other threads just return immediately:
+    if (reopenLock.tryAcquire()) {
+      try {
+        final G reference = acquire();
+        try {
+          G newReference = refreshIfNeeded(reference);
+          if (newReference != null) {
+            assert newReference != reference : "refreshIfNeeded should return null if refresh wasn't needed";
+            boolean success = false;
+            try {
+              swapReference(newReference);
+              success = true;
+            } finally {
+              if (!success) {
+                release(newReference);
+              }
+            }
+          }
+        } finally {
+          release(reference);
+        }
+        return true;
+      } finally {
+        reopenLock.release();
+      }
+    } else {
+      return false;
+    }
+  }
+
+  /**
+   * Release the refernce previously obtained via {@link #acquire()}.
+   * <p>
+   * <b>NOTE:</b> it's safe to call this after {@link #close()}.
+   */
+  public final void release(G reference) throws IOException {
+    assert reference != null;
+    decRef(reference);
+  }
+  
+}

Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java Tue Feb 14 12:55:43 2012
@@ -17,16 +17,10 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import java.io.Closeable;
 import java.io.IOException;
-import java.util.concurrent.Semaphore;
 
-import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.search.NRTManager; // javadocs
-import org.apache.lucene.search.IndexSearcher; // javadocs
-import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 
 /**
@@ -50,27 +44,19 @@ import org.apache.lucene.store.Directory
  * </pre>
  * 
  * <p>
- * In addition you should periodically call {@link #maybeReopen}. While it's
+ * In addition you should periodically call {@link #maybeRefresh}. While it's
  * possible to call this just before running each query, this is discouraged
  * since it penalizes the unlucky queries that do the reopen. It's better to use
  * a separate background thread, that periodically calls maybeReopen. Finally,
  * be sure to call {@link #close} once you are done.
  * 
- * <p>
- * <b>NOTE</b>: if you have an {@link IndexWriter}, it's better to use
- * {@link NRTManager} since that class pulls near-real-time readers from the
- * IndexWriter.
- * 
  * @see SearcherFactory
  * 
  * @lucene.experimental
  */
+public final class SearcherManager extends ReferenceManager<IndexSearcher> {
 
-public final class SearcherManager implements Closeable {
-
-  private volatile IndexSearcher currentSearcher;
   private final SearcherFactory searcherFactory;
-  private final Semaphore reopenLock = new Semaphore(1);
   
   /**
    * Creates and returns a new SearcherManager from the given {@link IndexWriter}. 
@@ -93,7 +79,27 @@ public final class SearcherManager imple
       searcherFactory = new SearcherFactory();
     }
     this.searcherFactory = searcherFactory;
-    currentSearcher = searcherFactory.newSearcher(IndexReader.open(writer, applyAllDeletes));
+    current = searcherFactory.newSearcher(IndexReader.open(writer, applyAllDeletes));
+  }
+  
+  @Override
+  protected void decRef(IndexSearcher reference) throws IOException {
+    reference.getIndexReader().decRef();
+  }
+  
+  @Override
+  protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException {
+    final IndexReader newReader = IndexReader.openIfChanged(referenceToRefresh.getIndexReader());
+    if (newReader == null) {
+      return null;
+    } else {
+      return searcherFactory.newSearcher(newReader);
+    }
+  }
+  
+  @Override
+  protected boolean tryIncRef(IndexSearcher reference) {
+    return reference.getIndexReader().tryIncRef();
   }
 
   /**
@@ -110,63 +116,12 @@ public final class SearcherManager imple
       searcherFactory = new SearcherFactory();
     }
     this.searcherFactory = searcherFactory;
-    currentSearcher = searcherFactory.newSearcher(IndexReader.open(dir));
+    current = searcherFactory.newSearcher(IndexReader.open(dir));
   }
 
-  /**
-   * You must call this, periodically, to perform a reopen. This calls
-   * {@link IndexReader#openIfChanged(IndexReader)} with the underlying reader, and if that returns a
-   * new reader, it's warmed (if you provided a {@link SearcherFactory} and then
-   * swapped into production.
-   * 
-   * <p>
-   * <b>Threads</b>: it's fine for more than one thread to call this at once.
-   * Only the first thread will attempt the reopen; subsequent threads will see
-   * that another thread is already handling reopen and will return immediately.
-   * Note that this means if another thread is already reopening then subsequent
-   * threads will return right away without waiting for the reader reopen to
-   * complete.
-   * </p>
-   * 
-   * <p>
-   * This method returns true if a new reader was in fact opened or 
-   * if the current searcher has no pending changes.
-   * </p>
-   */
-  public boolean maybeReopen() throws IOException {
-    ensureOpen();
-    // Ensure only 1 thread does reopen at once; other
-    // threads just return immediately:
-    if (reopenLock.tryAcquire()) {
-      try {
-        // IR.openIfChanged preserves NRT and applyDeletes
-        // in the newly returned reader:
-        final IndexReader newReader;
-        final IndexSearcher searcherToReopen = acquire();
-        try {
-          newReader = IndexReader.openIfChanged(searcherToReopen.getIndexReader());
-        } finally {
-          release(searcherToReopen);
-        }
-        if (newReader != null) {
-          final IndexSearcher newSearcher = searcherFactory.newSearcher(newReader);
-          boolean success = false;
-          try {
-            swapSearcher(newSearcher);
-            success = true;
-          } finally {
-            if (!success) {
-              release(newSearcher);
-            }
-          }
-        }
-        return true;
-      } finally {
-        reopenLock.release();
-      }
-    } else {
-      return false;
-    }
+  /** @deprecated see {@link #maybeRefresh()}. */
+  public final boolean maybeReopen() throws IOException {
+    return maybeRefresh();
   }
   
   /**
@@ -174,8 +129,7 @@ public final class SearcherManager imple
    * ie. reader was opened, otherwise <code>false</code>.
    * @see IndexReader#isCurrent() 
    */
-  public boolean isSearcherCurrent() throws CorruptIndexException,
-      IOException {
+  public boolean isSearcherCurrent() throws IOException {
     final IndexSearcher searcher = acquire();
     try {
       return searcher.getIndexReader().isCurrent();
@@ -183,56 +137,5 @@ public final class SearcherManager imple
       release(searcher);
     }
   }
-
-  /**
-   * Release the searcher previously obtained with {@link #acquire}.
-   * 
-   * <p>
-   * <b>NOTE</b>: it's safe to call this after {@link #close}.
-   */
-  public void release(IndexSearcher searcher) throws IOException {
-    assert searcher != null;
-    searcher.getIndexReader().decRef();
-  }
-
-  /**
-   * Close this SearcherManager to future searching. Any searches still in
-   * process in other threads won't be affected, and they should still call
-   * {@link #release} after they are done.
-   */
-  public synchronized void close() throws IOException {
-    if (currentSearcher != null) {
-      // make sure we can call this more than once
-      // closeable javadoc says:
-      // if this is already closed then invoking this method has no effect.
-      swapSearcher(null);
-    }
-  }
-
-  /**
-   * Obtain the current IndexSearcher. You must match every call to acquire with
-   * one call to {@link #release}; it's best to do so in a finally clause.
-   */
-  public IndexSearcher acquire() {
-    IndexSearcher searcher;
-    do {
-      if ((searcher = currentSearcher) == null) {
-        throw new AlreadyClosedException("this SearcherManager is closed");
-      }
-    } while (!searcher.getIndexReader().tryIncRef());
-    return searcher;
-  }
-
-  private void ensureOpen() {
-    if (currentSearcher == null) {
-      throw new AlreadyClosedException("this SearcherManager is closed");
-    }
-  }
-
-  private synchronized void swapSearcher(IndexSearcher newSearcher) throws IOException {
-    ensureOpen();
-    final IndexSearcher oldSearcher = currentSearcher;
-    currentSearcher = newSearcher;
-    release(oldSearcher);
-  }
+  
 }

Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1243906&r1=1243905&r2=1243906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java Tue Feb 14 12:55:43 2012
@@ -31,9 +31,9 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
-import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.NamedThreadFactory;
@@ -55,7 +55,7 @@ public class TestSearcherManager extends
     if (!isNRT) {
       writer.commit();
     }
-    assertTrue(mgr.maybeReopen() || mgr.isSearcherCurrent());
+    assertTrue(mgr.maybeRefresh() || mgr.isSearcherCurrent());
     return mgr.acquire();
   }
 
@@ -101,7 +101,7 @@ public class TestSearcherManager extends
             Thread.sleep(_TestUtil.nextInt(random, 1, 100));
             writer.commit();
             Thread.sleep(_TestUtil.nextInt(random, 1, 5));
-            if (mgr.maybeReopen()) {
+            if (mgr.maybeRefresh()) {
               lifetimeMGR.prune(pruner);
             }
           }
@@ -130,7 +130,7 @@ public class TestSearcherManager extends
       // synchronous to your search threads, but still we
       // test as apps will presumably do this for
       // simplicity:
-      if (mgr.maybeReopen()) {
+      if (mgr.maybeRefresh()) {
         lifetimeMGR.prune(pruner);
       }
     }
@@ -196,6 +196,7 @@ public class TestSearcherManager extends
     final AtomicBoolean triedReopen = new AtomicBoolean(false);
     final ExecutorService es = random.nextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
     final SearcherFactory factory = new SearcherFactory() {
+      @Override
       public IndexSearcher newSearcher(IndexReader r) throws IOException {
         try {
           if (triedReopen.get()) {
@@ -232,7 +233,7 @@ public class TestSearcherManager extends
           if (VERBOSE) {
             System.out.println("NOW call maybeReopen");
           }
-          searcherManager.maybeReopen();
+          searcherManager.maybeRefresh();
           success.set(true);
         } catch (AlreadyClosedException e) {
           // expected
@@ -275,4 +276,41 @@ public class TestSearcherManager extends
       es.awaitTermination(1, TimeUnit.SECONDS);
     }
   }
+  
+  public void testCloseTwice() throws Exception {
+    // test that we can close SM twice (per Closeable's contract).
+    Directory dir = newDirectory();
+    new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+    SearcherManager sm = new SearcherManager(dir, null);
+    sm.close();
+    sm.close();
+    dir.close();
+  }
+
+  public void testEnsureOpen() throws Exception {
+    Directory dir = newDirectory();
+    new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+    SearcherManager sm = new SearcherManager(dir, null);
+    IndexSearcher s = sm.acquire();
+    sm.close();
+    
+    // this should succeed;
+    sm.release(s);
+    
+    try {
+      // this should fail
+      sm.acquire();
+    } catch (AlreadyClosedException e) {
+      // ok
+    }
+    
+    try {
+      // this should fail
+      sm.maybeRefresh();
+    } catch (AlreadyClosedException e) {
+      // ok
+    }
+    dir.close();
+  }
+  
 }