You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/10/10 21:58:32 UTC

svn commit: r1181188 [2/8] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/grouping/ dev-tools/idea/solr/ dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java Mon Oct 10 19:58:24 2011
@@ -17,157 +17,163 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Semaphore;
 
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.NRTManager; // javadocs
+import org.apache.lucene.search.IndexSearcher; // javadocs
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 
-/** Utility class to safely share {@link IndexSearcher} instances
- *  across multiple threads, while periodically reopening.
- *  This class ensures each IndexSearcher instance is not
- *  closed until it is no longer needed.
- *
- *  <p>Use {@link #acquire} to obtain the current searcher, and
- *  {@link #release} to release it, like this:
- *
- *  <pre>
- *    IndexSearcher s = manager.acquire();
- *    try {
- *      // Do searching, doc retrieval, etc. with s
- *    } finally {
- *      manager.release(s);
- *    }
- *    // Do not use s after this!
- *    s = null;
- *  </pre>
- *
- *  <p>In addition you should periodically call {@link
- *  #maybeReopen}.  While it's possible to call this just
- *  before running each query, this is discouraged since it
- *  penalizes the unlucky queries that do the reopen.  It's
- *  better to  use a separate background thread, that
- *  periodically calls maybeReopen.  Finally, be sure to
- *  call {@link #close} once you are done.
- *
- *  <p><b>NOTE</b>: if you have an {@link IndexWriter}, it's
- *  better to use {@link NRTManager} since that class pulls
- *  near-real-time readers from the IndexWriter.
- *
- *  @lucene.experimental
+/**
+ * Utility class to safely share {@link IndexSearcher} instances across multiple
+ * threads, while periodically reopening. This class ensures each searcher is
+ * closed only once all threads have finished using it.
+ * 
+ * <p>
+ * Use {@link #acquire} to obtain the current searcher, and {@link #release} to
+ * release it, like this:
+ * 
+ * <pre>
+ * IndexSearcher s = manager.acquire();
+ * try {
+ *   // Do searching, doc retrieval, etc. with s
+ * } finally {
+ *   manager.release(s);
+ * }
+ * // Do not use s after this!
+ * s = null;
+ * </pre>
+ * 
+ * <p>
+ * In addition you should periodically call {@link #maybeReopen}. While it's
+ * possible to call this just before running each query, this is discouraged
+ * since it penalizes the unlucky queries that do the reopen. It's better to use
+ * a separate background thread, that periodically calls maybeReopen. Finally,
+ * be sure to call {@link #close} once you are done.
+ * 
+ * <p>
+ * <b>NOTE</b>: if you have an {@link IndexWriter}, it's better to use
+ * {@link NRTManager} since that class pulls near-real-time readers from the
+ * IndexWriter.
+ * 
+ * @lucene.experimental
  */
 
-public class SearcherManager implements Closeable {
+public abstract class SearcherManager {
 
-  // Current searcher
-  private volatile IndexSearcher currentSearcher;
-  private final SearcherWarmer warmer;
-  private final Semaphore reopening = new Semaphore(1);
-  private final ExecutorService es;
-
-  /** Opens an initial searcher from the Directory.
-   *
-   * @param dir Directory to open the searcher from
-   *
-   * @param warmer optional {@link SearcherWarmer}.  Pass
-   *        null if you don't require the searcher to warmed
-   *        before going live.
-   *
-   *  <p><b>NOTE</b>: the provided {@link SearcherWarmer} is
-   *  not invoked for the initial searcher; you should
-   *  warm it yourself if necessary.
-   */
-  public SearcherManager(Directory dir, SearcherWarmer warmer) throws IOException {
-    this(dir, warmer, null);
-  }
-
-  /** Opens an initial searcher from the Directory.
-   *
-   * @param dir Directory to open the searcher from
-   *
-   * @param warmer optional {@link SearcherWarmer}.  Pass
-   *        null if you don't require the searcher to warmed
-   *        before going live.
-   *
-   * @param es optional ExecutorService so different segments can
-   *        be searched concurrently (see {@link
-   *        IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}.  Pass null
-   *        to search segments sequentially.
-   *
-   *  <p><b>NOTE</b>: the provided {@link SearcherWarmer} is
-   *  not invoked for the initial searcher; you should
-   *  warm it yourself if necessary.
-   */
-  public SearcherManager(Directory dir, SearcherWarmer warmer, ExecutorService es) throws IOException {
+  protected volatile IndexSearcher currentSearcher;
+  protected final ExecutorService es;
+  protected final SearcherWarmer warmer;
+  protected final Semaphore reopenLock = new Semaphore(1);
+  
+  protected SearcherManager(IndexReader openedReader, SearcherWarmer warmer,
+      ExecutorService es) throws IOException {
     this.es = es;
-    currentSearcher = new IndexSearcher(IndexReader.open(dir), this.es);
     this.warmer = warmer;
+    currentSearcher = new IndexSearcher(openedReader, es);
   }
 
-  /** You must call this, periodically, to perform a
-   *  reopen.  This calls {@link IndexReader#openIfChanged} on the
-   *  underlying reader, and if that returns a new reader,
-   *  it's warmed (if you provided a {@link SearcherWarmer}
-   *  and then swapped into production.
-   *
-   *  <p><b>Threads</b>: it's fine for more than one thread to
-   *  call this at once.  Only the first thread will attempt
-   *  the reopen; subsequent threads will see that another
-   *  thread is already handling reopen and will return
-   *  immediately.  Note that this means if another thread
-   *  is already reopening then subsequent threads will
-   *  return right away without waiting for the reader
-   *  reopen to complete.</p>
-   *
-   *  <p>This method returns true if a new reader was in
-   *  fact opened.</p>
+  /**
+   * You must call this, periodically, to perform a reopen. This calls
+   * {@link #openIfChanged(IndexReader)} with the underlying reader, and if that returns a
+   * new reader, it's warmed (if you provided a {@link SearcherWarmer} and then
+   * swapped into production.
+   * 
+   * <p>
+   * <b>Threads</b>: it's fine for more than one thread to call this at once.
+   * Only the first thread will attempt the reopen; subsequent threads will see
+   * that another thread is already handling reopen and will return immediately.
+   * Note that this means if another thread is already reopening then subsequent
+   * threads will return right away without waiting for the reader reopen to
+   * complete.
+   * </p>
+   * 
+   * <p>
+   * This method returns true if a new reader was in fact opened.
+   * </p>
    */
-  public boolean maybeReopen()
-    throws  IOException {
-
-    if (currentSearcher == null) {
-      throw new AlreadyClosedException("this SearcherManager is closed");
-    }
-
+  public boolean maybeReopen() throws IOException {
+    ensureOpen();
     // Ensure only 1 thread does reopen at once; other
     // threads just return immediately:
-    if (reopening.tryAcquire()) {
+    if (reopenLock.tryAcquire()) {
       try {
-        IndexReader newReader = IndexReader.openIfChanged(currentSearcher.getIndexReader());
+        final IndexReader newReader = openIfChanged(currentSearcher.getIndexReader());
         if (newReader != null) {
-          IndexSearcher newSearcher = new IndexSearcher(newReader, es);
-          if (warmer != null) {
-            boolean success = false;
-            try {
+          final IndexSearcher newSearcher = new IndexSearcher(newReader, es);
+          boolean success = false;
+          try {
+            if (warmer != null) {
               warmer.warm(newSearcher);
-              success = true;
-            } finally {
-              if (!success) {
-                newReader.decRef();
-              }
+            }
+            swapSearcher(newSearcher);
+            success = true;
+          } finally {
+            if (!success) {
+              release(newSearcher);
             }
           }
-          swapSearcher(newSearcher);
           return true;
         } else {
           return false;
         }
       } finally {
-        reopening.release();
+        reopenLock.release();
       }
     } else {
       return false;
     }
   }
 
-  /** Obtain the current IndexSearcher.  You must match
-   *  every call to acquire with one call to {@link #release};
-   *  it's best to do so in a finally clause. */
+  /**
+   * Returns <code>true</code> if no changes have occured since this searcher
+   * ie. reader was opened, otherwise <code>false</code>.
+   * @see IndexReader#isCurrent() 
+   */
+  public boolean isSearcherCurrent() throws CorruptIndexException,
+      IOException {
+    final IndexSearcher searcher = acquire();
+    try {
+      return searcher.getIndexReader().isCurrent();
+    } finally {
+      release(searcher);
+    }
+  }
+
+  /**
+   * Release the searcher previously obtained with {@link #acquire}.
+   * 
+   * <p>
+   * <b>NOTE</b>: it's safe to call this after {@link #close}.
+   */
+  public void release(IndexSearcher searcher) throws IOException {
+    assert searcher != null;
+    searcher.getIndexReader().decRef();
+  }
+
+  /**
+   * Close this SearcherManager to future searching. Any searches still in
+   * process in other threads won't be affected, and they should still call
+   * {@link #release} after they are done.
+   */
+  public synchronized void close() throws IOException {
+    if (currentSearcher != null) {
+      // make sure we can call this more than once
+      // closeable javadoc says:
+      // if this is already closed then invoking this method has no effect.
+      swapSearcher(null);
+    }
+  }
+
+  /**
+   * Obtain the current IndexSearcher. You must match every call to acquire with
+   * one call to {@link #release}; it's best to do so in a finally clause.
+   */
   public IndexSearcher acquire() {
     IndexSearcher searcher;
     do {
@@ -176,35 +182,130 @@ public class SearcherManager implements 
       }
     } while (!searcher.getIndexReader().tryIncRef());
     return searcher;
-  }    
-
-  /** Release the searcher previously obtained with {@link
-   *  #acquire}.
-   *
-   *  <p><b>NOTE</b>: it's safe to call this after {@link
-   *  #close}. */
-  public void release(IndexSearcher searcher)
-    throws IOException {
-    searcher.getIndexReader().decRef();
   }
 
-  // Replaces old searcher with new one - needs to be synced to make close() work
-  private synchronized void swapSearcher(IndexSearcher newSearcher)
-    throws IOException {
-    IndexSearcher oldSearcher = currentSearcher;
-    if (oldSearcher == null) {
+  private void ensureOpen() {
+    if (currentSearcher == null) {
       throw new AlreadyClosedException("this SearcherManager is closed");
     }
+  }
+
+  protected synchronized void swapSearcher(IndexSearcher newSearcher) throws IOException {
+    ensureOpen();
+    final IndexSearcher oldSearcher = currentSearcher;
     currentSearcher = newSearcher;
     release(oldSearcher);
   }
 
-  /** Close this SearcherManager to future searching.  Any
-   *  searches still in process in other threads won't be
-   *  affected, and they should still call {@link #release}
-   *  after they are done. */
-  @Override
-  public void close() throws IOException {
-    swapSearcher(null);
+  protected abstract IndexReader openIfChanged(IndexReader oldReader)
+      throws IOException;
+
+  /**
+   * Creates and returns a new SearcherManager from the given {@link IndexWriter}. 
+   * @param writer the IndexWriter to open the IndexReader from.
+   * @param applyAllDeletes If <code>true</code>, all buffered deletes will
+   *        be applied (made visible) in the {@link IndexSearcher} / {@link IndexReader}.
+   *        If <code>false</code>, the deletes are not applied but remain buffered 
+   *        (in IndexWriter) so that they will be applied in the future.
+   *        Applying deletes can be costly, so if your app can tolerate deleted documents
+   *        being returned you might gain some performance by passing <code>false</code>.
+   * @param warmer An optional {@link SearcherWarmer}. Pass
+   *        <code>null</code> if you don't require the searcher to warmed
+   *        before going live.  If this is  <code>non-null</code> then a
+   *        merged segment warmer is installed on the
+   *        provided IndexWriter's config.
+   * @param es An optional {@link ExecutorService} so different segments can
+   *        be searched concurrently (see {@link
+   *        IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}.  Pass <code>null</code>
+   *        to search segments sequentially.
+   *        
+   * @see IndexReader#openIfChanged(IndexReader, IndexWriter, boolean)
+   * @throws IOException
+   */
+  public static SearcherManager open(IndexWriter writer, boolean applyAllDeletes,
+      SearcherWarmer warmer, ExecutorService es) throws IOException {
+    final IndexReader open = IndexReader.open(writer, true);
+    boolean success = false;
+    try {
+      SearcherManager manager = new NRTSearcherManager(writer, applyAllDeletes,
+          open, warmer, es);
+      success = true;
+      return manager;
+    } finally {
+      if (!success) {
+        open.close();
+      }
+    }
+  }
+
+  /**
+   * Creates and returns a new SearcherManager from the given {@link Directory}. 
+   * @param dir the directory to open the IndexReader on.
+   * @param warmer An optional {@link SearcherWarmer}.  Pass
+   *        <code>null</code> if you don't require the searcher to warmed
+   *        before going live.  If this is  <code>non-null</code> then a
+   *        merged segment warmer is installed on the
+   *        provided IndexWriter's config.
+   * @param es And optional {@link ExecutorService} so different segments can
+   *        be searched concurrently (see {@link
+   *        IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}.  Pass <code>null</code>
+   *        to search segments sequentially.
+   *        
+   * @throws IOException
+   */
+  public static SearcherManager open(Directory dir, SearcherWarmer warmer,
+      ExecutorService es) throws IOException {
+    final IndexReader open = IndexReader.open(dir, true);
+    boolean success = false;
+    try {
+      SearcherManager manager = new DirectorySearchManager(open, warmer, es);
+      success = true;
+      return manager;
+    } finally {
+      if (!success) {
+        open.close();
+      }
+    }
+  }
+
+  static final class NRTSearcherManager extends SearcherManager {
+    private final IndexWriter writer;
+    private final boolean applyDeletes;
+
+    NRTSearcherManager(final IndexWriter writer, final boolean applyDeletes,
+        final IndexReader openedReader, final SearcherWarmer warmer, final ExecutorService es)
+        throws IOException {
+      super(openedReader, warmer, es);
+      this.writer = writer;
+      this.applyDeletes = applyDeletes;
+      if (warmer != null) {
+        writer.getConfig().setMergedSegmentWarmer(
+            new IndexWriter.IndexReaderWarmer() {
+              @Override
+              public void warm(IndexReader reader) throws IOException {
+                warmer.warm(new IndexSearcher(reader, es));
+              }
+            });
+      }
+    }
+
+    @Override
+    protected IndexReader openIfChanged(IndexReader oldReader)
+        throws IOException {
+      return IndexReader.openIfChanged(oldReader, writer, applyDeletes);
+    }
+  }
+
+  static final class DirectorySearchManager extends SearcherManager {
+    DirectorySearchManager(IndexReader openedReader,
+        SearcherWarmer warmer, ExecutorService es) throws IOException {
+      super(openedReader, warmer, es);
+    }
+
+    @Override
+    protected IndexReader openIfChanged(IndexReader oldReader)
+        throws IOException {
+      return IndexReader.openIfChanged(oldReader, true);
+    }
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Mon Oct 10 19:58:24 2011
@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.search.SearcherWarmer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
@@ -41,7 +42,8 @@ public class TestNRTManager extends Thre
     if (VERBOSE) {
       System.out.println("TEST: finalSearcher maxGen=" + maxGen);
     }
-    return nrt.get(maxGen, true);
+    final SearcherManager manager = nrt.waitForGeneration(maxGen, true);
+    return manager.acquire();
   }
 
   @Override
@@ -67,14 +69,15 @@ public class TestNRTManager extends Thre
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
       }
-      final IndexSearcher s = nrt.get(gen, true);
+      SearcherManager manager = nrt.waitForGeneration(gen, true);
+      final IndexSearcher s = manager.acquire();
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
       }
       try {
         assertEquals(docs.size(), s.search(new TermQuery(id), 10).totalHits);
       } finally {
-        nrt.release(s);
+        manager.release(s);
       }
     }
     
@@ -89,14 +92,15 @@ public class TestNRTManager extends Thre
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
       }
-      final IndexSearcher s = nrt.get(gen, false);
+      final SearcherManager manager = nrt.waitForGeneration(gen, false);
+      final IndexSearcher s = manager.acquire();
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
       }
       try {
         assertEquals(docs.size(), s.search(new TermQuery(id), 10).totalHits);
       } finally {
-        nrt.release(s);
+        manager.release(s);
       }
     }
     lastGens.set(gen);
@@ -111,14 +115,15 @@ public class TestNRTManager extends Thre
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
       }
-      final IndexSearcher s = nrt.get(gen, false);
+      final SearcherManager manager = nrt.waitForGeneration(gen, false);
+      final IndexSearcher s = manager.acquire();
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
       }
       try {
         assertEquals(1, s.search(new TermQuery(id), 10).totalHits);
       } finally {
-        nrt.release(s);
+        manager.release(s);
       }
     }
     lastGens.set(gen);
@@ -132,14 +137,15 @@ public class TestNRTManager extends Thre
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
       }
-      final IndexSearcher s = nrt.get(gen, true);
+      final SearcherManager manager = nrt.waitForGeneration(gen, true);
+      final IndexSearcher s = manager.acquire();
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
       }
       try {
         assertEquals(1, s.search(new TermQuery(id), 10).totalHits);
       } finally {
-        nrt.release(s);
+        manager.release(s);
       }
     }
     lastGens.set(gen);
@@ -153,14 +159,15 @@ public class TestNRTManager extends Thre
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: verify del " + id);
       }
-      final IndexSearcher s = nrt.get(gen, true);
+      final SearcherManager manager = nrt.waitForGeneration(gen, true);
+      final IndexSearcher s = manager.acquire();
       if (VERBOSE) {
         System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
       }
       try {
         assertEquals(0, s.search(new TermQuery(id), 10).totalHits);
       } finally {
-        nrt.release(s);
+        manager.release(s);
       }
     }
     lastGens.set(gen);
@@ -168,7 +175,6 @@ public class TestNRTManager extends Thre
 
   private NRTManager nrt;
   private NRTManagerReopenThread nrtThread;
-
   @Override
   protected void doAfterWriter(ExecutorService es) throws Exception {
     final double minReopenSec = 0.01 + 0.05 * random.nextDouble();
@@ -185,7 +191,8 @@ public class TestNRTManager extends Thre
                              TestNRTManager.this.warmCalled = true;
                              s.search(new TermQuery(new Term("body", "united")), 10);
                            }
-                         });
+                         }, false);
+                         
     nrtThread = new NRTManagerReopenThread(nrt, maxReopenSec, minReopenSec);
     nrtThread.setName("NRT Reopen Thread");
     nrtThread.setPriority(Math.min(Thread.currentThread().getPriority()+2, Thread.MAX_PRIORITY));
@@ -214,12 +221,12 @@ public class TestNRTManager extends Thre
 
   @Override
   protected IndexSearcher getCurrentSearcher() throws Exception {
-    return nrt.get(random.nextBoolean());
+    return nrt.getSearcherManager(false).acquire();
   }
 
   @Override
   protected void releaseSearcher(IndexSearcher s) throws Exception {
-    nrt.release(s);
+    nrt.getSearcherManager(false).release(s);
   }
 
   @Override

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java Mon Oct 10 19:58:24 2011
@@ -18,10 +18,20 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
-
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.NamedThreadFactory;
 import org.apache.lucene.util._TestUtil;
 
 public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
@@ -34,25 +44,35 @@ public class TestSearcherManager extends
 
   @Override
   protected IndexSearcher getFinalSearcher() throws Exception  {
-    writer.commit();
-    mgr.maybeReopen();
+    if (!isNRT) {
+      writer.commit();
+    }
+    assertTrue(mgr.maybeReopen() || mgr.isSearcherCurrent());
     return mgr.acquire();
   }
 
   private SearcherManager mgr;
+  private boolean isNRT;
 
   @Override
   protected void doAfterWriter(ExecutorService es) throws Exception {
     // SearcherManager needs to see empty commit:
-    writer.commit();
-    mgr = new SearcherManager(dir,
-                              new SearcherWarmer() {
-                                @Override
-                                public void warm(IndexSearcher s) throws IOException {
-                                  TestSearcherManager.this.warmCalled = true;
-                                  s.search(new TermQuery(new Term("body", "united")), 10);
-                                }
-                              }, es);
+    final SearcherWarmer warmer = new SearcherWarmer() {
+      @Override
+      public void warm(IndexSearcher s) throws IOException {
+        TestSearcherManager.this.warmCalled = true;
+        s.search(new TermQuery(new Term("body", "united")), 10);
+      }
+    };
+    if (random.nextBoolean()) {
+      mgr = SearcherManager.open(writer, true, warmer, es);
+      isNRT = true;
+    } else {
+      writer.commit();
+      mgr = SearcherManager.open(dir, warmer, es);
+      isNRT = false;
+    }
+    
   }
 
   @Override
@@ -110,4 +130,78 @@ public class TestSearcherManager extends
     }
     mgr.close();
   }
+  
+  public void testIntermediateClose() throws IOException, InterruptedException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+        TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    writer.addDocument(new Document());
+    writer.commit();
+    final CountDownLatch awaitEnterWarm = new CountDownLatch(1);
+    final CountDownLatch awaitClose = new CountDownLatch(1);
+    final ExecutorService es = random.nextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
+    final SearcherWarmer warmer = new SearcherWarmer() {
+      @Override
+      public void warm(IndexSearcher s) throws IOException {
+        try {
+          awaitEnterWarm.countDown();
+          awaitClose.await();
+        } catch (InterruptedException e) {
+          //
+        }
+      }
+    };
+    final SearcherManager searcherManager = random.nextBoolean() ? SearcherManager.open(dir,
+        warmer, es) : SearcherManager.open(writer, random.nextBoolean(), warmer, es);
+    IndexSearcher searcher = searcherManager.acquire();
+    try {
+      assertEquals(1, searcher.getIndexReader().numDocs());
+    } finally {
+      searcherManager.release(searcher);
+    }
+    writer.addDocument(new Document());
+    writer.commit();
+    final AtomicBoolean success = new AtomicBoolean(false);
+    final AtomicBoolean triedReopen = new AtomicBoolean(false);
+    final Throwable[] exc = new Throwable[1];
+    Thread thread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          triedReopen.set(true);
+          searcherManager.maybeReopen();
+          success.set(true);
+        } catch (AlreadyClosedException e) {
+          // expected
+        } catch (Throwable e) {
+          exc[0] = e;
+          // use success as the barrier here to make sure we see the write
+          success.set(false);
+
+        }
+      }
+    });
+    thread.start();
+    awaitEnterWarm.await();
+    for (int i = 0; i < 2; i++) {
+      searcherManager.close();
+    }
+    awaitClose.countDown();
+    thread.join();
+    try {
+      searcherManager.acquire();
+      fail("already closed");
+    } catch (AlreadyClosedException ex) {
+      // expected
+    }
+    assertFalse(success.get());
+    assertTrue(triedReopen.get());
+    assertNull("" + exc[0], exc[0]);
+    writer.close();
+    dir.close();
+    if (es != null) {
+      es.shutdown();
+      es.awaitTermination(1, TimeUnit.SECONDS);
+    }
+  }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java Mon Oct 10 19:58:24 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.document;
 import java.io.Reader;
 import java.util.Comparator;
 
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.values.PerDocFieldValues;
 import org.apache.lucene.index.values.ValueType;
@@ -317,21 +316,34 @@ public class IndexDocValuesField extends
     final String value;
     switch (type) {
     case BYTES_FIXED_DEREF:
-    case BYTES_FIXED_SORTED:
     case BYTES_FIXED_STRAIGHT:
     case BYTES_VAR_DEREF:
-    case BYTES_VAR_SORTED:
     case BYTES_VAR_STRAIGHT:
-      value = "bytes:bytes.utf8ToString();";
+    case BYTES_FIXED_SORTED:
+    case BYTES_VAR_SORTED:
+      // don't use to unicode string this is not necessarily unicode here
+      value = "bytes: " + bytes.toString();
+      break;
+    case FIXED_INTS_16:
+      value = "int16: " + longValue;
+      break;
+    case FIXED_INTS_32:
+      value = "int32: " + longValue;
+      break;
+    case FIXED_INTS_64:
+      value = "int64: " + longValue;
+      break;
+    case FIXED_INTS_8:
+      value = "int8: " + longValue;
       break;
     case VAR_INTS:
-      value = "int:" + longValue;
+      value = "vint: " + longValue;
       break;
     case FLOAT_32:
-      value = "float32:" + doubleValue;
+      value = "float32: " + doubleValue;
       break;
     case FLOAT_64:
-      value = "float64:" + doubleValue;
+      value = "float64: " + doubleValue;
       break;
     default:
       throw new IllegalArgumentException("unknown type: " + type);
@@ -353,14 +365,18 @@ public class IndexDocValuesField extends
     final IndexDocValuesField valField = new IndexDocValuesField(field.name(), field.fieldType(), field.stringValue());
     switch (type) {
     case BYTES_FIXED_DEREF:
-    case BYTES_FIXED_SORTED:
     case BYTES_FIXED_STRAIGHT:
     case BYTES_VAR_DEREF:
-    case BYTES_VAR_SORTED:
     case BYTES_VAR_STRAIGHT:
+    case BYTES_FIXED_SORTED:
+    case BYTES_VAR_SORTED:
       BytesRef ref = field.isBinary() ? field.binaryValue() : new BytesRef(field.stringValue());
       valField.setBytes(ref, type);
       break;
+    case FIXED_INTS_16:
+    case FIXED_INTS_32:
+    case FIXED_INTS_64:
+    case FIXED_INTS_8:
     case VAR_INTS:
       valField.setInt(Long.parseLong(field.stringValue()));
       break;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java Mon Oct 10 19:58:24 2011
@@ -41,7 +41,7 @@ import java.util.Map;
 import org.apache.lucene.index.codecs.BlockTreeTermsReader;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.ValuesEnum;
+import org.apache.lucene.index.values.IndexDocValues.Source;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -1070,27 +1070,28 @@ public class CheckIndex {
           if (docValues == null) {
             continue;
           }
-          final ValuesEnum values = docValues.getEnum();
-          while (values.nextDoc() != ValuesEnum.NO_MORE_DOCS) {
+          final Source values = docValues.getDirectSource();
+          final int maxDoc = reader.maxDoc();
+          for (int i = 0; i < maxDoc; i++) {
             switch (fieldInfo.docValues) {
-            case BYTES_FIXED_DEREF:
             case BYTES_FIXED_SORTED:
+            case BYTES_VAR_SORTED:
+            case BYTES_FIXED_DEREF:
             case BYTES_FIXED_STRAIGHT:
             case BYTES_VAR_DEREF:
-            case BYTES_VAR_SORTED:
             case BYTES_VAR_STRAIGHT:
-              values.bytes();
+              values.getBytes(i, new BytesRef());
               break;
             case FLOAT_32:
             case FLOAT_64:
-              values.getFloat();
+              values.getFloat(i);
               break;
             case VAR_INTS:
             case FIXED_INTS_16:
             case FIXED_INTS_32:
             case FIXED_INTS_64:
             case FIXED_INTS_8:
-              values.getInt();
+              values.getInt(i);
               break;
             default:
               throw new IllegalArgumentException("Field: " + fieldInfo.name

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Mon Oct 10 19:58:24 2011
@@ -186,13 +186,41 @@ public abstract class DocumentsWriterPer
     if (numThreadStatesActive < perThreads.length) {
       final ThreadState threadState = perThreads[numThreadStatesActive];
       threadState.lock(); // lock so nobody else will get this ThreadState
-      numThreadStatesActive++; // increment will publish the ThreadState
-      threadState.perThread.initialize();
-      return threadState;
+      boolean unlock = true;
+      try {
+        if (threadState.isActive()) {
+          // unreleased thread states are deactivated during DW#close()
+          numThreadStatesActive++; // increment will publish the ThreadState
+          assert threadState.perThread != null;
+          threadState.perThread.initialize();
+          unlock = false;
+          return threadState;
+        }
+        // unlock since the threadstate is not active anymore - we are closed!
+        assert assertUnreleasedThreadStatesInactive();
+        return null;
+      } finally {
+        if (unlock) {
+          // in any case make sure we unlock if we fail 
+          threadState.unlock();
+        }
+      }
     }
     return null;
   }
   
+  private synchronized boolean assertUnreleasedThreadStatesInactive() {
+    for (int i = numThreadStatesActive; i < perThreads.length; i++) {
+      assert perThreads[i].tryLock() : "unreleased threadstate should not be locked";
+      try {
+        assert !perThreads[i].isActive() : "expected unreleased thread state to be inactive";
+      } finally {
+        perThreads[i].unlock();
+      }
+    }
+    return true;
+  }
+  
   /**
    * Deactivate all unreleased threadstates 
    */

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java Mon Oct 10 19:58:24 2011
@@ -86,7 +86,7 @@ public final class FieldInfo {
   public int getCodecId() {
     return codecId;
   }
-
+  
   @Override
   public Object clone() {
     FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
@@ -132,6 +132,12 @@ public final class FieldInfo {
     }
   }
   
+  public void resetDocValues(ValueType v) {
+    if (docValues != null) {
+      docValues = v;
+    }
+  }
+  
   public boolean hasDocValues() {
     return docValues != null;
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java Mon Oct 10 19:58:24 2011
@@ -652,31 +652,30 @@ public final class FieldInfos implements
         case BYTES_FIXED_DEREF:
           b = 5;
           break;
-        case BYTES_FIXED_SORTED:
+        case BYTES_VAR_STRAIGHT:
           b = 6;
           break;
-        case BYTES_VAR_STRAIGHT:
+        case BYTES_VAR_DEREF:
           b = 7;
           break;
-        case BYTES_VAR_DEREF:
+        case FIXED_INTS_16:
           b = 8;
           break;
-        case BYTES_VAR_SORTED:
+        case FIXED_INTS_32:
           b = 9;
           break;
-        case FIXED_INTS_16:
+        case FIXED_INTS_64:
           b = 10;
           break;
-        case FIXED_INTS_32:
+        case FIXED_INTS_8:
           b = 11;
           break;
-        case FIXED_INTS_64:
+        case BYTES_FIXED_SORTED:
           b = 12;
           break;
-        case FIXED_INTS_8:
+        case BYTES_VAR_SORTED:
           b = 13;
           break;
-       
         default:
           throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
         }
@@ -754,29 +753,29 @@ public final class FieldInfos implements
           docValuesType = ValueType.BYTES_FIXED_DEREF;
           break;
         case 6:
-          docValuesType = ValueType.BYTES_FIXED_SORTED;
+          docValuesType = ValueType.BYTES_VAR_STRAIGHT;
           break;
         case 7:
-          docValuesType = ValueType.BYTES_VAR_STRAIGHT;
+          docValuesType = ValueType.BYTES_VAR_DEREF;
           break;
         case 8:
-          docValuesType = ValueType.BYTES_VAR_DEREF;
+          docValuesType = ValueType.FIXED_INTS_16;
           break;
         case 9:
-          docValuesType = ValueType.BYTES_VAR_SORTED;
+          docValuesType = ValueType.FIXED_INTS_32;
           break;
         case 10:
-          docValuesType = ValueType.FIXED_INTS_16;
+          docValuesType = ValueType.FIXED_INTS_64;
           break;
         case 11:
-          docValuesType = ValueType.FIXED_INTS_32;
+          docValuesType = ValueType.FIXED_INTS_8;
           break;
         case 12:
-          docValuesType = ValueType.FIXED_INTS_64;
+          docValuesType = ValueType.BYTES_FIXED_SORTED;
           break;
         case 13:
-          docValuesType = ValueType.FIXED_INTS_8;
-          break;  
+          docValuesType = ValueType.BYTES_VAR_SORTED;
+          break;
         
         default:
           throw new IllegalStateException("unhandled indexValues type " + b);

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java Mon Oct 10 19:58:24 2011
@@ -128,7 +128,7 @@ public class MultiPerDocValues extends P
           if (docsUpto != start) {
             type = values.type();
             docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(
-                new MultiIndexDocValues.DummyDocValues(start, type), docsUpto, start
+                new MultiIndexDocValues.EmptyDocValues(start, type), docsUpto, start
                     - docsUpto));
           }
           docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(values, start,
@@ -137,7 +137,7 @@ public class MultiPerDocValues extends P
 
         } else if (i + 1 == subs.length && !docValuesIndex.isEmpty()) {
           docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(
-              new MultiIndexDocValues.DummyDocValues(start, type), docsUpto, start
+              new MultiIndexDocValues.EmptyDocValues(start, type), docsUpto, start
                   - docsUpto));
         }
       }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Mon Oct 10 19:58:24 2011
@@ -33,7 +33,6 @@ import org.apache.lucene.index.codecs.Fi
 import org.apache.lucene.index.codecs.FieldsWriter;
 import org.apache.lucene.index.codecs.MergeState;
 import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -141,6 +140,8 @@ final class SegmentMerger {
     if (fieldInfos.hasVectors()) {
       mergeVectors();
     }
+    // write FIS once merge is done. IDV might change types or drops fields
+    fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
     return mergedDocs;
   }
 
@@ -254,7 +255,6 @@ final class SegmentMerger {
       }
     }
     final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false);
-    fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
 
     int docCount = 0;
 
@@ -584,28 +584,11 @@ final class SegmentMerger {
   }
 
   private void mergePerDoc() throws IOException {
-    final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();    
-    final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
-    int docBase = 0;
-    for (MergeState.IndexReaderAndLiveDocs r : readers) {
-      final int maxDoc = r.reader.maxDoc();
-      final PerDocValues producer = r.reader.perDocValues();
-      if (producer != null) {
-        perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, perDocProducers
-            .size()));
-        perDocProducers.add(producer);
-      }
-      docBase += maxDoc;
-    }
-    if (!perDocSlices.isEmpty()) {
       final PerDocConsumer docsConsumer = codec
           .docsConsumer(new PerDocWriteState(segmentWriteState));
       boolean success = false;
       try {
-        final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(
-            perDocProducers.toArray(PerDocValues.EMPTY_ARRAY),
-            perDocSlices.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
-        docsConsumer.merge(mergeState, multiPerDocValues);
+        docsConsumer.merge(mergeState);
         success = true;
       } finally {
         if (success) {
@@ -614,11 +597,8 @@ final class SegmentMerger {
           IOUtils.closeWhileHandlingException(docsConsumer);
         }
       }
-    }
-    /* don't close the perDocProducers here since they are private segment producers
-     * and will be closed once the SegmentReader goes out of scope */ 
   }
-
+  
   private MergeState mergeState;
 
   public boolean getAnyNonBulkMerges() {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Mon Oct 10 19:58:24 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.Set;
 
 import org.apache.lucene.index.PerDocWriteState;
@@ -26,7 +25,6 @@ import org.apache.lucene.index.SegmentIn
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 
 /** @lucene.experimental */
 public abstract class Codec {
@@ -34,22 +32,9 @@ public abstract class Codec {
   /** Unique name that's used to retrieve this codec when
    *  reading the index */
   public final String name;
-  protected final boolean dvUseCompoundFile;
-  protected final Comparator<BytesRef> docValuesSortComparator;
   
   protected Codec(String name) {
-    this(name, true);
-  }
-  
-  protected Codec(String name, boolean docValuesUseCompoundFile) {
-    this(name, docValuesUseCompoundFile, BytesRef.getUTF8SortedAsUnicodeComparator());
-  }
-
-  protected Codec(String name, boolean docValuesUseCompoundFile,
-      Comparator<BytesRef> docValuesSortComparator) {
     this.name = name;
-    this.dvUseCompoundFile = docValuesUseCompoundFile;
-    this.docValuesSortComparator = docValuesSortComparator;
   }
 
   /** Writes a new segment */
@@ -87,38 +72,6 @@ public abstract class Codec {
 
   /** Records all file extensions this codec uses */
   public abstract void getExtensions(Set<String> extensions);
-
-  /**
-   * Returns <code>true</code> iff compound file should be used for
-   * IndexDocValues, otherwise <code>false</code>. The default is
-   * <code>true</code>.
-   * <p>
-   * NOTE: To change the default value you need to subclass a {@link Codec} with
-   * a distinct name since this value is final and should not be changed to
-   * prevent the risk of a index corruption. This setting is private to a
-   * {@link Codec}. If you intend to change this value on an existing
-   * {@link Codec} re-indexing is required.
-   * 
-   * @return <code>true</code> iff compound file should be used for
-   *         IndexDocValues, otherwise <code>false</code>.
-   */
-  public boolean getDocValuesUseCFS() {
-    return dvUseCompoundFile;
-  }
-
-  /**
-   * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
-   * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
-   * <p>
-   * NOTE: To change the default value you need to subclass a {@link Codec} with
-   * a distinct name since this value is final and should not be changed to
-   * prevent the risk of a index corruption. This setting is private to a
-   * {@link Codec}. If you intend to change this value on an existing
-   * {@link Codec} re-indexing is required.
-   */
-  public Comparator<BytesRef> getDocValuesSortComparator() {
-    return docValuesSortComparator;
-  }
   
   @Override
   public String toString() {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.Set;
 
 import org.apache.lucene.index.FieldInfo;
@@ -26,113 +25,51 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.codecs.DocValuesWriterBase;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Counter;
 
 /**
- * 
+ * Default PerDocConsumer implementation that uses compound file.
  * @lucene.experimental
  */
-public class DefaultDocValuesConsumer extends PerDocConsumer {
-  private final String segmentName;
-  private final int codecId;
+public class DefaultDocValuesConsumer extends DocValuesWriterBase {
   private final Directory directory;
-  private final Counter bytesUsed;
-  private final Comparator<BytesRef> comparator;
-  private boolean useCompoundFile;
-  private final IOContext context;
   
-  public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
-    this.segmentName = state.segmentName;
-    this.codecId = state.codecId;
-    this.bytesUsed = state.bytesUsed;
-    this.context = state.context;
+  public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException {
+    super(state);
     //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
-    this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory,
-        IndexFileNames.segmentFileName(segmentName, codecId,
-            IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory;
-    this.comparator = comparator;
-    this.useCompoundFile = useCompoundFile;
+    this.directory = new CompoundFileDirectory(state.directory,
+        IndexFileNames.segmentFileName(state.segmentName, state.codecId,
+            IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true);
   }
-
-  public void close() throws IOException {
-    if (useCompoundFile) {
-      this.directory.close();
-    }
+  
+  @Override
+  protected Directory getDirectory() {
+    return directory;
   }
 
   @Override
-  public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
-    return Writer.create(field.getDocValues(),
-        docValuesId(segmentName, codecId, field.number),
-        directory, comparator, bytesUsed, context);
+  public void close() throws IOException {
+    this.directory.close();
   }
-  
+
   @SuppressWarnings("fallthrough")
-  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
-      Set<String> files, boolean useCompoundFile) throws IOException {
+  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     FieldInfos fieldInfos = segmentInfo.getFieldInfos();
     for (FieldInfo fieldInfo : fieldInfos) {
       if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
-        String filename = docValuesId(segmentInfo.name, codecId,
-            fieldInfo.number);
-        if (useCompoundFile) {
-          files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
-          files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
-          assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); 
-          assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); 
-          return;
-        } else {
-          switch (fieldInfo.getDocValues()) {
-          case BYTES_FIXED_DEREF:
-          case BYTES_VAR_DEREF:
-          case BYTES_VAR_SORTED:
-          case BYTES_FIXED_SORTED:
-          case BYTES_VAR_STRAIGHT:
-            files.add(IndexFileNames.segmentFileName(filename, "",
-                Writer.INDEX_EXTENSION));
-            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
-                Writer.INDEX_EXTENSION));
-            // until here all types use an index
-          case BYTES_FIXED_STRAIGHT:
-          case FLOAT_32:
-          case FLOAT_64:
-          case VAR_INTS:
-          case FIXED_INTS_16:
-          case FIXED_INTS_32:
-          case FIXED_INTS_64:
-          case FIXED_INTS_8:
-            files.add(IndexFileNames.segmentFileName(filename, "",
-                Writer.DATA_EXTENSION));
-            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
-                Writer.DATA_EXTENSION));
-            break;
-        
-          default:
-            assert false;
-          }
-        }
+        files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+        files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+        assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); 
+        assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); 
+        return;
       }
     }
   }
   
-
-  static String docValuesId(String segmentsName, int codecID, int fieldId) {
-    return segmentsName + "_" + codecID + "-" + fieldId;
-  }
-  
-  public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
-    if (useCompoundFile) {
-      extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
-      extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
-    } else {
-      extensions.add(Writer.DATA_EXTENSION);
-      extensions.add(Writer.INDEX_EXTENSION);
-    }
+  public static void getExtensions(Set<String> extensions) {
+    extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
+    extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
   }
-
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Mon Oct 10 19:58:24 2011
@@ -16,184 +16,50 @@ package org.apache.lucene.index.codecs;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
+import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.values.Bytes;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.codecs.DocValuesReaderBase;
 import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.ValueType;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
 /**
- * Abstract base class for FieldsProducer implementations supporting
- * {@link IndexDocValues}.
- * 
+ * Default PerDocValues implementation that uses compound file.
  * @lucene.experimental
  */
-public class DefaultDocValuesProducer extends PerDocValues {
-
+public class DefaultDocValuesProducer extends DocValuesReaderBase {
   protected final TreeMap<String, IndexDocValues> docValues;
-  private final boolean useCompoundFile;
-  private final Closeable cfs;
-  private final Comparator<BytesRef> sortComparator;
+  private final Directory cfs;
 
   /**
-   * 
    * Creates a new {@link DefaultDocValuesProducer} instance and loads all
    * {@link IndexDocValues} instances for this segment and codec.
-   * 
-   * @param si
-   *          the segment info to load the {@link IndexDocValues} for.
-   * @param dir
-   *          the directory to load the {@link IndexDocValues} from.
-   * @param fieldInfo
-   *          the {@link FieldInfos}
-   * @param codecId
-   *          the codec ID
-   * @param useCompoundFile
-   *          if <code>true</code> this producer opens a compound file to read
-   *          IndexDocValues fields, otherwise each field defines its own set of
-   *          files.
-   * @param sortComparator
-   *          defines the sort order for sorted IndexDocValues variants
-   * @throws IOException
-   *           if an {@link IOException} occurs
    */
-  public DefaultDocValuesProducer(SegmentInfo si, Directory dir, 
-      FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
-    this.useCompoundFile = useCompoundFile;
-    this.sortComparator = sortComparator;
-    final Directory directory;
-    if (useCompoundFile) {
-      cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
-    } else {
-      cfs = null;
-      directory = dir;
-    }
-    docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context);
-  }
-
-  /**
-   * Returns a {@link IndexDocValues} instance for the given field name or
-   * <code>null</code> if this field has no {@link IndexDocValues}.
-   */
-  @Override
-  public IndexDocValues docValues(String field) throws IOException {
-    return docValues.get(field);
-  }
-
-  // Only opens files... doesn't actually load any values
-  protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
-      String segment, int docCount, Directory dir, int codecId, IOContext context)
-      throws IOException {
-    TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
-    boolean success = false;
-    try {
-
-      for (FieldInfo fieldInfo : fieldInfos) {
-        if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
-          final String field = fieldInfo.name;
-          // TODO can we have a compound file per segment and codec for
-          // docvalues?
-          final String id = DefaultDocValuesConsumer.docValuesId(segment,
-              codecId, fieldInfo.number);
-          values.put(field,
-              loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context));
-        }
-      }
-      success = true;
-    } finally {
-      if (!success) {
-        // if we fail we must close all opened resources if there are any
-        closeInternal(values.values());
-      }
-    }
-    return values;
+  public DefaultDocValuesProducer(SegmentReadState state) throws IOException {
+    cfs = new CompoundFileDirectory(state.dir, 
+        IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), 
+        state.context, false);
+    docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context);
   }
   
-
-  /**
-   * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
-   * Codecs that use different implementations for a certain {@link ValueType} can
-   * simply override this method and return their custom implementations.
-   * 
-   * @param docCount
-   *          number of documents in the segment
-   * @param dir
-   *          the {@link Directory} to load the {@link IndexDocValues} from
-   * @param id
-   *          the unique file ID within the segment
-   * @param type
-   *          the type to load
-   * @param sortComparator byte comparator used by sorted variants
-   * @return a {@link IndexDocValues} instance for the given type
-   * @throws IOException
-   *           if an {@link IOException} occurs
-   * @throws IllegalArgumentException
-   *           if the given {@link ValueType} is not supported
-   */
-  protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
-      ValueType type, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
-    switch (type) {
-    case FIXED_INTS_16:
-    case FIXED_INTS_32:
-    case FIXED_INTS_64:
-    case FIXED_INTS_8:
-    case VAR_INTS:
-      return Ints.getValues(dir, id, docCount, type, context);
-    case FLOAT_32:
-      return Floats.getValues(dir, id, docCount, context);
-    case FLOAT_64:
-      return Floats.getValues(dir, id, docCount, context);
-    case BYTES_FIXED_STRAIGHT:
-      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context);
-    case BYTES_FIXED_DEREF:
-      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context);
-    case BYTES_FIXED_SORTED:
-      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context);
-    case BYTES_VAR_STRAIGHT:
-      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context);
-    case BYTES_VAR_DEREF:
-      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context);
-    case BYTES_VAR_SORTED:
-      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context);
-    default:
-      throw new IllegalStateException("unrecognized index values mode " + type);
-    }
-  }
-
-  public void close() throws IOException {
-    closeInternal(docValues.values());
-  }
-
-  private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
-    final Collection<? extends Closeable> toClose;
-    if (useCompoundFile) {
-      final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
-      list.add(cfs);
-      toClose = list; 
-    } else {
-      toClose = closeables;
-    } 
-    IOUtils.close(toClose);
+  @Override
+  protected Map<String,IndexDocValues> docValues() {
+    return docValues;
   }
 
   @Override
-  public Collection<String> fields() {
-    return docValues.keySet();
+  protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
+    final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
+    list.add(cfs);
+    IOUtils.close(list);
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -92,48 +92,38 @@ public abstract class DocValuesConsumer 
    * 
    * @param mergeState
    *          the state to merge
-   * @param values
-   *          the docValues to merge in
+   * @param docValues docValues array containing one instance per reader (
+   *          {@link MergeState#readers}) or <code>null</code> if the reader has
+   *          no {@link IndexDocValues} instance.
    * @throws IOException
    *           if an {@link IOException} occurs
    */
-  public void merge(org.apache.lucene.index.codecs.MergeState mergeState,
-      IndexDocValues values) throws IOException {
+  public void merge(MergeState mergeState, IndexDocValues[] docValues) throws IOException {
     assert mergeState != null;
-    // TODO we need some kind of compatibility notation for values such
-    // that two slightly different segments can be merged eg. fixed vs.
-    // variable byte len or float32 vs. float64
-    boolean merged = false;
-    /*
-     * We ignore the given DocValues here and merge from the subReaders directly
-     * to support bulk copies on the DocValues Writer level. if this gets merged
-     * with MultiDocValues the writer can not optimize for bulk-copyable data
-     */
+    boolean hasMerged = false;
     for(int readerIDX=0;readerIDX<mergeState.readers.size();readerIDX++) {
       final org.apache.lucene.index.codecs.MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(readerIDX);
-      final IndexDocValues r = reader.reader.docValues(mergeState.fieldInfo.name);
-      if (r != null) {
-        merged = true;
-        merge(new Writer.MergeState(r, mergeState.docBase[readerIDX], reader.reader.maxDoc(),
+      if (docValues[readerIDX] != null) {
+        hasMerged = true;
+        merge(new Writer.SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
                                     reader.liveDocs));
       }
     }
-    if (merged) {
+    // only finish if no exception is thrown!
+    if (hasMerged) {
       finish(mergeState.mergedDocCount);
     }
   }
 
   /**
-   * Merges the given {@link MergeState} into this {@link DocValuesConsumer}.
-   * {@link MergeState#docBase} must always be increasing. Merging segments out
-   * of order is not supported.
+   * Merges the given {@link SingleSubMergeState} into this {@link DocValuesConsumer}.
    * 
    * @param mergeState
-   *          the {@link MergeState} to merge
+   *          the {@link SingleSubMergeState} to merge
    * @throws IOException
    *           if an {@link IOException} occurs
    */
-  protected abstract void merge(MergeState mergeState) throws IOException;
+  protected abstract void merge(SingleSubMergeState mergeState) throws IOException;
 
   /**
    * Specialized auxiliary MergeState is necessary since we don't want to
@@ -141,7 +131,7 @@ public abstract class DocValuesConsumer 
    * created for each merged low level {@link IndexReader} we are merging to
    * support low level bulk copies.
    */
-  public static class MergeState {
+  public static class SingleSubMergeState {
     /**
      * the source reader for this MergeState - merged values should be read from
      * this instance
@@ -154,7 +144,7 @@ public abstract class DocValuesConsumer 
     /** the not deleted bits for this MergeState */
     public final Bits liveDocs;
 
-    public MergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
+    public SingleSubMergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
       assert reader != null;
       this.reader = reader;
       this.docBase = docBase;

Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java Mon Oct 10 19:58:24 2011
@@ -58,11 +58,11 @@ public abstract class DocValuesReaderBas
   public Collection<String> fields() {
     return docValues().keySet();
   }
-
+  
   public Comparator<BytesRef> getComparator() throws IOException {
     return BytesRef.getUTF8SortedAsUnicodeComparator();
   }
-  
+
   // Only opens files... doesn't actually load any values
   protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
       String segment, int docCount, Directory dir, int codecId, IOContext context)
@@ -121,9 +121,9 @@ public abstract class DocValuesReaderBas
     case VAR_INTS:
       return Ints.getValues(dir, id, docCount, type, context);
     case FLOAT_32:
-      return Floats.getValues(dir, id, docCount, context);
+      return Floats.getValues(dir, id, docCount, context, type);
     case FLOAT_64:
-      return Floats.getValues(dir, id, docCount, context);
+      return Floats.getValues(dir, id, docCount, context, type);
     case BYTES_FIXED_STRAIGHT:
       return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context);
     case BYTES_FIXED_DEREF:

Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java Mon Oct 10 19:58:24 2011
@@ -54,7 +54,7 @@ public abstract class DocValuesWriterBas
   @Override
   public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
     return Writer.create(field.getDocValues(),
-        docValuesId(segmentName, codecId, field.number),
+        docValuesId(segmentName, codecId, field.number), 
         getDirectory(), getComparator(), bytesUsed, context);
   }
 
@@ -62,6 +62,7 @@ public abstract class DocValuesWriterBas
     return segmentsName + "_" + codecID + "-" + fieldId;
   }
   
+  
   public Comparator<BytesRef> getComparator() throws IOException {
     return BytesRef.getUTF8SortedAsUnicodeComparator();
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java Mon Oct 10 19:58:24 2011
@@ -19,7 +19,10 @@ import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.index.values.TypePromoter;
+import org.apache.lucene.index.values.ValueType;
 
 /**
  * Abstract API that consumes per document values. Concrete implementations of
@@ -40,28 +43,75 @@ public abstract class PerDocConsumer imp
    * Consumes and merges the given {@link PerDocValues} producer
    * into this consumers format.   
    */
-  public void merge(MergeState mergeState, PerDocValues producer)
+  public void merge(MergeState mergeState)
       throws IOException {
-    Iterable<String> fields = producer.fields();
-    for (String field : fields) {
-      mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
-      assert mergeState.fieldInfo != null : "FieldInfo for field is null: "
-          + field;
-      if (mergeState.fieldInfo.hasDocValues()) {
-        final IndexDocValues docValues = producer.docValues(field);
-        if (docValues == null) {
-          /*
-           * It is actually possible that a fieldInfo has a values type but no
-           * values are actually available. this can happen if there are already
-           * segments without values around.
-           */
+    final FieldInfos fieldInfos = mergeState.fieldInfos;
+    final IndexDocValues[] docValues = new IndexDocValues[mergeState.readers.size()];
+    final PerDocValues[] perDocValues = new PerDocValues[mergeState.readers.size()];
+    // pull all PerDocValues 
+    for (int i = 0; i < perDocValues.length; i++) {
+      perDocValues[i] =  mergeState.readers.get(i).reader.perDocValues();
+    }
+    for (FieldInfo fieldInfo : fieldInfos) {
+      mergeState.fieldInfo = fieldInfo;
+      TypePromoter currentPromoter = TypePromoter.getIdentityPromoter();
+      if (fieldInfo.hasDocValues()) {
+        for (int i = 0; i < perDocValues.length; i++) {
+          if (perDocValues[i] != null) { // get all IDV to merge
+            docValues[i] = perDocValues[i].docValues(fieldInfo.name);
+            if (docValues[i] != null) {
+              currentPromoter = promoteValueType(fieldInfo, docValues[i], currentPromoter);
+              if (currentPromoter == null) {
+                break;
+              }     
+            }
+          }
+        }
+        
+        if (currentPromoter == null) {
+          fieldInfo.resetDocValues(null);
           continue;
         }
+        assert currentPromoter != TypePromoter.getIdentityPromoter();
+        if (fieldInfo.getDocValues() != currentPromoter.type()) {
+          // reset the type if we got promoted
+          fieldInfo.resetDocValues(currentPromoter.type());
+        }
+        
         final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
         assert docValuesConsumer != null;
         docValuesConsumer.merge(mergeState, docValues);
       }
     }
+    /* NOTE: don't close the perDocProducers here since they are private segment producers
+     * and will be closed once the SegmentReader goes out of scope */ 
+  }
 
+  protected TypePromoter promoteValueType(final FieldInfo fieldInfo, final IndexDocValues docValues,
+      TypePromoter currentPromoter) {
+    assert currentPromoter != null;
+    final TypePromoter incomingPromoter = TypePromoter.create(docValues.type(),  docValues.getValueSize());
+    assert incomingPromoter != null;
+    final TypePromoter newPromoter = currentPromoter.promote(incomingPromoter);
+    return newPromoter == null ? handleIncompatibleValueType(fieldInfo, incomingPromoter, currentPromoter) : newPromoter;    
+  }
+
+  /**
+   * Resolves a conflicts of incompatible {@link TypePromoter}s. The default
+   * implementation promotes incompatible types to
+   * {@link ValueType#BYTES_VAR_STRAIGHT} and preserves all values. If this
+   * method returns <code>null</code> all docvalues for the given
+   * {@link FieldInfo} are dropped and all values are lost.
+   * 
+   * @param incomingPromoter
+   *          the incompatible incoming promoter
+   * @param currentPromoter
+   *          the current promoter
+   * @return a promoted {@link TypePromoter} or <code>null</code> iff this index
+   *         docvalues should be dropped for this field.
+   */
+  protected TypePromoter handleIncompatibleValueType(FieldInfo fieldInfo, TypePromoter incomingPromoter, TypePromoter currentPromoter) {
+    return TypePromoter.create(ValueType.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
   }
+  
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java Mon Oct 10 19:58:24 2011
@@ -51,5 +51,4 @@ public abstract class PerDocValues imple
    * Returns all fields this {@link PerDocValues} contains values for.
    */
   public abstract Collection<String> fields();
-  
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Mon Oct 10 19:58:24 2011
@@ -787,22 +787,22 @@ public class MemoryCodec extends Codec {
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
-    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(EXTENSION);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE);
+    return new DefaultDocValuesProducer(state);
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Mon Oct 10 19:58:24 2011
@@ -134,22 +134,22 @@ public class PulsingCodec extends Codec 
   public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, codecID, files);
     BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -56,9 +56,9 @@ public class SepDocValuesConsumer extend
         switch (fieldInfo.getDocValues()) {
           case BYTES_FIXED_DEREF:
           case BYTES_VAR_DEREF:
-          case BYTES_VAR_SORTED:
-          case BYTES_FIXED_SORTED:
           case BYTES_VAR_STRAIGHT:
+          case BYTES_FIXED_SORTED:
+          case BYTES_VAR_SORTED:
             files.add(IndexFileNames.segmentFileName(filename, "",
                 Writer.INDEX_EXTENSION));
             assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
@@ -77,7 +77,6 @@ public class SepDocValuesConsumer extend
             assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
                 Writer.DATA_EXTENSION));
             break;
-        
           default:
             assert false;
         }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Mon Oct 10 19:58:24 2011
@@ -69,23 +69,23 @@ public class SimpleTextCodec extends Cod
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(getPostingsFileName(segmentInfo.name, id));
-    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(POSTINGS_EXTENSION);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
   
   // TODO: would be great if these used a plain text impl
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Mon Oct 10 19:58:24 2011
@@ -110,19 +110,19 @@ public class StandardCodec extends Codec
   public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, codecID, files);
     BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getStandardExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
   }
 
   public static void getStandardExtensions(Set<String> extensions) {
     extensions.add(FREQ_EXTENSION);
     extensions.add(PROX_EXTENSION);
     BlockTreeTermsReader.getExtensions(extensions);
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
@@ -132,11 +132,11 @@ public class StandardCodec extends Codec
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }